2 * Copyright (C) 2012-2023 Free Software Foundation, Inc.
4 * This file is part of GNU lightning.
6 * GNU lightning is free software; you can redistribute it and/or modify it
7 * under the terms of the GNU Lesser General Public License as published
8 * by the Free Software Foundation; either version 3, or (at your option)
11 * GNU lightning is distributed in the hope that it will be useful, but
12 * WITHOUT ANY WARRANTY; without even the implied warranty of MERCHANTABILITY
13 * or FITNESS FOR A PARTICULAR PURPOSE. See the GNU Lesser General Public
14 * License for more details.
17 * Paulo Cesar Pereira de Andrade
20 #include <lightning.h>
21 #include <lightning/jit_private.h>
23 # include <sys/mman.h>
30 # define MAP_ANON MAP_ANONYMOUS
31 # ifndef MAP_ANONYMOUS
32 # define MAP_ANONYMOUS 0
36 #define jit_regload_reload 0 /* convert to reload */
37 #define jit_regload_delete 1 /* just remove node */
38 #define jit_regload_isdead 2 /* delete and unset live bit */
43 static jit_word_t hash_data(const void*, jit_word_t);
45 #define new_pool() _new_pool(_jit)
46 static void _new_pool(jit_state_t*);
48 #define new_node(u) _new_node(_jit, u)
49 static jit_node_t *_new_node(jit_state_t*, jit_code_t);
51 #define link_node(u) _link_node(_jit, u)
52 static inline jit_node_t *_link_node(jit_state_t*, jit_node_t*);
54 #define del_node(u, v) _del_node(_jit, u, v)
55 static inline void _del_node(jit_state_t*, jit_node_t*, jit_node_t*);
57 #define free_node(u) _free_node(_jit, u)
58 static inline void _free_node(jit_state_t*, jit_node_t*);
60 #define del_label(u, v) _del_label(_jit, u, v)
61 static void _del_label(jit_state_t*, jit_node_t*, jit_node_t*);
63 #define jit_dataset() _jit_dataset(_jit)
65 _jit_dataset(jit_state_t *_jit);
67 #define block_update_set(block, target) _block_update_set(_jit, block, target)
68 static jit_bool_t _block_update_set(jit_state_t*, jit_block_t*, jit_block_t*);
70 #define propagate_backward(block) _propagate_backward(_jit, block)
71 static void _propagate_backward(jit_state_t*, jit_block_t*);
73 #define check_block_again() _check_block_again(_jit)
74 static jit_bool_t _check_block_again(jit_state_t*);
76 #define do_setup() _do_setup(_jit)
77 static void _do_setup(jit_state_t*);
79 #define jit_setup(block) _jit_setup(_jit, block)
81 _jit_setup(jit_state_t *_jit, jit_block_t *block);
83 #define do_follow(always) _do_follow(_jit, always)
84 static void _do_follow(jit_state_t*, jit_bool_t);
86 #define jit_follow(block) _jit_follow(_jit, block)
88 _jit_follow(jit_state_t *_jit, jit_block_t *block);
90 #define jit_update(node, live, mask) _jit_update(_jit, node, live, mask)
92 _jit_update(jit_state_t *_jit, jit_node_t *node,
93 jit_regset_t *live, jit_regset_t *mask);
95 #define thread_jumps() _thread_jumps(_jit)
97 _thread_jumps(jit_state_t *_jit);
99 #define sequential_labels() _sequential_labels(_jit)
101 _sequential_labels(jit_state_t *_jit);
103 #define split_branches() _split_branches(_jit)
105 _split_branches(jit_state_t *_jit);
107 #define shortcut_jump(prev, node) _shortcut_jump(_jit, prev, node)
109 _shortcut_jump(jit_state_t *_jit, jit_node_t *prev, jit_node_t *node);
111 #define redundant_jump(prev, node) _redundant_jump(_jit, prev, node)
113 _redundant_jump(jit_state_t *_jit, jit_node_t *prev, jit_node_t *node);
116 reverse_jump_code(jit_code_t code);
118 #define reverse_jump(prev, node) _reverse_jump(_jit, prev, node)
120 _reverse_jump(jit_state_t *_jit, jit_node_t *prev, jit_node_t *node);
122 #define redundant_store(node, jump) _redundant_store(_jit, node, jump)
124 _redundant_store(jit_state_t *_jit, jit_node_t *node, jit_bool_t jump);
126 #define simplify_movr(p, n, k, s) _simplify_movr(_jit, p, n, k, s)
128 _simplify_movr(jit_state_t *_jit, jit_node_t *prev, jit_node_t *node,
129 jit_int32_t kind, jit_int32_t size);
131 #define simplify_movi(p, n, k, s) _simplify_movi(_jit, p, n, k, s)
133 _simplify_movi(jit_state_t *_jit, jit_node_t *prev, jit_node_t *node,
134 jit_int32_t kind, jit_int32_t size);
136 #define simplify_ldxi(prev, node) _simplify_ldxi(_jit, prev, node)
138 _simplify_ldxi(jit_state_t *_jit, jit_node_t *prev, jit_node_t *node);
140 #define simplify_stxi(prev, node) _simplify_stxi(_jit, prev, node)
142 _simplify_stxi(jit_state_t *_jit, jit_node_t *prev, jit_node_t *node);
144 #define simplify_spill(node, regno) _simplify_spill(_jit, node, regno)
146 _simplify_spill(jit_state_t *_jit, jit_node_t *node, jit_int32_t regno);
148 #define simplify() _simplify(_jit)
150 _simplify(jit_state_t *_jit);
152 #define jit_reg_undef -1
153 #define jit_reg_static 0
154 #define jit_reg_change 1
155 #define register_change_p(n, l, r) _register_change_p(_jit, n, l, r)
157 _register_change_p(jit_state_t *_jit, jit_node_t *node, jit_node_t *link,
160 #define spill_reglive_p(node, regno) _spill_reglive_p(_jit, node, regno)
162 _spill_reglive_p(jit_state_t *_jit, jit_node_t *node, jit_int32_t regno);
164 #define patch_registers() _patch_registers(_jit)
166 _patch_registers(jit_state_t *_jit);
168 #define patch_register(n,l,r,p) _patch_register(_jit,n,l,r,p)
170 _patch_register(jit_state_t *jit, jit_node_t *node, jit_node_t *link,
171 jit_int32_t regno, jit_int32_t patch);
184 init_jit_with_debug(const char *progname, FILE *dbg_out)
187 jit_init_debug(progname, dbg_out);
192 init_jit(const char *progname)
194 init_jit_with_debug(progname, stderr);
205 _jit_get_reg(jit_state_t *_jit, jit_int32_t regspec)
210 spec = regspec & ~(jit_class_chk|jit_class_nospill);
211 if (spec & jit_class_named) {
212 regno = jit_regno(spec);
213 if (jit_regset_tstbit(&_jitc->regsav, regno))
214 /* fail if register is spilled */
216 if (jit_regset_tstbit(&_jitc->regarg, regno))
217 /* fail if register is an argument to current instruction */
219 if (jit_regset_tstbit(&_jitc->reglive, regno)) {
220 if (regspec & jit_class_nospill)
221 /* fail if register is live and should not spill/reload */
225 jit_regset_setbit(&_jitc->regarg, regno);
229 assert(jit_class(spec) != 0);
232 /* search for a free register matching spec */
233 for (regno = 0; regno < _jitc->reglen; regno++) {
234 if ((jit_class(_rvs[regno].spec) & spec) == spec &&
235 !jit_regset_tstbit(&_jitc->regarg, regno) &&
236 !jit_regset_tstbit(&_jitc->reglive, regno)) {
237 if (jit_regset_tstbit(&_jitc->regmask, regno)) {
238 /* search further, attempting to find a truly known
239 * free register, not just one in unknown state. */
242 for (regfree = regno + 1;
243 regfree < _jitc->reglen; regfree++) {
244 if ((jit_class(_rvs[regfree].spec) & spec) == spec &&
245 !jit_regset_tstbit(&_jitc->regarg, regfree) &&
246 !jit_regset_tstbit(&_jitc->reglive, regfree) &&
247 !jit_regset_tstbit(&_jitc->regmask, regfree)) {
257 /* search for a register matching spec that is not an argument
258 * for the current instruction */
259 for (regno = 0; regno < _jitc->reglen; regno++) {
260 if ((jit_class(_rvs[regno].spec) & spec) == spec &&
261 !jit_regset_tstbit(&_jitc->regsav, regno) &&
262 !jit_regset_tstbit(&_jitc->regarg, regno) &&
263 !(regspec & jit_class_nospill)) {
265 assert(_jitc->function != NULL);
266 if (spec & jit_class_gpr) {
267 if (!_jitc->function->regoff[regno]) {
268 _jitc->function->regoff[regno] =
269 jit_allocai(sizeof(jit_word_t));
273 /* emit_stxi must not need temporary registers */
274 assert(!_jitc->getreg);
277 emit_stxi(_jitc->function->regoff[regno], JIT_FP, regno);
283 if (!_jitc->function->regoff[regno]) {
284 _jitc->function->regoff[regno] =
285 jit_allocai(sizeof(jit_float64_t));
289 /* emit_stxi must not need temporary registers */
290 assert(!_jitc->getreg);
293 emit_stxi_d(_jitc->function->regoff[regno], JIT_FP, regno);
298 jit_regset_setbit(&_jitc->regsav, regno);
300 jit_regset_setbit(&_jitc->regarg, regno);
301 if (jit_class(_rvs[regno].spec) & jit_class_sav) {
302 /* if will modify callee save registers without a
303 * function prolog, better patch this assertion */
304 assert(_jitc->function != NULL);
305 if (!jit_regset_tstbit(&_jitc->function->regset, regno)) {
306 jit_regset_setbit(&_jitc->function->regset, regno);
315 /* nospill hint only valid during emit" */
316 assert(!(regspec & jit_class_nospill));
317 for (regno = 0; regno < _jitc->reglen; regno++) {
318 if ((jit_class(_rvs[regno].spec) & spec) == spec &&
319 !jit_regset_tstbit(&_jitc->regsav, regno) &&
320 !jit_regset_tstbit(&_jitc->regarg, regno)) {
321 jit_regset_setbit(&_jitc->regarg, regno);
322 jit_regset_setbit(&_jitc->regsav, regno);
324 return (jit_regno_patch|regno);
329 /* Out of hardware registers */
331 assert(regspec & jit_class_chk);
336 _jit_unget_reg(jit_state_t *_jit, jit_int32_t regno)
338 regno = jit_regno(regno);
339 if (jit_regset_tstbit(&_jitc->regsav, regno)) {
342 /* emit_ldxi must not need a temporary register */
343 assert(!_jitc->getreg);
346 if (jit_class(_rvs[regno].spec) & jit_class_gpr)
347 emit_ldxi(regno, JIT_FP, _jitc->function->regoff[regno]);
349 emit_ldxi_d(regno, JIT_FP, _jitc->function->regoff[regno]);
351 /* emit_ldxi must not need a temporary register */
357 jit_regset_clrbit(&_jitc->regsav, regno);
359 #if defined(jit_carry)
360 assert((regno == jit_carry /*&& _NOREG != jit_carry*/) ||
361 jit_regset_tstbit(&_jitc->regarg, regno) != 0);
363 assert(jit_regset_tstbit(&_jitc->regarg, regno) != 0);
365 jit_regset_clrbit(&_jitc->regarg, regno);
369 _jit_callee_save_p(jit_state_t *_jit, jit_int32_t regno)
371 assert(regno >= 0 && regno < JIT_NOREG);
372 return (!!(_rvs[regno].spec & jit_class_sav));
376 _jit_pointer_p(jit_state_t *_jit, jit_pointer_t address)
378 return ((jit_uint8_t *)address >= _jit->code.ptr &&
379 (jit_word_t)address < _jit->pc.w);
384 jit_regset_com(jit_regset_t *u, jit_regset_t *v)
386 u->rl = ~v->rl; u->rh = ~v->rh;
387 u->fl = ~v->fl; u->fh = ~v->fh;
391 jit_regset_and(jit_regset_t *u, jit_regset_t *v, jit_regset_t *w)
393 u->rl = v->rl & w->rl; u->rh = v->rh & w->rh;
394 u->fl = v->fl & w->fl; u->fh = v->fh & w->fh;
398 jit_regset_ior(jit_regset_t *u, jit_regset_t *v, jit_regset_t *w)
400 u->rl = v->rl | w->rl; u->rh = v->rh | w->rh;
401 u->fl = v->fl | w->fl; u->fh = v->fh | w->fh;
405 jit_regset_xor(jit_regset_t *u, jit_regset_t *v, jit_regset_t *w)
407 u->rl = v->rl ^ w->rl; u->rh = v->rh ^ w->rh;
408 u->fl = v->fl ^ w->fl; u->fh = v->fh ^ w->fh;
412 jit_regset_set(jit_regset_t *u, jit_regset_t *v)
414 u->rl = v->rl; u->rh = v->rh;
415 u->fl = v->fl; u->fh = v->fh;
419 jit_regset_set_mask(jit_regset_t *u, jit_int32_t v)
421 jit_bool_t w = !!(v & (v - 1));
423 assert(v >= 0 && v <= 256);
425 u->rl = u->rh = u->fl = u->fh = -1LL;
427 u->rl = w ? (1LL << v) - 1 : -1LL;
428 u->rh = u->fl = u->fh = 0;
432 u->rh = w ? (1LL << (v - 64)) - 1 : -1LL;
436 u->rl = u->rh = -1LL;
437 u->fl = w ? (1LL << (v - 128)) - 1 : -1LL;
441 u->rl = u->rh = u->fl = -1LL;
442 u->fh = w ? (1LL << (v - 128)) - 1 : -1LL;
447 jit_regset_cmp_ui(jit_regset_t *u, jit_word_t v)
449 return !((u->rl == v && u->rh == 0 && u->fl == 0 && u->fh == 0));
453 jit_regset_set_ui(jit_regset_t *u, jit_word_t v)
456 u->rh = u->fl = u->fh = 0;
460 jit_regset_set_p(jit_regset_t *u)
462 return (u->rl || u->rh || u->fl || u->fh);
466 jit_regset_clrbit(jit_regset_t *set, jit_int32_t bit)
468 assert(bit >= 0 && bit <= 255);
470 set->rl &= ~(1LL << bit);
472 set->rh &= ~(1LL << (bit - 64));
474 set->fl &= ~(1LL << (bit - 128));
476 set->fh &= ~(1LL << (bit - 192));
480 jit_regset_setbit(jit_regset_t *set, jit_int32_t bit)
482 assert(bit >= 0 && bit <= 255);
484 set->rl |= 1LL << bit;
486 set->rh |= 1LL << (bit - 64);
488 set->fl |= 1LL << (bit - 128);
490 set->fh |= 1LL << (bit - 192);
494 jit_regset_tstbit(jit_regset_t *set, jit_int32_t bit)
496 assert(bit >= 0 && bit <= 255);
498 return (!!(set->rl & (1LL << bit)));
500 return (!!(set->rh & (1LL << (bit - 64))));
502 return (!!(set->fl & (1LL << (bit - 128))));
503 return (!!(set->fh & (1LL << (bit - 192))));
507 jit_regset_scan1(jit_regset_t *set, jit_int32_t offset)
509 assert(offset >= 0 && offset <= 255);
510 for (; offset < 64; offset++) {
511 if (set->rl & (1LL << offset))
514 for (; offset < 128; offset++) {
515 if (set->rh & (1LL << (offset - 64)))
518 for (; offset < 192; offset++) {
519 if (set->fl & (1LL << (offset - 128)))
522 for (; offset < 256; offset++) {
523 if (set->fh & (1LL << (offset - 192)))
529 #elif __sparc__ && __WORDSIZE == 64
531 jit_regset_com(jit_regset_t *u, jit_regset_t *v)
533 u->rl = ~v->rl; u->rh = ~v->rh;
537 jit_regset_and(jit_regset_t *u, jit_regset_t *v, jit_regset_t *w)
539 u->rl = v->rl & w->rl; u->rh = v->rh & w->rh;
543 jit_regset_ior(jit_regset_t *u, jit_regset_t *v, jit_regset_t *w)
545 u->rl = v->rl | w->rl; u->rh = v->rh | w->rh;
549 jit_regset_xor(jit_regset_t *u, jit_regset_t *v, jit_regset_t *w)
551 u->rl = v->rl ^ w->rl; u->rh = v->rh ^ w->rh;
555 jit_regset_set(jit_regset_t *u, jit_regset_t *v)
557 u->rl = v->rl; u->rh = v->rh;
561 jit_regset_set_mask(jit_regset_t *u, jit_int32_t v)
563 jit_bool_t w = !!(v & (v - 1));
565 assert(v >= 0 && v <= 128);
567 u->rl = u->rh = -1LL;
569 u->rl = w ? (1LL << v) - 1 : -1LL;
574 u->rh = w ? (1LL << (v - 64)) - 1 : -1LL;
579 jit_regset_cmp_ui(jit_regset_t *u, jit_word_t v)
581 return !((u->rl == v && u->rh == 0));
585 jit_regset_set_ui(jit_regset_t *u, jit_word_t v)
592 jit_regset_set_p(jit_regset_t *u)
594 return (u->rl || u->rh);
598 jit_regset_clrbit(jit_regset_t *set, jit_int32_t bit)
600 assert(bit >= 0 && bit <= 128);
602 set->rl &= ~(1LL << bit);
604 set->rh &= ~(1LL << (bit - 64));
608 jit_regset_setbit(jit_regset_t *set, jit_int32_t bit)
610 assert(bit >= 0 && bit <= 127);
612 set->rl |= 1LL << bit;
614 set->rh |= 1LL << (bit - 64);
618 jit_regset_tstbit(jit_regset_t *set, jit_int32_t bit)
620 assert(bit >= 0 && bit <= 127);
622 return (!!(set->rl & (1LL << bit)));
624 return (!!(set->rh & (1LL << (bit - 64))));
628 jit_regset_scan1(jit_regset_t *set, jit_int32_t offset)
630 assert(offset >= 0 && offset <= 127);
631 for (; offset < 64; offset++) {
632 if (set->rl & (1LL << offset))
635 for (; offset < 128; offset++) {
636 if (set->rh & (1LL << (offset - 64)))
644 jit_regset_scan1(jit_regset_t *set, jit_int32_t offset)
647 assert(offset >= 0 && offset <= 63);
648 if ((mask = *set >> offset)) {
661 _jit_save(jit_state_t *_jit, jit_int32_t reg)
663 reg = jit_regno(reg);
664 assert(!_jitc->realize);
665 _jitc->spill[reg] = jit_new_node_w(jit_code_save, reg);
669 _jit_load(jit_state_t *_jit, jit_int32_t reg)
673 reg = jit_regno(reg);
674 assert(!_jitc->realize);
675 assert(_jitc->spill[reg] != NULL);
676 node = jit_new_node_w(jit_code_load, reg);
677 /* create a path to flag the save/load is not required */
678 node->link = _jitc->spill[reg];
679 node->link->link = node;
680 _jitc->spill[reg] = NULL;
684 hash_data(const void *data, jit_word_t length)
686 const jit_uint8_t *ptr;
688 for (i = key = 0, ptr = data; i < length; i++)
689 key = (key << (key & 1)) ^ ptr[i];
694 _jit_address(jit_state_t *_jit, jit_node_t *node)
697 assert(node != NULL &&
698 /* If a node type that is documented to be a fixed marker */
699 (node->code == jit_code_note || node->code == jit_code_name ||
700 /* If another special fixed marker, returned by jit_indirect() */
701 (node->code == jit_code_label && (node->flag & jit_flag_use) != 0)));
702 return ((jit_pointer_t)node->u.w);
706 _jit_data(jit_state_t *_jit, const void *data,
707 jit_word_t length, jit_int32_t align)
712 assert(!_jitc->realize);
714 /* Ensure there is space even if asking for a duplicate */
715 if (((_jitc->data.offset + 7) & -8) + length > _jit->data.length) {
718 size = (_jit->data.length + length + 4096) & - 4095;
719 assert(size >= _jit->data.length);
720 if (_jitc->data.ptr == NULL)
721 jit_alloc((jit_pointer_t *)&_jitc->data.ptr, size);
723 jit_realloc((jit_pointer_t *)&_jitc->data.ptr,
724 _jit->data.length, size);
725 _jit->data.length = size;
727 if (_jitc->data.table == NULL)
728 jit_alloc((jit_pointer_t *)&_jitc->data.table,
729 (_jitc->data.size = 16) * sizeof(jit_node_t*));
731 key = hash_data(data, length) & (_jitc->data.size - 1);
732 node = _jitc->data.table[key];
733 for (; node; node = node->next) {
734 if (node->v.w == length &&
735 memcmp(_jitc->data.ptr + node->u.w, data, length) == 0)
740 node = jit_new_node_no_link(jit_code_data);
747 _jitc->data.offset = (_jitc->data.offset + 1) & -2;
750 _jitc->data.offset = (_jitc->data.offset + 3) & -4;
753 _jitc->data.offset = (_jitc->data.offset + 7) & -8;
756 node->u.w = _jitc->data.offset;
758 jit_memcpy(_jitc->data.ptr + _jitc->data.offset, data, length);
759 _jitc->data.offset += length;
761 node->next = _jitc->data.table[key];
762 _jitc->data.table[key] = node;
765 /* Rehash if more than 75% used table */
766 if (_jitc->data.count >
767 (_jitc->data.size >> 1) + (_jitc->data.size >> 2) &&
768 (_jitc->data.size << 1) > _jitc->data.size) {
774 jit_alloc((jit_pointer_t *)&hash,
775 (_jitc->data.size << 1) * sizeof(jit_node_t*));
776 for (i = 0; i < _jitc->data.size; i++) {
777 temp = _jitc->data.table[i];
778 for (; temp; temp = next) {
780 key = hash_data(_jitc->data.ptr + temp->u.w, temp->v.w) &
781 ((_jitc->data.size << 1) - 1);
782 temp->next = hash[key];
786 jit_free((jit_pointer_t *)&_jitc->data.table);
787 _jitc->data.table = hash;
788 _jitc->data.size <<= 1;
796 _new_pool(jit_state_t *_jit)
801 if (_jitc->pool.offset >= _jitc->pool.length) {
804 length = _jitc->pool.length + 16;
805 jit_realloc((jit_pointer_t *)&_jitc->pool.ptr,
806 _jitc->pool.length * sizeof(jit_node_t *),
807 length * sizeof(jit_node_t *));
808 _jitc->pool.length = length;
810 jit_alloc((jit_pointer_t *)(_jitc->pool.ptr + _jitc->pool.offset),
811 sizeof(jit_node_t) * 1024);
812 list = _jitc->pool.ptr[_jitc->pool.offset];
813 for (offset = 1; offset < 1024; offset++, list++)
814 list->next = list + 1;
815 list->next = _jitc->list;
816 _jitc->list = _jitc->pool.ptr[_jitc->pool.offset];
817 ++_jitc->pool.offset;
821 _new_node(jit_state_t *_jit, jit_code_t code)
825 if (_jitc->list == NULL)
828 _jitc->list = node->next;
830 node->flag |= jit_flag_synth;
837 static inline jit_node_t *
838 _link_node(jit_state_t *_jit, jit_node_t *node)
841 _jitc->tail->next = node;
844 return (_jitc->tail = node);
848 _del_node(jit_state_t *_jit, jit_node_t *prev, jit_node_t *node)
851 assert(prev == _jitc->head);
852 _jitc->head = node->next;
855 prev->next = node->next;
856 memset(node, 0, sizeof(jit_node_t));
857 node->next = _jitc->list;
862 _free_node(jit_state_t *_jit, jit_node_t *node)
864 memset(node, 0, sizeof(jit_node_t));
865 node->next = _jitc->list;
870 _del_label(jit_state_t *_jit, jit_node_t *prev, jit_node_t *node)
874 /* only allow call to del_label on linked labels */
875 block = _jitc->blocks.ptr + node->v.w;
876 assert(block->label == node);
878 /* del_label() should only be called when optimizing.
879 * This will leave an empty block index */
880 jit_regset_del(&block->reglive);
881 jit_regset_del(&block->regmask);
884 /* redundant, should be already true */
885 assert(node->link == NULL);
886 del_node(prev, node);
894 jit_alloc((jit_pointer_t *)&_jit, sizeof(jit_state_t));
895 jit_alloc((jit_pointer_t *)&_jitc, sizeof(jit_compiler_t));
896 jit_regset_new(&_jitc->regarg);
897 jit_regset_new(&_jitc->regsav);
898 jit_regset_new(&_jitc->reglive);
899 jit_regset_new(&_jitc->regmask);
900 jit_regset_new(&_jitc->explive);
904 jit_alloc((jit_pointer_t *)&_jitc->spill,
905 _jitc->reglen * sizeof(jit_node_t*));
906 jit_alloc((jit_pointer_t *)&_jitc->gen,
907 _jitc->reglen * sizeof(jit_int32_t));
908 jit_alloc((jit_pointer_t *)&_jitc->values,
909 _jitc->reglen * sizeof(jit_value_t));
911 jit_alloc((jit_pointer_t *)&_jitc->patches.ptr,
912 (_jitc->patches.length = 1024) * sizeof(jit_patch_t));
913 jit_alloc((jit_pointer_t *)&_jitc->functions.ptr,
914 (_jitc->functions.length = 16) * sizeof(jit_function_t));
915 jit_alloc((jit_pointer_t *)&_jitc->pool.ptr,
916 (_jitc->pool.length = 16) * sizeof(jit_node_t*));
917 jit_alloc((jit_pointer_t *)&_jitc->blocks.ptr,
918 (_jitc->blocks.length = 16) * sizeof(jit_block_t));
919 #if __arm__ && DISASSEMBLER
920 jit_alloc((jit_pointer_t *)&_jitc->data_info.ptr,
921 (_jitc->data_info.length = 1024) * sizeof(jit_data_info_t));
924 /* allocate at most one extra note in case jit_name() is
925 * never called, or called after adding at least one note */
926 _jit->note.length = 1;
927 _jitc->note.size = sizeof(jit_note_t);
933 _jit_clear_state(jit_state_t *_jit)
935 #if DEVEL_DISASSEMBLER
936 # define jit_really_clear_state() _jit_really_clear_state(_jit)
939 void _jit_really_clear_state(jit_state_t *_jit)
943 jit_function_t *function;
945 /* release memory not required at jit execution time and set
946 * pointers to NULL to explicitly know they are released */
947 _jitc->head = _jitc->tail = NULL;
949 jit_free((jit_pointer_t *)&_jitc->data.table);
950 _jitc->data.size = _jitc->data.count = 0;
952 jit_free((jit_pointer_t *)&_jitc->spill);
953 jit_free((jit_pointer_t *)&_jitc->gen);
954 jit_free((jit_pointer_t *)&_jitc->values);
956 jit_free((jit_pointer_t *)&_jitc->blocks.ptr);
958 jit_free((jit_pointer_t *)&_jitc->patches.ptr);
959 _jitc->patches.offset = _jitc->patches.length = 0;
961 for (offset = 0; offset < _jitc->functions.offset; offset++) {
962 function = _jitc->functions.ptr + offset;
963 jit_free((jit_pointer_t *)&function->regoff);
965 jit_free((jit_pointer_t *)&_jitc->functions.ptr);
966 _jitc->functions.offset = _jitc->functions.length = 0;
967 _jitc->function = NULL;
969 for (offset = 0; offset < _jitc->pool.offset; offset++)
970 jit_free((jit_pointer_t *)(_jitc->pool.ptr + offset));
971 jit_free((jit_pointer_t *)&_jitc->pool.ptr);
972 _jitc->pool.offset = _jitc->pool.length = 0;
975 _jitc->note.head = _jitc->note.tail =
976 _jitc->note.name = _jitc->note.note = NULL;
977 _jitc->note.base = NULL;
979 #if __arm__ && DISASSEMBLER
980 jit_free((jit_pointer_t *)&_jitc->data_info.ptr);
983 #if (__powerpc__ && _CALL_AIXDESC) || __ia64__
984 jit_free((jit_pointer_t *)&_jitc->prolog.ptr);
988 jit_regset_del(&_jitc->regs);
991 jit_free((jit_pointer_t *)&_jitc);
995 _jit_destroy_state(jit_state_t *_jit)
997 #if DEVEL_DISASSEMBLER
998 jit_really_clear_state();
1001 if (!_jit->user_code)
1002 munmap(_jit->code.ptr, _jit->code.length);
1003 if (!_jit->user_data)
1004 munmap(_jit->data.ptr, _jit->data.length);
1006 jit_free((jit_pointer_t *)&_jit);
1010 _jit_synth_inc(jit_state_t *_jit)
1012 assert(_jitc->synth < 8);
1017 _jit_new_node(jit_state_t *_jit, jit_code_t code)
1019 assert(!_jitc->realize);
1020 return (link_node(new_node(code)));
1024 _jit_new_node_no_link(jit_state_t *_jit, jit_code_t code)
1026 assert(!_jitc->realize);
1027 return (new_node(code));
1031 _jit_link_node(jit_state_t *_jit, jit_node_t *node)
1033 assert(!_jitc->realize);
1038 _jit_synth_dec(jit_state_t *_jit)
1040 assert(_jitc->synth > 0);
1045 _jit_new_node_w(jit_state_t *_jit, jit_code_t code,
1048 jit_node_t *node = new_node(code);
1049 assert(!_jitc->realize);
1051 return (link_node(node));
1055 _jit_new_node_f(jit_state_t *_jit, jit_code_t code,
1058 jit_node_t *node = new_node(code);
1059 assert(!_jitc->realize);
1061 return (link_node(node));
1065 _jit_new_node_d(jit_state_t *_jit, jit_code_t code,
1068 jit_node_t *node = new_node(code);
1069 assert(!_jitc->realize);
1071 return (link_node(node));
1075 _jit_new_node_p(jit_state_t *_jit, jit_code_t code,
1078 jit_node_t *node = new_node(code);
1079 assert(!_jitc->realize);
1081 return (link_node(node));
1085 _jit_new_node_ww(jit_state_t *_jit, jit_code_t code,
1086 jit_word_t u, jit_word_t v)
1088 jit_node_t *node = new_node(code);
1089 assert(!_jitc->realize);
1092 return (link_node(node));
1096 _jit_new_node_wp(jit_state_t *_jit, jit_code_t code,
1097 jit_word_t u, jit_pointer_t v)
1099 return (jit_new_node_ww(code, u, (jit_word_t)v));
1103 _jit_new_node_fp(jit_state_t *_jit, jit_code_t code,
1104 jit_float32_t u, jit_pointer_t v)
1106 jit_node_t *node = new_node(code);
1107 assert(!_jitc->realize);
1109 node->v.w = (jit_word_t)v;
1110 return (link_node(node));
1114 _jit_new_node_dp(jit_state_t *_jit, jit_code_t code,
1115 jit_float64_t u, jit_pointer_t v)
1117 jit_node_t *node = new_node(code);
1118 assert(!_jitc->realize);
1120 node->v.w = (jit_word_t)v;
1121 return (link_node(node));
1125 _jit_new_node_pw(jit_state_t *_jit, jit_code_t code,
1126 jit_pointer_t u, jit_word_t v)
1128 return (jit_new_node_ww(code, (jit_word_t)u, v));
1132 _jit_new_node_wf(jit_state_t *_jit, jit_code_t code,
1133 jit_word_t u, jit_float32_t v)
1135 jit_node_t *node = new_node(code);
1136 assert(!_jitc->realize);
1139 return (link_node(node));
1143 _jit_new_node_wd(jit_state_t *_jit, jit_code_t code,
1144 jit_word_t u, jit_float64_t v)
1146 jit_node_t *node = new_node(code);
1147 assert(!_jitc->realize);
1150 return (link_node(node));
1154 _jit_new_node_www(jit_state_t *_jit, jit_code_t code,
1155 jit_word_t u, jit_word_t v, jit_word_t w)
1157 jit_node_t *node = new_node(code);
1158 assert(!_jitc->realize);
1162 return (link_node(node));
1166 _jit_new_node_qww(jit_state_t *_jit, jit_code_t code,
1167 jit_int32_t l, jit_int32_t h,
1168 jit_word_t v, jit_word_t w)
1170 jit_node_t *node = new_node(code);
1171 assert(!_jitc->realize);
1177 return (link_node(node));
1181 _jit_new_node_wqw(jit_state_t *_jit, jit_code_t code,
1182 jit_word_t u, jit_int32_t l,
1183 jit_int32_t h, jit_word_t w)
1185 jit_node_t *node = new_node(code);
1186 assert(!_jitc->realize);
1192 return (link_node(node));
1196 _jit_new_node_wwq(jit_state_t *_jit, jit_code_t code,
1197 jit_word_t u, jit_word_t v,
1198 jit_int32_t l, jit_int32_t h)
1200 jit_node_t *node = new_node(code);
1201 assert(!_jitc->realize);
1206 return (link_node(node));
1210 _jit_new_node_wwf(jit_state_t *_jit, jit_code_t code,
1211 jit_word_t u, jit_word_t v, jit_float32_t w)
1213 jit_node_t *node = new_node(code);
1214 assert(!_jitc->realize);
1218 return (link_node(node));
1222 _jit_new_node_wqf(jit_state_t *_jit, jit_code_t code,
1223 jit_word_t u, jit_int32_t l,
1224 jit_int32_t h, jit_float32_t w)
1226 jit_node_t *node = new_node(code);
1227 assert(!_jitc->realize);
1233 return (link_node(node));
1237 _jit_new_node_wwd(jit_state_t *_jit, jit_code_t code,
1238 jit_word_t u, jit_word_t v, jit_float64_t w)
1240 jit_node_t *node = new_node(code);
1241 assert(!_jitc->realize);
1245 return (link_node(node));
1249 _jit_new_node_wqd(jit_state_t *_jit, jit_code_t code,
1250 jit_word_t u, jit_int32_t l,
1251 jit_int32_t h, jit_float64_t w)
1253 jit_node_t *node = new_node(code);
1254 assert(!_jitc->realize);
1260 return (link_node(node));
1264 _jit_new_node_pww(jit_state_t *_jit, jit_code_t code,
1265 jit_pointer_t u, jit_word_t v, jit_word_t w)
1267 jit_node_t *node = new_node(code);
1268 assert(!_jitc->realize);
1272 return (link_node(node));
1276 _jit_new_node_pwf(jit_state_t *_jit, jit_code_t code,
1277 jit_pointer_t u, jit_word_t v, jit_float32_t w)
1279 jit_node_t *node = new_node(code);
1280 assert(!_jitc->realize);
1284 return (link_node(node));
1288 _jit_new_node_pwd(jit_state_t *_jit, jit_code_t code,
1289 jit_pointer_t u, jit_word_t v, jit_float64_t w)
1291 jit_node_t *node = new_node(code);
1292 assert(!_jitc->realize);
1296 return (link_node(node));
1300 _jit_label(jit_state_t *_jit)
1304 if (!(node = _jitc->tail) || node->code != jit_code_label) {
1305 node = jit_forward();
1313 _jit_forward(jit_state_t *_jit)
1315 return (jit_new_node_no_link(jit_code_label));
1319 _jit_indirect(jit_state_t *_jit)
1324 node->flag |= jit_flag_use;
1330 _jit_link(jit_state_t *_jit, jit_node_t *node)
1334 assert((node->code == jit_code_label ||
1335 node->code == jit_code_prolog ||
1336 node->code == jit_code_epilog) && !node->next);
1337 jit_link_node(node);
1338 if (_jitc->blocks.offset >= _jitc->blocks.length) {
1341 length = _jitc->blocks.length + 16;
1342 jit_realloc((jit_pointer_t *)&_jitc->blocks.ptr,
1343 _jitc->blocks.length * sizeof(jit_block_t),
1344 length * sizeof(jit_block_t));
1345 _jitc->blocks.length = length;
1347 block = _jitc->blocks.ptr + _jitc->blocks.offset;
1348 block->label = node;
1349 node->v.w = _jitc->blocks.offset;
1350 jit_regset_new(&block->reglive);
1351 jit_regset_new(&block->regmask);
1352 ++_jitc->blocks.offset;
1356 _jit_forward_p(jit_state_t *_jit, jit_node_t *node)
1358 return (node->code == jit_code_label && !node->next && node != _jitc->tail);
1362 _jit_indirect_p(jit_state_t *_jit, jit_node_t *node)
1364 return (node->code == jit_code_label && !!(node->flag & jit_flag_use));
1368 _jit_target_p(jit_state_t *_jit, jit_node_t *node)
1370 return (node->code == jit_code_label && !!node->link);
1374 _jit_prepare(jit_state_t *_jit)
1376 assert(_jitc->function != NULL);
1377 _jitc->function->call.call = jit_call_default;
1378 _jitc->function->call.argi =
1379 _jitc->function->call.argf =
1380 _jitc->function->call.size = 0;
1381 _jitc->prepare = jit_new_node(jit_code_prepare);
1385 _jit_patch(jit_state_t* _jit, jit_node_t *instr)
1389 if (!(label = _jitc->tail) || label->code != jit_code_label)
1390 label = jit_label();
1391 jit_patch_at(instr, label);
1395 _jit_classify(jit_state_t *_jit, jit_code_t code)
1400 case jit_code_data: case jit_code_save: case jit_code_load:
1401 case jit_code_name: case jit_code_label: case jit_code_note:
1402 case jit_code_prolog: case jit_code_ellipsis: case jit_code_va_push:
1403 case jit_code_epilog: case jit_code_ret: case jit_code_prepare:
1406 case jit_code_live: case jit_code_va_end:
1407 case jit_code_retr_c: case jit_code_retr_uc:
1408 case jit_code_retr_s: case jit_code_retr_us:
1409 case jit_code_retr_i: case jit_code_retr_ui:
1410 case jit_code_retr_l:
1411 case jit_code_retr_f: case jit_code_retr_d:
1412 case jit_code_pushargr_c:
1413 case jit_code_pushargr_uc:
1414 case jit_code_pushargr_s:
1415 case jit_code_pushargr_us:
1416 case jit_code_pushargr_i:
1417 case jit_code_pushargr_ui:
1418 case jit_code_pushargr_l:
1419 case jit_code_pushargr_f:
1420 case jit_code_pushargr_d:
1421 case jit_code_finishr: /* synthesized will set jit_cc_a0_jmp */
1422 mask = jit_cc_a0_reg;
1424 case jit_code_align: case jit_code_skip:
1425 case jit_code_reti_c: case jit_code_reti_uc:
1426 case jit_code_reti_s: case jit_code_reti_us:
1427 case jit_code_reti_i: case jit_code_reti_ui:
1428 case jit_code_reti_l:
1429 case jit_code_pushargi_c:
1430 case jit_code_pushargi_uc:
1431 case jit_code_pushargi_s:
1432 case jit_code_pushargi_us:
1433 case jit_code_pushargi_i:
1434 case jit_code_pushargi_ui:
1435 case jit_code_pushargi_l:
1436 case jit_code_finishi: /* synthesized will set jit_cc_a0_jmp */
1437 mask = jit_cc_a0_int;
1439 case jit_code_reti_f: case jit_code_pushargi_f:
1440 mask = jit_cc_a0_flt;
1442 case jit_code_reti_d: case jit_code_pushargi_d:
1443 mask = jit_cc_a0_dbl;
1445 case jit_code_allocai:
1446 mask = jit_cc_a0_int|jit_cc_a1_int;
1448 case jit_code_arg_c: case jit_code_arg_s:
1449 case jit_code_arg_i: case jit_code_arg_l:
1450 case jit_code_arg_f: case jit_code_arg_d:
1451 mask = jit_cc_a0_int|jit_cc_a0_arg;
1453 case jit_code_calli: case jit_code_jmpi:
1454 mask = jit_cc_a0_jmp;
1456 case jit_code_callr: case jit_code_jmpr:
1457 mask = jit_cc_a0_reg|jit_cc_a0_jmp;
1459 case jit_code_retval_c: case jit_code_retval_uc:
1460 case jit_code_retval_s: case jit_code_retval_us:
1461 case jit_code_retval_i: case jit_code_retval_ui:
1462 case jit_code_retval_l:
1463 case jit_code_retval_f: case jit_code_retval_d:
1464 case jit_code_va_start:
1465 mask = jit_cc_a0_reg|jit_cc_a0_chg;
1467 case jit_code_getarg_c: case jit_code_getarg_uc:
1468 case jit_code_getarg_s: case jit_code_getarg_us:
1469 case jit_code_getarg_i: case jit_code_getarg_ui:
1470 case jit_code_getarg_l:
1471 case jit_code_getarg_f: case jit_code_getarg_d:
1472 mask = jit_cc_a0_reg|jit_cc_a0_chg|jit_cc_a1_arg;
1474 case jit_code_putargr_c:case jit_code_putargr_uc:
1475 case jit_code_putargr_s:case jit_code_putargr_us:
1476 case jit_code_putargr_i:case jit_code_putargr_ui:
1477 case jit_code_putargr_l:
1478 case jit_code_putargr_f:case jit_code_putargr_d:
1479 mask = jit_cc_a0_reg|jit_cc_a1_arg;
1481 case jit_code_putargi_c:case jit_code_putargi_uc:
1482 case jit_code_putargi_s:case jit_code_putargi_us:
1483 case jit_code_putargi_i:case jit_code_putargi_ui:
1484 case jit_code_putargi_l:
1485 mask = jit_cc_a0_int|jit_cc_a1_arg;
1487 case jit_code_putargi_f:
1488 mask = jit_cc_a0_flt|jit_cc_a1_arg;
1490 case jit_code_putargi_d:
1491 mask = jit_cc_a0_dbl|jit_cc_a1_arg;
1493 case jit_code_movi: case jit_code_ldi_c: case jit_code_ldi_uc:
1494 case jit_code_ldi_s: case jit_code_ldi_us: case jit_code_ldi_i:
1495 case jit_code_ldi_ui: case jit_code_ldi_l: case jit_code_ldi_f:
1496 case jit_code_ldi_d:
1497 case jit_code_negi: case jit_code_comi:
1498 case jit_code_exti_c: case jit_code_exti_uc:
1499 case jit_code_exti_s: case jit_code_exti_us:
1500 case jit_code_exti_i: case jit_code_exti_ui:
1501 case jit_code_bswapi_us:case jit_code_bswapi_ui:
1502 case jit_code_bswapi_ul:
1503 case jit_code_htoni_us: case jit_code_htoni_ui:
1504 case jit_code_htoni_ul:
1505 case jit_code_cloi: case jit_code_clzi:
1506 case jit_code_ctoi: case jit_code_ctzi:
1507 case jit_code_rbiti: case jit_code_popcnti:
1508 case jit_code_movi_w_f: case jit_code_movi_w_d:
1509 mask = jit_cc_a0_reg|jit_cc_a0_chg|jit_cc_a1_int;
1511 case jit_code_unldi: case jit_code_unldi_u: case jit_code_unldi_x:
1512 mask = jit_cc_a0_reg|jit_cc_a0_chg|jit_cc_a1_int|jit_cc_a2_int;
1514 case jit_code_movi_ww_d:
1515 mask = jit_cc_a0_reg|jit_cc_a0_chg|jit_cc_a1_int|jit_cc_a2_int;
1517 case jit_code_movi_f: case jit_code_movi_f_w:
1518 case jit_code_negi_f: case jit_code_absi_f:
1519 case jit_code_sqrti_f:
1520 mask = jit_cc_a0_reg|jit_cc_a0_chg|jit_cc_a1_flt;
1522 case jit_code_movi_d: case jit_code_movi_d_w:
1523 case jit_code_negi_d: case jit_code_absi_d:
1524 case jit_code_sqrti_d:
1525 mask = jit_cc_a0_reg|jit_cc_a0_chg|jit_cc_a1_dbl;
1527 case jit_code_movi_d_ww:
1528 mask = jit_cc_a0_reg|jit_cc_a0_chg|jit_cc_a1_reg|jit_cc_a1_chg|
1531 case jit_code_negr: case jit_code_comr: case jit_code_movr:
1532 case jit_code_extr_c: case jit_code_extr_uc: case jit_code_extr_s:
1533 case jit_code_extr_us: case jit_code_extr_i: case jit_code_extr_ui:
1534 case jit_code_truncr_f_i: case jit_code_truncr_f_l:
1535 case jit_code_truncr_d_i: case jit_code_truncr_d_l:
1536 case jit_code_htonr_us: case jit_code_htonr_ui: case jit_code_htonr_ul:
1537 case jit_code_bswapr_us: case jit_code_bswapr_ui: case jit_code_bswapr_ul:
1538 case jit_code_ldr_c: case jit_code_ldr_uc:
1539 case jit_code_ldr_s: case jit_code_ldr_us: case jit_code_ldr_i:
1540 case jit_code_ldr_ui: case jit_code_ldr_l: case jit_code_negr_f:
1541 case jit_code_absr_f: case jit_code_sqrtr_f: case jit_code_movr_f:
1542 case jit_code_extr_f: case jit_code_extr_d_f: case jit_code_ldr_f:
1543 case jit_code_negr_d: case jit_code_absr_d: case jit_code_sqrtr_d:
1544 case jit_code_movr_d: case jit_code_extr_d: case jit_code_extr_f_d:
1545 case jit_code_ldr_d: case jit_code_rbitr: case jit_code_popcntr:
1546 case jit_code_clor: case jit_code_clzr:
1547 case jit_code_ctor: case jit_code_ctzr:
1548 case jit_code_movr_w_f: case jit_code_movr_f_w:
1549 case jit_code_movr_w_d: case jit_code_movr_d_w:
1550 case jit_code_va_arg: case jit_code_va_arg_d:
1551 mask = jit_cc_a0_reg|jit_cc_a0_chg|jit_cc_a1_reg;
1553 case jit_code_movr_d_ww:
1554 mask = jit_cc_a0_reg|jit_cc_a0_chg|jit_cc_a1_reg|jit_cc_a1_chg|
1557 case jit_code_addi: case jit_code_addxi: case jit_code_addci:
1558 case jit_code_subi: case jit_code_subxi: case jit_code_subci:
1560 case jit_code_muli: case jit_code_hmuli: case jit_code_hmuli_u:
1561 case jit_code_divi: case jit_code_divi_u:
1562 case jit_code_remi: case jit_code_remi_u: case jit_code_andi:
1563 case jit_code_ori: case jit_code_xori: case jit_code_lshi:
1564 case jit_code_rshi: case jit_code_rshi_u: case jit_code_lroti:
1565 case jit_code_rroti: case jit_code_lti:
1566 case jit_code_lti_u: case jit_code_lei: case jit_code_lei_u:
1567 case jit_code_eqi: case jit_code_gei: case jit_code_gei_u:
1568 case jit_code_gti: case jit_code_gti_u: case jit_code_nei:
1569 case jit_code_ldxi_c: case jit_code_ldxi_uc: case jit_code_ldxi_s:
1570 case jit_code_ldxi_us: case jit_code_ldxi_i: case jit_code_ldxi_ui:
1571 case jit_code_ldxi_l: case jit_code_ldxi_f: case jit_code_ldxi_d:
1572 case jit_code_unldr: case jit_code_unldr_u: case jit_code_unldr_x:
1573 mask = jit_cc_a0_reg|jit_cc_a0_chg|jit_cc_a1_reg|jit_cc_a2_int;
1575 case jit_code_qmuli: case jit_code_qmuli_u:
1576 case jit_code_qdivi: case jit_code_qdivi_u:
1577 case jit_code_qlshi: case jit_code_qlshi_u:
1578 case jit_code_qrshi: case jit_code_qrshi_u:
1579 mask = jit_cc_a0_reg|jit_cc_a0_rlh|jit_cc_a0_chg|
1580 jit_cc_a1_reg|jit_cc_a2_int;
1582 case jit_code_addi_f: case jit_code_subi_f: case jit_code_rsbi_f:
1583 case jit_code_muli_f: case jit_code_divi_f: case jit_code_lti_f:
1584 case jit_code_lei_f: case jit_code_eqi_f: case jit_code_gei_f:
1585 case jit_code_gti_f: case jit_code_nei_f: case jit_code_unlti_f:
1586 case jit_code_unlei_f: case jit_code_uneqi_f: case jit_code_ungei_f:
1587 case jit_code_ungti_f: case jit_code_ltgti_f: case jit_code_ordi_f:
1588 case jit_code_unordi_f:
1589 mask = jit_cc_a0_reg|jit_cc_a0_chg|jit_cc_a1_reg|jit_cc_a2_flt;
1591 case jit_code_addi_d: case jit_code_subi_d: case jit_code_rsbi_d:
1592 case jit_code_muli_d: case jit_code_divi_d: case jit_code_lti_d:
1593 case jit_code_lei_d: case jit_code_eqi_d: case jit_code_gei_d:
1594 case jit_code_gti_d: case jit_code_nei_d: case jit_code_unlti_d:
1595 case jit_code_unlei_d: case jit_code_uneqi_d: case jit_code_ungei_d:
1596 case jit_code_ungti_d: case jit_code_ltgti_d: case jit_code_ordi_d:
1597 case jit_code_unordi_d:
1598 mask = jit_cc_a0_reg|jit_cc_a0_chg|jit_cc_a1_reg|jit_cc_a2_dbl;
1600 case jit_code_addr: case jit_code_addxr: case jit_code_addcr:
1601 case jit_code_subr: case jit_code_subxr: case jit_code_subcr:
1602 case jit_code_mulr: case jit_code_hmulr: case jit_code_hmulr_u:
1603 case jit_code_divr: case jit_code_divr_u:
1604 case jit_code_remr: case jit_code_remr_u: case jit_code_andr:
1605 case jit_code_orr: case jit_code_xorr: case jit_code_lshr:
1606 case jit_code_rshr: case jit_code_rshr_u: case jit_code_lrotr:
1607 case jit_code_rrotr: case jit_code_ltr:
1608 case jit_code_ltr_u: case jit_code_ler: case jit_code_ler_u:
1609 case jit_code_eqr: case jit_code_ger: case jit_code_ger_u:
1610 case jit_code_gtr: case jit_code_gtr_u: case jit_code_ner:
1611 case jit_code_ldxr_c: case jit_code_ldxr_uc: case jit_code_ldxr_s:
1612 case jit_code_ldxr_us: case jit_code_ldxr_i: case jit_code_ldxr_ui:
1613 case jit_code_ldxr_l: case jit_code_addr_f: case jit_code_subr_f:
1614 case jit_code_mulr_f: case jit_code_divr_f: case jit_code_ltr_f:
1615 case jit_code_ler_f: case jit_code_eqr_f: case jit_code_ger_f:
1616 case jit_code_gtr_f: case jit_code_ner_f: case jit_code_unltr_f:
1617 case jit_code_unler_f: case jit_code_uneqr_f: case jit_code_unger_f:
1618 case jit_code_ungtr_f: case jit_code_ltgtr_f: case jit_code_ordr_f:
1619 case jit_code_unordr_f: case jit_code_ldxr_f: case jit_code_addr_d:
1620 case jit_code_subr_d: case jit_code_mulr_d: case jit_code_divr_d:
1621 case jit_code_ltr_d: case jit_code_ler_d: case jit_code_eqr_d:
1622 case jit_code_ger_d: case jit_code_gtr_d: case jit_code_ner_d:
1623 case jit_code_unltr_d: case jit_code_unler_d: case jit_code_uneqr_d:
1624 case jit_code_unger_d: case jit_code_ungtr_d: case jit_code_ltgtr_d:
1625 case jit_code_ordr_d: case jit_code_unordr_d: case jit_code_ldxr_d:
1626 case jit_code_movr_ww_d:
1627 mask = jit_cc_a0_reg|jit_cc_a0_chg|jit_cc_a1_reg|jit_cc_a2_reg;
1629 case jit_code_qmulr: case jit_code_qmulr_u:
1630 case jit_code_qdivr: case jit_code_qdivr_u:
1631 case jit_code_qlshr: case jit_code_qlshr_u:
1632 case jit_code_qrshr: case jit_code_qrshr_u:
1633 mask = jit_cc_a0_reg|jit_cc_a0_rlh|jit_cc_a0_chg|
1634 jit_cc_a1_reg|jit_cc_a2_reg;
1636 case jit_code_sti_c: case jit_code_sti_s: case jit_code_sti_i:
1637 case jit_code_sti_l: case jit_code_sti_f: case jit_code_sti_d:
1638 mask = jit_cc_a0_int|jit_cc_a1_reg;
1640 case jit_code_unsti: case jit_code_unsti_x:
1641 mask = jit_cc_a0_int|jit_cc_a1_reg|jit_cc_a2_int;
1643 case jit_code_blti: case jit_code_blti_u: case jit_code_blei:
1644 case jit_code_blei_u: case jit_code_beqi: case jit_code_bgei:
1645 case jit_code_bgei_u: case jit_code_bgti: case jit_code_bgti_u:
1646 case jit_code_bnei: case jit_code_bmsi: case jit_code_bmci:
1647 mask = jit_cc_a0_jmp|jit_cc_a1_reg|jit_cc_a2_int;
1649 case jit_code_blti_f: case jit_code_blei_f: case jit_code_beqi_f:
1650 case jit_code_bgei_f: case jit_code_bgti_f: case jit_code_bnei_f:
1651 case jit_code_bunlti_f: case jit_code_bunlei_f: case jit_code_buneqi_f:
1652 case jit_code_bungei_f: case jit_code_bungti_f: case jit_code_bltgti_f:
1653 case jit_code_bordi_f: case jit_code_bunordi_f:
1654 mask = jit_cc_a0_jmp|jit_cc_a1_reg|jit_cc_a2_flt;
1656 case jit_code_blti_d: case jit_code_blei_d: case jit_code_beqi_d:
1657 case jit_code_bgei_d: case jit_code_bgti_d: case jit_code_bnei_d:
1658 case jit_code_bunlti_d: case jit_code_bunlei_d: case jit_code_buneqi_d:
1659 case jit_code_bungei_d: case jit_code_bungti_d: case jit_code_bltgti_d:
1660 case jit_code_bordi_d: case jit_code_bunordi_d:
1661 mask = jit_cc_a0_jmp|jit_cc_a1_reg|jit_cc_a2_dbl;
1663 case jit_code_allocar: /* synthesized instructions make it
1664 * equivalent to jit_cc_a0_chg */
1665 case jit_code_str_c: case jit_code_str_s: case jit_code_str_i:
1666 case jit_code_str_l: case jit_code_str_f: case jit_code_str_d:
1667 mask = jit_cc_a0_reg|jit_cc_a1_reg;
1669 case jit_code_unstr: case jit_code_unstr_x:
1670 mask = jit_cc_a0_reg|jit_cc_a1_reg|jit_cc_a2_int;
1672 case jit_code_stxi_c: case jit_code_stxi_s: case jit_code_stxi_i:
1673 case jit_code_stxi_l: case jit_code_stxi_f: case jit_code_stxi_d:
1674 mask = jit_cc_a0_int|jit_cc_a1_reg|jit_cc_a2_reg;
1676 case jit_code_bltr: case jit_code_bltr_u: case jit_code_bler:
1677 case jit_code_bler_u: case jit_code_beqr: case jit_code_bger:
1678 case jit_code_bger_u: case jit_code_bgtr: case jit_code_bgtr_u:
1679 case jit_code_bner: case jit_code_bmsr: case jit_code_bmcr:
1680 case jit_code_bltr_f: case jit_code_bler_f: case jit_code_beqr_f:
1681 case jit_code_bger_f: case jit_code_bgtr_f: case jit_code_bner_f:
1682 case jit_code_bunltr_f: case jit_code_bunler_f: case jit_code_buneqr_f:
1683 case jit_code_bunger_f: case jit_code_bungtr_f: case jit_code_bltgtr_f:
1684 case jit_code_bordr_f: case jit_code_bunordr_f:case jit_code_bltr_d:
1685 case jit_code_bler_d: case jit_code_beqr_d: case jit_code_bger_d:
1686 case jit_code_bgtr_d: case jit_code_bner_d: case jit_code_bunltr_d:
1687 case jit_code_bunler_d: case jit_code_buneqr_d: case jit_code_bunger_d:
1688 case jit_code_bungtr_d: case jit_code_bltgtr_d: case jit_code_bordr_d:
1689 case jit_code_bunordr_d:
1690 mask = jit_cc_a0_jmp|jit_cc_a1_reg|jit_cc_a2_reg;
1692 case jit_code_boaddi: case jit_code_boaddi_u: case jit_code_bxaddi:
1693 case jit_code_bxaddi_u: case jit_code_bosubi: case jit_code_bosubi_u:
1694 case jit_code_bxsubi: case jit_code_bxsubi_u:
1695 mask = jit_cc_a0_jmp|jit_cc_a1_reg|jit_cc_a1_chg|jit_cc_a2_int;
1697 case jit_code_stxr_c: case jit_code_stxr_s: case jit_code_stxr_i:
1698 case jit_code_stxr_l: case jit_code_stxr_f: case jit_code_stxr_d:
1699 mask = jit_cc_a0_reg|jit_cc_a1_reg|jit_cc_a2_reg;
1701 case jit_code_boaddr: case jit_code_boaddr_u: case jit_code_bxaddr:
1702 case jit_code_bxaddr_u: case jit_code_bosubr: case jit_code_bosubr_u:
1703 case jit_code_bxsubr: case jit_code_bxsubr_u:
1704 mask = jit_cc_a0_jmp|jit_cc_a1_reg|jit_cc_a1_chg|jit_cc_a2_reg;
1706 case jit_code_movnr: case jit_code_movzr:
1707 mask = jit_cc_a0_reg|jit_cc_a0_cnd|jit_cc_a1_reg|jit_cc_a2_reg;
1710 mask = jit_cc_a0_reg|jit_cc_a0_chg|jit_cc_a1_reg|
1711 jit_cc_a2_reg|jit_cc_a2_rlh;
1714 mask = jit_cc_a0_reg|jit_cc_a0_chg|jit_cc_a1_int|
1715 jit_cc_a2_reg|jit_cc_a2_rlh;
1717 case jit_code_extr: case jit_code_extr_u: case jit_code_depr:
1718 mask = jit_cc_a0_reg|jit_cc_a0_chg|jit_cc_a1_reg|jit_cc_a2_rlh;
1720 case jit_code_exti: case jit_code_exti_u: case jit_code_depi:
1721 mask = jit_cc_a0_reg|jit_cc_a0_chg|jit_cc_a1_int|jit_cc_a2_rlh;
1723 case jit_code_fmar_f: case jit_code_fmar_d:
1724 case jit_code_fmsr_f: case jit_code_fmsr_d:
1725 case jit_code_fnmar_f: case jit_code_fnmar_d:
1726 case jit_code_fnmsr_f: case jit_code_fnmsr_d:
1727 mask = jit_cc_a0_reg|jit_cc_a0_chg|
1728 jit_cc_a1_reg|jit_cc_a1_rlh|jit_cc_a2_reg;
1730 case jit_code_fmai_f: case jit_code_fmsi_f:
1731 case jit_code_fnmai_f: case jit_code_fnmsi_f:
1732 mask = jit_cc_a0_reg|jit_cc_a0_chg|
1733 jit_cc_a1_reg|jit_cc_a1_rlh|jit_cc_a2_flt;
1735 case jit_code_fmai_d: case jit_code_fmsi_d:
1736 case jit_code_fnmai_d: case jit_code_fnmsi_d:
1737 mask = jit_cc_a0_reg|jit_cc_a0_chg|
1738 jit_cc_a1_reg|jit_cc_a1_rlh|jit_cc_a2_dbl;
1748 _jit_patch_abs(jit_state_t *_jit, jit_node_t *instr, jit_pointer_t address)
1753 switch (instr->code) {
1754 case jit_code_movi: case jit_code_ldi_c: case jit_code_ldi_uc:
1755 case jit_code_ldi_s: case jit_code_ldi_us: case jit_code_ldi_i:
1756 case jit_code_ldi_ui: case jit_code_ldi_l: case jit_code_ldi_f:
1757 case jit_code_ldi_d:
1758 instr->v.p = address;
1760 case jit_code_sti_c: case jit_code_sti_s: case jit_code_sti_i:
1761 case jit_code_sti_l: case jit_code_sti_f: case jit_code_sti_d:
1762 instr->u.p = address;
1766 mask = jit_classify(instr->code);
1768 assert((mask & (jit_cc_a0_reg|jit_cc_a0_jmp)) == jit_cc_a0_jmp);
1769 instr->u.p = address;
1774 _jit_patch_at(jit_state_t *_jit, jit_node_t *instr, jit_node_t *label)
1779 assert(!(instr->flag & jit_flag_node));
1780 instr->flag |= jit_flag_node;
1781 switch (instr->code) {
1783 assert(label->code == jit_code_label ||
1784 label->code == jit_code_data);
1786 if (label->code == jit_code_data)
1787 instr->flag |= jit_flag_data;
1790 assert(label->code == jit_code_label ||
1791 label->code == jit_code_epilog);
1796 mask = jit_classify(instr->code);
1798 assert((mask & (jit_cc_a0_reg|jit_cc_a0_jmp)) == jit_cc_a0_jmp);
1799 assert(label->code == jit_code_label);
1803 /* link field is used as list of nodes associated with a given label */
1804 instr->link = label->link;
1805 label->link = instr;
1809 _do_setup(jit_state_t *_jit)
1814 /* create initial mapping of live register values
1815 * at the start of a basic block */
1816 for (offset = 0; offset < _jitc->blocks.offset; offset++) {
1817 block = _jitc->blocks.ptr + offset;
1820 if (block->label->code == jit_code_epilog) {
1821 jit_regset_setbit(&block->reglive, JIT_RET);
1822 jit_regset_setbit(&block->reglive, JIT_FRET);
1823 jit_regset_com(&block->regmask, &block->reglive);
1831 _block_update_set(jit_state_t *_jit,
1832 jit_block_t *block, jit_block_t *target)
1834 jit_regset_t regmask;
1836 jit_regset_ior(®mask, &block->reglive, &target->reglive);
1837 jit_regset_and(®mask, ®mask, &block->regmask);
1838 if (jit_regset_set_p(®mask)) {
1839 jit_regset_ior(&block->reglive, &block->reglive, ®mask);
1840 jit_regset_and(®mask, &block->reglive, &block->regmask);
1841 jit_regset_com(®mask, ®mask);
1842 jit_regset_and(&block->regmask, &block->regmask, ®mask);
1850 _propagate_backward(jit_state_t *_jit, jit_block_t *block)
1855 for (offset = block->label->v.w - 1;
1856 offset >= 0; --offset) {
1857 prev = _jitc->blocks.ptr + offset;
1858 if (!block_update_set(prev, block) ||
1859 !(prev->label->flag & jit_flag_head))
1865 _check_block_again(jit_state_t *_jit)
1869 jit_node_t *node, *label;
1870 jit_block_t *block, *target;
1873 for (offset = 0; offset < _jitc->blocks.offset; offset++) {
1874 block = _jitc->blocks.ptr + offset;
1880 /* If no block changed state */
1887 for (node = _jitc->head; node; node = node->next) {
1888 /* Special jumps that match jit_cc_a0_jmp */
1889 if (node->code == jit_code_calli || node->code == jit_code_callr)
1892 /* Remember current label */
1893 if (node->code == jit_code_label ||
1894 node->code == jit_code_prolog ||
1895 node->code == jit_code_epilog) {
1897 /* If previous block does not pass through */
1898 if (!(node->flag & jit_flag_head))
1901 target = _jitc->blocks.ptr + node->v.w;
1902 if (block && target->again && block_update_set(block, target)) {
1903 propagate_backward(block);
1908 /* If not the first jmpi */
1910 /* If a jump to dynamic address or if a jump to raw address */
1911 if (!(jit_classify(node->code) & jit_cc_a0_jmp) ||
1912 !(node->flag & jit_flag_node))
1915 /* Mark predecessor needs updating due to target change */
1916 target = _jitc->blocks.ptr + label->v.w;
1917 if (target->again && block_update_set(block, target)) {
1918 propagate_backward(block);
1930 _do_follow(jit_state_t *_jit, jit_bool_t always)
1935 /* set live state of registers not referenced in a block, but
1936 * referenced in a jump target or normal flow */
1937 for (offset = 0; offset < _jitc->blocks.offset; offset++) {
1938 block = _jitc->blocks.ptr + offset;
1939 if (!block->label || block->label->code == jit_code_epilog)
1941 if (always || block->again) {
1949 _jit_optimize(jit_state_t *_jit)
1957 jit_regset_t regmask;
1960 _jitc->function = NULL;
1963 sequential_labels();
1968 jit_regset_set_ui(®mask, 0);
1969 for (offset = 0; offset < _jitc->reglen; offset++) {
1970 if ((jit_class(_rvs[offset].spec) & (jit_class_gpr|jit_class_fpr)) &&
1971 (jit_class(_rvs[offset].spec) & jit_class_sav) == jit_class_sav)
1972 jit_regset_setbit(®mask, offset);
1975 /* Figure out labels that are only reached with a jump */
1977 for (node = _jitc->head; node; node = node->next) {
1978 switch (node->code) {
1979 case jit_code_label:
1981 node->flag |= jit_flag_head;
1983 /* Block is dead code or only reachable with an
1984 * indirect jumps. In such condition, must assume
1985 * all callee save registers are live. */
1986 block = _jitc->blocks.ptr + node->v.w;
1987 jit_regset_ior(&block->reglive,
1988 &block->reglive, ®mask);
1989 /* Cleanup regmask */
1990 block_update_set(block, block);
1994 case jit_code_jmpi: case jit_code_jmpr:
1995 case jit_code_epilog:
1998 case jit_code_data: case jit_code_note:
2010 for (node = _jitc->head; node; node = node->next) {
2011 mask = jit_classify(node->code);
2012 if (mask & jit_cc_a0_reg)
2013 node->u.w &= ~jit_regno_patch;
2014 if (mask & jit_cc_a1_reg)
2015 node->v.w &= ~jit_regno_patch;
2016 if (mask & jit_cc_a2_reg)
2017 node->w.w &= ~jit_regno_patch;
2018 if (node->code == jit_code_beqi) {
2019 if (redundant_store(node, 1)) {
2020 block = _jitc->blocks.ptr + ((jit_node_t *)node->u.n)->v.w;
2025 else if (node->code == jit_code_bnei) {
2026 if (redundant_store(node, 0)) {
2027 block = _jitc->blocks.ptr + ((jit_node_t *)node->u.n)->v.w;
2035 todo = check_block_again();
2037 /* If instructions were removed or first pass did modify the entry
2038 * state of any block */
2044 /* If any block again has the entry state modified. */
2045 todo = check_block_again();
2049 for (node = _jitc->head; node; node = node->next) {
2050 mask = jit_classify(node->code);
2051 if (mask & jit_cc_a0_reg)
2052 node->u.w &= ~jit_regno_patch;
2053 if (mask & jit_cc_a1_reg)
2054 node->v.w &= ~jit_regno_patch;
2055 if (mask & jit_cc_a2_reg)
2056 node->w.w &= ~jit_regno_patch;
2057 if (node->code == jit_code_prolog)
2058 _jitc->function = _jitc->functions.ptr + node->w.w;
2059 else if(node->code == jit_code_epilog)
2060 _jitc->function = NULL;
2063 if (mask & jit_cc_a0_flt) {
2064 node->u.p = jit_data(&node->u.f, sizeof(jit_float32_t), 4);
2065 node->flag |= jit_flag_node | jit_flag_data;
2067 else if (mask & jit_cc_a0_dbl) {
2068 node->u.p = jit_data(&node->u.d, sizeof(jit_float64_t), 8);
2069 node->flag |= jit_flag_node | jit_flag_data;
2071 else if (mask & jit_cc_a1_flt) {
2072 node->v.p = jit_data(&node->v.f, sizeof(jit_float32_t), 4);
2073 node->flag |= jit_flag_node | jit_flag_data;
2075 else if (mask & jit_cc_a1_dbl) {
2076 node->v.p = jit_data(&node->v.d, sizeof(jit_float64_t), 8);
2077 node->flag |= jit_flag_node | jit_flag_data;
2079 else if (mask & jit_cc_a2_flt) {
2080 node->w.p = jit_data(&node->w.f, sizeof(jit_float32_t), 4);
2081 node->flag |= jit_flag_node | jit_flag_data;
2083 else if (mask & jit_cc_a2_dbl) {
2084 node->w.p = jit_data(&node->w.d, sizeof(jit_float64_t), 8);
2085 node->flag |= jit_flag_node | jit_flag_data;
2088 if (_jitc->function) {
2089 if ((mask & (jit_cc_a0_reg|jit_cc_a0_chg)) ==
2090 (jit_cc_a0_reg|jit_cc_a0_chg)) {
2091 if (mask & jit_cc_a0_rlh) {
2092 jit_regset_setbit(&_jitc->function->regset,
2093 jit_regno(node->u.q.l));
2094 jit_regset_setbit(&_jitc->function->regset,
2095 jit_regno(node->u.q.h));
2098 jit_regset_setbit(&_jitc->function->regset,
2099 jit_regno(node->u.w));
2101 if ((mask & (jit_cc_a1_reg|jit_cc_a1_chg)) ==
2102 (jit_cc_a1_reg|jit_cc_a1_chg))
2103 jit_regset_setbit(&_jitc->function->regset,
2104 jit_regno(node->v.w));
2105 if ((mask & (jit_cc_a2_reg|jit_cc_a2_chg)) ==
2106 (jit_cc_a2_reg|jit_cc_a2_chg))
2107 jit_regset_setbit(&_jitc->function->regset,
2108 jit_regno(node->w.w));
2115 _jit_reglive(jit_state_t *_jit, jit_node_t *node)
2121 switch (node->code) {
2122 case jit_code_label: case jit_code_prolog: case jit_code_epilog:
2123 block = _jitc->blocks.ptr + node->v.w;
2124 jit_regset_set(&_jitc->reglive, &block->reglive);
2125 jit_regset_set_ui(&_jitc->explive, 0);
2128 jit_regset_setbit(&_jitc->explive, node->u.w);
2130 case jit_code_callr:
2131 value = jit_regno(node->u.w);
2132 if (!(node->u.w & jit_regno_patch)) {
2133 jit_regset_setbit(&_jitc->reglive, value);
2135 case jit_code_calli:
2136 for (value = 0; value < _jitc->reglen; value++) {
2137 spec = jit_class(_rvs[value].spec);
2138 if ((spec & jit_class_arg) && jit_regarg_p(node, value))
2139 jit_regset_setbit(&_jitc->reglive, value);
2140 else if (!(spec & jit_class_sav))
2141 jit_regset_clrbit(&_jitc->reglive, value);
2145 value = jit_classify(node->code);
2146 if (value & jit_cc_a0_reg) {
2147 if (value & jit_cc_a0_rlh) {
2148 if (!(node->u.q.l & jit_regno_patch)) {
2149 if (value & jit_cc_a0_chg) {
2150 jit_regset_clrbit(&_jitc->reglive, node->u.q.l);
2151 jit_regset_setbit(&_jitc->regmask, node->u.q.l);
2154 jit_regset_setbit(&_jitc->reglive, node->u.q.l);
2156 if (!(node->u.q.h & jit_regno_patch)) {
2157 if (value & jit_cc_a0_chg) {
2158 jit_regset_clrbit(&_jitc->reglive, node->u.q.h);
2159 jit_regset_setbit(&_jitc->regmask, node->u.q.h);
2162 jit_regset_setbit(&_jitc->reglive, node->u.q.h);
2166 if (!(node->u.w & jit_regno_patch)) {
2167 if (value & jit_cc_a0_chg) {
2168 jit_regset_clrbit(&_jitc->reglive, node->u.w);
2169 jit_regset_setbit(&_jitc->regmask, node->u.w);
2172 jit_regset_setbit(&_jitc->reglive, node->u.w);
2176 if (value & jit_cc_a1_reg) {
2177 if (value & jit_cc_a1_rlh) {
2178 /* Assume registers are not changed */
2179 if (!(node->v.q.l & jit_regno_patch))
2180 jit_regset_setbit(&_jitc->reglive, node->v.q.l);
2181 if (!(node->v.q.h & jit_regno_patch))
2182 jit_regset_setbit(&_jitc->reglive, node->v.q.h);
2184 else if (!(node->v.w & jit_regno_patch)) {
2185 if (value & jit_cc_a1_chg) {
2186 jit_regset_clrbit(&_jitc->reglive, node->v.w);
2187 jit_regset_setbit(&_jitc->regmask, node->v.w);
2190 jit_regset_setbit(&_jitc->reglive, node->v.w);
2193 if (value & jit_cc_a2_reg) {
2194 if (value & jit_cc_a2_rlh) {
2195 /* Assume registers are not changed */
2196 if (!(node->w.q.l & jit_regno_patch))
2197 jit_regset_setbit(&_jitc->reglive, node->w.q.l);
2198 if (!(node->w.q.h & jit_regno_patch))
2199 jit_regset_setbit(&_jitc->reglive, node->w.q.h);
2202 if (!(node->w.w & jit_regno_patch)) {
2203 if (value & jit_cc_a2_chg) {
2204 jit_regset_clrbit(&_jitc->reglive, node->w.w);
2205 jit_regset_setbit(&_jitc->regmask, node->w.w);
2208 jit_regset_setbit(&_jitc->reglive, node->w.w);
2212 if (jit_regset_set_p(&_jitc->regmask)) {
2213 jit_update(node->next, &_jitc->reglive, &_jitc->regmask);
2214 if (jit_regset_set_p(&_jitc->regmask)) {
2215 /* any unresolved live state is considered as live */
2216 jit_regset_ior(&_jitc->reglive,
2217 &_jitc->reglive, &_jitc->regmask);
2218 jit_regset_set_ui(&_jitc->regmask, 0);
2226 _jit_regarg_set(jit_state_t *_jit, jit_node_t *node, jit_int32_t value)
2231 if (value & jit_cc_a0_reg) {
2232 if (value & jit_cc_a0_rlh) {
2233 jit_regset_setbit(&_jitc->regarg, jit_regno(node->u.q.l));
2234 jit_regset_setbit(&_jitc->regarg, jit_regno(node->u.q.h));
2237 jit_regset_setbit(&_jitc->regarg, jit_regno(node->u.w));
2239 if (value & jit_cc_a1_reg) {
2240 if (value & jit_cc_a1_rlh) {
2241 jit_regset_setbit(&_jitc->regarg, jit_regno(node->v.q.l));
2242 jit_regset_setbit(&_jitc->regarg, jit_regno(node->v.q.h));
2245 jit_regset_setbit(&_jitc->regarg, jit_regno(node->v.w));
2247 if (value & jit_cc_a2_reg) {
2248 if (value & jit_cc_a2_rlh) {
2249 jit_regset_setbit(&_jitc->regarg, jit_regno(node->w.q.l));
2250 jit_regset_setbit(&_jitc->regarg, jit_regno(node->w.q.h));
2253 jit_regset_setbit(&_jitc->regarg, jit_regno(node->w.w));
2255 /* Prevent incorrect detection of running out of registers
2256 * if will need to patch jump, and all registers have been
2257 * used in the current block. */
2258 if (node->code == jit_code_jmpi && (node->flag & jit_flag_node)) {
2259 jit_node_t *label = node->u.n;
2260 jit_block_t *block = _jitc->blocks.ptr + label->v.w;
2261 jit_regset_set(&_jitc->reglive, &block->reglive);
2262 jit_regset_set(&_jitc->regmask, &block->regmask);
2263 if (jit_regset_set_p(&_jitc->explive)) {
2264 jit_regset_ior(&_jitc->reglive, &block->reglive, &_jitc->explive);
2265 jit_regset_xor(&_jitc->regmask, &_jitc->regmask, &_jitc->explive);
2271 _jit_regarg_clr(jit_state_t *_jit, jit_node_t *node, jit_int32_t value)
2274 jit_size_collect(node);
2276 if (value & jit_cc_a0_reg) {
2277 if (value & jit_cc_a0_rlh) {
2278 jit_regset_clrbit(&_jitc->regarg, jit_regno(node->u.q.l));
2279 jit_regset_clrbit(&_jitc->regarg, jit_regno(node->u.q.h));
2282 jit_regset_clrbit(&_jitc->regarg, jit_regno(node->u.w));
2284 if (value & jit_cc_a1_reg) {
2285 if (value & jit_cc_a1_rlh) {
2286 jit_regset_clrbit(&_jitc->regarg, jit_regno(node->v.q.l));
2287 jit_regset_clrbit(&_jitc->regarg, jit_regno(node->v.q.h));
2290 jit_regset_clrbit(&_jitc->regarg, jit_regno(node->v.w));
2292 if (value & jit_cc_a2_reg) {
2293 if (value & jit_cc_a2_rlh) {
2294 jit_regset_clrbit(&_jitc->regarg, jit_regno(node->w.q.l));
2295 jit_regset_clrbit(&_jitc->regarg, jit_regno(node->w.q.h));
2298 jit_regset_clrbit(&_jitc->regarg, jit_regno(node->w.w));
2303 _jit_realize(jit_state_t *_jit)
2305 assert(!_jitc->realize);
2306 if (_jitc->function)
2311 /* ensure it is aligned */
2312 _jitc->data.offset = (_jitc->data.offset + 7) & -8;
2315 /* Heuristic to guess code buffer size */
2317 _jit->code.length = _jitc->pool.length * 1024 * _jitc->mult;
2319 _jit->code.length = jit_get_size();
2324 _jit_dataset(jit_state_t *_jit)
2333 assert(!_jitc->dataset);
2335 assert(_jit->user_data);
2337 if (!_jit->user_data) {
2339 /* create read only data buffer */
2340 _jit->data.length = (_jitc->data.offset +
2341 /* reserve space for annotations */
2342 _jitc->note.size + 4095) & -4096;
2344 mmap_fd = open("/dev/zero", O_RDWR);
2346 _jit->data.ptr = mmap(NULL, _jit->data.length,
2347 PROT_READ | PROT_WRITE,
2348 MAP_PRIVATE | MAP_ANON, mmap_fd, 0);
2349 assert(_jit->data.ptr != MAP_FAILED);
2354 #endif /* !HAVE_MMAP */
2356 if (!_jitc->no_data)
2357 jit_memcpy(_jit->data.ptr, _jitc->data.ptr, _jitc->data.offset);
2359 if (_jitc->no_note) {
2360 /* Space for one note is always allocated, so revert it here
2361 * if after jit_new_state was called, it is also requested to
2362 * not generate annotation information */
2363 _jit->note.length = 0;
2364 _jitc->note.size = 0;
2367 _jitc->note.base = _jit->data.ptr;
2368 if (!_jitc->no_data)
2369 _jitc->note.base += _jitc->data.offset;
2370 memset(_jitc->note.base, 0, _jitc->note.size);
2373 if (_jit->user_data)
2374 /* Need the temporary hashed data until jit_emit is finished */
2375 ptr = _jitc->no_data ? _jitc->data.ptr : _jit->data.ptr;
2377 ptr = _jit->data.ptr;
2378 /* Temporary hashed data no longer required */
2379 jit_free((jit_pointer_t *)&_jitc->data.ptr);
2382 for (offset = 0; offset < _jitc->data.size; offset++) {
2383 for (node = _jitc->data.table[offset]; node; node = node->next) {
2384 node->flag |= jit_flag_patch;
2385 node->u.w = (jit_word_t)(ptr + node->u.w);
2393 _jit_get_code(jit_state_t *_jit, jit_word_t *length)
2395 assert(_jitc->realize);
2398 /* If code already generated, return exact size of code */
2399 *length = _jit->pc.uc - _jit->code.ptr;
2401 /* Else return current size of the code buffer */
2402 *length = _jit->code.length;
2405 return (_jit->code.ptr);
2409 _jit_set_code(jit_state_t *_jit, jit_pointer_t ptr, jit_word_t length)
2411 assert(_jitc->realize);
2412 _jit->code.ptr = ptr;
2413 _jit->code.length = length;
2414 _jit->user_code = 1;
2418 _jit_get_data(jit_state_t *_jit, jit_word_t *data_size, jit_word_t *note_size)
2420 assert(_jitc->realize);
2422 *data_size = _jitc->data.offset;
2424 *note_size = _jitc->note.size;
2425 return (_jit->data.ptr);
2429 _jit_set_data(jit_state_t *_jit, jit_pointer_t ptr,
2430 jit_word_t length, jit_word_t flags)
2432 assert(_jitc->realize);
2433 if (flags & JIT_DISABLE_DATA)
2436 assert(length >= _jitc->data.offset);
2437 if (flags & JIT_DISABLE_NOTE)
2440 if (flags & JIT_DISABLE_DATA)
2441 assert(length >= _jitc->note.size);
2443 assert(length >= _jitc->data.offset + _jitc->note.size);
2445 _jit->data.ptr = ptr;
2446 _jit->data.length = length;
2447 _jit->user_data = 1;
2451 _jit_emit(jit_state_t *_jit)
2462 int mmap_prot, mmap_flags;
2464 if (!_jitc->realize)
2467 if (!_jitc->dataset)
2473 assert(_jit->user_code);
2475 if (!_jit->user_code) {
2476 mmap_prot = PROT_READ | PROT_WRITE;
2477 #if !(__OpenBSD__ || __APPLE__)
2478 mmap_prot |= PROT_EXEC;
2481 mmap_prot = PROT_MPROTECT(mmap_prot);
2484 mmap_flags = MAP_PRIVATE;
2486 mmap_flags |= MAP_ANON;
2488 mmap_fd = open("/dev/zero", O_RDWR);
2490 _jit->code.ptr = mmap(NULL, _jit->code.length,
2491 mmap_prot, mmap_flags, mmap_fd, 0);
2492 assert(_jit->code.ptr != MAP_FAILED);
2494 #endif /* !HAVE_MMAP */
2495 _jitc->code.end = _jit->code.ptr + _jit->code.length -
2496 jit_get_max_instr();
2497 _jit->pc.uc = _jit->code.ptr;
2504 mprotect(_jit->code.ptr, _jit->code.length, PROT_READ | PROT_WRITE);
2505 assert(result == 0);
2507 if ((code = emit_code()) == NULL) {
2508 _jitc->patches.offset = 0;
2509 for (node = _jitc->head; node; node = node->next) {
2511 (node->code == jit_code_label ||
2512 node->code == jit_code_epilog))
2513 node->flag &= ~jit_flag_patch;
2516 assert(_jit->user_code);
2518 if (_jit->user_code)
2522 length = _jitc->pool.length * 1024 * _jitc->mult;
2524 /* Should only happen on very special cases */
2525 length = _jit->code.length + 4096;
2529 munmap(_jit->code.ptr, _jit->code.length);
2534 _jit->code.ptr = mremap(_jit->code.ptr, _jit->code.length,
2535 _jit->code.ptr, length, 0);
2537 _jit->code.ptr = mremap(_jit->code.ptr, _jit->code.length,
2538 length, MREMAP_MAYMOVE, NULL);
2541 _jit->code.ptr = mmap(NULL, length,
2542 mmap_prot, mmap_flags, mmap_fd, 0);
2545 assert(_jit->code.ptr != MAP_FAILED);
2546 _jit->code.length = length;
2547 _jitc->code.end = _jit->code.ptr + _jit->code.length -
2548 jit_get_max_instr();
2549 _jit->pc.uc = _jit->code.ptr;
2550 #endif /* !HAVE_MMAP */
2557 if (!_jit->user_code)
2562 if (!_jitc->no_note)
2565 if (_jit->user_data)
2566 jit_free((jit_pointer_t *)&_jitc->data.ptr);
2572 mprotect(_jit->data.ptr, _jit->data.length, PROT_READ);
2573 assert(result == 0);
2575 if (!_jit->user_code) {
2576 _jit->code.protect = _jit->pc.uc - _jit->code.ptr;
2577 # if __riscv && __WORDSIZE == 64
2578 /* FIXME should start adding consts at a page boundary */
2579 _jit->code.protect -= _jitc->consts.hash.count * sizeof(jit_word_t);
2583 mprotect(_jit->code.ptr, _jit->code.protect, PROT_READ | PROT_EXEC);
2585 assert(result == 0);
2587 #endif /* HAVE_MMAP */
2589 return (_jit->code.ptr);
2595 _jit_protect(jit_state_t *_jit)
2598 assert (_jit->user_code);
2603 if (_jit->user_code) return;
2607 mprotect (_jit->code.ptr, _jit->code.protect, PROT_READ | PROT_EXEC);
2608 assert (result == 0);
2613 _jit_unprotect(jit_state_t *_jit)
2616 assert (_jit->user_code);
2621 if (_jit->user_code) return;
2625 mprotect (_jit->code.ptr, _jit->code.protect, PROT_READ | PROT_WRITE);
2626 assert (result == 0);
2631 _jit_frame(jit_state_t *_jit, jit_int32_t frame)
2633 jit_trampoline(frame, 1);
2637 _jit_tramp(jit_state_t *_jit, jit_int32_t frame)
2639 jit_trampoline(frame, 0);
2643 _jit_trampoline(jit_state_t *_jit, jit_int32_t frame, jit_bool_t prolog)
2647 /* Must be called after prolog, actually, just to simplify
2648 * tests and know there is a current function and that
2649 * _jitc->function->self.aoff is at the before any alloca value */
2650 assert(_jitc->tail && _jitc->tail->code == jit_code_prolog);
2652 /* + 24 for 3 possible spilled temporaries (that could be a double) */
2654 #if defined(__hppa__)
2655 frame += _jitc->function->self.aoff;
2657 frame -= _jitc->function->self.aoff;
2659 _jitc->function->frame = frame;
2661 _jitc->function->define_frame = 1;
2663 _jitc->function->assume_frame = 1;
2664 for (regno = 0; regno < _jitc->reglen; regno++)
2665 if (jit_class(_rvs[regno].spec) & jit_class_sav)
2666 jit_regset_setbit(&_jitc->function->regset, regno);
2669 /* Compute initial reglive and regmask set values of a basic block.
2670 * reglive is the set of known live registers
2671 * regmask is the set of registers not referenced in the block
2672 * Registers in regmask might be live.
2675 _jit_setup(jit_state_t *_jit, jit_block_t *block)
2679 unsigned long value;
2681 jit_regset_set_mask(&block->regmask, _jitc->reglen);
2682 for (value = 0; value < _jitc->reglen; ++value)
2683 if (!(jit_class(_rvs[value].spec) & (jit_class_gpr|jit_class_fpr)))
2684 jit_regset_clrbit(&block->regmask, value);
2686 for (node = block->label->next; node; node = node->next) {
2687 switch (node->code) {
2688 case jit_code_label: case jit_code_prolog:
2689 case jit_code_epilog:
2692 /* Check argument registers in reverse order to properly
2693 * handle registers that are both, argument and result */
2694 value = jit_classify(node->code);
2695 if (value & jit_cc_a2_reg) {
2696 live = !(value & jit_cc_a2_chg);
2697 if (value & jit_cc_a2_rlh) {
2698 /* Assume will not modify a pair in second argument */
2700 if (!(node->w.q.l & jit_regno_patch) &&
2701 jit_regset_tstbit(&block->regmask, node->w.q.l)) {
2702 jit_regset_clrbit(&block->regmask, node->w.q.l);
2704 if (!(node->w.q.h & jit_regno_patch) &&
2705 jit_regset_tstbit(&block->regmask, node->w.q.h)) {
2706 jit_regset_clrbit(&block->regmask, node->w.q.h);
2710 if (!(node->w.w & jit_regno_patch) &&
2711 jit_regset_tstbit(&block->regmask, node->w.w)) {
2712 jit_regset_clrbit(&block->regmask, node->w.w);
2714 jit_regset_setbit(&block->reglive, node->w.w);
2718 if (value & jit_cc_a1_reg) {
2719 live = !(value & jit_cc_a1_chg);
2720 if (value & jit_cc_a1_rlh) {
2721 /* Assume will not modify a pair in second argument */
2723 if (!(node->v.q.l & jit_regno_patch) &&
2724 jit_regset_tstbit(&block->regmask, node->v.q.l)) {
2725 jit_regset_clrbit(&block->regmask, node->v.q.l);
2727 if (!(node->v.q.h & jit_regno_patch) &&
2728 jit_regset_tstbit(&block->regmask, node->v.q.h)) {
2729 jit_regset_clrbit(&block->regmask, node->v.q.h);
2733 if (!(node->v.w & jit_regno_patch) &&
2734 jit_regset_tstbit(&block->regmask, node->v.w)) {
2735 jit_regset_clrbit(&block->regmask, node->v.w);
2737 jit_regset_setbit(&block->reglive, node->v.w);
2741 if (value & jit_cc_a0_reg) {
2742 live = !(value & jit_cc_a0_chg);
2743 if (value & jit_cc_a0_rlh) {
2744 if (!(node->u.q.l & jit_regno_patch) &&
2745 jit_regset_tstbit(&block->regmask, node->u.q.l)) {
2746 jit_regset_clrbit(&block->regmask, node->u.q.l);
2748 jit_regset_setbit(&block->reglive, node->u.q.l);
2750 if (!(node->u.q.h & jit_regno_patch) &&
2751 jit_regset_tstbit(&block->regmask, node->u.q.h)) {
2752 jit_regset_clrbit(&block->regmask, node->u.q.h);
2754 jit_regset_setbit(&block->reglive, node->u.q.h);
2758 if (!(node->u.w & jit_regno_patch) &&
2759 jit_regset_tstbit(&block->regmask, node->u.w)) {
2760 jit_regset_clrbit(&block->regmask, node->u.w);
2762 jit_regset_setbit(&block->reglive, node->u.w);
2771 /* Update regmask and reglive of blocks at entry point of branch targets
2772 * or normal flow that have a live register not used in this block.
2775 _jit_follow(jit_state_t *_jit, jit_block_t *block)
2781 unsigned long value;
2783 jit_regset_t reglive;
2784 jit_regset_t regmask;
2785 jit_regset_t regtemp;
2787 jit_regset_set(®live, &block->reglive);
2788 jit_regset_set(®mask, &block->regmask);
2789 for (node = block->label->next; node; node = node->next) {
2790 switch (node->code) {
2791 case jit_code_label:
2792 /* Do not consider jmpi and jmpr cannot jump to the
2793 * next instruction. */
2794 next = _jitc->blocks.ptr + node->v.w;
2795 /* Set of live registers in next block that are at unknown
2796 * state in this block. */
2797 jit_regset_and(®temp, ®mask, &next->reglive);
2798 if (jit_regset_set_p(®temp)) {
2799 /* Add live state of next block to current block. */
2800 jit_regset_ior(&block->reglive, &block->reglive, ®temp);
2801 /* Remove from unknown state bitmask. */
2802 jit_regset_com(®temp, ®temp);
2803 jit_regset_and(&block->regmask, &block->regmask, ®temp);
2806 case jit_code_prolog:
2807 case jit_code_epilog:
2809 case jit_code_callr:
2810 value = jit_regno(node->u.w);
2811 if (!(node->u.w & jit_regno_patch)) {
2812 if (jit_regset_tstbit(®mask, value)) {
2813 jit_regset_clrbit(®mask, value);
2814 jit_regset_setbit(®live, value);
2817 case jit_code_calli:
2818 for (value = 0; value < _jitc->reglen; ++value) {
2819 value = jit_regset_scan1(®mask, value);
2820 if (value >= _jitc->reglen)
2822 spec = jit_class(_rvs[value].spec);
2823 if (!(spec & jit_class_sav))
2824 jit_regset_clrbit(®mask, value);
2825 if ((spec & jit_class_arg) && jit_regarg_p(node, value))
2826 jit_regset_setbit(®live, value);
2830 value = jit_classify(node->code);
2831 if (value & jit_cc_a2_reg) {
2832 if (value & jit_cc_a2_rlh) {
2833 if (!(node->w.q.l & jit_regno_patch)) {
2834 /* Assume register is not changed */
2835 if (jit_regset_tstbit(®mask, node->w.q.l))
2836 jit_regset_clrbit(®mask, node->w.q.l);
2838 if (!(node->w.q.h & jit_regno_patch)) {
2839 if (jit_regset_tstbit(®mask, node->w.q.h))
2840 jit_regset_clrbit(®mask, node->w.q.h);
2844 if (!(node->w.w & jit_regno_patch)) {
2845 if (jit_regset_tstbit(®mask, node->w.w)) {
2846 jit_regset_clrbit(®mask, node->w.w);
2847 if (!(value & jit_cc_a2_chg))
2848 jit_regset_setbit(®live, node->w.w);
2853 if (value & jit_cc_a1_reg) {
2854 if (value & jit_cc_a1_rlh) {
2855 if (!(node->v.q.l & jit_regno_patch)) {
2856 /* Assume register is not changed */
2857 if (jit_regset_tstbit(®mask, node->v.q.l))
2858 jit_regset_clrbit(®mask, node->v.q.l);
2860 if (!(node->v.q.h & jit_regno_patch)) {
2861 if (jit_regset_tstbit(®mask, node->v.q.h))
2862 jit_regset_clrbit(®mask, node->v.q.h);
2866 if (!(node->v.w & jit_regno_patch)) {
2867 if (jit_regset_tstbit(®mask, node->v.w)) {
2868 jit_regset_clrbit(®mask, node->v.w);
2869 if (!(value & jit_cc_a1_chg))
2870 jit_regset_setbit(®live, node->v.w);
2875 if (value & jit_cc_a0_reg) {
2876 if (value & jit_cc_a0_rlh) {
2877 if (!(node->u.q.l & jit_regno_patch)) {
2878 if (jit_regset_tstbit(®mask, node->u.q.l)) {
2879 jit_regset_clrbit(®mask, node->u.q.l);
2880 if (!(value & jit_cc_a0_chg))
2881 jit_regset_setbit(®live, node->u.q.l);
2884 if (!(node->u.q.h & jit_regno_patch)) {
2885 if (jit_regset_tstbit(®mask, node->u.q.h)) {
2886 jit_regset_clrbit(®mask, node->u.q.h);
2887 if (!(value & jit_cc_a0_chg))
2888 jit_regset_setbit(®live, node->u.q.h);
2893 if (!(node->u.w & jit_regno_patch)) {
2894 if (jit_regset_tstbit(®mask, node->u.w)) {
2895 jit_regset_clrbit(®mask, node->u.w);
2896 if (!(value & jit_cc_a0_chg))
2897 jit_regset_setbit(®live, node->u.w);
2902 if (value & jit_cc_a0_jmp) {
2903 if (node->flag & jit_flag_node) {
2905 /* Do not consider jmpi and jmpr cannot jump to the
2906 * next instruction. */
2907 next = _jitc->blocks.ptr + label->v.w;
2908 jit_regset_and(®temp, ®mask, &next->reglive);
2909 if (jit_regset_set_p(®temp)) {
2910 /* Add live state. */
2911 jit_regset_ior(&block->reglive,
2912 &block->reglive, ®temp);
2913 /* Remove from unknown state bitmask. */
2914 jit_regset_com(®temp, ®temp);
2915 jit_regset_and(&block->regmask,
2916 &block->regmask, ®temp);
2921 /* Jump to unknown location.
2922 * This is a pitfall of the implementation.
2923 * Only jmpi to not a jit code should reach here,
2924 * or a jmpr of a computed address.
2925 * Because the implementation needs jit_class_nospill
2926 * registers, must treat jmpr as a function call. This
2927 * means that only JIT_Vn registers can be trusted on
2930 jit_regset_set_ui(®mask, 0);
2931 for (regno = 0; regno < _jitc->reglen; regno++) {
2932 spec = jit_class(_rvs[regno].spec);
2933 if ((spec & (jit_class_gpr|jit_class_fpr)) &&
2934 (spec & jit_class_sav))
2935 jit_regset_setbit(®mask, regno);
2937 /* Assume non callee save registers are live due
2938 * to jump to unknown location. */
2939 /* Treat all callee save as live. */
2940 jit_regset_ior(&block->reglive, ®live, ®mask);
2941 /* Treat anything else as dead. */
2950 /* Follow code generation up to finding a label or end of code.
2951 * When finding a label, update the set of live registers.
2952 * On branches, update based on taken branch or normal flow.
2955 _jit_update(jit_state_t *_jit, jit_node_t *node,
2956 jit_regset_t *live, jit_regset_t *mask)
2960 unsigned long value;
2963 jit_regset_t regtemp;
2965 for (; node; node = node->next) {
2966 if (jit_regset_set_p(mask) == 0)
2968 switch (node->code) {
2969 case jit_code_label:
2970 block = _jitc->blocks.ptr + node->v.w;
2971 jit_regset_and(®temp, mask, &block->reglive);
2972 if (jit_regset_set_p(®temp)) {
2973 /* Add live state. */
2974 jit_regset_ior(live, live, ®temp);
2975 /* Remove from unknown state bitmask. */
2976 jit_regset_com(®temp, ®temp);
2977 jit_regset_and(mask, mask, ®temp);
2980 case jit_code_prolog:
2981 jit_regset_set_ui(mask, 0);
2983 case jit_code_epilog:
2984 jit_regset_set_ui(mask, 0);
2986 case jit_code_callr:
2987 value = jit_regno(node->u.w);
2988 if (!(node->u.w & jit_regno_patch)) {
2989 if (jit_regset_tstbit(mask, value)) {
2990 jit_regset_clrbit(mask, value);
2991 jit_regset_setbit(live, value);
2994 case jit_code_calli:
2995 for (value = 0; value < _jitc->reglen; ++value) {
2996 value = jit_regset_scan1(mask, value);
2997 if (value >= _jitc->reglen)
2999 spec = jit_class(_rvs[value].spec);
3000 if (!(spec & jit_class_sav))
3001 jit_regset_clrbit(mask, value);
3002 if ((spec & jit_class_arg) && jit_regarg_p(node, value))
3003 jit_regset_setbit(live, value);
3007 value = jit_classify(node->code);
3008 if (value & jit_cc_a2_reg) {
3009 if (value & jit_cc_a2_rlh) {
3010 if (!(node->w.q.l & jit_regno_patch)) {
3011 /* Assume register is not changed */
3012 if (jit_regset_tstbit(mask, node->w.q.l))
3013 jit_regset_clrbit(mask, node->w.q.l);
3015 if (!(node->w.q.h & jit_regno_patch)) {
3016 if (jit_regset_tstbit(mask, node->w.q.h))
3017 jit_regset_clrbit(mask, node->w.q.h);
3021 if (!(node->w.w & jit_regno_patch)) {
3022 if (jit_regset_tstbit(mask, node->w.w)) {
3023 jit_regset_clrbit(mask, node->w.w);
3024 if (!(value & jit_cc_a2_chg))
3025 jit_regset_setbit(live, node->w.w);
3030 if (value & jit_cc_a1_reg) {
3031 if (value & jit_cc_a1_rlh) {
3032 if (!(node->v.q.l & jit_regno_patch)) {
3033 /* Assume register is not changed */
3034 if (jit_regset_tstbit(mask, node->v.q.l))
3035 jit_regset_clrbit(mask, node->v.q.l);
3037 if (!(node->v.q.h & jit_regno_patch)) {
3038 if (jit_regset_tstbit(mask, node->v.q.h))
3039 jit_regset_clrbit(mask, node->v.q.h);
3043 if (!(node->v.w & jit_regno_patch)) {
3044 if (jit_regset_tstbit(mask, node->v.w)) {
3045 jit_regset_clrbit(mask, node->v.w);
3046 if (!(value & jit_cc_a1_chg))
3047 jit_regset_setbit(live, node->v.w);
3052 if (value & jit_cc_a0_reg) {
3053 if (value & jit_cc_a0_rlh) {
3054 if (!(node->u.q.l & jit_regno_patch)) {
3055 if (jit_regset_tstbit(mask, node->u.q.l)) {
3056 jit_regset_clrbit(mask, node->u.q.l);
3057 if (!(value & jit_cc_a0_chg))
3058 jit_regset_setbit(live, node->u.q.l);
3061 if (!(node->u.q.h & jit_regno_patch)) {
3062 if (jit_regset_tstbit(mask, node->u.q.h)) {
3063 jit_regset_clrbit(mask, node->u.q.h);
3064 if (!(value & jit_cc_a0_chg))
3065 jit_regset_setbit(live, node->u.q.h);
3070 if (!(node->u.w & jit_regno_patch)) {
3071 if (jit_regset_tstbit(mask, node->u.w)) {
3072 jit_regset_clrbit(mask, node->u.w);
3073 if (!(value & jit_cc_a0_chg))
3074 jit_regset_setbit(live, node->u.w);
3079 if (value & jit_cc_a0_jmp) {
3080 if (node->flag & jit_flag_node) {
3082 /* Do not consider jmpi and jmpr cannot jump to the
3083 * next instruction. */
3084 block = _jitc->blocks.ptr + label->v.w;
3085 jit_regset_and(®temp, mask, &block->reglive);
3086 if (jit_regset_set_p(®temp)) {
3087 /* Add live state. */
3088 jit_regset_ior(live, live, ®temp);
3089 /* Remove from unknown state bitmask. */
3090 jit_regset_com(®temp, ®temp);
3091 jit_regset_and(mask, mask, ®temp);
3095 /* Jump to unknown location.
3096 * This is a pitfall of the implementation.
3097 * Only jmpi to not a jit code should reach here,
3098 * or a jmpr of a computed address.
3099 * Because the implementation needs jit_class_nospill
3100 * registers, must treat jmpr as a function call. This
3101 * means that only JIT_Vn registers can be trusted on
3104 jit_regset_set_ui(mask, 0);
3105 for (regno = 0; regno < _jitc->reglen; regno++) {
3106 spec = jit_class(_rvs[regno].spec);
3107 if ((spec & (jit_class_gpr|jit_class_fpr)) &&
3108 (spec & jit_class_sav))
3109 jit_regset_setbit(mask, regno);
3111 /* Assume non callee save registers are live due
3112 * to jump to unknown location. */
3113 /* Treat all callee save as live. */
3114 jit_regset_ior(live, live, mask);
3115 /* Prevent explicitly set as live registers to
3116 * be used as a temporary for the jmpi. */
3117 jit_regset_ior(live, live, &_jitc->explive);
3118 /* Treat anything else as dead. */
3128 _thread_jumps(jit_state_t *_jit)
3135 for (prev = node = _jitc->head; node;) {
3137 switch (node->code) {
3139 if (redundant_jump(prev, node)) {
3143 if (shortcut_jump(prev, node))
3147 case jit_code_callr: case jit_code_calli:
3148 /* non optimizable jump like code */
3151 mask = jit_classify(node->code);
3152 if (mask & jit_cc_a0_jmp) {
3153 if (reverse_jump(prev, node) ||
3154 shortcut_jump(prev, node))
3165 _sequential_labels(jit_state_t *_jit)
3173 for (prev = node = _jitc->head; node; node = next) {
3175 if (node->code == jit_code_label) {
3178 del_label(prev, node);
3181 if (prev != node && prev->code == jit_code_label) {
3182 if ((jump = node->link)) {
3183 for (; jump; jump = link) {
3185 if (jump->code == jit_code_movi)
3189 jump->link = prev->link;
3194 del_label(prev, node);
3198 if (next && next->code == jit_code_label && !next->flag) {
3199 if ((jump = next->link)) {
3200 for (; jump; jump = link) {
3202 if (jump->code == jit_code_movi)
3206 jump->link = node->link;
3211 del_label(node, next);
3221 _split_branches(jit_state_t *_jit)
3227 jit_block_t *blocks;
3231 length = _jitc->blocks.length;
3232 jit_alloc((jit_pointer_t *)&blocks, length * sizeof(jit_block_t));
3233 if ((node = _jitc->head) &&
3234 (node->code == jit_code_label || node->code == jit_code_prolog)) {
3235 block = _jitc->blocks.ptr + node->v.w;
3236 memcpy(blocks, block, sizeof(jit_block_t));
3242 for (node = _jitc->head; node; node = next) {
3243 if ((next = node->next)) {
3244 if (next->code == jit_code_label ||
3245 next->code == jit_code_prolog ||
3246 next->code == jit_code_epilog) {
3247 if (offset >= length) {
3248 jit_realloc((jit_pointer_t *)&blocks,
3249 length * sizeof(jit_block_t),
3250 (length + 16) * sizeof(jit_block_t));
3253 block = _jitc->blocks.ptr + next->v.w;
3254 memcpy(blocks + offset, block, sizeof(jit_block_t));
3255 next->v.w = offset++;
3257 /* split block on branches */
3258 else if (jit_classify(node->code) & jit_cc_a0_jmp) {
3259 label = new_node(jit_code_label);
3262 if (offset >= length) {
3263 jit_realloc((jit_pointer_t *)&blocks,
3264 length * sizeof(jit_block_t),
3265 (length + 16) * sizeof(jit_block_t));
3268 block = blocks + offset;
3269 block->label = label;
3270 label->v.w = offset++;
3271 jit_regset_new(&block->reglive);
3272 jit_regset_new(&block->regmask);
3276 jit_free((jit_pointer_t *)&_jitc->blocks.ptr);
3277 _jitc->blocks.ptr = blocks;
3278 _jitc->blocks.offset = offset;
3279 _jitc->blocks.length = length;
3283 _shortcut_jump(jit_state_t *_jit, jit_node_t *prev, jit_node_t *node)
3290 if (!(node->flag & jit_flag_node))
3292 assert(node->code != jit_code_jmpr);
3293 cond = node->code != jit_code_jmpi;
3295 for (next = jump->next; next; next = next->next) {
3296 switch (next->code) {
3298 if (!(next->flag & jit_flag_node))
3300 if (jump->link == node)
3301 jump->link = node->link;
3303 for (temp = jump->link;
3306 assert(temp != NULL);
3307 temp->link = node->link;
3311 node->link = jump->link;
3317 node->code = jit_code_jmpr;
3318 node->u.w = next->u.w;
3320 node->flag &= ~jit_flag_node;
3322 case jit_code_note: case jit_code_label:
3332 _redundant_jump(jit_state_t *_jit, jit_node_t *prev, jit_node_t *node)
3334 jit_node_t *local_prev;
3335 jit_node_t *local_next;
3337 if (!(node->flag & jit_flag_node))
3339 for (local_prev = node, local_next = node->next;
3341 local_prev = local_next, local_next = local_next->next) {
3343 switch (local_next->code) {
3344 case jit_code_label: case jit_code_epilog:
3345 if (node->u.n == local_next) {
3346 if (local_next->link == node)
3347 local_next->link = node->link;
3349 for (local_prev = local_next->link;
3350 local_prev->link != node;
3351 local_prev = local_prev->link)
3352 assert(local_prev != NULL);
3353 local_prev->link = node->link;
3355 del_node(prev, node);
3359 case jit_code_name: case jit_code_note:
3369 reverse_jump_code(jit_code_t code)
3372 case jit_code_bltr: return (jit_code_bger);
3373 case jit_code_blti: return (jit_code_bgei);
3374 case jit_code_bltr_u: return (jit_code_bger_u);
3375 case jit_code_blti_u: return (jit_code_bgei_u);
3376 case jit_code_bler: return (jit_code_bgtr);
3377 case jit_code_blei: return (jit_code_bgti);
3378 case jit_code_bler_u: return (jit_code_bgtr_u);
3379 case jit_code_blei_u: return (jit_code_bgti_u);
3380 case jit_code_beqr: return (jit_code_bner);
3381 case jit_code_beqi: return (jit_code_bnei);
3382 case jit_code_bger: return (jit_code_bltr);
3383 case jit_code_bgei: return (jit_code_blti);
3384 case jit_code_bger_u: return (jit_code_bltr_u);
3385 case jit_code_bgei_u: return (jit_code_blti_u);
3386 case jit_code_bgtr: return (jit_code_bler);
3387 case jit_code_bgti: return (jit_code_blei);
3388 case jit_code_bgtr_u: return (jit_code_bler_u);
3389 case jit_code_bgti_u: return (jit_code_blei_u);
3390 case jit_code_bner: return (jit_code_beqr);
3391 case jit_code_bnei: return (jit_code_beqi);
3392 case jit_code_bmsr: return (jit_code_bmcr);
3393 case jit_code_bmsi: return (jit_code_bmci);
3394 case jit_code_bmcr: return (jit_code_bmsr);
3395 case jit_code_bmci: return (jit_code_bmsi);
3396 case jit_code_bltr_f: return (jit_code_bunger_f);
3397 case jit_code_blti_f: return (jit_code_bungei_f);
3398 case jit_code_bler_f: return (jit_code_bungtr_f);
3399 case jit_code_blei_f: return (jit_code_bungti_f);
3401 case jit_code_beqr_f: return (jit_code_bner_f);
3402 case jit_code_beqi_f: return (jit_code_bnei_f);
3404 case jit_code_bger_f: return (jit_code_bunltr_f);
3405 case jit_code_bgei_f: return (jit_code_bunlti_f);
3406 case jit_code_bgtr_f: return (jit_code_bunler_f);
3407 case jit_code_bgti_f: return (jit_code_bunlei_f);
3409 case jit_code_bner_f: return (jit_code_beqr_f);
3410 case jit_code_bnei_f: return (jit_code_beqi_f);
3412 case jit_code_bunltr_f: return (jit_code_bger_f);
3413 case jit_code_bunlti_f: return (jit_code_bgei_f);
3414 case jit_code_bunler_f: return (jit_code_bgtr_f);
3415 case jit_code_bunlei_f: return (jit_code_bgti_f);
3417 case jit_code_buneqr_f: return (jit_code_bltgtr_f);
3418 case jit_code_buneqi_f: return (jit_code_bltgti_f);
3420 case jit_code_bunger_f: return (jit_code_bltr_f);
3421 case jit_code_bungei_f: return (jit_code_blti_f);
3422 case jit_code_bungtr_f: return (jit_code_bler_f);
3423 case jit_code_bungti_f: return (jit_code_blei_f);
3425 case jit_code_bltgtr_f: return (jit_code_buneqr_f);
3426 case jit_code_bltgti_f: return (jit_code_buneqi_f);
3428 case jit_code_bordr_f: return (jit_code_bunordr_f);
3429 case jit_code_bordi_f: return (jit_code_bunordi_f);
3430 case jit_code_bunordr_f:return (jit_code_bordr_f);
3431 case jit_code_bunordi_f:return (jit_code_bordi_f);
3432 case jit_code_bltr_d: return (jit_code_bunger_d);
3433 case jit_code_blti_d: return (jit_code_bungei_d);
3434 case jit_code_bler_d: return (jit_code_bungtr_d);
3435 case jit_code_blei_d: return (jit_code_bungti_d);
3437 case jit_code_beqr_d: return (jit_code_bner_d);
3438 case jit_code_beqi_d: return (jit_code_bnei_d);
3440 case jit_code_bger_d: return (jit_code_bunltr_d);
3441 case jit_code_bgei_d: return (jit_code_bunlti_d);
3442 case jit_code_bgtr_d: return (jit_code_bunler_d);
3443 case jit_code_bgti_d: return (jit_code_bunlei_d);
3445 case jit_code_bner_d: return (jit_code_beqr_d);
3446 case jit_code_bnei_d: return (jit_code_beqi_d);
3448 case jit_code_bunltr_d: return (jit_code_bger_d);
3449 case jit_code_bunlti_d: return (jit_code_bgei_d);
3450 case jit_code_bunler_d: return (jit_code_bgtr_d);
3451 case jit_code_bunlei_d: return (jit_code_bgti_d);
3453 case jit_code_buneqr_d: return (jit_code_bltgtr_d);
3454 case jit_code_buneqi_d: return (jit_code_bltgti_d);
3456 case jit_code_bunger_d: return (jit_code_bltr_d);
3457 case jit_code_bungei_d: return (jit_code_blti_d);
3458 case jit_code_bungtr_d: return (jit_code_bler_d);
3459 case jit_code_bungti_d: return (jit_code_blei_d);
3461 case jit_code_bltgtr_d: return (jit_code_buneqr_d);
3462 case jit_code_bltgti_d: return (jit_code_buneqi_d);
3464 case jit_code_bordr_d: return (jit_code_bunordr_d);
3465 case jit_code_bordi_d: return (jit_code_bunordi_d);
3466 case jit_code_bunordr_d:return (jit_code_bordr_d);
3467 case jit_code_bunordi_d:return (jit_code_bordi_d);
3468 case jit_code_boaddr: return (jit_code_bxaddr);
3469 case jit_code_boaddi: return (jit_code_bxaddi);
3470 case jit_code_boaddr_u: return (jit_code_bxaddr_u);
3471 case jit_code_boaddi_u: return (jit_code_bxaddi_u);
3472 case jit_code_bxaddr: return (jit_code_boaddr);
3473 case jit_code_bxaddi: return (jit_code_boaddi);
3474 case jit_code_bxaddr_u: return (jit_code_boaddr_u);
3475 case jit_code_bxaddi_u: return (jit_code_boaddi_u);
3476 case jit_code_bosubr: return (jit_code_bxsubr);
3477 case jit_code_bosubi: return (jit_code_bxsubi);
3478 case jit_code_bosubr_u: return (jit_code_bxsubr_u);
3479 case jit_code_bosubi_u: return (jit_code_bxsubi_u);
3480 case jit_code_bxsubr: return (jit_code_bosubr);
3481 case jit_code_bxsubi: return (jit_code_bosubi);
3482 case jit_code_bxsubr_u: return (jit_code_bosubr_u);
3483 case jit_code_bxsubi_u: return (jit_code_bosubi_u);
3484 default: abort(); /* invalid jump code */
3489 * change common pattern:
3490 * <cond_jump L0> <jump L1> <label L0>
3492 * <reverse_cond_jump L1>
3495 _reverse_jump(jit_state_t *_jit, jit_node_t *prev, jit_node_t *node)
3497 jit_node_t *local_prev;
3498 jit_node_t *local_next;
3499 jit_node_t *local_jump;
3501 if (!(node->flag & jit_flag_node))
3503 /* =><cond_jump L0> <jump L1> <label L0> */
3504 local_next = node->next;
3505 if (local_next->code != jit_code_jmpi ||
3506 !(local_next->flag & jit_flag_node))
3508 /* <cond_jump L0> =><jump L1> <label L0> */
3510 local_jump = local_next->u.n;
3511 for (local_prev = local_next, local_next = local_next->next;
3513 local_prev = local_next, local_next = local_next->next) {
3514 switch (local_next->code) {
3515 case jit_code_label: case jit_code_epilog:
3516 if (node->u.n == local_next) {
3517 if (local_next->link == node)
3518 local_next->link = node->link;
3520 for (local_prev = local_next->link;
3521 local_prev->link != node;
3522 local_prev = local_prev->link)
3523 assert(local_prev != NULL);
3524 local_prev->link = node->link;
3526 del_node(node, node->next);
3527 node->code = reverse_jump_code(node->code);
3528 node->u.n = local_jump;
3529 node->link = local_jump->link;
3530 local_jump->link = node;
3544 _redundant_store(jit_state_t *_jit, jit_node_t *node, jit_bool_t jump)
3555 if (prev->code == jit_code_epilog)
3557 assert(prev->code == jit_code_label);
3558 if ((prev->flag & jit_flag_head) || node->link || prev->link != node)
3559 /* multiple sources */
3561 /* if there are sequential labels it will return below */
3567 regno = jit_regno(node->v.w);
3568 for (iter = prev->next; iter; prev = iter, iter = iter->next) {
3569 switch (iter->code) {
3570 case jit_code_label: case jit_code_prolog:
3571 case jit_code_epilog:
3574 if (regno == jit_regno(iter->u.w)) {
3575 if (iter->flag || iter->v.w != word)
3578 del_node(prev, iter);
3583 spec = jit_classify(iter->code);
3584 if (spec & jit_cc_a0_jmp)
3586 if ((spec & (jit_cc_a0_reg|jit_cc_a0_chg)) ==
3587 (jit_cc_a0_reg|jit_cc_a0_chg)) {
3588 if (spec & jit_cc_a0_rlh) {
3589 if (regno == jit_regno(iter->u.q.l) ||
3590 regno == jit_regno(iter->u.q.h))
3594 if (regno == jit_regno(iter->u.w))
3598 if ((spec & (jit_cc_a1_reg|jit_cc_a1_chg)) ==
3599 (jit_cc_a1_reg|jit_cc_a1_chg)) {
3600 if (regno == jit_regno(iter->v.w))
3603 if ((spec & (jit_cc_a2_reg|jit_cc_a2_chg)) ==
3604 (jit_cc_a2_reg|jit_cc_a2_chg)) {
3605 if (regno == jit_regno(iter->w.w))
3616 _simplify_movr(jit_state_t *_jit, jit_node_t *prev, jit_node_t *node,
3617 jit_int32_t kind, jit_int32_t size)
3624 regno = jit_regno(node->u.w);
3625 right = jit_regno(node->v.w);
3626 value = _jitc->values + regno;
3628 spec = jit_class(_rvs[regno].spec);
3629 if (!(spec & (jit_class_gpr | jit_class_fpr))) {
3630 /* reserved register */
3632 ++_jitc->gen[regno];
3636 if ((value->kind == jit_kind_register &&
3637 jit_regno(value->base.q.l) == right &&
3638 value->base.q.h == _jitc->gen[right]) ||
3639 (value->kind == kind && _jitc->values[right].kind == kind &&
3640 memcmp(&value->base.w, &_jitc->values[right].base.w, size) == 0)) {
3641 del_node(prev, node);
3644 if (_jitc->values[right].kind == jit_kind_word)
3645 jit_memcpy(value, _jitc->values + right, sizeof(jit_value_t));
3647 value->kind = jit_kind_register;
3648 value->base.q.l = right;
3649 value->base.q.h = _jitc->gen[right];
3651 ++_jitc->gen[regno];
3657 _simplify_movi(jit_state_t *_jit, jit_node_t *prev, jit_node_t *node,
3658 jit_int32_t kind, jit_int32_t size)
3665 regno = jit_regno(node->u.w);
3666 value = _jitc->values + regno;
3667 if (node->flag & jit_flag_node) {
3668 /* set to undefined if value will be patched */
3670 ++_jitc->gen[regno];
3673 spec = jit_class(_rvs[regno].spec);
3674 if (!(spec & (jit_class_gpr | jit_class_fpr))) {
3675 /* reserved register */
3677 ++_jitc->gen[regno];
3680 if (value->kind == kind) {
3681 if (memcmp(&node->v.w, &value->base.w, size) == 0) {
3682 del_node(prev, node);
3685 if (kind == jit_kind_word)
3686 spec &= jit_class_gpr;
3688 spec &= (jit_class_xpr | jit_class_fpr);
3689 for (offset = 0; offset < _jitc->reglen; offset++) {
3690 if (_jitc->values[offset].kind == kind &&
3691 memcmp(&node->v.w, &_jitc->values[offset].base.w, size) == 0 &&
3692 (jit_class(_rvs[offset].spec) & spec) == spec) {
3693 if (kind == jit_kind_word)
3694 node->code = jit_code_movr;
3695 else if (kind == jit_kind_float32)
3696 node->code = jit_code_movr_f;
3698 node->code = jit_code_movr_d;
3700 jit_memcpy(value, _jitc->values + offset, sizeof(jit_value_t));
3701 ++_jitc->gen[regno];
3707 jit_memcpy(&value->base.w, &node->v.w, size);
3708 ++_jitc->gen[regno];
3713 /* simple/safe redundandy test not checking if another register
3714 * holds the same value
3717 _simplify_ldxi(jit_state_t *_jit, jit_node_t *prev, jit_node_t *node)
3723 regno = jit_regno(node->u.w);
3724 right = jit_regno(node->v.w);
3725 value = _jitc->values + regno;
3726 if (regno != right &&
3727 value->kind == jit_kind_code && value->code == node->code &&
3728 value->base.q.l == right && value->base.q.h == _jitc->gen[right] &&
3729 node->w.w == value->disp.w) {
3730 del_node(prev, node);
3733 value->kind = jit_kind_code;
3734 value->code = node->code;
3735 value->base.q.l = right;
3736 value->base.q.h = _jitc->gen[right];
3737 value->disp.w = node->w.w;
3738 ++_jitc->gen[regno];
3744 _simplify_stxi(jit_state_t *_jit, jit_node_t *prev, jit_node_t *node)
3751 regno = jit_regno(node->w.w);
3752 right = jit_regno(node->v.w);
3753 value = _jitc->values + regno;
3755 /* check for redundant store after load */
3756 if (regno != right &&
3757 value->kind == jit_kind_code && value->code == node->code &&
3758 value->base.q.l == right && value->base.q.h == _jitc->gen[right] &&
3759 node->u.w == value->disp.w) {
3760 del_node(prev, node);
3764 /* assume anything can alias, and invalidate tracked values */
3765 for (offset = 0; offset < _jitc->reglen; offset++) {
3766 if (_jitc->values[offset].kind == jit_kind_code) {
3767 _jitc->values[offset].kind = 0;
3768 ++_jitc->gen[offset];
3772 /* no multiple information, so, if set to a constant,
3773 * prefer to keep that information */
3774 if (value->kind == 0) {
3775 switch (node->code) {
3776 /* no information about signed/unsigned either */
3777 case jit_code_stxi_c: value->code = jit_code_ldxi_c; break;
3778 case jit_code_stxi_s: value->code = jit_code_ldxi_s; break;
3779 case jit_code_stxi_i: value->code = jit_code_ldxi_i; break;
3780 case jit_code_stxi_l: value->code = jit_code_ldxi_l; break;
3781 case jit_code_stxi_f: value->code = jit_code_ldxi_f; break;
3782 case jit_code_stxi_d: value->code = jit_code_ldxi_d; break;
3785 value->kind = jit_kind_code;
3786 value->base.q.l = right;
3787 value->base.q.h = _jitc->gen[right];
3788 value->disp.w = node->u.w;
3794 /* usually there should be only one store in the
3795 * jit_get_reg/jit_unget_reg, but properly handle
3796 * multiple ones by moving the save node */
3798 _simplify_spill(jit_state_t *_jit, jit_node_t *node, jit_int32_t regno)
3803 if ((temp = _jitc->spill[regno]) && (save = temp->next) != node) {
3804 temp->next = save->next;
3805 save->next = node->next;
3807 _jitc->spill[regno] = node;
3811 /* checks for simple cases where a register is set more than
3812 * once to the same value, and is a common pattern of calls
3813 * to jit_pushargi and jit_pushargr
3816 _simplify(jit_state_t *_jit)
3826 for (prev = NULL, node = _jitc->head; node; prev = node, node = next) {
3828 switch (node->code) {
3829 case jit_code_label: case jit_code_prolog:
3830 case jit_code_callr: case jit_code_calli:
3832 memset(_jitc->gen, 0, sizeof(jit_int32_t) * _jitc->reglen);
3833 memset(_jitc->values, 0, sizeof(jit_value_t) * _jitc->reglen);
3836 _jitc->spill[jit_regno(node->u.w)] = prev;
3839 regno = jit_regno(node->u.w);
3840 if (register_change_p(node->link->next, node, regno) !=
3842 /* spill not required due to optimizing common
3843 * redundancy case of calling jit_get_reg/jit_unget_reg
3844 * and then setting the register to the value it is
3845 * already holding */
3846 patch_register(node->link->next, node,
3847 jit_regno_patch|regno, regno);
3849 del_node(_jitc->spill[regno], node->link);
3850 del_node(prev, node);
3853 _jitc->spill[regno] = NULL;
3856 regno = jit_regno(node->u.w);
3857 if (simplify_movr(prev, node,
3858 jit_kind_word, sizeof(jit_word_t))) {
3860 simplify_spill(node = prev, regno);
3864 regno = jit_regno(node->u.w);
3865 if (simplify_movi(prev, node,
3866 jit_kind_word, sizeof(jit_word_t))) {
3868 simplify_spill(node = prev, regno);
3871 case jit_code_movr_f:
3872 regno = jit_regno(node->u.w);
3873 if (simplify_movr(prev, node,
3874 jit_kind_float32, sizeof(jit_float32_t))) {
3876 simplify_spill(node = prev, regno);
3879 case jit_code_movi_f:
3880 regno = jit_regno(node->u.w);
3881 if (simplify_movi(prev, node,
3882 jit_kind_float32, sizeof(jit_float32_t))) {
3884 simplify_spill(node = prev, regno);
3887 case jit_code_movr_d:
3888 regno = jit_regno(node->u.w);
3889 if (simplify_movr(prev, node,
3890 jit_kind_float64, sizeof(jit_float64_t))) {
3892 simplify_spill(node = prev, regno);
3895 case jit_code_movi_d:
3896 regno = jit_regno(node->u.w);
3897 if (simplify_movi(prev, node,
3898 jit_kind_float64, sizeof(jit_float64_t))) {
3900 simplify_spill(node = prev, regno);
3903 case jit_code_ldxi_c: case jit_code_ldxi_uc:
3904 case jit_code_ldxi_s: case jit_code_ldxi_us:
3905 case jit_code_ldxi_i: case jit_code_ldxi_ui:
3906 case jit_code_ldxi_l:
3907 case jit_code_ldxi_f: case jit_code_ldxi_d:
3908 regno = jit_regno(node->u.w);
3909 if (simplify_ldxi(prev, node)) {
3911 simplify_spill(node = prev, regno);
3914 case jit_code_stxi_c: case jit_code_stxi_s:
3915 case jit_code_stxi_i: case jit_code_stxi_l:
3916 case jit_code_stxi_f: case jit_code_stxi_d:
3917 regno = jit_regno(node->u.w);
3918 if (simplify_stxi(prev, node)) {
3920 simplify_spill(node = prev, regno);
3924 info = jit_classify(node->code);
3925 if (info & jit_cc_a0_jmp)
3926 /* labels are not implicitly added when not taking
3927 * a conditional branch */
3929 if (info & jit_cc_a0_chg) {
3930 if (info & jit_cc_a0_rlh) {
3931 regno = jit_regno(node->u.q.l);
3932 _jitc->values[regno].kind = 0;
3933 ++_jitc->gen[regno];
3934 regno = jit_regno(node->u.q.h);
3935 _jitc->values[regno].kind = 0;
3936 ++_jitc->gen[regno];
3939 regno = jit_regno(node->u.w);
3940 _jitc->values[regno].kind = 0;
3941 ++_jitc->gen[regno];
3944 if (info & jit_cc_a1_chg) {
3946 /* Assume registers are not changed */
3947 if (info & jit_cc_a1_rlh) {
3948 regno = jit_regno(node->v.q.l);
3949 _jitc->values[regno].kind = 0;
3950 ++_jitc->gen[regno];
3951 regno = jit_regno(node->v.q.h);
3952 _jitc->values[regno].kind = 0;
3953 ++_jitc->gen[regno];
3957 regno = jit_regno(node->v.w);
3958 _jitc->values[regno].kind = 0;
3959 ++_jitc->gen[regno];
3964 if (info & jit_cc_a2_chg) {
3966 /* Assume registers are not changed */
3967 if (info & jit_cc_a2_rlh) {
3968 regno = jit_regno(node->w.q.l);
3969 _jitc->values[regno].kind = 0;
3970 ++_jitc->gen[regno];
3971 regno = jit_regno(node->w.q.h);
3972 _jitc->values[regno].kind = 0;
3973 ++_jitc->gen[regno];
3977 regno = jit_regno(node->w.w);
3978 _jitc->values[regno].kind = 0;
3979 ++_jitc->gen[regno];
3991 _register_change_p(jit_state_t *_jit, jit_node_t *node, jit_node_t *link,
3995 jit_bool_t use, change;
3997 for (; node != link; node = node->next) {
3998 switch (node->code) {
3999 case jit_code_label: case jit_code_prolog:
4000 /* lack of extra information so assume it is live */
4001 return (jit_reg_static);
4002 case jit_code_callr: case jit_code_calli:
4003 if (!(jit_class(_rvs[regno].spec) & jit_class_sav))
4004 return (jit_reg_undef);
4007 value = jit_classify(node->code);
4009 if (value & jit_cc_a0_rlh) {
4010 if (node->u.q.l == regno || node->u.q.h == regno) {
4011 if (value & jit_cc_a0_chg)
4016 else if ((value & jit_cc_a0_reg) && node->u.w == regno) {
4017 if (value & jit_cc_a0_chg)
4022 (value & jit_cc_a1_reg) && node->v.w == regno) {
4023 if (value & jit_cc_a1_chg)
4028 (value & jit_cc_a2_reg) && node->w.w == regno) {
4029 if (value & jit_cc_a2_chg)
4033 /* lack of extra information */
4034 if (!use && (value & (jit_cc_a0_jmp|jit_cc_a0_cnd)) &&
4035 /* In case of indirect branches, always consider
4036 * callee save registers as live. */
4037 !(jit_class(_rvs[regno].spec) & jit_class_sav))
4040 return (jit_reg_change);
4044 return (jit_reg_static);
4047 /* most of this could be done at the same time as generating jit, but
4048 * avoid complications on different cpu backends and patch spill/loads
4049 * here, by simulating jit generation */
4051 _spill_reglive_p(jit_state_t *_jit, jit_node_t *node, jit_int32_t regno)
4053 if (!jit_regset_tstbit(&_jitc->reglive, regno)) {
4054 jit_regset_setbit(&_jitc->regmask, regno);
4055 jit_update(node->next, &_jitc->reglive, &_jitc->regmask);
4056 if (!jit_regset_tstbit(&_jitc->reglive, regno) &&
4057 register_change_p(node->next, node->link, regno) != jit_reg_change)
4065 _patch_registers(jit_state_t *_jit)
4075 _jitc->function = NULL;
4077 jit_reglive_setup();
4078 for (prev = NULL, node = _jitc->head; node; node = next) {
4081 info = jit_classify(node->code);
4082 jit_regarg_set(node, info);
4084 switch (node->code) {
4086 regno = jit_regno(node->u.w);
4087 if (!spill_reglive_p(node, regno)) {
4088 /* register is not live, just remove spill/reload */
4089 jit_regarg_clr(node, info);
4090 node->link->v.w = jit_regload_delete;
4091 del_node(prev, node);
4095 /* try to find a free register of the same class */
4096 spec = jit_class(_rvs[regno].spec) & ~jit_class_arg;
4097 for (value = 0; value < _jitc->reglen; value++) {
4098 if (value != regno &&
4099 ((jit_class(_rvs[value].spec) & spec) &
4100 ~jit_class_arg) == spec &&
4101 !jit_regset_tstbit(&_jitc->regarg, value) &&
4102 !spill_reglive_p(node, value))
4105 if (value < _jitc->reglen) {
4106 jit_regarg_clr(node, info);
4107 patch_register(node->next, node->link,
4108 jit_regno_patch|node->u.w,
4109 jit_regno_patch|value);
4110 /* mark as live just in case there are nested
4111 * register patches, so that next patch will
4112 * not want to use the same register */
4113 jit_regset_setbit(&_jitc->reglive, value);
4114 /* register is not live, just remove spill/reload */
4115 node->link->v.w = jit_regload_isdead;
4116 del_node(prev, node);
4120 /* failed to find a free register */
4121 if (spec & jit_class_gpr) {
4122 if (!_jitc->function->regoff[regno])
4123 _jitc->function->regoff[regno] =
4124 jit_allocai(sizeof(jit_word_t));
4125 #if __WORDSIZE == 32
4126 node->code = jit_code_stxi_i;
4128 node->code = jit_code_stxi_l;
4132 node->code = jit_code_stxi_d;
4133 if (!_jitc->function->regoff[regno])
4134 _jitc->function->regoff[regno] =
4135 jit_allocai(sizeof(jit_float64_t));
4137 node->u.w = _jitc->function->regoff[regno];
4145 regno = jit_regno(node->u.w);
4147 if (node->v.w == jit_regload_isdead)
4148 jit_regset_clrbit(&_jitc->reglive, regno);
4149 del_node(prev, node);
4152 spec = jit_class(_rvs[regno].spec);
4153 if (spec & jit_class_gpr) {
4154 #if __WORDSIZE == 32
4155 node->code = jit_code_ldxi_i;
4157 node->code = jit_code_ldxi_l;
4161 node->code = jit_code_ldxi_d;
4164 node->w.w = _jitc->function->regoff[regno];
4167 case jit_code_prolog:
4168 _jitc->function = _jitc->functions.ptr + node->w.w;
4170 case jit_code_epilog:
4171 _jitc->function = NULL;
4177 jit_regarg_clr(node, info);
4178 /* update register live state */
4185 _patch_register(jit_state_t *_jit, jit_node_t *node, jit_node_t *link,
4186 jit_int32_t regno, jit_int32_t patch)
4190 for (; node != link; node = node->next) {
4191 value = jit_classify(node->code);
4192 if (value & jit_cc_a0_reg) {
4193 if (value & jit_cc_a0_rlh) {
4194 if (node->u.q.l == regno)
4195 node->u.q.l = patch;
4196 if (node->u.q.h == regno)
4197 node->u.q.h = patch;
4200 if (node->u.w == regno)
4204 if (value & jit_cc_a1_reg) {
4205 if (value & jit_cc_a1_rlh) {
4206 if (node->v.q.l == regno)
4207 node->v.q.l = patch;
4208 if (node->v.q.h == regno)
4209 node->v.q.h = patch;
4212 if (node->v.w == regno)
4216 if (value & jit_cc_a2_reg) {
4217 if (value & jit_cc_a2_rlh) {
4218 if (node->w.q.l == regno)
4219 node->w.q.l = patch;
4220 if (node->w.q.h == regno)
4221 node->w.q.h = patch;
4224 if (node->w.w == regno)
4231 #if __BYTE_ORDER == __LITTLE_ENDIAN
4232 # define htonr_us(r0,r1) bswapr_us(r0,r1)
4233 # define htonr_ui(r0,r1) bswapr_ui(r0,r1)
4234 # if __WORDSIZE == 64
4235 # define htonr_ul(r0,r1) bswapr_ul(r0,r1)
4238 # define htonr_us(r0,r1) extr_us(r0,r1)
4239 # if __WORDSIZE == 32
4240 # define htonr_ui(r0,r1) movr(r0,r1)
4242 # define htonr_ui(r0,r1) extr_ui(r0,r1)
4243 # define htonr_ul(r0,r1) movr(r0,r1)
4247 static maybe_unused void
4248 generic_bswapr_us(jit_state_t *_jit, jit_int32_t r0, jit_int32_t r1);
4249 static maybe_unused void
4250 generic_bswapr_ui(jit_state_t *_jit, jit_int32_t r0, jit_int32_t r1);
4251 #if __WORDSIZE == 64
4252 static maybe_unused void
4253 generic_bswapr_ul(jit_state_t *_jit, jit_int32_t r0, jit_int32_t r1);
4256 #define depi(r0, i0, i1, i2) _depi(_jit, r0, i0, i1, i2)
4257 static void _depi(jit_state_t*,jit_int32_t, jit_word_t, jit_word_t, jit_word_t);
4258 #define negi(r0, i0) _negi(_jit, r0, i0)
4259 static void _negi(jit_state_t*, jit_int32_t, jit_word_t);
4260 #define comi(r0, i0) _comi(_jit, r0, i0)
4261 static void _comi(jit_state_t*, jit_int32_t, jit_word_t);
4262 #define exti_c(r0, i0) _exti_c(_jit, r0, i0)
4263 static void _exti_c(jit_state_t*, jit_int32_t, jit_word_t);
4264 #define exti_uc(r0, i0) _exti_uc(_jit, r0, i0)
4265 static void _exti_uc(jit_state_t*, jit_int32_t, jit_word_t);
4266 #define exti_s(r0, i0) _exti_s(_jit, r0, i0)
4267 static void _exti_s(jit_state_t*, jit_int32_t, jit_word_t);
4268 #define exti_us(r0, i0) _exti_us(_jit, r0, i0)
4269 static void _exti_us(jit_state_t*, jit_int32_t, jit_word_t);
4270 #if __WORDSIZE == 64
4271 #define exti_i(r0, i0) _exti_i(_jit, r0, i0)
4272 static void _exti_i(jit_state_t*, jit_int32_t, jit_word_t);
4273 #define exti_ui(r0, i0) _exti_ui(_jit, r0, i0)
4274 static void _exti_ui(jit_state_t*, jit_int32_t, jit_word_t);
4276 #define bswapi_us(r0, i0) _bswapi_us(_jit, r0, i0)
4277 static void _bswapi_us(jit_state_t*, jit_int32_t, jit_word_t);
4278 #define bswapi_ui(r0, i0) _bswapi_ui(_jit, r0, i0)
4279 static void _bswapi_ui(jit_state_t*, jit_int32_t, jit_word_t);
4280 #if __WORDSIZE == 64
4281 # define bswapi_ul(r0, i0) _bswapi_ul(_jit, r0, i0)
4282 static void _bswapi_ul(jit_state_t*, jit_int32_t, jit_word_t);
4284 #define htoni_us(r0, i0) _htoni_us(_jit, r0, i0)
4285 static void _htoni_us(jit_state_t*, jit_int32_t, jit_word_t);
4286 #define htoni_ui(r0, i0) _htoni_ui(_jit, r0, i0)
4287 static void _htoni_ui(jit_state_t*, jit_int32_t, jit_word_t);
4288 #if __WORDSIZE == 64
4289 # define htoni_ul(r0, i0) _htoni_ul(_jit, r0, i0)
4290 static void _htoni_ul(jit_state_t*, jit_int32_t, jit_word_t);
4292 # define movi_f_w(r0, i0) _movi_f_w(_jit, r0, i0)
4293 static void _movi_f_w(jit_state_t*, jit_int32_t, jit_float32_t);
4294 #if __WORDSIZE == 32 && !(defined(__mips__) && NEW_ABI)
4295 # define movi_d_ww(r0, r1, i0) _movi_d_ww(_jit, r0, r1, i0)
4296 static void _movi_d_ww(jit_state_t*, jit_int32_t, jit_int32_t, jit_float64_t);
4298 # define movi_d_w(r0, i0) _movi_d_w(_jit, r0, i0)
4299 static void _movi_d_w(jit_state_t*, jit_int32_t, jit_float64_t);
4301 #define cloi(r0, i0) _cloi(_jit, r0, i0)
4302 static void _cloi(jit_state_t*, jit_int32_t, jit_word_t);
4303 #define clzi(r0, i0) _clzi(_jit, r0, i0)
4304 static void _clzi(jit_state_t*, jit_int32_t, jit_word_t);
4305 #define ctoi(r0, i0) _ctoi(_jit, r0, i0)
4306 static void _ctoi(jit_state_t*, jit_int32_t, jit_word_t);
4307 #define ctzi(r0, i0) _ctzi(_jit, r0, i0)
4308 static void _ctzi(jit_state_t*, jit_int32_t, jit_word_t);
4309 #define rbiti(r0, i0) _rbiti(_jit, r0, i0)
4310 static void _rbiti(jit_state_t*, jit_int32_t, jit_word_t);
4311 #define popcnti(r0, i0) _popcnti(_jit, r0, i0)
4312 static void _popcnti(jit_state_t*, jit_int32_t, jit_word_t);
4313 #define exti(r0, i0, i1, i2) _exti(_jit, r0, i0, i1, i2)
4314 static void _exti(jit_state_t*,jit_int32_t, jit_word_t, jit_word_t, jit_word_t);
4315 #define exti_u(r0, i0, i1, i2) _exti_u(_jit, r0, i0, i1, i2)
4316 static void _exti_u(jit_state_t*,
4317 jit_int32_t, jit_word_t, jit_word_t, jit_word_t);
4318 #define generic_unldr(r0, r1, i0) _generic_unldr(_jit, r0, r1, i0)
4319 static void _generic_unldr(jit_state_t*, jit_int32_t, jit_int32_t, jit_word_t);
4320 #define generic_unldi(r0, i0, i1) _generic_unldi(_jit, r0, i0, i1)
4321 static void _generic_unldi(jit_state_t*, jit_int32_t, jit_word_t, jit_word_t);
4322 #define generic_unldr_u(r0, r1, i0) _generic_unldr_u(_jit, r0, r1, i0)
4323 static void _generic_unldr_u(jit_state_t*,
4324 jit_int32_t, jit_int32_t, jit_word_t);
4325 #define generic_unldi_u(r0, i0, i1) _generic_unldi_u(_jit, r0, i0, i1)
4326 static void _generic_unldi_u(jit_state_t*, jit_int32_t, jit_word_t, jit_word_t);
4327 #define generic_unstr(r0, r1, i0) _generic_unstr(_jit, r0, r1, i0)
4328 static void _generic_unstr(jit_state_t*, jit_int32_t, jit_int32_t, jit_word_t);
4329 #define generic_unsti(i0, r0, i1) _generic_unsti(_jit, i0, r0, i1)
4330 static void _generic_unsti(jit_state_t*, jit_word_t, jit_int32_t, jit_word_t);
4331 #if !defined(__i386__) && !defined(__x86_64__)
4332 # define generic_unldr_x(r0, r1, i0) _generic_unldr_x(_jit, r0, r1, i0)
4333 static void _generic_unldr_x(jit_state_t*,
4334 jit_int32_t, jit_int32_t, jit_word_t);
4335 # define generic_unldi_x(r0, i0, i1) _generic_unldi_x(_jit, r0, i0, i1)
4336 static void _generic_unldi_x(jit_state_t*, jit_int32_t, jit_word_t, jit_word_t);
4337 # define generic_unstr_x(r0, r1, i0) _generic_unstr_x(_jit, r0, r1, i0)
4338 static void _generic_unstr_x(jit_state_t*,
4339 jit_int32_t, jit_int32_t, jit_word_t);
4340 # define generic_unsti_x(i0, r0, i1) _generic_unsti_x(_jit, i0, r0, i1)
4341 static void _generic_unsti_x(jit_state_t*, jit_word_t, jit_int32_t, jit_word_t);
4343 #define patch_alist(revert) _patch_alist(_jit, revert)
4344 static maybe_unused void _patch_alist(jit_state_t *_jit, jit_bool_t revert);
4346 #if defined(__i386__) || defined(__x86_64__)
4347 # include "jit_x86.c"
4348 #elif defined(__mips__)
4349 # include "jit_mips.c"
4350 #elif defined(__arm__)
4351 # include "jit_arm.c"
4352 #elif defined(__powerpc__)
4353 # include "jit_ppc.c"
4354 #elif defined(__sparc__)
4355 # include "jit_sparc.c"
4356 #elif defined(__ia64__)
4357 # include "jit_ia64.c"
4358 #elif defined(__hppa__)
4359 # include "jit_hppa.c"
4360 #elif defined(__aarch64__)
4361 # include "jit_aarch64.c"
4362 #elif defined(__s390__) || defined(__s390x__)
4363 # include "jit_s390.c"
4364 #elif defined(__alpha__)
4365 # include "jit_alpha.c"
4366 #elif defined(__riscv)
4367 # include "jit_riscv.c"
4368 #elif defined(__loongarch__)
4369 # include "jit_loongarch.c"
4372 static maybe_unused void
4373 generic_bswapr_us(jit_state_t *_jit, jit_int32_t r0, jit_int32_t r1)
4375 jit_int32_t reg = jit_get_reg(jit_class_gpr);
4377 rshi(rn(reg), r1, 8);
4379 andi(rn(reg), rn(reg), 0xff);
4381 orr(r0, r0, rn(reg));
4386 static maybe_unused void
4387 generic_bswapr_ui(jit_state_t *_jit, jit_int32_t r0, jit_int32_t r1)
4389 jit_int32_t reg = jit_get_reg(jit_class_gpr);
4391 rshi(rn(reg), r1, 16);
4393 bswapr_us(rn(reg), rn(reg));
4395 orr(r0, r0, rn(reg));
4400 #if __WORDSIZE == 64
4401 static maybe_unused void
4402 generic_bswapr_ul(jit_state_t *_jit, jit_int32_t r0, jit_int32_t r1)
4404 jit_int32_t reg = jit_get_reg(jit_class_gpr);
4406 rshi_u(rn(reg), r1, 32);
4408 bswapr_ui(rn(reg), rn(reg));
4410 orr(r0, r0, rn(reg));
4417 _depi(jit_state_t *_jit,
4418 jit_int32_t r0, jit_word_t i0, jit_word_t i1, jit_word_t i2)
4421 reg = jit_get_reg(jit_class_gpr);
4423 depr(r0, rn(reg), i1, i2);
4428 _negi(jit_state_t *_jit, jit_int32_t r0, jit_word_t i0)
4434 _comi(jit_state_t *_jit, jit_int32_t r0, jit_word_t i0)
4440 _exti_c(jit_state_t *_jit, jit_int32_t r0, jit_word_t i0)
4442 movi(r0, (jit_int8_t)i0);
4446 _exti_uc(jit_state_t *_jit, jit_int32_t r0, jit_word_t i0)
4448 movi(r0, (jit_uint8_t)i0);
4452 _exti_s(jit_state_t *_jit, jit_int32_t r0, jit_word_t i0)
4454 movi(r0, (jit_int16_t)i0);
4458 _exti_us(jit_state_t *_jit, jit_int32_t r0, jit_word_t i0)
4460 movi(r0, (jit_uint16_t)i0);
4463 #if __WORDSIZE == 64
4465 _exti_i(jit_state_t *_jit, jit_int32_t r0, jit_word_t i0)
4467 movi(r0, (jit_int32_t)i0);
4471 _exti_ui(jit_state_t *_jit, jit_int32_t r0, jit_word_t i0)
4473 movi(r0, (jit_uint32_t)i0);
4478 _bswapi_us(jit_state_t *_jit, jit_int32_t r0, jit_word_t i0)
4491 _bswapi_ui(jit_state_t *_jit, jit_int32_t r0, jit_word_t i0)
4505 #if __WORDSIZE == 64
4507 _bswapi_ul(jit_state_t *_jit, jit_int32_t r0, jit_word_t i0)
4527 _htoni_us(jit_state_t *_jit, jit_int32_t r0, jit_word_t i0)
4529 #if __BYTE_ORDER == __LITTLE_ENDIAN
4537 _htoni_ui(jit_state_t *_jit, jit_int32_t r0, jit_word_t i0)
4539 #if __BYTE_ORDER == __LITTLE_ENDIAN
4542 # if __WORDSIZE == 32
4550 #if __WORDSIZE == 64
4552 _htoni_ul(jit_state_t *_jit, jit_int32_t r0, jit_word_t i0)
4554 # if __BYTE_ORDER == __LITTLE_ENDIAN
4563 _movi_f_w(jit_state_t *_jit, jit_int32_t r0, jit_float32_t i0)
4570 #if defined(__ia64__)
4571 /* Should be used only in this case (with out0 == 120) */
4573 r0 = _jitc->rout + (r0 - 120);
4578 #if __WORDSIZE == 32 && !(defined(__mips__) && NEW_ABI)
4580 _movi_d_ww(jit_state_t *_jit, jit_int32_t r0, jit_int32_t r1, jit_float64_t i0)
4587 /* Mips does not change byte order of double values */
4588 # if __BYTE_ORDER == __LITTLE_ENDIAN || defined(__mips__)
4589 movi(r0, data.i[0]);
4590 movi(r1, data.i[1]);
4592 movi(r1, data.i[0]);
4593 movi(r0, data.i[1]);
4599 _movi_d_w(jit_state_t *_jit, jit_int32_t r0, jit_float64_t i0)
4606 # if defined(__ia64__)
4607 /* Should be used only in this case (with out0 == 120) */
4609 r0 = _jitc->rout + (r0 - 120);
4616 _jit_negi_f(jit_state_t *_jit, jit_fpr_t u, jit_float32_t v)
4618 jit_inc_synth_wf(negi_f, u, v);
4625 _jit_absi_f(jit_state_t *_jit, jit_fpr_t u, jit_float32_t v)
4627 jit_inc_synth_wf(absi_f, u, v);
4634 _jit_sqrti_f(jit_state_t *_jit, jit_fpr_t u, jit_float32_t v)
4636 jit_inc_synth_wf(sqrti_f, u, v);
4643 _jit_fmai_f(jit_state_t *_jit,
4644 jit_fpr_t u, jit_fpr_t v, jit_fpr_t w, jit_float32_t x)
4647 jit_inc_synth_wqf(fmai_f, u, v, w, x);
4648 if (u != v && u != w) {
4650 jit_fmar_f(u, v, w, u);
4653 y = jit_get_reg(jit_class_fpr);
4655 jit_fmar_f(u, v, w, y);
4662 _jit_fmsi_f(jit_state_t *_jit,
4663 jit_fpr_t u, jit_fpr_t v, jit_fpr_t w, jit_float32_t x)
4666 jit_inc_synth_wqf(fmai_f, u, v, w, x);
4667 if (u != v && u != w) {
4669 jit_fmsr_f(u, v, w, u);
4672 y = jit_get_reg(jit_class_fpr);
4674 jit_fmsr_f(u, v, w, y);
4681 _jit_fnmai_f(jit_state_t *_jit,
4682 jit_fpr_t u, jit_fpr_t v, jit_fpr_t w, jit_float32_t x)
4685 jit_inc_synth_wqf(fmai_f, u, v, w, x);
4686 if (u != v && u != w) {
4688 jit_fnmar_f(u, v, w, u);
4691 y = jit_get_reg(jit_class_fpr);
4693 jit_fnmar_f(u, v, w, y);
4700 _jit_fnmsi_f(jit_state_t *_jit,
4701 jit_fpr_t u, jit_fpr_t v, jit_fpr_t w, jit_float32_t x)
4704 jit_inc_synth_wqf(fmai_f, u, v, w, x);
4705 if (u != v && u != w) {
4707 jit_fnmsr_f(u, v, w, u);
4710 y = jit_get_reg(jit_class_fpr);
4712 jit_fnmsr_f(u, v, w, y);
4719 _jit_negi_d(jit_state_t *_jit, jit_fpr_t u, jit_float64_t v)
4721 jit_inc_synth_wd(negi_d, u, v);
4728 _jit_absi_d(jit_state_t *_jit, jit_fpr_t u, jit_float64_t v)
4730 jit_inc_synth_wd(absi_d, u, v);
4737 _jit_sqrti_d(jit_state_t *_jit, jit_fpr_t u, jit_float64_t v)
4739 jit_inc_synth_wd(sqrti_d, u, v);
4746 _jit_fmai_d(jit_state_t *_jit,
4747 jit_fpr_t u, jit_fpr_t v, jit_fpr_t w, jit_float64_t x)
4750 jit_inc_synth_wqd(fmai_d, u, v, w, x);
4751 if (u != v && u != w) {
4753 jit_fmar_d(u, v, w, u);
4756 y = jit_get_reg(jit_class_fpr);
4758 jit_fmar_d(u, v, w, y);
4765 _jit_fmsi_d(jit_state_t *_jit,
4766 jit_fpr_t u, jit_fpr_t v, jit_fpr_t w, jit_float64_t x)
4769 jit_inc_synth_wqd(fmai_d, u, v, w, x);
4770 if (u != v && u != w) {
4772 jit_fmsr_d(u, v, w, u);
4775 y = jit_get_reg(jit_class_fpr);
4777 jit_fmsr_d(u, v, w, y);
4784 _jit_fnmai_d(jit_state_t *_jit,
4785 jit_fpr_t u, jit_fpr_t v, jit_fpr_t w, jit_float64_t x)
4788 jit_inc_synth_wqd(fmai_d, u, v, w, x);
4789 if (u != v && u != w) {
4791 jit_fnmar_d(u, v, w, u);
4794 y = jit_get_reg(jit_class_fpr);
4796 jit_fnmar_d(u, v, w, y);
4803 _jit_fnmsi_d(jit_state_t *_jit,
4804 jit_fpr_t u, jit_fpr_t v, jit_fpr_t w, jit_float64_t x)
4807 jit_inc_synth_wqd(fmai_d, u, v, w, x);
4808 if (u != v && u != w) {
4810 jit_fnmsr_d(u, v, w, u);
4813 y = jit_get_reg(jit_class_fpr);
4815 jit_fnmsr_d(u, v, w, y);
4822 _cloi(jit_state_t *_jit, jit_int32_t r0, jit_word_t i0)
4828 _clzi(jit_state_t *_jit, jit_int32_t r0, jit_word_t i0)
4830 #if __WORDSIZE == 64 && _WIN32
4831 movi(r0, (i0) ? __builtin_clzll(i0) : __WORDSIZE);
4833 movi(r0, (i0) ? __builtin_clzl(i0) : __WORDSIZE);
4838 _ctoi(jit_state_t *_jit, jit_int32_t r0, jit_word_t i0)
4844 _ctzi(jit_state_t *_jit, jit_int32_t r0, jit_word_t i0)
4846 #if __WORDSIZE == 64 && _WIN32
4847 movi(r0, (i0) ? __builtin_ctzll(i0) : __WORDSIZE);
4849 movi(r0, (i0) ? __builtin_ctzl(i0) : __WORDSIZE);
4854 _rbiti(jit_state_t *_jit, jit_int32_t r0, jit_word_t i0)
4859 jit_uint8_t v[__WORDSIZE >> 3];
4861 static const unsigned char swap_tab[256] = {
4862 0, 128, 64, 192, 32, 160, 96, 224,
4863 16, 144, 80, 208, 48, 176, 112, 240,
4864 8, 136, 72, 200, 40, 168, 104, 232,
4865 24, 152, 88, 216 ,56, 184, 120, 248,
4866 4, 132, 68, 196, 36, 164, 100, 228,
4867 20, 148, 84, 212, 52, 180, 116, 244,
4868 12, 140, 76, 204, 44, 172, 108, 236,
4869 28, 156, 92, 220, 60, 188, 124, 252,
4870 2, 130, 66, 194, 34, 162, 98, 226,
4871 18, 146, 82, 210, 50, 178, 114, 242,
4872 10, 138, 74, 202, 42, 170, 106, 234,
4873 26, 154, 90, 218, 58, 186, 122, 250,
4874 6, 134, 70, 198, 38, 166, 102, 230,
4875 22, 150, 86, 214, 54, 182, 118, 246,
4876 14, 142, 78, 206, 46, 174, 110, 238,
4877 30, 158, 94, 222, 62, 190, 126, 254,
4878 1, 129, 65, 193, 33, 161, 97, 225,
4879 17, 145, 81, 209, 49, 177, 113, 241,
4880 9, 137, 73, 201, 41, 169, 105, 233,
4881 25, 153, 89, 217, 57, 185, 121, 249,
4882 5, 133, 69, 197, 37, 165, 101, 229,
4883 21, 149, 85, 213, 53, 181, 117, 245,
4884 13, 141, 77, 205, 45, 173, 109, 237,
4885 29, 157, 93, 221, 61, 189, 125, 253,
4886 3, 131, 67, 195, 35, 163, 99, 227,
4887 19, 147, 83, 211, 51, 179, 115, 243,
4888 11, 139, 75, 203, 43, 171, 107, 235,
4889 27, 155, 91, 219, 59, 187, 123, 251,
4890 7, 135, 71, 199, 39, 167, 103, 231,
4891 23, 151, 87, 215, 55, 183, 119, 247,
4892 15, 143, 79, 207, 47, 175, 111, 239,
4893 31, 159, 95, 223, 63, 191, 127, 255
4896 for (i = 0; i < sizeof(jit_word_t); ++i)
4897 v.v[i] = swap_tab[u.v[sizeof(jit_word_t) - i - 1]];
4902 _popcnti(jit_state_t *_jit, jit_int32_t r0, jit_word_t i0)
4904 #if __WORDSIZE == 64 && _WIN32
4905 movi(r0, (i0) ? __builtin_popcountll(i0) : __WORDSIZE);
4907 movi(r0, (i0) ? __builtin_popcountl(i0) : __WORDSIZE);
4911 static void _exti(jit_state_t *_jit,
4912 jit_int32_t r0, jit_word_t i0, jit_word_t i1, jit_word_t i2)
4914 #if __BYTE_ORDER == __BIG_ENDIAN
4915 i1 = __WORDSIZE - (i1 + i2);
4917 i0 <<= __WORDSIZE - (i1 + i2);
4918 i0 >>= __WORDSIZE - i2;
4922 static void _exti_u(jit_state_t *_jit,
4923 jit_int32_t r0, jit_word_t i0, jit_word_t i1, jit_word_t i2)
4926 #if __BYTE_ORDER == __BIG_ENDIAN
4927 i1 = __WORDSIZE - (i1 + i2);
4930 i0 >>= __WORDSIZE - i2;
4931 #if __WORDSIZE == 64 && !_WIN32
4932 i0 &= (1L << i2) - 1;
4934 i0 &= (1LL << i2) - 1;
4940 _generic_unldr(jit_state_t *_jit,
4941 jit_int32_t r0, jit_int32_t r1, jit_word_t i0)
4944 assert(i0 >= 1 && i0 <= sizeof(jit_word_t));
4945 if (i0 & (i0 - 1)) {
4946 t0 = jit_get_reg(jit_class_gpr);
4958 #if __BYTE_ORDER == __LITTLE_ENDIAN
4968 #if __WORDSIZE == 32
4977 # if __BYTE_ORDER == __LITTLE_ENDIAN
4988 # if __BYTE_ORDER == __LITTLE_ENDIAN
4999 # if __BYTE_ORDER == __LITTLE_ENDIAN
5016 if (i0 & (i0 - 1)) {
5023 _generic_unldi(jit_state_t *_jit,
5024 jit_int32_t r0, jit_word_t i0, jit_word_t i1)
5027 assert(i1 >= 1 && i1 <= sizeof(jit_word_t));
5028 if (i1 & (i1 - 1)) {
5029 t0 = jit_get_reg(jit_class_gpr);
5040 # if __BYTE_ORDER == __LITTLE_ENDIAN
5050 # if __WORDSIZE == 32
5059 # if __BYTE_ORDER == __LITTLE_ENDIAN
5070 # if __BYTE_ORDER == __LITTLE_ENDIAN
5081 # if __BYTE_ORDER == __LITTLE_ENDIAN
5098 if (i1 & (i1 - 1)) {
5105 _generic_unldr_u(jit_state_t *_jit,
5106 jit_int32_t r0, jit_int32_t r1, jit_word_t i0)
5109 assert(i0 >= 1 && i0 <= sizeof(jit_word_t));
5110 if (i0 & (i0 - 1)) {
5111 t0 = jit_get_reg(jit_class_gpr);
5123 # if __BYTE_ORDER == __LITTLE_ENDIAN
5133 # if __WORDSIZE == 32
5142 # if __BYTE_ORDER == __LITTLE_ENDIAN
5153 # if __BYTE_ORDER == __LITTLE_ENDIAN
5164 # if __BYTE_ORDER == __LITTLE_ENDIAN
5181 if (i0 & (i0 - 1)) {
5188 _generic_unldi_u(jit_state_t *_jit,
5189 jit_int32_t r0, jit_word_t i0, jit_word_t i1)
5192 assert(i1 >= 1 && i1 <= sizeof(jit_word_t));
5193 if (i1 & (i1 - 1)) {
5194 t0 = jit_get_reg(jit_class_gpr);
5205 # if __BYTE_ORDER == __LITTLE_ENDIAN
5215 # if __WORDSIZE == 32
5224 # if __BYTE_ORDER == __LITTLE_ENDIAN
5235 # if __BYTE_ORDER == __LITTLE_ENDIAN
5246 # if __BYTE_ORDER == __LITTLE_ENDIAN
5263 if (i1 & (i1 - 1)) {
5270 _generic_unstr(jit_state_t *_jit, jit_int32_t r0, jit_int32_t r1, jit_word_t i0)
5273 assert(i0 > 0 && i0 <= sizeof(jit_word_t));
5274 if (i0 & (i0 - 1)) {
5275 t0 = jit_get_reg(jit_class_gpr);
5286 # if __BYTE_ORDER == __LITTLE_ENDIAN
5296 # if __WORDSIZE == 32
5305 # if __BYTE_ORDER == __LITTLE_ENDIAN
5316 # if __BYTE_ORDER == __LITTLE_ENDIAN
5327 # if __BYTE_ORDER == __LITTLE_ENDIAN
5351 _generic_unsti(jit_state_t *_jit, jit_word_t i0, jit_int32_t r0, jit_word_t i1)
5354 assert(i1 > 0 && i1 <= sizeof(jit_word_t));
5355 if (i1 & (i1 - 1)) {
5356 t0 = jit_get_reg(jit_class_gpr);
5367 # if __BYTE_ORDER == __LITTLE_ENDIAN
5377 # if __WORDSIZE == 32
5386 # if __BYTE_ORDER == __LITTLE_ENDIAN
5397 # if __BYTE_ORDER == __LITTLE_ENDIAN
5408 # if __BYTE_ORDER == __LITTLE_ENDIAN
5431 #if !defined(__i386__) && !defined(__x86_64__) && !defined(__arm__)
5433 _generic_unldr_x(jit_state_t *_jit,
5434 jit_int32_t r0, jit_int32_t r1, jit_word_t i0)
5436 assert(i0 == 4 || i0 == 8);
5444 _generic_unldi_x(jit_state_t *_jit,
5445 jit_int32_t r0, jit_word_t i0, jit_word_t i1)
5447 assert(i1 == 4 || i1 == 8);
5455 _generic_unstr_x(jit_state_t *_jit,
5456 jit_int32_t r0, jit_int32_t r1, jit_word_t i0)
5458 assert(i0 == 4 || i0 == 8);
5466 _generic_unsti_x(jit_state_t *_jit,
5467 jit_word_t i0, jit_int32_t r0, jit_word_t i1)
5469 assert(i1 == 4 || i1 == 8);
5477 #if defined(stack_framesize)
5478 static maybe_unused void
5479 _patch_alist(jit_state_t *_jit, jit_bool_t revert)
5483 diff = jit_diffsize();
5487 for (node = _jitc->function->alist; node; node = node->link) {
5488 switch (node->code) {
5489 case jit_code_ldxi_c: case jit_code_ldxi_uc:
5490 case jit_code_ldxi_s: case jit_code_ldxi_us:
5491 case jit_code_ldxi_i:
5492 #if __WORDSIZE == 64
5493 case jit_code_ldxi_ui: case jit_code_ldxi_l:
5495 case jit_code_ldxi_f: case jit_code_ldxi_d:
5498 case jit_code_stxi_c: case jit_code_stxi_s:
5499 case jit_code_stxi_i:
5500 #if __WORDSIZE == 64
5501 case jit_code_stxi_l:
5503 case jit_code_stxi_f: case jit_code_stxi_d: