2 * Copyright (C) 2012-2023 Free Software Foundation, Inc.
4 * This file is part of GNU lightning.
6 * GNU lightning is free software; you can redistribute it and/or modify it
7 * under the terms of the GNU Lesser General Public License as published
8 * by the Free Software Foundation; either version 3, or (at your option)
11 * GNU lightning is distributed in the hope that it will be useful, but
12 * WITHOUT ANY WARRANTY; without even the implied warranty of MERCHANTABILITY
13 * or FITNESS FOR A PARTICULAR PURPOSE. See the GNU Lesser General Public
14 * License for more details.
17 * Paulo Cesar Pereira de Andrade
20 #include <lightning.h>
21 #include <lightning/jit_private.h>
23 # include <sys/mman.h>
30 # define MAP_ANON MAP_ANONYMOUS
31 # ifndef MAP_ANONYMOUS
32 # define MAP_ANONYMOUS 0
36 #define jit_regload_reload 0 /* convert to reload */
37 #define jit_regload_delete 1 /* just remove node */
38 #define jit_regload_isdead 2 /* delete and unset live bit */
43 static jit_word_t hash_data(const void*, jit_word_t);
45 #define new_pool() _new_pool(_jit)
46 static void _new_pool(jit_state_t*);
48 #define new_node(u) _new_node(_jit, u)
49 static jit_node_t *_new_node(jit_state_t*, jit_code_t);
51 #define link_node(u) _link_node(_jit, u)
52 static inline jit_node_t *_link_node(jit_state_t*, jit_node_t*);
54 #define del_node(u, v) _del_node(_jit, u, v)
55 static inline void _del_node(jit_state_t*, jit_node_t*, jit_node_t*);
57 #define free_node(u) _free_node(_jit, u)
58 static inline void _free_node(jit_state_t*, jit_node_t*);
60 #define del_label(u, v) _del_label(_jit, u, v)
61 static void _del_label(jit_state_t*, jit_node_t*, jit_node_t*);
63 #define jit_dataset() _jit_dataset(_jit)
65 _jit_dataset(jit_state_t *_jit);
67 #define block_update_set(block, target) _block_update_set(_jit, block, target)
68 static jit_bool_t _block_update_set(jit_state_t*, jit_block_t*, jit_block_t*);
70 #define propagate_backward(block) _propagate_backward(_jit, block)
71 static void _propagate_backward(jit_state_t*, jit_block_t*);
73 #define check_block_again() _check_block_again(_jit)
74 static jit_bool_t _check_block_again(jit_state_t*);
76 #define do_setup() _do_setup(_jit)
77 static void _do_setup(jit_state_t*);
79 #define jit_setup(block) _jit_setup(_jit, block)
81 _jit_setup(jit_state_t *_jit, jit_block_t *block);
83 #define do_follow(always) _do_follow(_jit, always)
84 static void _do_follow(jit_state_t*, jit_bool_t);
86 #define jit_follow(block) _jit_follow(_jit, block)
88 _jit_follow(jit_state_t *_jit, jit_block_t *block);
90 #define jit_update(node, live, mask) _jit_update(_jit, node, live, mask)
92 _jit_update(jit_state_t *_jit, jit_node_t *node,
93 jit_regset_t *live, jit_regset_t *mask);
95 #define thread_jumps() _thread_jumps(_jit)
97 _thread_jumps(jit_state_t *_jit);
99 #define sequential_labels() _sequential_labels(_jit)
101 _sequential_labels(jit_state_t *_jit);
103 #define split_branches() _split_branches(_jit)
105 _split_branches(jit_state_t *_jit);
107 #define shortcut_jump(prev, node) _shortcut_jump(_jit, prev, node)
109 _shortcut_jump(jit_state_t *_jit, jit_node_t *prev, jit_node_t *node);
111 #define redundant_jump(prev, node) _redundant_jump(_jit, prev, node)
113 _redundant_jump(jit_state_t *_jit, jit_node_t *prev, jit_node_t *node);
116 reverse_jump_code(jit_code_t code);
118 #define reverse_jump(prev, node) _reverse_jump(_jit, prev, node)
120 _reverse_jump(jit_state_t *_jit, jit_node_t *prev, jit_node_t *node);
122 #define redundant_store(node, jump) _redundant_store(_jit, node, jump)
124 _redundant_store(jit_state_t *_jit, jit_node_t *node, jit_bool_t jump);
126 #define simplify_movr(p, n, k, s) _simplify_movr(_jit, p, n, k, s)
128 _simplify_movr(jit_state_t *_jit, jit_node_t *prev, jit_node_t *node,
129 jit_int32_t kind, jit_int32_t size);
131 #define simplify_movi(p, n, k, s) _simplify_movi(_jit, p, n, k, s)
133 _simplify_movi(jit_state_t *_jit, jit_node_t *prev, jit_node_t *node,
134 jit_int32_t kind, jit_int32_t size);
136 #define simplify_ldxi(prev, node) _simplify_ldxi(_jit, prev, node)
138 _simplify_ldxi(jit_state_t *_jit, jit_node_t *prev, jit_node_t *node);
140 #define simplify_stxi(prev, node) _simplify_stxi(_jit, prev, node)
142 _simplify_stxi(jit_state_t *_jit, jit_node_t *prev, jit_node_t *node);
144 #define simplify_spill(node, regno) _simplify_spill(_jit, node, regno)
146 _simplify_spill(jit_state_t *_jit, jit_node_t *node, jit_int32_t regno);
148 #define simplify() _simplify(_jit)
150 _simplify(jit_state_t *_jit);
152 #define jit_reg_undef -1
153 #define jit_reg_static 0
154 #define jit_reg_change 1
155 #define register_change_p(n, l, r) _register_change_p(_jit, n, l, r)
157 _register_change_p(jit_state_t *_jit, jit_node_t *node, jit_node_t *link,
160 #define spill_reglive_p(node, regno) _spill_reglive_p(_jit, node, regno)
162 _spill_reglive_p(jit_state_t *_jit, jit_node_t *node, jit_int32_t regno);
164 #define patch_registers() _patch_registers(_jit)
166 _patch_registers(jit_state_t *_jit);
168 #define patch_register(n,l,r,p) _patch_register(_jit,n,l,r,p)
170 _patch_register(jit_state_t *jit, jit_node_t *node, jit_node_t *link,
171 jit_int32_t regno, jit_int32_t patch);
184 init_jit(const char *progname)
187 jit_init_debug(progname);
199 _jit_get_reg(jit_state_t *_jit, jit_int32_t regspec)
204 spec = regspec & ~(jit_class_chk|jit_class_nospill);
205 if (spec & jit_class_named) {
206 regno = jit_regno(spec);
207 if (jit_regset_tstbit(&_jitc->regsav, regno))
208 /* fail if register is spilled */
210 if (jit_regset_tstbit(&_jitc->regarg, regno))
211 /* fail if register is an argument to current instruction */
213 if (jit_regset_tstbit(&_jitc->reglive, regno)) {
214 if (regspec & jit_class_nospill)
215 /* fail if register is live and should not spill/reload */
219 jit_regset_setbit(&_jitc->regarg, regno);
223 assert(jit_class(spec) != 0);
226 /* search for a free register matching spec */
227 for (regno = 0; regno < _jitc->reglen; regno++) {
228 if ((jit_class(_rvs[regno].spec) & spec) == spec &&
229 !jit_regset_tstbit(&_jitc->regarg, regno) &&
230 !jit_regset_tstbit(&_jitc->reglive, regno)) {
231 if (jit_regset_tstbit(&_jitc->regmask, regno)) {
232 /* search further, attempting to find a truly known
233 * free register, not just one in unknown state. */
236 for (regfree = regno + 1;
237 regfree < _jitc->reglen; regfree++) {
238 if ((jit_class(_rvs[regfree].spec) & spec) == spec &&
239 !jit_regset_tstbit(&_jitc->regarg, regfree) &&
240 !jit_regset_tstbit(&_jitc->reglive, regfree) &&
241 !jit_regset_tstbit(&_jitc->regmask, regfree)) {
251 /* search for a register matching spec that is not an argument
252 * for the current instruction */
253 for (regno = 0; regno < _jitc->reglen; regno++) {
254 if ((jit_class(_rvs[regno].spec) & spec) == spec &&
255 !jit_regset_tstbit(&_jitc->regsav, regno) &&
256 !jit_regset_tstbit(&_jitc->regarg, regno) &&
257 !(regspec & jit_class_nospill)) {
259 assert(_jitc->function != NULL);
260 if (spec & jit_class_gpr) {
261 if (!_jitc->function->regoff[regno]) {
262 _jitc->function->regoff[regno] =
263 jit_allocai(sizeof(jit_word_t));
267 /* emit_stxi must not need temporary registers */
268 assert(!_jitc->getreg);
271 emit_stxi(_jitc->function->regoff[regno], JIT_FP, regno);
277 if (!_jitc->function->regoff[regno]) {
278 _jitc->function->regoff[regno] =
279 jit_allocai(sizeof(jit_float64_t));
283 /* emit_stxi must not need temporary registers */
284 assert(!_jitc->getreg);
287 emit_stxi_d(_jitc->function->regoff[regno], JIT_FP, regno);
292 jit_regset_setbit(&_jitc->regsav, regno);
294 jit_regset_setbit(&_jitc->regarg, regno);
295 if (jit_class(_rvs[regno].spec) & jit_class_sav) {
296 /* if will modify callee save registers without a
297 * function prolog, better patch this assertion */
298 assert(_jitc->function != NULL);
299 if (!jit_regset_tstbit(&_jitc->function->regset, regno)) {
300 jit_regset_setbit(&_jitc->function->regset, regno);
309 /* nospill hint only valid during emit" */
310 assert(!(regspec & jit_class_nospill));
311 for (regno = 0; regno < _jitc->reglen; regno++) {
312 if ((jit_class(_rvs[regno].spec) & spec) == spec &&
313 !jit_regset_tstbit(&_jitc->regsav, regno) &&
314 !jit_regset_tstbit(&_jitc->regarg, regno)) {
315 jit_regset_setbit(&_jitc->regarg, regno);
316 jit_regset_setbit(&_jitc->regsav, regno);
318 return (jit_regno_patch|regno);
323 /* Out of hardware registers */
325 assert(regspec & jit_class_chk);
330 _jit_unget_reg(jit_state_t *_jit, jit_int32_t regno)
332 regno = jit_regno(regno);
333 if (jit_regset_tstbit(&_jitc->regsav, regno)) {
336 /* emit_ldxi must not need a temporary register */
337 assert(!_jitc->getreg);
340 if (jit_class(_rvs[regno].spec) & jit_class_gpr)
341 emit_ldxi(regno, JIT_FP, _jitc->function->regoff[regno]);
343 emit_ldxi_d(regno, JIT_FP, _jitc->function->regoff[regno]);
345 /* emit_ldxi must not need a temporary register */
351 jit_regset_clrbit(&_jitc->regsav, regno);
353 #if defined(jit_carry)
354 assert((regno == jit_carry /*&& _NOREG != jit_carry*/) ||
355 jit_regset_tstbit(&_jitc->regarg, regno) != 0);
357 assert(jit_regset_tstbit(&_jitc->regarg, regno) != 0);
359 jit_regset_clrbit(&_jitc->regarg, regno);
363 _jit_callee_save_p(jit_state_t *_jit, jit_int32_t regno)
365 assert(regno >= 0 && regno < JIT_NOREG);
366 return (!!(_rvs[regno].spec & jit_class_sav));
370 _jit_pointer_p(jit_state_t *_jit, jit_pointer_t address)
372 return ((jit_uint8_t *)address >= _jit->code.ptr &&
373 (jit_word_t)address < _jit->pc.w);
378 jit_regset_com(jit_regset_t *u, jit_regset_t *v)
380 u->rl = ~v->rl; u->rh = ~v->rh;
381 u->fl = ~v->fl; u->fh = ~v->fh;
385 jit_regset_and(jit_regset_t *u, jit_regset_t *v, jit_regset_t *w)
387 u->rl = v->rl & w->rl; u->rh = v->rh & w->rh;
388 u->fl = v->fl & w->fl; u->fh = v->fh & w->fh;
392 jit_regset_ior(jit_regset_t *u, jit_regset_t *v, jit_regset_t *w)
394 u->rl = v->rl | w->rl; u->rh = v->rh | w->rh;
395 u->fl = v->fl | w->fl; u->fh = v->fh | w->fh;
399 jit_regset_xor(jit_regset_t *u, jit_regset_t *v, jit_regset_t *w)
401 u->rl = v->rl ^ w->rl; u->rh = v->rh ^ w->rh;
402 u->fl = v->fl ^ w->fl; u->fh = v->fh ^ w->fh;
406 jit_regset_set(jit_regset_t *u, jit_regset_t *v)
408 u->rl = v->rl; u->rh = v->rh;
409 u->fl = v->fl; u->fh = v->fh;
413 jit_regset_set_mask(jit_regset_t *u, jit_int32_t v)
415 jit_bool_t w = !!(v & (v - 1));
417 assert(v >= 0 && v <= 256);
419 u->rl = u->rh = u->fl = u->fh = -1LL;
421 u->rl = w ? (1LL << v) - 1 : -1LL;
422 u->rh = u->fl = u->fh = 0;
426 u->rh = w ? (1LL << (v - 64)) - 1 : -1LL;
430 u->rl = u->rh = -1LL;
431 u->fl = w ? (1LL << (v - 128)) - 1 : -1LL;
435 u->rl = u->rh = u->fl = -1LL;
436 u->fh = w ? (1LL << (v - 128)) - 1 : -1LL;
441 jit_regset_cmp_ui(jit_regset_t *u, jit_word_t v)
443 return !((u->rl == v && u->rh == 0 && u->fl == 0 && u->fh == 0));
447 jit_regset_set_ui(jit_regset_t *u, jit_word_t v)
450 u->rh = u->fl = u->fh = 0;
454 jit_regset_set_p(jit_regset_t *u)
456 return (u->rl || u->rh || u->fl || u->fh);
460 jit_regset_clrbit(jit_regset_t *set, jit_int32_t bit)
462 assert(bit >= 0 && bit <= 255);
464 set->rl &= ~(1LL << bit);
466 set->rh &= ~(1LL << (bit - 64));
468 set->fl &= ~(1LL << (bit - 128));
470 set->fh &= ~(1LL << (bit - 192));
474 jit_regset_setbit(jit_regset_t *set, jit_int32_t bit)
476 assert(bit >= 0 && bit <= 255);
478 set->rl |= 1LL << bit;
480 set->rh |= 1LL << (bit - 64);
482 set->fl |= 1LL << (bit - 128);
484 set->fh |= 1LL << (bit - 192);
488 jit_regset_tstbit(jit_regset_t *set, jit_int32_t bit)
490 assert(bit >= 0 && bit <= 255);
492 return (!!(set->rl & (1LL << bit)));
494 return (!!(set->rh & (1LL << (bit - 64))));
496 return (!!(set->fl & (1LL << (bit - 128))));
497 return (!!(set->fh & (1LL << (bit - 192))));
501 jit_regset_scan1(jit_regset_t *set, jit_int32_t offset)
503 assert(offset >= 0 && offset <= 255);
504 for (; offset < 64; offset++) {
505 if (set->rl & (1LL << offset))
508 for (; offset < 128; offset++) {
509 if (set->rh & (1LL << (offset - 64)))
512 for (; offset < 192; offset++) {
513 if (set->fl & (1LL << (offset - 128)))
516 for (; offset < 256; offset++) {
517 if (set->fh & (1LL << (offset - 192)))
523 #elif __sparc__ && __WORDSIZE == 64
525 jit_regset_com(jit_regset_t *u, jit_regset_t *v)
527 u->rl = ~v->rl; u->rh = ~v->rh;
531 jit_regset_and(jit_regset_t *u, jit_regset_t *v, jit_regset_t *w)
533 u->rl = v->rl & w->rl; u->rh = v->rh & w->rh;
537 jit_regset_ior(jit_regset_t *u, jit_regset_t *v, jit_regset_t *w)
539 u->rl = v->rl | w->rl; u->rh = v->rh | w->rh;
543 jit_regset_xor(jit_regset_t *u, jit_regset_t *v, jit_regset_t *w)
545 u->rl = v->rl ^ w->rl; u->rh = v->rh ^ w->rh;
549 jit_regset_set(jit_regset_t *u, jit_regset_t *v)
551 u->rl = v->rl; u->rh = v->rh;
555 jit_regset_set_mask(jit_regset_t *u, jit_int32_t v)
557 jit_bool_t w = !!(v & (v - 1));
559 assert(v >= 0 && v <= 128);
561 u->rl = u->rh = -1LL;
563 u->rl = w ? (1LL << v) - 1 : -1LL;
568 u->rh = w ? (1LL << (v - 64)) - 1 : -1LL;
573 jit_regset_cmp_ui(jit_regset_t *u, jit_word_t v)
575 return !((u->rl == v && u->rh == 0));
579 jit_regset_set_ui(jit_regset_t *u, jit_word_t v)
586 jit_regset_set_p(jit_regset_t *u)
588 return (u->rl || u->rh);
592 jit_regset_clrbit(jit_regset_t *set, jit_int32_t bit)
594 assert(bit >= 0 && bit <= 128);
596 set->rl &= ~(1LL << bit);
598 set->rh &= ~(1LL << (bit - 64));
602 jit_regset_setbit(jit_regset_t *set, jit_int32_t bit)
604 assert(bit >= 0 && bit <= 127);
606 set->rl |= 1LL << bit;
608 set->rh |= 1LL << (bit - 64);
612 jit_regset_tstbit(jit_regset_t *set, jit_int32_t bit)
614 assert(bit >= 0 && bit <= 127);
616 return (!!(set->rl & (1LL << bit)));
618 return (!!(set->rh & (1LL << (bit - 64))));
622 jit_regset_scan1(jit_regset_t *set, jit_int32_t offset)
624 assert(offset >= 0 && offset <= 127);
625 for (; offset < 64; offset++) {
626 if (set->rl & (1LL << offset))
629 for (; offset < 128; offset++) {
630 if (set->rh & (1LL << (offset - 64)))
638 jit_regset_scan1(jit_regset_t *set, jit_int32_t offset)
641 assert(offset >= 0 && offset <= 63);
642 if ((mask = *set >> offset)) {
655 _jit_save(jit_state_t *_jit, jit_int32_t reg)
657 reg = jit_regno(reg);
658 assert(!_jitc->realize);
659 _jitc->spill[reg] = jit_new_node_w(jit_code_save, reg);
663 _jit_load(jit_state_t *_jit, jit_int32_t reg)
667 reg = jit_regno(reg);
668 assert(!_jitc->realize);
669 assert(_jitc->spill[reg] != NULL);
670 node = jit_new_node_w(jit_code_load, reg);
671 /* create a path to flag the save/load is not required */
672 node->link = _jitc->spill[reg];
673 node->link->link = node;
674 _jitc->spill[reg] = NULL;
678 hash_data(const void *data, jit_word_t length)
680 const jit_uint8_t *ptr;
682 for (i = key = 0, ptr = data; i < length; i++)
683 key = (key << (key & 1)) ^ ptr[i];
688 _jit_address(jit_state_t *_jit, jit_node_t *node)
691 assert(node != NULL &&
692 /* If a node type that is documented to be a fixed marker */
693 (node->code == jit_code_note || node->code == jit_code_name ||
694 /* If another special fixed marker, returned by jit_indirect() */
695 (node->code == jit_code_label && (node->flag & jit_flag_use) != 0)));
696 return ((jit_pointer_t)node->u.w);
700 _jit_data(jit_state_t *_jit, const void *data,
701 jit_word_t length, jit_int32_t align)
706 assert(!_jitc->realize);
708 /* Ensure there is space even if asking for a duplicate */
709 if (((_jitc->data.offset + 7) & -8) + length > _jit->data.length) {
712 size = (_jit->data.length + length + 4096) & - 4095;
713 assert(size >= _jit->data.length);
714 if (_jitc->data.ptr == NULL)
715 jit_alloc((jit_pointer_t *)&_jitc->data.ptr, size);
717 jit_realloc((jit_pointer_t *)&_jitc->data.ptr,
718 _jit->data.length, size);
719 _jit->data.length = size;
721 if (_jitc->data.table == NULL)
722 jit_alloc((jit_pointer_t *)&_jitc->data.table,
723 (_jitc->data.size = 16) * sizeof(jit_node_t*));
725 key = hash_data(data, length) & (_jitc->data.size - 1);
726 node = _jitc->data.table[key];
727 for (; node; node = node->next) {
728 if (node->v.w == length &&
729 memcmp(_jitc->data.ptr + node->u.w, data, length) == 0)
734 node = jit_new_node_no_link(jit_code_data);
741 _jitc->data.offset = (_jitc->data.offset + 1) & -2;
744 _jitc->data.offset = (_jitc->data.offset + 3) & -4;
747 _jitc->data.offset = (_jitc->data.offset + 7) & -8;
750 node->u.w = _jitc->data.offset;
752 jit_memcpy(_jitc->data.ptr + _jitc->data.offset, data, length);
753 _jitc->data.offset += length;
755 node->next = _jitc->data.table[key];
756 _jitc->data.table[key] = node;
759 /* Rehash if more than 75% used table */
760 if (_jitc->data.count >
761 (_jitc->data.size >> 1) + (_jitc->data.size >> 2) &&
762 (_jitc->data.size << 1) > _jitc->data.size) {
768 jit_alloc((jit_pointer_t *)&hash,
769 (_jitc->data.size << 1) * sizeof(jit_node_t*));
770 for (i = 0; i < _jitc->data.size; i++) {
771 temp = _jitc->data.table[i];
772 for (; temp; temp = next) {
774 key = hash_data(_jitc->data.ptr + temp->u.w, temp->v.w) &
775 ((_jitc->data.size << 1) - 1);
776 temp->next = hash[key];
780 jit_free((jit_pointer_t *)&_jitc->data.table);
781 _jitc->data.table = hash;
782 _jitc->data.size <<= 1;
790 _new_pool(jit_state_t *_jit)
795 if (_jitc->pool.offset >= _jitc->pool.length) {
798 length = _jitc->pool.length + 16;
799 jit_realloc((jit_pointer_t *)&_jitc->pool.ptr,
800 _jitc->pool.length * sizeof(jit_node_t *),
801 length * sizeof(jit_node_t *));
802 _jitc->pool.length = length;
804 jit_alloc((jit_pointer_t *)(_jitc->pool.ptr + _jitc->pool.offset),
805 sizeof(jit_node_t) * 1024);
806 list = _jitc->pool.ptr[_jitc->pool.offset];
807 for (offset = 1; offset < 1024; offset++, list++)
808 list->next = list + 1;
809 list->next = _jitc->list;
810 _jitc->list = _jitc->pool.ptr[_jitc->pool.offset];
811 ++_jitc->pool.offset;
815 _new_node(jit_state_t *_jit, jit_code_t code)
819 if (_jitc->list == NULL)
822 _jitc->list = node->next;
824 node->flag |= jit_flag_synth;
831 static inline jit_node_t *
832 _link_node(jit_state_t *_jit, jit_node_t *node)
835 _jitc->tail->next = node;
838 return (_jitc->tail = node);
842 _del_node(jit_state_t *_jit, jit_node_t *prev, jit_node_t *node)
845 assert(prev == _jitc->head);
846 _jitc->head = node->next;
849 prev->next = node->next;
850 memset(node, 0, sizeof(jit_node_t));
851 node->next = _jitc->list;
856 _free_node(jit_state_t *_jit, jit_node_t *node)
858 memset(node, 0, sizeof(jit_node_t));
859 node->next = _jitc->list;
864 _del_label(jit_state_t *_jit, jit_node_t *prev, jit_node_t *node)
868 /* only allow call to del_label on linked labels */
869 block = _jitc->blocks.ptr + node->v.w;
870 assert(block->label == node);
872 /* del_label() should only be called when optimizing.
873 * This will leave an empty block index */
874 jit_regset_del(&block->reglive);
875 jit_regset_del(&block->regmask);
878 /* redundant, should be already true */
879 assert(node->link == NULL);
880 del_node(prev, node);
888 jit_alloc((jit_pointer_t *)&_jit, sizeof(jit_state_t));
889 jit_alloc((jit_pointer_t *)&_jitc, sizeof(jit_compiler_t));
890 jit_regset_new(&_jitc->regarg);
891 jit_regset_new(&_jitc->regsav);
892 jit_regset_new(&_jitc->reglive);
893 jit_regset_new(&_jitc->regmask);
894 jit_regset_new(&_jitc->explive);
898 jit_alloc((jit_pointer_t *)&_jitc->spill,
899 _jitc->reglen * sizeof(jit_node_t*));
900 jit_alloc((jit_pointer_t *)&_jitc->gen,
901 _jitc->reglen * sizeof(jit_int32_t));
902 jit_alloc((jit_pointer_t *)&_jitc->values,
903 _jitc->reglen * sizeof(jit_value_t));
905 jit_alloc((jit_pointer_t *)&_jitc->patches.ptr,
906 (_jitc->patches.length = 1024) * sizeof(jit_patch_t));
907 jit_alloc((jit_pointer_t *)&_jitc->functions.ptr,
908 (_jitc->functions.length = 16) * sizeof(jit_function_t));
909 jit_alloc((jit_pointer_t *)&_jitc->pool.ptr,
910 (_jitc->pool.length = 16) * sizeof(jit_node_t*));
911 jit_alloc((jit_pointer_t *)&_jitc->blocks.ptr,
912 (_jitc->blocks.length = 16) * sizeof(jit_block_t));
913 #if __arm__ && DISASSEMBLER
914 jit_alloc((jit_pointer_t *)&_jitc->data_info.ptr,
915 (_jitc->data_info.length = 1024) * sizeof(jit_data_info_t));
918 /* allocate at most one extra note in case jit_name() is
919 * never called, or called after adding at least one note */
920 _jit->note.length = 1;
921 _jitc->note.size = sizeof(jit_note_t);
927 _jit_clear_state(jit_state_t *_jit)
929 #if DEVEL_DISASSEMBLER
930 # define jit_really_clear_state() _jit_really_clear_state(_jit)
933 void _jit_really_clear_state(jit_state_t *_jit)
937 jit_function_t *function;
939 /* release memory not required at jit execution time and set
940 * pointers to NULL to explicitly know they are released */
941 _jitc->head = _jitc->tail = NULL;
943 jit_free((jit_pointer_t *)&_jitc->data.table);
944 _jitc->data.size = _jitc->data.count = 0;
946 jit_free((jit_pointer_t *)&_jitc->spill);
947 jit_free((jit_pointer_t *)&_jitc->gen);
948 jit_free((jit_pointer_t *)&_jitc->values);
950 jit_free((jit_pointer_t *)&_jitc->blocks.ptr);
952 jit_free((jit_pointer_t *)&_jitc->patches.ptr);
953 _jitc->patches.offset = _jitc->patches.length = 0;
955 for (offset = 0; offset < _jitc->functions.offset; offset++) {
956 function = _jitc->functions.ptr + offset;
957 jit_free((jit_pointer_t *)&function->regoff);
959 jit_free((jit_pointer_t *)&_jitc->functions.ptr);
960 _jitc->functions.offset = _jitc->functions.length = 0;
961 _jitc->function = NULL;
963 for (offset = 0; offset < _jitc->pool.offset; offset++)
964 jit_free((jit_pointer_t *)(_jitc->pool.ptr + offset));
965 jit_free((jit_pointer_t *)&_jitc->pool.ptr);
966 _jitc->pool.offset = _jitc->pool.length = 0;
969 _jitc->note.head = _jitc->note.tail =
970 _jitc->note.name = _jitc->note.note = NULL;
971 _jitc->note.base = NULL;
973 #if __arm__ && DISASSEMBLER
974 jit_free((jit_pointer_t *)&_jitc->data_info.ptr);
977 #if (__powerpc__ && _CALL_AIXDESC) || __ia64__
978 jit_free((jit_pointer_t *)&_jitc->prolog.ptr);
982 jit_regset_del(&_jitc->regs);
985 jit_free((jit_pointer_t *)&_jitc);
989 _jit_destroy_state(jit_state_t *_jit)
991 #if DEVEL_DISASSEMBLER
992 jit_really_clear_state();
995 if (!_jit->user_code)
996 munmap(_jit->code.ptr, _jit->code.length);
997 if (!_jit->user_data)
998 munmap(_jit->data.ptr, _jit->data.length);
1000 jit_free((jit_pointer_t *)&_jit);
1004 _jit_synth_inc(jit_state_t *_jit)
1006 assert(_jitc->synth < 8);
1011 _jit_new_node(jit_state_t *_jit, jit_code_t code)
1013 assert(!_jitc->realize);
1014 return (link_node(new_node(code)));
1018 _jit_new_node_no_link(jit_state_t *_jit, jit_code_t code)
1020 assert(!_jitc->realize);
1021 return (new_node(code));
1025 _jit_link_node(jit_state_t *_jit, jit_node_t *node)
1027 assert(!_jitc->realize);
1032 _jit_synth_dec(jit_state_t *_jit)
1034 assert(_jitc->synth > 0);
1039 _jit_new_node_w(jit_state_t *_jit, jit_code_t code,
1042 jit_node_t *node = new_node(code);
1043 assert(!_jitc->realize);
1045 return (link_node(node));
1049 _jit_new_node_f(jit_state_t *_jit, jit_code_t code,
1052 jit_node_t *node = new_node(code);
1053 assert(!_jitc->realize);
1055 return (link_node(node));
1059 _jit_new_node_d(jit_state_t *_jit, jit_code_t code,
1062 jit_node_t *node = new_node(code);
1063 assert(!_jitc->realize);
1065 return (link_node(node));
1069 _jit_new_node_p(jit_state_t *_jit, jit_code_t code,
1072 jit_node_t *node = new_node(code);
1073 assert(!_jitc->realize);
1075 return (link_node(node));
1079 _jit_new_node_ww(jit_state_t *_jit, jit_code_t code,
1080 jit_word_t u, jit_word_t v)
1082 jit_node_t *node = new_node(code);
1083 assert(!_jitc->realize);
1086 return (link_node(node));
1090 _jit_new_node_wp(jit_state_t *_jit, jit_code_t code,
1091 jit_word_t u, jit_pointer_t v)
1093 return (jit_new_node_ww(code, u, (jit_word_t)v));
1097 _jit_new_node_fp(jit_state_t *_jit, jit_code_t code,
1098 jit_float32_t u, jit_pointer_t v)
1100 jit_node_t *node = new_node(code);
1101 assert(!_jitc->realize);
1103 node->v.w = (jit_word_t)v;
1104 return (link_node(node));
1108 _jit_new_node_dp(jit_state_t *_jit, jit_code_t code,
1109 jit_float64_t u, jit_pointer_t v)
1111 jit_node_t *node = new_node(code);
1112 assert(!_jitc->realize);
1114 node->v.w = (jit_word_t)v;
1115 return (link_node(node));
1119 _jit_new_node_pw(jit_state_t *_jit, jit_code_t code,
1120 jit_pointer_t u, jit_word_t v)
1122 return (jit_new_node_ww(code, (jit_word_t)u, v));
1126 _jit_new_node_wf(jit_state_t *_jit, jit_code_t code,
1127 jit_word_t u, jit_float32_t v)
1129 jit_node_t *node = new_node(code);
1130 assert(!_jitc->realize);
1133 return (link_node(node));
1137 _jit_new_node_wd(jit_state_t *_jit, jit_code_t code,
1138 jit_word_t u, jit_float64_t v)
1140 jit_node_t *node = new_node(code);
1141 assert(!_jitc->realize);
1144 return (link_node(node));
1148 _jit_new_node_www(jit_state_t *_jit, jit_code_t code,
1149 jit_word_t u, jit_word_t v, jit_word_t w)
1151 jit_node_t *node = new_node(code);
1152 assert(!_jitc->realize);
1156 return (link_node(node));
1160 _jit_new_node_qww(jit_state_t *_jit, jit_code_t code,
1161 jit_int32_t l, jit_int32_t h,
1162 jit_word_t v, jit_word_t w)
1164 jit_node_t *node = new_node(code);
1165 assert(!_jitc->realize);
1171 return (link_node(node));
1175 _jit_new_node_wqw(jit_state_t *_jit, jit_code_t code,
1176 jit_word_t u, jit_int32_t l,
1177 jit_int32_t h, jit_word_t w)
1179 jit_node_t *node = new_node(code);
1180 assert(!_jitc->realize);
1186 return (link_node(node));
1190 _jit_new_node_wwq(jit_state_t *_jit, jit_code_t code,
1191 jit_word_t u, jit_word_t v,
1192 jit_int32_t l, jit_int32_t h)
1194 jit_node_t *node = new_node(code);
1195 assert(!_jitc->realize);
1200 return (link_node(node));
1204 _jit_new_node_wwf(jit_state_t *_jit, jit_code_t code,
1205 jit_word_t u, jit_word_t v, jit_float32_t w)
1207 jit_node_t *node = new_node(code);
1208 assert(!_jitc->realize);
1212 return (link_node(node));
1216 _jit_new_node_wqf(jit_state_t *_jit, jit_code_t code,
1217 jit_word_t u, jit_int32_t l,
1218 jit_int32_t h, jit_float32_t w)
1220 jit_node_t *node = new_node(code);
1221 assert(!_jitc->realize);
1227 return (link_node(node));
1231 _jit_new_node_wwd(jit_state_t *_jit, jit_code_t code,
1232 jit_word_t u, jit_word_t v, jit_float64_t w)
1234 jit_node_t *node = new_node(code);
1235 assert(!_jitc->realize);
1239 return (link_node(node));
1243 _jit_new_node_wqd(jit_state_t *_jit, jit_code_t code,
1244 jit_word_t u, jit_int32_t l,
1245 jit_int32_t h, jit_float64_t w)
1247 jit_node_t *node = new_node(code);
1248 assert(!_jitc->realize);
1254 return (link_node(node));
1258 _jit_new_node_pww(jit_state_t *_jit, jit_code_t code,
1259 jit_pointer_t u, jit_word_t v, jit_word_t w)
1261 jit_node_t *node = new_node(code);
1262 assert(!_jitc->realize);
1266 return (link_node(node));
1270 _jit_new_node_pwf(jit_state_t *_jit, jit_code_t code,
1271 jit_pointer_t u, jit_word_t v, jit_float32_t w)
1273 jit_node_t *node = new_node(code);
1274 assert(!_jitc->realize);
1278 return (link_node(node));
1282 _jit_new_node_pwd(jit_state_t *_jit, jit_code_t code,
1283 jit_pointer_t u, jit_word_t v, jit_float64_t w)
1285 jit_node_t *node = new_node(code);
1286 assert(!_jitc->realize);
1290 return (link_node(node));
1294 _jit_label(jit_state_t *_jit)
1298 if (!(node = _jitc->tail) || node->code != jit_code_label) {
1299 node = jit_forward();
1307 _jit_forward(jit_state_t *_jit)
1309 return (jit_new_node_no_link(jit_code_label));
1313 _jit_indirect(jit_state_t *_jit)
1318 node->flag |= jit_flag_use;
1324 _jit_link(jit_state_t *_jit, jit_node_t *node)
1328 assert((node->code == jit_code_label ||
1329 node->code == jit_code_prolog ||
1330 node->code == jit_code_epilog) && !node->next);
1331 jit_link_node(node);
1332 if (_jitc->blocks.offset >= _jitc->blocks.length) {
1335 length = _jitc->blocks.length + 16;
1336 jit_realloc((jit_pointer_t *)&_jitc->blocks.ptr,
1337 _jitc->blocks.length * sizeof(jit_block_t),
1338 length * sizeof(jit_block_t));
1339 _jitc->blocks.length = length;
1341 block = _jitc->blocks.ptr + _jitc->blocks.offset;
1342 block->label = node;
1343 node->v.w = _jitc->blocks.offset;
1344 jit_regset_new(&block->reglive);
1345 jit_regset_new(&block->regmask);
1346 ++_jitc->blocks.offset;
1350 _jit_forward_p(jit_state_t *_jit, jit_node_t *node)
1352 return (node->code == jit_code_label && !node->next && node != _jitc->tail);
1356 _jit_indirect_p(jit_state_t *_jit, jit_node_t *node)
1358 return (node->code == jit_code_label && !!(node->flag & jit_flag_use));
1362 _jit_target_p(jit_state_t *_jit, jit_node_t *node)
1364 return (node->code == jit_code_label && !!node->link);
1368 _jit_prepare(jit_state_t *_jit)
1370 assert(_jitc->function != NULL);
1371 _jitc->function->call.call = jit_call_default;
1372 _jitc->function->call.argi =
1373 _jitc->function->call.argf =
1374 _jitc->function->call.size = 0;
1375 _jitc->prepare = jit_new_node(jit_code_prepare);
1379 _jit_patch(jit_state_t* _jit, jit_node_t *instr)
1383 if (!(label = _jitc->tail) || label->code != jit_code_label)
1384 label = jit_label();
1385 jit_patch_at(instr, label);
1389 _jit_classify(jit_state_t *_jit, jit_code_t code)
1394 case jit_code_data: case jit_code_save: case jit_code_load:
1395 case jit_code_name: case jit_code_label: case jit_code_note:
1396 case jit_code_prolog: case jit_code_ellipsis: case jit_code_va_push:
1397 case jit_code_epilog: case jit_code_ret: case jit_code_prepare:
1400 case jit_code_live: case jit_code_va_end:
1401 case jit_code_retr_c: case jit_code_retr_uc:
1402 case jit_code_retr_s: case jit_code_retr_us:
1403 case jit_code_retr_i: case jit_code_retr_ui:
1404 case jit_code_retr_l:
1405 case jit_code_retr_f: case jit_code_retr_d:
1406 case jit_code_pushargr_c:
1407 case jit_code_pushargr_uc:
1408 case jit_code_pushargr_s:
1409 case jit_code_pushargr_us:
1410 case jit_code_pushargr_i:
1411 case jit_code_pushargr_ui:
1412 case jit_code_pushargr_l:
1413 case jit_code_pushargr_f:
1414 case jit_code_pushargr_d:
1415 case jit_code_finishr: /* synthesized will set jit_cc_a0_jmp */
1416 mask = jit_cc_a0_reg;
1418 case jit_code_align: case jit_code_skip:
1419 case jit_code_reti_c: case jit_code_reti_uc:
1420 case jit_code_reti_s: case jit_code_reti_us:
1421 case jit_code_reti_i: case jit_code_reti_ui:
1422 case jit_code_reti_l:
1423 case jit_code_pushargi_c:
1424 case jit_code_pushargi_uc:
1425 case jit_code_pushargi_s:
1426 case jit_code_pushargi_us:
1427 case jit_code_pushargi_i:
1428 case jit_code_pushargi_ui:
1429 case jit_code_pushargi_l:
1430 case jit_code_finishi: /* synthesized will set jit_cc_a0_jmp */
1431 mask = jit_cc_a0_int;
1433 case jit_code_reti_f: case jit_code_pushargi_f:
1434 mask = jit_cc_a0_flt;
1436 case jit_code_reti_d: case jit_code_pushargi_d:
1437 mask = jit_cc_a0_dbl;
1439 case jit_code_allocai:
1440 mask = jit_cc_a0_int|jit_cc_a1_int;
1442 case jit_code_arg_c: case jit_code_arg_s:
1443 case jit_code_arg_i: case jit_code_arg_l:
1444 case jit_code_arg_f: case jit_code_arg_d:
1445 mask = jit_cc_a0_int|jit_cc_a0_arg;
1447 case jit_code_calli: case jit_code_jmpi:
1448 mask = jit_cc_a0_jmp;
1450 case jit_code_callr: case jit_code_jmpr:
1451 mask = jit_cc_a0_reg|jit_cc_a0_jmp;
1453 case jit_code_retval_c: case jit_code_retval_uc:
1454 case jit_code_retval_s: case jit_code_retval_us:
1455 case jit_code_retval_i: case jit_code_retval_ui:
1456 case jit_code_retval_l:
1457 case jit_code_retval_f: case jit_code_retval_d:
1458 case jit_code_va_start:
1459 mask = jit_cc_a0_reg|jit_cc_a0_chg;
1461 case jit_code_getarg_c: case jit_code_getarg_uc:
1462 case jit_code_getarg_s: case jit_code_getarg_us:
1463 case jit_code_getarg_i: case jit_code_getarg_ui:
1464 case jit_code_getarg_l:
1465 case jit_code_getarg_f: case jit_code_getarg_d:
1466 mask = jit_cc_a0_reg|jit_cc_a0_chg|jit_cc_a1_arg;
1468 case jit_code_putargr_c:case jit_code_putargr_uc:
1469 case jit_code_putargr_s:case jit_code_putargr_us:
1470 case jit_code_putargr_i:case jit_code_putargr_ui:
1471 case jit_code_putargr_l:
1472 case jit_code_putargr_f:case jit_code_putargr_d:
1473 mask = jit_cc_a0_reg|jit_cc_a1_arg;
1475 case jit_code_putargi_c:case jit_code_putargi_uc:
1476 case jit_code_putargi_s:case jit_code_putargi_us:
1477 case jit_code_putargi_i:case jit_code_putargi_ui:
1478 case jit_code_putargi_l:
1479 mask = jit_cc_a0_int|jit_cc_a1_arg;
1481 case jit_code_putargi_f:
1482 mask = jit_cc_a0_flt|jit_cc_a1_arg;
1484 case jit_code_putargi_d:
1485 mask = jit_cc_a0_dbl|jit_cc_a1_arg;
1487 case jit_code_movi: case jit_code_ldi_c: case jit_code_ldi_uc:
1488 case jit_code_ldi_s: case jit_code_ldi_us: case jit_code_ldi_i:
1489 case jit_code_ldi_ui: case jit_code_ldi_l: case jit_code_ldi_f:
1490 case jit_code_ldi_d:
1491 case jit_code_negi: case jit_code_comi:
1492 case jit_code_exti_c: case jit_code_exti_uc:
1493 case jit_code_exti_s: case jit_code_exti_us:
1494 case jit_code_exti_i: case jit_code_exti_ui:
1495 case jit_code_bswapi_us:case jit_code_bswapi_ui:
1496 case jit_code_bswapi_ul:
1497 case jit_code_htoni_us: case jit_code_htoni_ui:
1498 case jit_code_htoni_ul:
1499 case jit_code_cloi: case jit_code_clzi:
1500 case jit_code_ctoi: case jit_code_ctzi:
1501 case jit_code_rbiti: case jit_code_popcnti:
1502 case jit_code_movi_w_f: case jit_code_movi_w_d:
1503 mask = jit_cc_a0_reg|jit_cc_a0_chg|jit_cc_a1_int;
1505 case jit_code_unldi: case jit_code_unldi_u: case jit_code_unldi_x:
1506 mask = jit_cc_a0_reg|jit_cc_a0_chg|jit_cc_a1_int|jit_cc_a2_int;
1508 case jit_code_movi_ww_d:
1509 mask = jit_cc_a0_reg|jit_cc_a0_chg|jit_cc_a1_int|jit_cc_a2_int;
1511 case jit_code_movi_f: case jit_code_movi_f_w:
1512 case jit_code_negi_f: case jit_code_absi_f:
1513 case jit_code_sqrti_f:
1514 mask = jit_cc_a0_reg|jit_cc_a0_chg|jit_cc_a1_flt;
1516 case jit_code_movi_d: case jit_code_movi_d_w:
1517 case jit_code_negi_d: case jit_code_absi_d:
1518 case jit_code_sqrti_d:
1519 mask = jit_cc_a0_reg|jit_cc_a0_chg|jit_cc_a1_dbl;
1521 case jit_code_movi_d_ww:
1522 mask = jit_cc_a0_reg|jit_cc_a0_chg|jit_cc_a1_reg|jit_cc_a1_chg|
1525 case jit_code_negr: case jit_code_comr: case jit_code_movr:
1526 case jit_code_extr_c: case jit_code_extr_uc: case jit_code_extr_s:
1527 case jit_code_extr_us: case jit_code_extr_i: case jit_code_extr_ui:
1528 case jit_code_truncr_f_i: case jit_code_truncr_f_l:
1529 case jit_code_truncr_d_i: case jit_code_truncr_d_l:
1530 case jit_code_htonr_us: case jit_code_htonr_ui: case jit_code_htonr_ul:
1531 case jit_code_bswapr_us: case jit_code_bswapr_ui: case jit_code_bswapr_ul:
1532 case jit_code_ldr_c: case jit_code_ldr_uc:
1533 case jit_code_ldr_s: case jit_code_ldr_us: case jit_code_ldr_i:
1534 case jit_code_ldr_ui: case jit_code_ldr_l: case jit_code_negr_f:
1535 case jit_code_absr_f: case jit_code_sqrtr_f: case jit_code_movr_f:
1536 case jit_code_extr_f: case jit_code_extr_d_f: case jit_code_ldr_f:
1537 case jit_code_negr_d: case jit_code_absr_d: case jit_code_sqrtr_d:
1538 case jit_code_movr_d: case jit_code_extr_d: case jit_code_extr_f_d:
1539 case jit_code_ldr_d: case jit_code_rbitr: case jit_code_popcntr:
1540 case jit_code_clor: case jit_code_clzr:
1541 case jit_code_ctor: case jit_code_ctzr:
1542 case jit_code_movr_w_f: case jit_code_movr_f_w:
1543 case jit_code_movr_w_d: case jit_code_movr_d_w:
1544 case jit_code_va_arg: case jit_code_va_arg_d:
1545 mask = jit_cc_a0_reg|jit_cc_a0_chg|jit_cc_a1_reg;
1547 case jit_code_movr_d_ww:
1548 mask = jit_cc_a0_reg|jit_cc_a0_chg|jit_cc_a1_reg|jit_cc_a1_chg|
1551 case jit_code_addi: case jit_code_addxi: case jit_code_addci:
1552 case jit_code_subi: case jit_code_subxi: case jit_code_subci:
1554 case jit_code_muli: case jit_code_hmuli: case jit_code_hmuli_u:
1555 case jit_code_divi: case jit_code_divi_u:
1556 case jit_code_remi: case jit_code_remi_u: case jit_code_andi:
1557 case jit_code_ori: case jit_code_xori: case jit_code_lshi:
1558 case jit_code_rshi: case jit_code_rshi_u: case jit_code_lroti:
1559 case jit_code_rroti: case jit_code_lti:
1560 case jit_code_lti_u: case jit_code_lei: case jit_code_lei_u:
1561 case jit_code_eqi: case jit_code_gei: case jit_code_gei_u:
1562 case jit_code_gti: case jit_code_gti_u: case jit_code_nei:
1563 case jit_code_ldxi_c: case jit_code_ldxi_uc: case jit_code_ldxi_s:
1564 case jit_code_ldxi_us: case jit_code_ldxi_i: case jit_code_ldxi_ui:
1565 case jit_code_ldxi_l: case jit_code_ldxi_f: case jit_code_ldxi_d:
1566 case jit_code_unldr: case jit_code_unldr_u: case jit_code_unldr_x:
1567 mask = jit_cc_a0_reg|jit_cc_a0_chg|jit_cc_a1_reg|jit_cc_a2_int;
1569 case jit_code_qmuli: case jit_code_qmuli_u:
1570 case jit_code_qdivi: case jit_code_qdivi_u:
1571 case jit_code_qlshi: case jit_code_qlshi_u:
1572 case jit_code_qrshi: case jit_code_qrshi_u:
1573 mask = jit_cc_a0_reg|jit_cc_a0_rlh|jit_cc_a0_chg|
1574 jit_cc_a1_reg|jit_cc_a2_int;
1576 case jit_code_addi_f: case jit_code_subi_f: case jit_code_rsbi_f:
1577 case jit_code_muli_f: case jit_code_divi_f: case jit_code_lti_f:
1578 case jit_code_lei_f: case jit_code_eqi_f: case jit_code_gei_f:
1579 case jit_code_gti_f: case jit_code_nei_f: case jit_code_unlti_f:
1580 case jit_code_unlei_f: case jit_code_uneqi_f: case jit_code_ungei_f:
1581 case jit_code_ungti_f: case jit_code_ltgti_f: case jit_code_ordi_f:
1582 case jit_code_unordi_f:
1583 mask = jit_cc_a0_reg|jit_cc_a0_chg|jit_cc_a1_reg|jit_cc_a2_flt;
1585 case jit_code_addi_d: case jit_code_subi_d: case jit_code_rsbi_d:
1586 case jit_code_muli_d: case jit_code_divi_d: case jit_code_lti_d:
1587 case jit_code_lei_d: case jit_code_eqi_d: case jit_code_gei_d:
1588 case jit_code_gti_d: case jit_code_nei_d: case jit_code_unlti_d:
1589 case jit_code_unlei_d: case jit_code_uneqi_d: case jit_code_ungei_d:
1590 case jit_code_ungti_d: case jit_code_ltgti_d: case jit_code_ordi_d:
1591 case jit_code_unordi_d:
1592 mask = jit_cc_a0_reg|jit_cc_a0_chg|jit_cc_a1_reg|jit_cc_a2_dbl;
1594 case jit_code_addr: case jit_code_addxr: case jit_code_addcr:
1595 case jit_code_subr: case jit_code_subxr: case jit_code_subcr:
1596 case jit_code_mulr: case jit_code_hmulr: case jit_code_hmulr_u:
1597 case jit_code_divr: case jit_code_divr_u:
1598 case jit_code_remr: case jit_code_remr_u: case jit_code_andr:
1599 case jit_code_orr: case jit_code_xorr: case jit_code_lshr:
1600 case jit_code_rshr: case jit_code_rshr_u: case jit_code_lrotr:
1601 case jit_code_rrotr: case jit_code_ltr:
1602 case jit_code_ltr_u: case jit_code_ler: case jit_code_ler_u:
1603 case jit_code_eqr: case jit_code_ger: case jit_code_ger_u:
1604 case jit_code_gtr: case jit_code_gtr_u: case jit_code_ner:
1605 case jit_code_ldxr_c: case jit_code_ldxr_uc: case jit_code_ldxr_s:
1606 case jit_code_ldxr_us: case jit_code_ldxr_i: case jit_code_ldxr_ui:
1607 case jit_code_ldxr_l: case jit_code_addr_f: case jit_code_subr_f:
1608 case jit_code_mulr_f: case jit_code_divr_f: case jit_code_ltr_f:
1609 case jit_code_ler_f: case jit_code_eqr_f: case jit_code_ger_f:
1610 case jit_code_gtr_f: case jit_code_ner_f: case jit_code_unltr_f:
1611 case jit_code_unler_f: case jit_code_uneqr_f: case jit_code_unger_f:
1612 case jit_code_ungtr_f: case jit_code_ltgtr_f: case jit_code_ordr_f:
1613 case jit_code_unordr_f: case jit_code_ldxr_f: case jit_code_addr_d:
1614 case jit_code_subr_d: case jit_code_mulr_d: case jit_code_divr_d:
1615 case jit_code_ltr_d: case jit_code_ler_d: case jit_code_eqr_d:
1616 case jit_code_ger_d: case jit_code_gtr_d: case jit_code_ner_d:
1617 case jit_code_unltr_d: case jit_code_unler_d: case jit_code_uneqr_d:
1618 case jit_code_unger_d: case jit_code_ungtr_d: case jit_code_ltgtr_d:
1619 case jit_code_ordr_d: case jit_code_unordr_d: case jit_code_ldxr_d:
1620 case jit_code_movr_ww_d:
1621 mask = jit_cc_a0_reg|jit_cc_a0_chg|jit_cc_a1_reg|jit_cc_a2_reg;
1623 case jit_code_qmulr: case jit_code_qmulr_u:
1624 case jit_code_qdivr: case jit_code_qdivr_u:
1625 case jit_code_qlshr: case jit_code_qlshr_u:
1626 case jit_code_qrshr: case jit_code_qrshr_u:
1627 mask = jit_cc_a0_reg|jit_cc_a0_rlh|jit_cc_a0_chg|
1628 jit_cc_a1_reg|jit_cc_a2_reg;
1630 case jit_code_sti_c: case jit_code_sti_s: case jit_code_sti_i:
1631 case jit_code_sti_l: case jit_code_sti_f: case jit_code_sti_d:
1632 mask = jit_cc_a0_int|jit_cc_a1_reg;
1634 case jit_code_unsti: case jit_code_unsti_x:
1635 mask = jit_cc_a0_int|jit_cc_a1_reg|jit_cc_a2_int;
1637 case jit_code_blti: case jit_code_blti_u: case jit_code_blei:
1638 case jit_code_blei_u: case jit_code_beqi: case jit_code_bgei:
1639 case jit_code_bgei_u: case jit_code_bgti: case jit_code_bgti_u:
1640 case jit_code_bnei: case jit_code_bmsi: case jit_code_bmci:
1641 mask = jit_cc_a0_jmp|jit_cc_a1_reg|jit_cc_a2_int;
1643 case jit_code_blti_f: case jit_code_blei_f: case jit_code_beqi_f:
1644 case jit_code_bgei_f: case jit_code_bgti_f: case jit_code_bnei_f:
1645 case jit_code_bunlti_f: case jit_code_bunlei_f: case jit_code_buneqi_f:
1646 case jit_code_bungei_f: case jit_code_bungti_f: case jit_code_bltgti_f:
1647 case jit_code_bordi_f: case jit_code_bunordi_f:
1648 mask = jit_cc_a0_jmp|jit_cc_a1_reg|jit_cc_a2_flt;
1650 case jit_code_blti_d: case jit_code_blei_d: case jit_code_beqi_d:
1651 case jit_code_bgei_d: case jit_code_bgti_d: case jit_code_bnei_d:
1652 case jit_code_bunlti_d: case jit_code_bunlei_d: case jit_code_buneqi_d:
1653 case jit_code_bungei_d: case jit_code_bungti_d: case jit_code_bltgti_d:
1654 case jit_code_bordi_d: case jit_code_bunordi_d:
1655 mask = jit_cc_a0_jmp|jit_cc_a1_reg|jit_cc_a2_dbl;
1657 case jit_code_allocar: /* synthesized instructions make it
1658 * equivalent to jit_cc_a0_chg */
1659 case jit_code_str_c: case jit_code_str_s: case jit_code_str_i:
1660 case jit_code_str_l: case jit_code_str_f: case jit_code_str_d:
1661 mask = jit_cc_a0_reg|jit_cc_a1_reg;
1663 case jit_code_unstr: case jit_code_unstr_x:
1664 mask = jit_cc_a0_reg|jit_cc_a1_reg|jit_cc_a2_int;
1666 case jit_code_stxi_c: case jit_code_stxi_s: case jit_code_stxi_i:
1667 case jit_code_stxi_l: case jit_code_stxi_f: case jit_code_stxi_d:
1668 mask = jit_cc_a0_int|jit_cc_a1_reg|jit_cc_a2_reg;
1670 case jit_code_bltr: case jit_code_bltr_u: case jit_code_bler:
1671 case jit_code_bler_u: case jit_code_beqr: case jit_code_bger:
1672 case jit_code_bger_u: case jit_code_bgtr: case jit_code_bgtr_u:
1673 case jit_code_bner: case jit_code_bmsr: case jit_code_bmcr:
1674 case jit_code_bltr_f: case jit_code_bler_f: case jit_code_beqr_f:
1675 case jit_code_bger_f: case jit_code_bgtr_f: case jit_code_bner_f:
1676 case jit_code_bunltr_f: case jit_code_bunler_f: case jit_code_buneqr_f:
1677 case jit_code_bunger_f: case jit_code_bungtr_f: case jit_code_bltgtr_f:
1678 case jit_code_bordr_f: case jit_code_bunordr_f:case jit_code_bltr_d:
1679 case jit_code_bler_d: case jit_code_beqr_d: case jit_code_bger_d:
1680 case jit_code_bgtr_d: case jit_code_bner_d: case jit_code_bunltr_d:
1681 case jit_code_bunler_d: case jit_code_buneqr_d: case jit_code_bunger_d:
1682 case jit_code_bungtr_d: case jit_code_bltgtr_d: case jit_code_bordr_d:
1683 case jit_code_bunordr_d:
1684 mask = jit_cc_a0_jmp|jit_cc_a1_reg|jit_cc_a2_reg;
1686 case jit_code_boaddi: case jit_code_boaddi_u: case jit_code_bxaddi:
1687 case jit_code_bxaddi_u: case jit_code_bosubi: case jit_code_bosubi_u:
1688 case jit_code_bxsubi: case jit_code_bxsubi_u:
1689 mask = jit_cc_a0_jmp|jit_cc_a1_reg|jit_cc_a1_chg|jit_cc_a2_int;
1691 case jit_code_stxr_c: case jit_code_stxr_s: case jit_code_stxr_i:
1692 case jit_code_stxr_l: case jit_code_stxr_f: case jit_code_stxr_d:
1693 mask = jit_cc_a0_reg|jit_cc_a1_reg|jit_cc_a2_reg;
1695 case jit_code_boaddr: case jit_code_boaddr_u: case jit_code_bxaddr:
1696 case jit_code_bxaddr_u: case jit_code_bosubr: case jit_code_bosubr_u:
1697 case jit_code_bxsubr: case jit_code_bxsubr_u:
1698 mask = jit_cc_a0_jmp|jit_cc_a1_reg|jit_cc_a1_chg|jit_cc_a2_reg;
1700 case jit_code_movnr: case jit_code_movzr:
1701 mask = jit_cc_a0_reg|jit_cc_a0_cnd|jit_cc_a1_reg|jit_cc_a2_reg;
1704 mask = jit_cc_a0_reg|jit_cc_a0_chg|jit_cc_a1_reg|
1705 jit_cc_a2_reg|jit_cc_a2_rlh;
1708 mask = jit_cc_a0_reg|jit_cc_a0_chg|jit_cc_a1_int|
1709 jit_cc_a2_reg|jit_cc_a2_rlh;
1711 case jit_code_extr: case jit_code_extr_u: case jit_code_depr:
1712 mask = jit_cc_a0_reg|jit_cc_a0_chg|jit_cc_a1_reg|jit_cc_a2_rlh;
1714 case jit_code_exti: case jit_code_exti_u: case jit_code_depi:
1715 mask = jit_cc_a0_reg|jit_cc_a0_chg|jit_cc_a1_int|jit_cc_a2_rlh;
1717 case jit_code_fmar_f: case jit_code_fmar_d:
1718 case jit_code_fmsr_f: case jit_code_fmsr_d:
1719 case jit_code_fnmar_f: case jit_code_fnmar_d:
1720 case jit_code_fnmsr_f: case jit_code_fnmsr_d:
1721 mask = jit_cc_a0_reg|jit_cc_a0_chg|
1722 jit_cc_a1_reg|jit_cc_a1_rlh|jit_cc_a2_reg;
1724 case jit_code_fmai_f: case jit_code_fmsi_f:
1725 case jit_code_fnmai_f: case jit_code_fnmsi_f:
1726 mask = jit_cc_a0_reg|jit_cc_a0_chg|
1727 jit_cc_a1_reg|jit_cc_a1_rlh|jit_cc_a2_flt;
1729 case jit_code_fmai_d: case jit_code_fmsi_d:
1730 case jit_code_fnmai_d: case jit_code_fnmsi_d:
1731 mask = jit_cc_a0_reg|jit_cc_a0_chg|
1732 jit_cc_a1_reg|jit_cc_a1_rlh|jit_cc_a2_dbl;
1742 _jit_patch_abs(jit_state_t *_jit, jit_node_t *instr, jit_pointer_t address)
1747 switch (instr->code) {
1748 case jit_code_movi: case jit_code_ldi_c: case jit_code_ldi_uc:
1749 case jit_code_ldi_s: case jit_code_ldi_us: case jit_code_ldi_i:
1750 case jit_code_ldi_ui: case jit_code_ldi_l: case jit_code_ldi_f:
1751 case jit_code_ldi_d:
1752 instr->v.p = address;
1754 case jit_code_sti_c: case jit_code_sti_s: case jit_code_sti_i:
1755 case jit_code_sti_l: case jit_code_sti_f: case jit_code_sti_d:
1756 instr->u.p = address;
1760 mask = jit_classify(instr->code);
1762 assert((mask & (jit_cc_a0_reg|jit_cc_a0_jmp)) == jit_cc_a0_jmp);
1763 instr->u.p = address;
1768 _jit_patch_at(jit_state_t *_jit, jit_node_t *instr, jit_node_t *label)
1773 assert(!(instr->flag & jit_flag_node));
1774 instr->flag |= jit_flag_node;
1775 switch (instr->code) {
1777 assert(label->code == jit_code_label ||
1778 label->code == jit_code_data);
1780 if (label->code == jit_code_data)
1781 instr->flag |= jit_flag_data;
1784 assert(label->code == jit_code_label ||
1785 label->code == jit_code_epilog);
1790 mask = jit_classify(instr->code);
1792 assert((mask & (jit_cc_a0_reg|jit_cc_a0_jmp)) == jit_cc_a0_jmp);
1793 assert(label->code == jit_code_label);
1797 /* link field is used as list of nodes associated with a given label */
1798 instr->link = label->link;
1799 label->link = instr;
1803 _do_setup(jit_state_t *_jit)
1808 /* create initial mapping of live register values
1809 * at the start of a basic block */
1810 for (offset = 0; offset < _jitc->blocks.offset; offset++) {
1811 block = _jitc->blocks.ptr + offset;
1814 if (block->label->code == jit_code_epilog) {
1815 jit_regset_setbit(&block->reglive, JIT_RET);
1816 jit_regset_setbit(&block->reglive, JIT_FRET);
1817 jit_regset_com(&block->regmask, &block->reglive);
1825 _block_update_set(jit_state_t *_jit,
1826 jit_block_t *block, jit_block_t *target)
1828 jit_regset_t regmask;
1830 jit_regset_ior(®mask, &block->reglive, &target->reglive);
1831 jit_regset_and(®mask, ®mask, &block->regmask);
1832 if (jit_regset_set_p(®mask)) {
1833 jit_regset_ior(&block->reglive, &block->reglive, ®mask);
1834 jit_regset_and(®mask, &block->reglive, &block->regmask);
1835 jit_regset_com(®mask, ®mask);
1836 jit_regset_and(&block->regmask, &block->regmask, ®mask);
1844 _propagate_backward(jit_state_t *_jit, jit_block_t *block)
1849 for (offset = block->label->v.w - 1;
1850 offset >= 0; --offset) {
1851 prev = _jitc->blocks.ptr + offset;
1852 if (!block_update_set(prev, block) ||
1853 !(prev->label->flag & jit_flag_head))
1859 _check_block_again(jit_state_t *_jit)
1863 jit_node_t *node, *label;
1864 jit_block_t *block, *target;
1867 for (offset = 0; offset < _jitc->blocks.offset; offset++) {
1868 block = _jitc->blocks.ptr + offset;
1874 /* If no block changed state */
1881 for (node = _jitc->head; node; node = node->next) {
1882 /* Special jumps that match jit_cc_a0_jmp */
1883 if (node->code == jit_code_calli || node->code == jit_code_callr)
1886 /* Remember current label */
1887 if (node->code == jit_code_label ||
1888 node->code == jit_code_prolog ||
1889 node->code == jit_code_epilog) {
1891 /* If previous block does not pass through */
1892 if (!(node->flag & jit_flag_head))
1895 target = _jitc->blocks.ptr + node->v.w;
1896 if (block && target->again && block_update_set(block, target)) {
1897 propagate_backward(block);
1902 /* If not the first jmpi */
1904 /* If a jump to dynamic address or if a jump to raw address */
1905 if (!(jit_classify(node->code) & jit_cc_a0_jmp) ||
1906 !(node->flag & jit_flag_node))
1909 /* Mark predecessor needs updating due to target change */
1910 target = _jitc->blocks.ptr + label->v.w;
1911 if (target->again && block_update_set(block, target)) {
1912 propagate_backward(block);
1924 _do_follow(jit_state_t *_jit, jit_bool_t always)
1929 /* set live state of registers not referenced in a block, but
1930 * referenced in a jump target or normal flow */
1931 for (offset = 0; offset < _jitc->blocks.offset; offset++) {
1932 block = _jitc->blocks.ptr + offset;
1933 if (!block->label || block->label->code == jit_code_epilog)
1935 if (always || block->again) {
1943 _jit_optimize(jit_state_t *_jit)
1951 jit_regset_t regmask;
1954 _jitc->function = NULL;
1957 sequential_labels();
1962 jit_regset_set_ui(®mask, 0);
1963 for (offset = 0; offset < _jitc->reglen; offset++) {
1964 if ((jit_class(_rvs[offset].spec) & (jit_class_gpr|jit_class_fpr)) &&
1965 (jit_class(_rvs[offset].spec) & jit_class_sav) == jit_class_sav)
1966 jit_regset_setbit(®mask, offset);
1969 /* Figure out labels that are only reached with a jump */
1971 for (node = _jitc->head; node; node = node->next) {
1972 switch (node->code) {
1973 case jit_code_label:
1975 node->flag |= jit_flag_head;
1977 /* Block is dead code or only reachable with an
1978 * indirect jumps. In such condition, must assume
1979 * all callee save registers are live. */
1980 block = _jitc->blocks.ptr + node->v.w;
1981 jit_regset_ior(&block->reglive,
1982 &block->reglive, ®mask);
1983 /* Cleanup regmask */
1984 block_update_set(block, block);
1988 case jit_code_jmpi: case jit_code_jmpr:
1989 case jit_code_epilog:
1992 case jit_code_data: case jit_code_note:
2004 for (node = _jitc->head; node; node = node->next) {
2005 mask = jit_classify(node->code);
2006 if (mask & jit_cc_a0_reg)
2007 node->u.w &= ~jit_regno_patch;
2008 if (mask & jit_cc_a1_reg)
2009 node->v.w &= ~jit_regno_patch;
2010 if (mask & jit_cc_a2_reg)
2011 node->w.w &= ~jit_regno_patch;
2012 if (node->code == jit_code_beqi) {
2013 if (redundant_store(node, 1)) {
2014 block = _jitc->blocks.ptr + ((jit_node_t *)node->u.n)->v.w;
2019 else if (node->code == jit_code_bnei) {
2020 if (redundant_store(node, 0)) {
2021 block = _jitc->blocks.ptr + ((jit_node_t *)node->u.n)->v.w;
2029 todo = check_block_again();
2031 /* If instructions were removed or first pass did modify the entry
2032 * state of any block */
2038 /* If any block again has the entry state modified. */
2039 todo = check_block_again();
2043 for (node = _jitc->head; node; node = node->next) {
2044 mask = jit_classify(node->code);
2045 if (mask & jit_cc_a0_reg)
2046 node->u.w &= ~jit_regno_patch;
2047 if (mask & jit_cc_a1_reg)
2048 node->v.w &= ~jit_regno_patch;
2049 if (mask & jit_cc_a2_reg)
2050 node->w.w &= ~jit_regno_patch;
2051 if (node->code == jit_code_prolog)
2052 _jitc->function = _jitc->functions.ptr + node->w.w;
2053 else if(node->code == jit_code_epilog)
2054 _jitc->function = NULL;
2057 if (mask & jit_cc_a0_flt) {
2058 node->u.p = jit_data(&node->u.f, sizeof(jit_float32_t), 4);
2059 node->flag |= jit_flag_node | jit_flag_data;
2061 else if (mask & jit_cc_a0_dbl) {
2062 node->u.p = jit_data(&node->u.d, sizeof(jit_float64_t), 8);
2063 node->flag |= jit_flag_node | jit_flag_data;
2065 else if (mask & jit_cc_a1_flt) {
2066 node->v.p = jit_data(&node->v.f, sizeof(jit_float32_t), 4);
2067 node->flag |= jit_flag_node | jit_flag_data;
2069 else if (mask & jit_cc_a1_dbl) {
2070 node->v.p = jit_data(&node->v.d, sizeof(jit_float64_t), 8);
2071 node->flag |= jit_flag_node | jit_flag_data;
2073 else if (mask & jit_cc_a2_flt) {
2074 node->w.p = jit_data(&node->w.f, sizeof(jit_float32_t), 4);
2075 node->flag |= jit_flag_node | jit_flag_data;
2077 else if (mask & jit_cc_a2_dbl) {
2078 node->w.p = jit_data(&node->w.d, sizeof(jit_float64_t), 8);
2079 node->flag |= jit_flag_node | jit_flag_data;
2082 if (_jitc->function) {
2083 if ((mask & (jit_cc_a0_reg|jit_cc_a0_chg)) ==
2084 (jit_cc_a0_reg|jit_cc_a0_chg)) {
2085 if (mask & jit_cc_a0_rlh) {
2086 jit_regset_setbit(&_jitc->function->regset,
2087 jit_regno(node->u.q.l));
2088 jit_regset_setbit(&_jitc->function->regset,
2089 jit_regno(node->u.q.h));
2092 jit_regset_setbit(&_jitc->function->regset,
2093 jit_regno(node->u.w));
2095 if ((mask & (jit_cc_a1_reg|jit_cc_a1_chg)) ==
2096 (jit_cc_a1_reg|jit_cc_a1_chg))
2097 jit_regset_setbit(&_jitc->function->regset,
2098 jit_regno(node->v.w));
2099 if ((mask & (jit_cc_a2_reg|jit_cc_a2_chg)) ==
2100 (jit_cc_a2_reg|jit_cc_a2_chg))
2101 jit_regset_setbit(&_jitc->function->regset,
2102 jit_regno(node->w.w));
2109 _jit_reglive(jit_state_t *_jit, jit_node_t *node)
2115 switch (node->code) {
2116 case jit_code_label: case jit_code_prolog: case jit_code_epilog:
2117 block = _jitc->blocks.ptr + node->v.w;
2118 jit_regset_set(&_jitc->reglive, &block->reglive);
2119 jit_regset_set_ui(&_jitc->explive, 0);
2122 jit_regset_setbit(&_jitc->explive, node->u.w);
2124 case jit_code_callr:
2125 value = jit_regno(node->u.w);
2126 if (!(node->u.w & jit_regno_patch)) {
2127 jit_regset_setbit(&_jitc->reglive, value);
2129 case jit_code_calli:
2130 for (value = 0; value < _jitc->reglen; value++) {
2131 spec = jit_class(_rvs[value].spec);
2132 if ((spec & jit_class_arg) && jit_regarg_p(node, value))
2133 jit_regset_setbit(&_jitc->reglive, value);
2134 else if (!(spec & jit_class_sav))
2135 jit_regset_clrbit(&_jitc->reglive, value);
2139 value = jit_classify(node->code);
2140 if (value & jit_cc_a0_reg) {
2141 if (value & jit_cc_a0_rlh) {
2142 if (!(node->u.q.l & jit_regno_patch)) {
2143 if (value & jit_cc_a0_chg) {
2144 jit_regset_clrbit(&_jitc->reglive, node->u.q.l);
2145 jit_regset_setbit(&_jitc->regmask, node->u.q.l);
2148 jit_regset_setbit(&_jitc->reglive, node->u.q.l);
2150 if (!(node->u.q.h & jit_regno_patch)) {
2151 if (value & jit_cc_a0_chg) {
2152 jit_regset_clrbit(&_jitc->reglive, node->u.q.h);
2153 jit_regset_setbit(&_jitc->regmask, node->u.q.h);
2156 jit_regset_setbit(&_jitc->reglive, node->u.q.h);
2160 if (!(node->u.w & jit_regno_patch)) {
2161 if (value & jit_cc_a0_chg) {
2162 jit_regset_clrbit(&_jitc->reglive, node->u.w);
2163 jit_regset_setbit(&_jitc->regmask, node->u.w);
2166 jit_regset_setbit(&_jitc->reglive, node->u.w);
2170 if (value & jit_cc_a1_reg) {
2171 if (value & jit_cc_a1_rlh) {
2172 /* Assume registers are not changed */
2173 if (!(node->v.q.l & jit_regno_patch))
2174 jit_regset_setbit(&_jitc->reglive, node->v.q.l);
2175 if (!(node->v.q.h & jit_regno_patch))
2176 jit_regset_setbit(&_jitc->reglive, node->v.q.h);
2178 else if (!(node->v.w & jit_regno_patch)) {
2179 if (value & jit_cc_a1_chg) {
2180 jit_regset_clrbit(&_jitc->reglive, node->v.w);
2181 jit_regset_setbit(&_jitc->regmask, node->v.w);
2184 jit_regset_setbit(&_jitc->reglive, node->v.w);
2187 if (value & jit_cc_a2_reg) {
2188 if (value & jit_cc_a2_rlh) {
2189 /* Assume registers are not changed */
2190 if (!(node->w.q.l & jit_regno_patch))
2191 jit_regset_setbit(&_jitc->reglive, node->w.q.l);
2192 if (!(node->w.q.h & jit_regno_patch))
2193 jit_regset_setbit(&_jitc->reglive, node->w.q.h);
2196 if (!(node->w.w & jit_regno_patch)) {
2197 if (value & jit_cc_a2_chg) {
2198 jit_regset_clrbit(&_jitc->reglive, node->w.w);
2199 jit_regset_setbit(&_jitc->regmask, node->w.w);
2202 jit_regset_setbit(&_jitc->reglive, node->w.w);
2206 if (jit_regset_set_p(&_jitc->regmask)) {
2207 jit_update(node->next, &_jitc->reglive, &_jitc->regmask);
2208 if (jit_regset_set_p(&_jitc->regmask)) {
2209 /* any unresolved live state is considered as live */
2210 jit_regset_ior(&_jitc->reglive,
2211 &_jitc->reglive, &_jitc->regmask);
2212 jit_regset_set_ui(&_jitc->regmask, 0);
2220 _jit_regarg_set(jit_state_t *_jit, jit_node_t *node, jit_int32_t value)
2225 if (value & jit_cc_a0_reg) {
2226 if (value & jit_cc_a0_rlh) {
2227 jit_regset_setbit(&_jitc->regarg, jit_regno(node->u.q.l));
2228 jit_regset_setbit(&_jitc->regarg, jit_regno(node->u.q.h));
2231 jit_regset_setbit(&_jitc->regarg, jit_regno(node->u.w));
2233 if (value & jit_cc_a1_reg) {
2234 if (value & jit_cc_a1_rlh) {
2235 jit_regset_setbit(&_jitc->regarg, jit_regno(node->v.q.l));
2236 jit_regset_setbit(&_jitc->regarg, jit_regno(node->v.q.h));
2239 jit_regset_setbit(&_jitc->regarg, jit_regno(node->v.w));
2241 if (value & jit_cc_a2_reg) {
2242 if (value & jit_cc_a2_rlh) {
2243 jit_regset_setbit(&_jitc->regarg, jit_regno(node->w.q.l));
2244 jit_regset_setbit(&_jitc->regarg, jit_regno(node->w.q.h));
2247 jit_regset_setbit(&_jitc->regarg, jit_regno(node->w.w));
2249 /* Prevent incorrect detection of running out of registers
2250 * if will need to patch jump, and all registers have been
2251 * used in the current block. */
2252 if (node->code == jit_code_jmpi && (node->flag & jit_flag_node)) {
2253 jit_node_t *label = node->u.n;
2254 jit_block_t *block = _jitc->blocks.ptr + label->v.w;
2255 jit_regset_set(&_jitc->reglive, &block->reglive);
2256 jit_regset_set(&_jitc->regmask, &block->regmask);
2257 if (jit_regset_set_p(&_jitc->explive)) {
2258 jit_regset_ior(&_jitc->reglive, &block->reglive, &_jitc->explive);
2259 jit_regset_xor(&_jitc->regmask, &_jitc->regmask, &_jitc->explive);
2265 _jit_regarg_clr(jit_state_t *_jit, jit_node_t *node, jit_int32_t value)
2268 jit_size_collect(node);
2270 if (value & jit_cc_a0_reg) {
2271 if (value & jit_cc_a0_rlh) {
2272 jit_regset_clrbit(&_jitc->regarg, jit_regno(node->u.q.l));
2273 jit_regset_clrbit(&_jitc->regarg, jit_regno(node->u.q.h));
2276 jit_regset_clrbit(&_jitc->regarg, jit_regno(node->u.w));
2278 if (value & jit_cc_a1_reg) {
2279 if (value & jit_cc_a1_rlh) {
2280 jit_regset_clrbit(&_jitc->regarg, jit_regno(node->v.q.l));
2281 jit_regset_clrbit(&_jitc->regarg, jit_regno(node->v.q.h));
2284 jit_regset_clrbit(&_jitc->regarg, jit_regno(node->v.w));
2286 if (value & jit_cc_a2_reg) {
2287 if (value & jit_cc_a2_rlh) {
2288 jit_regset_clrbit(&_jitc->regarg, jit_regno(node->w.q.l));
2289 jit_regset_clrbit(&_jitc->regarg, jit_regno(node->w.q.h));
2292 jit_regset_clrbit(&_jitc->regarg, jit_regno(node->w.w));
2297 _jit_realize(jit_state_t *_jit)
2299 assert(!_jitc->realize);
2300 if (_jitc->function)
2305 /* ensure it is aligned */
2306 _jitc->data.offset = (_jitc->data.offset + 7) & -8;
2309 /* Heuristic to guess code buffer size */
2311 _jit->code.length = _jitc->pool.length * 1024 * _jitc->mult;
2313 _jit->code.length = jit_get_size();
2318 _jit_dataset(jit_state_t *_jit)
2327 assert(!_jitc->dataset);
2329 assert(_jit->user_data);
2331 if (!_jit->user_data) {
2333 /* create read only data buffer */
2334 _jit->data.length = (_jitc->data.offset +
2335 /* reserve space for annotations */
2336 _jitc->note.size + 4095) & -4096;
2338 mmap_fd = open("/dev/zero", O_RDWR);
2340 _jit->data.ptr = mmap(NULL, _jit->data.length,
2341 PROT_READ | PROT_WRITE,
2342 MAP_PRIVATE | MAP_ANON, mmap_fd, 0);
2343 assert(_jit->data.ptr != MAP_FAILED);
2348 #endif /* !HAVE_MMAP */
2350 if (!_jitc->no_data)
2351 jit_memcpy(_jit->data.ptr, _jitc->data.ptr, _jitc->data.offset);
2353 if (_jitc->no_note) {
2354 /* Space for one note is always allocated, so revert it here
2355 * if after jit_new_state was called, it is also requested to
2356 * not generate annotation information */
2357 _jit->note.length = 0;
2358 _jitc->note.size = 0;
2361 _jitc->note.base = _jit->data.ptr;
2362 if (!_jitc->no_data)
2363 _jitc->note.base += _jitc->data.offset;
2364 memset(_jitc->note.base, 0, _jitc->note.size);
2367 if (_jit->user_data)
2368 /* Need the temporary hashed data until jit_emit is finished */
2369 ptr = _jitc->no_data ? _jitc->data.ptr : _jit->data.ptr;
2371 ptr = _jit->data.ptr;
2372 /* Temporary hashed data no longer required */
2373 jit_free((jit_pointer_t *)&_jitc->data.ptr);
2376 for (offset = 0; offset < _jitc->data.size; offset++) {
2377 for (node = _jitc->data.table[offset]; node; node = node->next) {
2378 node->flag |= jit_flag_patch;
2379 node->u.w = (jit_word_t)(ptr + node->u.w);
2387 _jit_get_code(jit_state_t *_jit, jit_word_t *length)
2389 assert(_jitc->realize);
2392 /* If code already generated, return exact size of code */
2393 *length = _jit->pc.uc - _jit->code.ptr;
2395 /* Else return current size of the code buffer */
2396 *length = _jit->code.length;
2399 return (_jit->code.ptr);
2403 _jit_set_code(jit_state_t *_jit, jit_pointer_t ptr, jit_word_t length)
2405 assert(_jitc->realize);
2406 _jit->code.ptr = ptr;
2407 _jit->code.length = length;
2408 _jit->user_code = 1;
2412 _jit_get_data(jit_state_t *_jit, jit_word_t *data_size, jit_word_t *note_size)
2414 assert(_jitc->realize);
2416 *data_size = _jitc->data.offset;
2418 *note_size = _jitc->note.size;
2419 return (_jit->data.ptr);
2423 _jit_set_data(jit_state_t *_jit, jit_pointer_t ptr,
2424 jit_word_t length, jit_word_t flags)
2426 assert(_jitc->realize);
2427 if (flags & JIT_DISABLE_DATA)
2430 assert(length >= _jitc->data.offset);
2431 if (flags & JIT_DISABLE_NOTE)
2434 if (flags & JIT_DISABLE_DATA)
2435 assert(length >= _jitc->note.size);
2437 assert(length >= _jitc->data.offset + _jitc->note.size);
2439 _jit->data.ptr = ptr;
2440 _jit->data.length = length;
2441 _jit->user_data = 1;
2445 _jit_emit(jit_state_t *_jit)
2456 int mmap_prot, mmap_flags;
2458 if (!_jitc->realize)
2461 if (!_jitc->dataset)
2467 assert(_jit->user_code);
2469 if (!_jit->user_code) {
2470 mmap_prot = PROT_READ | PROT_WRITE;
2471 #if !(__OpenBSD__ || __APPLE__)
2472 mmap_prot |= PROT_EXEC;
2475 mmap_prot = PROT_MPROTECT(mmap_prot);
2478 mmap_flags = MAP_PRIVATE;
2480 mmap_flags |= MAP_ANON;
2482 mmap_fd = open("/dev/zero", O_RDWR);
2484 _jit->code.ptr = mmap(NULL, _jit->code.length,
2485 mmap_prot, mmap_flags, mmap_fd, 0);
2486 assert(_jit->code.ptr != MAP_FAILED);
2488 #endif /* !HAVE_MMAP */
2489 _jitc->code.end = _jit->code.ptr + _jit->code.length -
2490 jit_get_max_instr();
2491 _jit->pc.uc = _jit->code.ptr;
2498 mprotect(_jit->code.ptr, _jit->code.length, PROT_READ | PROT_WRITE);
2499 assert(result == 0);
2501 if ((code = emit_code()) == NULL) {
2502 _jitc->patches.offset = 0;
2503 for (node = _jitc->head; node; node = node->next) {
2505 (node->code == jit_code_label ||
2506 node->code == jit_code_epilog))
2507 node->flag &= ~jit_flag_patch;
2510 assert(_jit->user_code);
2512 if (_jit->user_code)
2516 length = _jitc->pool.length * 1024 * _jitc->mult;
2518 /* Should only happen on very special cases */
2519 length = _jit->code.length + 4096;
2523 munmap(_jit->code.ptr, _jit->code.length);
2528 _jit->code.ptr = mremap(_jit->code.ptr, _jit->code.length,
2529 _jit->code.ptr, length, 0);
2531 _jit->code.ptr = mremap(_jit->code.ptr, _jit->code.length,
2532 length, MREMAP_MAYMOVE, NULL);
2535 _jit->code.ptr = mmap(NULL, length,
2536 mmap_prot, mmap_flags, mmap_fd, 0);
2539 assert(_jit->code.ptr != MAP_FAILED);
2540 _jit->code.length = length;
2541 _jitc->code.end = _jit->code.ptr + _jit->code.length -
2542 jit_get_max_instr();
2543 _jit->pc.uc = _jit->code.ptr;
2544 #endif /* !HAVE_MMAP */
2551 if (!_jit->user_code)
2556 if (!_jitc->no_note)
2559 if (_jit->user_data)
2560 jit_free((jit_pointer_t *)&_jitc->data.ptr);
2566 mprotect(_jit->data.ptr, _jit->data.length, PROT_READ);
2567 assert(result == 0);
2569 if (!_jit->user_code) {
2570 _jit->code.protect = _jit->pc.uc - _jit->code.ptr;
2571 # if __riscv && __WORDSIZE == 64
2572 /* FIXME should start adding consts at a page boundary */
2573 _jit->code.protect -= _jitc->consts.hash.count * sizeof(jit_word_t);
2577 mprotect(_jit->code.ptr, _jit->code.protect, PROT_READ | PROT_EXEC);
2579 assert(result == 0);
2581 #endif /* HAVE_MMAP */
2583 return (_jit->code.ptr);
2589 _jit_protect(jit_state_t *_jit)
2592 assert (_jit->user_code);
2597 if (_jit->user_code) return;
2601 mprotect (_jit->code.ptr, _jit->code.protect, PROT_READ | PROT_EXEC);
2602 assert (result == 0);
2607 _jit_unprotect(jit_state_t *_jit)
2610 assert (_jit->user_code);
2615 if (_jit->user_code) return;
2619 mprotect (_jit->code.ptr, _jit->code.protect, PROT_READ | PROT_WRITE);
2620 assert (result == 0);
2625 _jit_frame(jit_state_t *_jit, jit_int32_t frame)
2627 jit_trampoline(frame, 1);
2631 _jit_tramp(jit_state_t *_jit, jit_int32_t frame)
2633 jit_trampoline(frame, 0);
2637 _jit_trampoline(jit_state_t *_jit, jit_int32_t frame, jit_bool_t prolog)
2641 /* Must be called after prolog, actually, just to simplify
2642 * tests and know there is a current function and that
2643 * _jitc->function->self.aoff is at the before any alloca value */
2644 assert(_jitc->tail && _jitc->tail->code == jit_code_prolog);
2646 /* + 24 for 3 possible spilled temporaries (that could be a double) */
2648 #if defined(__hppa__)
2649 frame += _jitc->function->self.aoff;
2651 frame -= _jitc->function->self.aoff;
2653 _jitc->function->frame = frame;
2655 _jitc->function->define_frame = 1;
2657 _jitc->function->assume_frame = 1;
2658 for (regno = 0; regno < _jitc->reglen; regno++)
2659 if (jit_class(_rvs[regno].spec) & jit_class_sav)
2660 jit_regset_setbit(&_jitc->function->regset, regno);
2663 /* Compute initial reglive and regmask set values of a basic block.
2664 * reglive is the set of known live registers
2665 * regmask is the set of registers not referenced in the block
2666 * Registers in regmask might be live.
2669 _jit_setup(jit_state_t *_jit, jit_block_t *block)
2673 unsigned long value;
2675 jit_regset_set_mask(&block->regmask, _jitc->reglen);
2676 for (value = 0; value < _jitc->reglen; ++value)
2677 if (!(jit_class(_rvs[value].spec) & (jit_class_gpr|jit_class_fpr)))
2678 jit_regset_clrbit(&block->regmask, value);
2680 for (node = block->label->next; node; node = node->next) {
2681 switch (node->code) {
2682 case jit_code_label: case jit_code_prolog:
2683 case jit_code_epilog:
2686 /* Check argument registers in reverse order to properly
2687 * handle registers that are both, argument and result */
2688 value = jit_classify(node->code);
2689 if (value & jit_cc_a2_reg) {
2690 live = !(value & jit_cc_a2_chg);
2691 if (value & jit_cc_a2_rlh) {
2692 /* Assume will not modify a pair in second argument */
2694 if (!(node->w.q.l & jit_regno_patch) &&
2695 jit_regset_tstbit(&block->regmask, node->w.q.l)) {
2696 jit_regset_clrbit(&block->regmask, node->w.q.l);
2698 if (!(node->w.q.h & jit_regno_patch) &&
2699 jit_regset_tstbit(&block->regmask, node->w.q.h)) {
2700 jit_regset_clrbit(&block->regmask, node->w.q.h);
2704 if (!(node->w.w & jit_regno_patch) &&
2705 jit_regset_tstbit(&block->regmask, node->w.w)) {
2706 jit_regset_clrbit(&block->regmask, node->w.w);
2708 jit_regset_setbit(&block->reglive, node->w.w);
2712 if (value & jit_cc_a1_reg) {
2713 live = !(value & jit_cc_a1_chg);
2714 if (value & jit_cc_a1_rlh) {
2715 /* Assume will not modify a pair in second argument */
2717 if (!(node->v.q.l & jit_regno_patch) &&
2718 jit_regset_tstbit(&block->regmask, node->v.q.l)) {
2719 jit_regset_clrbit(&block->regmask, node->v.q.l);
2721 if (!(node->v.q.h & jit_regno_patch) &&
2722 jit_regset_tstbit(&block->regmask, node->v.q.h)) {
2723 jit_regset_clrbit(&block->regmask, node->v.q.h);
2727 if (!(node->v.w & jit_regno_patch) &&
2728 jit_regset_tstbit(&block->regmask, node->v.w)) {
2729 jit_regset_clrbit(&block->regmask, node->v.w);
2731 jit_regset_setbit(&block->reglive, node->v.w);
2735 if (value & jit_cc_a0_reg) {
2736 live = !(value & jit_cc_a0_chg);
2737 if (value & jit_cc_a0_rlh) {
2738 if (!(node->u.q.l & jit_regno_patch) &&
2739 jit_regset_tstbit(&block->regmask, node->u.q.l)) {
2740 jit_regset_clrbit(&block->regmask, node->u.q.l);
2742 jit_regset_setbit(&block->reglive, node->u.q.l);
2744 if (!(node->u.q.h & jit_regno_patch) &&
2745 jit_regset_tstbit(&block->regmask, node->u.q.h)) {
2746 jit_regset_clrbit(&block->regmask, node->u.q.h);
2748 jit_regset_setbit(&block->reglive, node->u.q.h);
2752 if (!(node->u.w & jit_regno_patch) &&
2753 jit_regset_tstbit(&block->regmask, node->u.w)) {
2754 jit_regset_clrbit(&block->regmask, node->u.w);
2756 jit_regset_setbit(&block->reglive, node->u.w);
2765 /* Update regmask and reglive of blocks at entry point of branch targets
2766 * or normal flow that have a live register not used in this block.
2769 _jit_follow(jit_state_t *_jit, jit_block_t *block)
2775 unsigned long value;
2777 jit_regset_t reglive;
2778 jit_regset_t regmask;
2779 jit_regset_t regtemp;
2781 jit_regset_set(®live, &block->reglive);
2782 jit_regset_set(®mask, &block->regmask);
2783 for (node = block->label->next; node; node = node->next) {
2784 switch (node->code) {
2785 case jit_code_label:
2786 /* Do not consider jmpi and jmpr cannot jump to the
2787 * next instruction. */
2788 next = _jitc->blocks.ptr + node->v.w;
2789 /* Set of live registers in next block that are at unknown
2790 * state in this block. */
2791 jit_regset_and(®temp, ®mask, &next->reglive);
2792 if (jit_regset_set_p(®temp)) {
2793 /* Add live state of next block to current block. */
2794 jit_regset_ior(&block->reglive, &block->reglive, ®temp);
2795 /* Remove from unknown state bitmask. */
2796 jit_regset_com(®temp, ®temp);
2797 jit_regset_and(&block->regmask, &block->regmask, ®temp);
2800 case jit_code_prolog:
2801 case jit_code_epilog:
2803 case jit_code_callr:
2804 value = jit_regno(node->u.w);
2805 if (!(node->u.w & jit_regno_patch)) {
2806 if (jit_regset_tstbit(®mask, value)) {
2807 jit_regset_clrbit(®mask, value);
2808 jit_regset_setbit(®live, value);
2811 case jit_code_calli:
2812 for (value = 0; value < _jitc->reglen; ++value) {
2813 value = jit_regset_scan1(®mask, value);
2814 if (value >= _jitc->reglen)
2816 spec = jit_class(_rvs[value].spec);
2817 if (!(spec & jit_class_sav))
2818 jit_regset_clrbit(®mask, value);
2819 if ((spec & jit_class_arg) && jit_regarg_p(node, value))
2820 jit_regset_setbit(®live, value);
2824 value = jit_classify(node->code);
2825 if (value & jit_cc_a2_reg) {
2826 if (value & jit_cc_a2_rlh) {
2827 if (!(node->w.q.l & jit_regno_patch)) {
2828 /* Assume register is not changed */
2829 if (jit_regset_tstbit(®mask, node->w.q.l))
2830 jit_regset_clrbit(®mask, node->w.q.l);
2832 if (!(node->w.q.h & jit_regno_patch)) {
2833 if (jit_regset_tstbit(®mask, node->w.q.h))
2834 jit_regset_clrbit(®mask, node->w.q.h);
2838 if (!(node->w.w & jit_regno_patch)) {
2839 if (jit_regset_tstbit(®mask, node->w.w)) {
2840 jit_regset_clrbit(®mask, node->w.w);
2841 if (!(value & jit_cc_a2_chg))
2842 jit_regset_setbit(®live, node->w.w);
2847 if (value & jit_cc_a1_reg) {
2848 if (value & jit_cc_a1_rlh) {
2849 if (!(node->v.q.l & jit_regno_patch)) {
2850 /* Assume register is not changed */
2851 if (jit_regset_tstbit(®mask, node->v.q.l))
2852 jit_regset_clrbit(®mask, node->v.q.l);
2854 if (!(node->v.q.h & jit_regno_patch)) {
2855 if (jit_regset_tstbit(®mask, node->v.q.h))
2856 jit_regset_clrbit(®mask, node->v.q.h);
2860 if (!(node->v.w & jit_regno_patch)) {
2861 if (jit_regset_tstbit(®mask, node->v.w)) {
2862 jit_regset_clrbit(®mask, node->v.w);
2863 if (!(value & jit_cc_a1_chg))
2864 jit_regset_setbit(®live, node->v.w);
2869 if (value & jit_cc_a0_reg) {
2870 if (value & jit_cc_a0_rlh) {
2871 if (!(node->u.q.l & jit_regno_patch)) {
2872 if (jit_regset_tstbit(®mask, node->u.q.l)) {
2873 jit_regset_clrbit(®mask, node->u.q.l);
2874 if (!(value & jit_cc_a0_chg))
2875 jit_regset_setbit(®live, node->u.q.l);
2878 if (!(node->u.q.h & jit_regno_patch)) {
2879 if (jit_regset_tstbit(®mask, node->u.q.h)) {
2880 jit_regset_clrbit(®mask, node->u.q.h);
2881 if (!(value & jit_cc_a0_chg))
2882 jit_regset_setbit(®live, node->u.q.h);
2887 if (!(node->u.w & jit_regno_patch)) {
2888 if (jit_regset_tstbit(®mask, node->u.w)) {
2889 jit_regset_clrbit(®mask, node->u.w);
2890 if (!(value & jit_cc_a0_chg))
2891 jit_regset_setbit(®live, node->u.w);
2896 if (value & jit_cc_a0_jmp) {
2897 if (node->flag & jit_flag_node) {
2899 /* Do not consider jmpi and jmpr cannot jump to the
2900 * next instruction. */
2901 next = _jitc->blocks.ptr + label->v.w;
2902 jit_regset_and(®temp, ®mask, &next->reglive);
2903 if (jit_regset_set_p(®temp)) {
2904 /* Add live state. */
2905 jit_regset_ior(&block->reglive,
2906 &block->reglive, ®temp);
2907 /* Remove from unknown state bitmask. */
2908 jit_regset_com(®temp, ®temp);
2909 jit_regset_and(&block->regmask,
2910 &block->regmask, ®temp);
2915 /* Jump to unknown location.
2916 * This is a pitfall of the implementation.
2917 * Only jmpi to not a jit code should reach here,
2918 * or a jmpr of a computed address.
2919 * Because the implementation needs jit_class_nospill
2920 * registers, must treat jmpr as a function call. This
2921 * means that only JIT_Vn registers can be trusted on
2924 jit_regset_set_ui(®mask, 0);
2925 for (regno = 0; regno < _jitc->reglen; regno++) {
2926 spec = jit_class(_rvs[regno].spec);
2927 if ((spec & (jit_class_gpr|jit_class_fpr)) &&
2928 (spec & jit_class_sav))
2929 jit_regset_setbit(®mask, regno);
2931 /* Assume non callee save registers are live due
2932 * to jump to unknown location. */
2933 /* Treat all callee save as live. */
2934 jit_regset_ior(&block->reglive, ®live, ®mask);
2935 /* Treat anything else as dead. */
2944 /* Follow code generation up to finding a label or end of code.
2945 * When finding a label, update the set of live registers.
2946 * On branches, update based on taken branch or normal flow.
2949 _jit_update(jit_state_t *_jit, jit_node_t *node,
2950 jit_regset_t *live, jit_regset_t *mask)
2954 unsigned long value;
2957 jit_regset_t regtemp;
2959 for (; node; node = node->next) {
2960 if (jit_regset_set_p(mask) == 0)
2962 switch (node->code) {
2963 case jit_code_label:
2964 block = _jitc->blocks.ptr + node->v.w;
2965 jit_regset_and(®temp, mask, &block->reglive);
2966 if (jit_regset_set_p(®temp)) {
2967 /* Add live state. */
2968 jit_regset_ior(live, live, ®temp);
2969 /* Remove from unknown state bitmask. */
2970 jit_regset_com(®temp, ®temp);
2971 jit_regset_and(mask, mask, ®temp);
2974 case jit_code_prolog:
2975 jit_regset_set_ui(mask, 0);
2977 case jit_code_epilog:
2978 jit_regset_set_ui(mask, 0);
2980 case jit_code_callr:
2981 value = jit_regno(node->u.w);
2982 if (!(node->u.w & jit_regno_patch)) {
2983 if (jit_regset_tstbit(mask, value)) {
2984 jit_regset_clrbit(mask, value);
2985 jit_regset_setbit(live, value);
2988 case jit_code_calli:
2989 for (value = 0; value < _jitc->reglen; ++value) {
2990 value = jit_regset_scan1(mask, value);
2991 if (value >= _jitc->reglen)
2993 spec = jit_class(_rvs[value].spec);
2994 if (!(spec & jit_class_sav))
2995 jit_regset_clrbit(mask, value);
2996 if ((spec & jit_class_arg) && jit_regarg_p(node, value))
2997 jit_regset_setbit(live, value);
3001 value = jit_classify(node->code);
3002 if (value & jit_cc_a2_reg) {
3003 if (value & jit_cc_a2_rlh) {
3004 if (!(node->w.q.l & jit_regno_patch)) {
3005 /* Assume register is not changed */
3006 if (jit_regset_tstbit(mask, node->w.q.l))
3007 jit_regset_clrbit(mask, node->w.q.l);
3009 if (!(node->w.q.h & jit_regno_patch)) {
3010 if (jit_regset_tstbit(mask, node->w.q.h))
3011 jit_regset_clrbit(mask, node->w.q.h);
3015 if (!(node->w.w & jit_regno_patch)) {
3016 if (jit_regset_tstbit(mask, node->w.w)) {
3017 jit_regset_clrbit(mask, node->w.w);
3018 if (!(value & jit_cc_a2_chg))
3019 jit_regset_setbit(live, node->w.w);
3024 if (value & jit_cc_a1_reg) {
3025 if (value & jit_cc_a1_rlh) {
3026 if (!(node->v.q.l & jit_regno_patch)) {
3027 /* Assume register is not changed */
3028 if (jit_regset_tstbit(mask, node->v.q.l))
3029 jit_regset_clrbit(mask, node->v.q.l);
3031 if (!(node->v.q.h & jit_regno_patch)) {
3032 if (jit_regset_tstbit(mask, node->v.q.h))
3033 jit_regset_clrbit(mask, node->v.q.h);
3037 if (!(node->v.w & jit_regno_patch)) {
3038 if (jit_regset_tstbit(mask, node->v.w)) {
3039 jit_regset_clrbit(mask, node->v.w);
3040 if (!(value & jit_cc_a1_chg))
3041 jit_regset_setbit(live, node->v.w);
3046 if (value & jit_cc_a0_reg) {
3047 if (value & jit_cc_a0_rlh) {
3048 if (!(node->u.q.l & jit_regno_patch)) {
3049 if (jit_regset_tstbit(mask, node->u.q.l)) {
3050 jit_regset_clrbit(mask, node->u.q.l);
3051 if (!(value & jit_cc_a0_chg))
3052 jit_regset_setbit(live, node->u.q.l);
3055 if (!(node->u.q.h & jit_regno_patch)) {
3056 if (jit_regset_tstbit(mask, node->u.q.h)) {
3057 jit_regset_clrbit(mask, node->u.q.h);
3058 if (!(value & jit_cc_a0_chg))
3059 jit_regset_setbit(live, node->u.q.h);
3064 if (!(node->u.w & jit_regno_patch)) {
3065 if (jit_regset_tstbit(mask, node->u.w)) {
3066 jit_regset_clrbit(mask, node->u.w);
3067 if (!(value & jit_cc_a0_chg))
3068 jit_regset_setbit(live, node->u.w);
3073 if (value & jit_cc_a0_jmp) {
3074 if (node->flag & jit_flag_node) {
3076 /* Do not consider jmpi and jmpr cannot jump to the
3077 * next instruction. */
3078 block = _jitc->blocks.ptr + label->v.w;
3079 jit_regset_and(®temp, mask, &block->reglive);
3080 if (jit_regset_set_p(®temp)) {
3081 /* Add live state. */
3082 jit_regset_ior(live, live, ®temp);
3083 /* Remove from unknown state bitmask. */
3084 jit_regset_com(®temp, ®temp);
3085 jit_regset_and(mask, mask, ®temp);
3089 /* Jump to unknown location.
3090 * This is a pitfall of the implementation.
3091 * Only jmpi to not a jit code should reach here,
3092 * or a jmpr of a computed address.
3093 * Because the implementation needs jit_class_nospill
3094 * registers, must treat jmpr as a function call. This
3095 * means that only JIT_Vn registers can be trusted on
3098 jit_regset_set_ui(mask, 0);
3099 for (regno = 0; regno < _jitc->reglen; regno++) {
3100 spec = jit_class(_rvs[regno].spec);
3101 if ((spec & (jit_class_gpr|jit_class_fpr)) &&
3102 (spec & jit_class_sav))
3103 jit_regset_setbit(mask, regno);
3105 /* Assume non callee save registers are live due
3106 * to jump to unknown location. */
3107 /* Treat all callee save as live. */
3108 jit_regset_ior(live, live, mask);
3109 /* Prevent explicitly set as live registers to
3110 * be used as a temporary for the jmpi. */
3111 jit_regset_ior(live, live, &_jitc->explive);
3112 /* Treat anything else as dead. */
3122 _thread_jumps(jit_state_t *_jit)
3129 for (prev = node = _jitc->head; node;) {
3131 switch (node->code) {
3133 if (redundant_jump(prev, node)) {
3137 if (shortcut_jump(prev, node))
3141 case jit_code_callr: case jit_code_calli:
3142 /* non optimizable jump like code */
3145 mask = jit_classify(node->code);
3146 if (mask & jit_cc_a0_jmp) {
3147 if (reverse_jump(prev, node) ||
3148 shortcut_jump(prev, node))
3159 _sequential_labels(jit_state_t *_jit)
3167 for (prev = node = _jitc->head; node; node = next) {
3169 if (node->code == jit_code_label) {
3172 del_label(prev, node);
3175 if (prev != node && prev->code == jit_code_label) {
3176 if ((jump = node->link)) {
3177 for (; jump; jump = link) {
3179 if (jump->code == jit_code_movi)
3183 jump->link = prev->link;
3188 del_label(prev, node);
3192 if (next && next->code == jit_code_label && !next->flag) {
3193 if ((jump = next->link)) {
3194 for (; jump; jump = link) {
3196 if (jump->code == jit_code_movi)
3200 jump->link = node->link;
3205 del_label(node, next);
3215 _split_branches(jit_state_t *_jit)
3221 jit_block_t *blocks;
3225 length = _jitc->blocks.length;
3226 jit_alloc((jit_pointer_t *)&blocks, length * sizeof(jit_block_t));
3227 if ((node = _jitc->head) &&
3228 (node->code == jit_code_label || node->code == jit_code_prolog)) {
3229 block = _jitc->blocks.ptr + node->v.w;
3230 memcpy(blocks, block, sizeof(jit_block_t));
3236 for (node = _jitc->head; node; node = next) {
3237 if ((next = node->next)) {
3238 if (next->code == jit_code_label ||
3239 next->code == jit_code_prolog ||
3240 next->code == jit_code_epilog) {
3241 if (offset >= length) {
3242 jit_realloc((jit_pointer_t *)&blocks,
3243 length * sizeof(jit_block_t),
3244 (length + 16) * sizeof(jit_block_t));
3247 block = _jitc->blocks.ptr + next->v.w;
3248 memcpy(blocks + offset, block, sizeof(jit_block_t));
3249 next->v.w = offset++;
3251 /* split block on branches */
3252 else if (jit_classify(node->code) & jit_cc_a0_jmp) {
3253 label = new_node(jit_code_label);
3256 if (offset >= length) {
3257 jit_realloc((jit_pointer_t *)&blocks,
3258 length * sizeof(jit_block_t),
3259 (length + 16) * sizeof(jit_block_t));
3262 block = blocks + offset;
3263 block->label = label;
3264 label->v.w = offset++;
3265 jit_regset_new(&block->reglive);
3266 jit_regset_new(&block->regmask);
3270 jit_free((jit_pointer_t *)&_jitc->blocks.ptr);
3271 _jitc->blocks.ptr = blocks;
3272 _jitc->blocks.offset = offset;
3273 _jitc->blocks.length = length;
3277 _shortcut_jump(jit_state_t *_jit, jit_node_t *prev, jit_node_t *node)
3284 if (!(node->flag & jit_flag_node))
3286 assert(node->code != jit_code_jmpr);
3287 cond = node->code != jit_code_jmpi;
3289 for (next = jump->next; next; next = next->next) {
3290 switch (next->code) {
3292 if (!(next->flag & jit_flag_node))
3294 if (jump->link == node)
3295 jump->link = node->link;
3297 for (temp = jump->link;
3300 assert(temp != NULL);
3301 temp->link = node->link;
3305 node->link = jump->link;
3311 node->code = jit_code_jmpr;
3312 node->u.w = next->u.w;
3314 node->flag &= ~jit_flag_node;
3316 case jit_code_note: case jit_code_label:
3326 _redundant_jump(jit_state_t *_jit, jit_node_t *prev, jit_node_t *node)
3328 jit_node_t *local_prev;
3329 jit_node_t *local_next;
3331 if (!(node->flag & jit_flag_node))
3333 for (local_prev = node, local_next = node->next;
3335 local_prev = local_next, local_next = local_next->next) {
3337 switch (local_next->code) {
3338 case jit_code_label: case jit_code_epilog:
3339 if (node->u.n == local_next) {
3340 if (local_next->link == node)
3341 local_next->link = node->link;
3343 for (local_prev = local_next->link;
3344 local_prev->link != node;
3345 local_prev = local_prev->link)
3346 assert(local_prev != NULL);
3347 local_prev->link = node->link;
3349 del_node(prev, node);
3353 case jit_code_name: case jit_code_note:
3363 reverse_jump_code(jit_code_t code)
3366 case jit_code_bltr: return (jit_code_bger);
3367 case jit_code_blti: return (jit_code_bgei);
3368 case jit_code_bltr_u: return (jit_code_bger_u);
3369 case jit_code_blti_u: return (jit_code_bgei_u);
3370 case jit_code_bler: return (jit_code_bgtr);
3371 case jit_code_blei: return (jit_code_bgti);
3372 case jit_code_bler_u: return (jit_code_bgtr_u);
3373 case jit_code_blei_u: return (jit_code_bgti_u);
3374 case jit_code_beqr: return (jit_code_bner);
3375 case jit_code_beqi: return (jit_code_bnei);
3376 case jit_code_bger: return (jit_code_bltr);
3377 case jit_code_bgei: return (jit_code_blti);
3378 case jit_code_bger_u: return (jit_code_bltr_u);
3379 case jit_code_bgei_u: return (jit_code_blti_u);
3380 case jit_code_bgtr: return (jit_code_bler);
3381 case jit_code_bgti: return (jit_code_blei);
3382 case jit_code_bgtr_u: return (jit_code_bler_u);
3383 case jit_code_bgti_u: return (jit_code_blei_u);
3384 case jit_code_bner: return (jit_code_beqr);
3385 case jit_code_bnei: return (jit_code_beqi);
3386 case jit_code_bmsr: return (jit_code_bmcr);
3387 case jit_code_bmsi: return (jit_code_bmci);
3388 case jit_code_bmcr: return (jit_code_bmsr);
3389 case jit_code_bmci: return (jit_code_bmsi);
3390 case jit_code_bltr_f: return (jit_code_bunger_f);
3391 case jit_code_blti_f: return (jit_code_bungei_f);
3392 case jit_code_bler_f: return (jit_code_bungtr_f);
3393 case jit_code_blei_f: return (jit_code_bungti_f);
3395 case jit_code_beqr_f: return (jit_code_bner_f);
3396 case jit_code_beqi_f: return (jit_code_bnei_f);
3398 case jit_code_bger_f: return (jit_code_bunltr_f);
3399 case jit_code_bgei_f: return (jit_code_bunlti_f);
3400 case jit_code_bgtr_f: return (jit_code_bunler_f);
3401 case jit_code_bgti_f: return (jit_code_bunlei_f);
3403 case jit_code_bner_f: return (jit_code_beqr_f);
3404 case jit_code_bnei_f: return (jit_code_beqi_f);
3406 case jit_code_bunltr_f: return (jit_code_bger_f);
3407 case jit_code_bunlti_f: return (jit_code_bgei_f);
3408 case jit_code_bunler_f: return (jit_code_bgtr_f);
3409 case jit_code_bunlei_f: return (jit_code_bgti_f);
3411 case jit_code_buneqr_f: return (jit_code_bltgtr_f);
3412 case jit_code_buneqi_f: return (jit_code_bltgti_f);
3414 case jit_code_bunger_f: return (jit_code_bltr_f);
3415 case jit_code_bungei_f: return (jit_code_blti_f);
3416 case jit_code_bungtr_f: return (jit_code_bler_f);
3417 case jit_code_bungti_f: return (jit_code_blei_f);
3419 case jit_code_bltgtr_f: return (jit_code_buneqr_f);
3420 case jit_code_bltgti_f: return (jit_code_buneqi_f);
3422 case jit_code_bordr_f: return (jit_code_bunordr_f);
3423 case jit_code_bordi_f: return (jit_code_bunordi_f);
3424 case jit_code_bunordr_f:return (jit_code_bordr_f);
3425 case jit_code_bunordi_f:return (jit_code_bordi_f);
3426 case jit_code_bltr_d: return (jit_code_bunger_d);
3427 case jit_code_blti_d: return (jit_code_bungei_d);
3428 case jit_code_bler_d: return (jit_code_bungtr_d);
3429 case jit_code_blei_d: return (jit_code_bungti_d);
3431 case jit_code_beqr_d: return (jit_code_bner_d);
3432 case jit_code_beqi_d: return (jit_code_bnei_d);
3434 case jit_code_bger_d: return (jit_code_bunltr_d);
3435 case jit_code_bgei_d: return (jit_code_bunlti_d);
3436 case jit_code_bgtr_d: return (jit_code_bunler_d);
3437 case jit_code_bgti_d: return (jit_code_bunlei_d);
3439 case jit_code_bner_d: return (jit_code_beqr_d);
3440 case jit_code_bnei_d: return (jit_code_beqi_d);
3442 case jit_code_bunltr_d: return (jit_code_bger_d);
3443 case jit_code_bunlti_d: return (jit_code_bgei_d);
3444 case jit_code_bunler_d: return (jit_code_bgtr_d);
3445 case jit_code_bunlei_d: return (jit_code_bgti_d);
3447 case jit_code_buneqr_d: return (jit_code_bltgtr_d);
3448 case jit_code_buneqi_d: return (jit_code_bltgti_d);
3450 case jit_code_bunger_d: return (jit_code_bltr_d);
3451 case jit_code_bungei_d: return (jit_code_blti_d);
3452 case jit_code_bungtr_d: return (jit_code_bler_d);
3453 case jit_code_bungti_d: return (jit_code_blei_d);
3455 case jit_code_bltgtr_d: return (jit_code_buneqr_d);
3456 case jit_code_bltgti_d: return (jit_code_buneqi_d);
3458 case jit_code_bordr_d: return (jit_code_bunordr_d);
3459 case jit_code_bordi_d: return (jit_code_bunordi_d);
3460 case jit_code_bunordr_d:return (jit_code_bordr_d);
3461 case jit_code_bunordi_d:return (jit_code_bordi_d);
3462 case jit_code_boaddr: return (jit_code_bxaddr);
3463 case jit_code_boaddi: return (jit_code_bxaddi);
3464 case jit_code_boaddr_u: return (jit_code_bxaddr_u);
3465 case jit_code_boaddi_u: return (jit_code_bxaddi_u);
3466 case jit_code_bxaddr: return (jit_code_boaddr);
3467 case jit_code_bxaddi: return (jit_code_boaddi);
3468 case jit_code_bxaddr_u: return (jit_code_boaddr_u);
3469 case jit_code_bxaddi_u: return (jit_code_boaddi_u);
3470 case jit_code_bosubr: return (jit_code_bxsubr);
3471 case jit_code_bosubi: return (jit_code_bxsubi);
3472 case jit_code_bosubr_u: return (jit_code_bxsubr_u);
3473 case jit_code_bosubi_u: return (jit_code_bxsubi_u);
3474 case jit_code_bxsubr: return (jit_code_bosubr);
3475 case jit_code_bxsubi: return (jit_code_bosubi);
3476 case jit_code_bxsubr_u: return (jit_code_bosubr_u);
3477 case jit_code_bxsubi_u: return (jit_code_bosubi_u);
3478 default: abort(); /* invalid jump code */
3483 * change common pattern:
3484 * <cond_jump L0> <jump L1> <label L0>
3486 * <reverse_cond_jump L1>
3489 _reverse_jump(jit_state_t *_jit, jit_node_t *prev, jit_node_t *node)
3491 jit_node_t *local_prev;
3492 jit_node_t *local_next;
3493 jit_node_t *local_jump;
3495 if (!(node->flag & jit_flag_node))
3497 /* =><cond_jump L0> <jump L1> <label L0> */
3498 local_next = node->next;
3499 if (local_next->code != jit_code_jmpi ||
3500 !(local_next->flag & jit_flag_node))
3502 /* <cond_jump L0> =><jump L1> <label L0> */
3504 local_jump = local_next->u.n;
3505 for (local_prev = local_next, local_next = local_next->next;
3507 local_prev = local_next, local_next = local_next->next) {
3508 switch (local_next->code) {
3509 case jit_code_label: case jit_code_epilog:
3510 if (node->u.n == local_next) {
3511 if (local_next->link == node)
3512 local_next->link = node->link;
3514 for (local_prev = local_next->link;
3515 local_prev->link != node;
3516 local_prev = local_prev->link)
3517 assert(local_prev != NULL);
3518 local_prev->link = node->link;
3520 del_node(node, node->next);
3521 node->code = reverse_jump_code(node->code);
3522 node->u.n = local_jump;
3523 node->link = local_jump->link;
3524 local_jump->link = node;
3538 _redundant_store(jit_state_t *_jit, jit_node_t *node, jit_bool_t jump)
3549 if (prev->code == jit_code_epilog)
3551 assert(prev->code == jit_code_label);
3552 if ((prev->flag & jit_flag_head) || node->link || prev->link != node)
3553 /* multiple sources */
3555 /* if there are sequential labels it will return below */
3561 regno = jit_regno(node->v.w);
3562 for (iter = prev->next; iter; prev = iter, iter = iter->next) {
3563 switch (iter->code) {
3564 case jit_code_label: case jit_code_prolog:
3565 case jit_code_epilog:
3568 if (regno == jit_regno(iter->u.w)) {
3569 if (iter->flag || iter->v.w != word)
3572 del_node(prev, iter);
3577 spec = jit_classify(iter->code);
3578 if (spec & jit_cc_a0_jmp)
3580 if ((spec & (jit_cc_a0_reg|jit_cc_a0_chg)) ==
3581 (jit_cc_a0_reg|jit_cc_a0_chg)) {
3582 if (spec & jit_cc_a0_rlh) {
3583 if (regno == jit_regno(iter->u.q.l) ||
3584 regno == jit_regno(iter->u.q.h))
3588 if (regno == jit_regno(iter->u.w))
3592 if ((spec & (jit_cc_a1_reg|jit_cc_a1_chg)) ==
3593 (jit_cc_a1_reg|jit_cc_a1_chg)) {
3594 if (regno == jit_regno(iter->v.w))
3597 if ((spec & (jit_cc_a2_reg|jit_cc_a2_chg)) ==
3598 (jit_cc_a2_reg|jit_cc_a2_chg)) {
3599 if (regno == jit_regno(iter->w.w))
3610 _simplify_movr(jit_state_t *_jit, jit_node_t *prev, jit_node_t *node,
3611 jit_int32_t kind, jit_int32_t size)
3618 regno = jit_regno(node->u.w);
3619 right = jit_regno(node->v.w);
3620 value = _jitc->values + regno;
3622 spec = jit_class(_rvs[regno].spec);
3623 if (!(spec & (jit_class_gpr | jit_class_fpr))) {
3624 /* reserved register */
3626 ++_jitc->gen[regno];
3630 if ((value->kind == jit_kind_register &&
3631 jit_regno(value->base.q.l) == right &&
3632 value->base.q.h == _jitc->gen[right]) ||
3633 (value->kind == kind && _jitc->values[right].kind == kind &&
3634 memcmp(&value->base.w, &_jitc->values[right].base.w, size) == 0)) {
3635 del_node(prev, node);
3638 if (_jitc->values[right].kind == jit_kind_word)
3639 jit_memcpy(value, _jitc->values + right, sizeof(jit_value_t));
3641 value->kind = jit_kind_register;
3642 value->base.q.l = right;
3643 value->base.q.h = _jitc->gen[right];
3645 ++_jitc->gen[regno];
3651 _simplify_movi(jit_state_t *_jit, jit_node_t *prev, jit_node_t *node,
3652 jit_int32_t kind, jit_int32_t size)
3659 regno = jit_regno(node->u.w);
3660 value = _jitc->values + regno;
3661 if (node->flag & jit_flag_node) {
3662 /* set to undefined if value will be patched */
3664 ++_jitc->gen[regno];
3667 spec = jit_class(_rvs[regno].spec);
3668 if (!(spec & (jit_class_gpr | jit_class_fpr))) {
3669 /* reserved register */
3671 ++_jitc->gen[regno];
3674 if (value->kind == kind) {
3675 if (memcmp(&node->v.w, &value->base.w, size) == 0) {
3676 del_node(prev, node);
3679 if (kind == jit_kind_word)
3680 spec &= jit_class_gpr;
3682 spec &= (jit_class_xpr | jit_class_fpr);
3683 for (offset = 0; offset < _jitc->reglen; offset++) {
3684 if (_jitc->values[offset].kind == kind &&
3685 memcmp(&node->v.w, &_jitc->values[offset].base.w, size) == 0 &&
3686 (jit_class(_rvs[offset].spec) & spec) == spec) {
3687 if (kind == jit_kind_word)
3688 node->code = jit_code_movr;
3689 else if (kind == jit_kind_float32)
3690 node->code = jit_code_movr_f;
3692 node->code = jit_code_movr_d;
3694 jit_memcpy(value, _jitc->values + offset, sizeof(jit_value_t));
3695 ++_jitc->gen[regno];
3701 jit_memcpy(&value->base.w, &node->v.w, size);
3702 ++_jitc->gen[regno];
3707 /* simple/safe redundandy test not checking if another register
3708 * holds the same value
3711 _simplify_ldxi(jit_state_t *_jit, jit_node_t *prev, jit_node_t *node)
3717 regno = jit_regno(node->u.w);
3718 right = jit_regno(node->v.w);
3719 value = _jitc->values + regno;
3720 if (regno != right &&
3721 value->kind == jit_kind_code && value->code == node->code &&
3722 value->base.q.l == right && value->base.q.h == _jitc->gen[right] &&
3723 node->w.w == value->disp.w) {
3724 del_node(prev, node);
3727 value->kind = jit_kind_code;
3728 value->code = node->code;
3729 value->base.q.l = right;
3730 value->base.q.h = _jitc->gen[right];
3731 value->disp.w = node->w.w;
3732 ++_jitc->gen[regno];
3738 _simplify_stxi(jit_state_t *_jit, jit_node_t *prev, jit_node_t *node)
3745 regno = jit_regno(node->w.w);
3746 right = jit_regno(node->v.w);
3747 value = _jitc->values + regno;
3749 /* check for redundant store after load */
3750 if (regno != right &&
3751 value->kind == jit_kind_code && value->code == node->code &&
3752 value->base.q.l == right && value->base.q.h == _jitc->gen[right] &&
3753 node->u.w == value->disp.w) {
3754 del_node(prev, node);
3758 /* assume anything can alias, and invalidate tracked values */
3759 for (offset = 0; offset < _jitc->reglen; offset++) {
3760 if (_jitc->values[offset].kind == jit_kind_code) {
3761 _jitc->values[offset].kind = 0;
3762 ++_jitc->gen[offset];
3766 /* no multiple information, so, if set to a constant,
3767 * prefer to keep that information */
3768 if (value->kind == 0) {
3769 switch (node->code) {
3770 /* no information about signed/unsigned either */
3771 case jit_code_stxi_c: value->code = jit_code_ldxi_c; break;
3772 case jit_code_stxi_s: value->code = jit_code_ldxi_s; break;
3773 case jit_code_stxi_i: value->code = jit_code_ldxi_i; break;
3774 case jit_code_stxi_l: value->code = jit_code_ldxi_l; break;
3775 case jit_code_stxi_f: value->code = jit_code_ldxi_f; break;
3776 case jit_code_stxi_d: value->code = jit_code_ldxi_d; break;
3779 value->kind = jit_kind_code;
3780 value->base.q.l = right;
3781 value->base.q.h = _jitc->gen[right];
3782 value->disp.w = node->u.w;
3788 /* usually there should be only one store in the
3789 * jit_get_reg/jit_unget_reg, but properly handle
3790 * multiple ones by moving the save node */
3792 _simplify_spill(jit_state_t *_jit, jit_node_t *node, jit_int32_t regno)
3797 if ((temp = _jitc->spill[regno]) && (save = temp->next) != node) {
3798 temp->next = save->next;
3799 save->next = node->next;
3801 _jitc->spill[regno] = node;
3805 /* checks for simple cases where a register is set more than
3806 * once to the same value, and is a common pattern of calls
3807 * to jit_pushargi and jit_pushargr
3810 _simplify(jit_state_t *_jit)
3820 for (prev = NULL, node = _jitc->head; node; prev = node, node = next) {
3822 switch (node->code) {
3823 case jit_code_label: case jit_code_prolog:
3824 case jit_code_callr: case jit_code_calli:
3826 memset(_jitc->gen, 0, sizeof(jit_int32_t) * _jitc->reglen);
3827 memset(_jitc->values, 0, sizeof(jit_value_t) * _jitc->reglen);
3830 _jitc->spill[jit_regno(node->u.w)] = prev;
3833 regno = jit_regno(node->u.w);
3834 if (register_change_p(node->link->next, node, regno) !=
3836 /* spill not required due to optimizing common
3837 * redundancy case of calling jit_get_reg/jit_unget_reg
3838 * and then setting the register to the value it is
3839 * already holding */
3840 patch_register(node->link->next, node,
3841 jit_regno_patch|regno, regno);
3843 del_node(_jitc->spill[regno], node->link);
3844 del_node(prev, node);
3847 _jitc->spill[regno] = NULL;
3850 regno = jit_regno(node->u.w);
3851 if (simplify_movr(prev, node,
3852 jit_kind_word, sizeof(jit_word_t))) {
3854 simplify_spill(node = prev, regno);
3858 regno = jit_regno(node->u.w);
3859 if (simplify_movi(prev, node,
3860 jit_kind_word, sizeof(jit_word_t))) {
3862 simplify_spill(node = prev, regno);
3865 case jit_code_movr_f:
3866 regno = jit_regno(node->u.w);
3867 if (simplify_movr(prev, node,
3868 jit_kind_float32, sizeof(jit_float32_t))) {
3870 simplify_spill(node = prev, regno);
3873 case jit_code_movi_f:
3874 regno = jit_regno(node->u.w);
3875 if (simplify_movi(prev, node,
3876 jit_kind_float32, sizeof(jit_float32_t))) {
3878 simplify_spill(node = prev, regno);
3881 case jit_code_movr_d:
3882 regno = jit_regno(node->u.w);
3883 if (simplify_movr(prev, node,
3884 jit_kind_float64, sizeof(jit_float64_t))) {
3886 simplify_spill(node = prev, regno);
3889 case jit_code_movi_d:
3890 regno = jit_regno(node->u.w);
3891 if (simplify_movi(prev, node,
3892 jit_kind_float64, sizeof(jit_float64_t))) {
3894 simplify_spill(node = prev, regno);
3897 case jit_code_ldxi_c: case jit_code_ldxi_uc:
3898 case jit_code_ldxi_s: case jit_code_ldxi_us:
3899 case jit_code_ldxi_i: case jit_code_ldxi_ui:
3900 case jit_code_ldxi_l:
3901 case jit_code_ldxi_f: case jit_code_ldxi_d:
3902 regno = jit_regno(node->u.w);
3903 if (simplify_ldxi(prev, node)) {
3905 simplify_spill(node = prev, regno);
3908 case jit_code_stxi_c: case jit_code_stxi_s:
3909 case jit_code_stxi_i: case jit_code_stxi_l:
3910 case jit_code_stxi_f: case jit_code_stxi_d:
3911 regno = jit_regno(node->u.w);
3912 if (simplify_stxi(prev, node)) {
3914 simplify_spill(node = prev, regno);
3918 info = jit_classify(node->code);
3919 if (info & jit_cc_a0_jmp)
3920 /* labels are not implicitly added when not taking
3921 * a conditional branch */
3923 if (info & jit_cc_a0_chg) {
3924 if (info & jit_cc_a0_rlh) {
3925 regno = jit_regno(node->u.q.l);
3926 _jitc->values[regno].kind = 0;
3927 ++_jitc->gen[regno];
3928 regno = jit_regno(node->u.q.h);
3929 _jitc->values[regno].kind = 0;
3930 ++_jitc->gen[regno];
3933 regno = jit_regno(node->u.w);
3934 _jitc->values[regno].kind = 0;
3935 ++_jitc->gen[regno];
3938 if (info & jit_cc_a1_chg) {
3940 /* Assume registers are not changed */
3941 if (info & jit_cc_a1_rlh) {
3942 regno = jit_regno(node->v.q.l);
3943 _jitc->values[regno].kind = 0;
3944 ++_jitc->gen[regno];
3945 regno = jit_regno(node->v.q.h);
3946 _jitc->values[regno].kind = 0;
3947 ++_jitc->gen[regno];
3951 regno = jit_regno(node->v.w);
3952 _jitc->values[regno].kind = 0;
3953 ++_jitc->gen[regno];
3958 if (info & jit_cc_a2_chg) {
3960 /* Assume registers are not changed */
3961 if (info & jit_cc_a2_rlh) {
3962 regno = jit_regno(node->w.q.l);
3963 _jitc->values[regno].kind = 0;
3964 ++_jitc->gen[regno];
3965 regno = jit_regno(node->w.q.h);
3966 _jitc->values[regno].kind = 0;
3967 ++_jitc->gen[regno];
3971 regno = jit_regno(node->w.w);
3972 _jitc->values[regno].kind = 0;
3973 ++_jitc->gen[regno];
3985 _register_change_p(jit_state_t *_jit, jit_node_t *node, jit_node_t *link,
3989 jit_bool_t use, change;
3991 for (; node != link; node = node->next) {
3992 switch (node->code) {
3993 case jit_code_label: case jit_code_prolog:
3994 /* lack of extra information so assume it is live */
3995 return (jit_reg_static);
3996 case jit_code_callr: case jit_code_calli:
3997 if (!(jit_class(_rvs[regno].spec) & jit_class_sav))
3998 return (jit_reg_undef);
4001 value = jit_classify(node->code);
4003 if (value & jit_cc_a0_rlh) {
4004 if (node->u.q.l == regno || node->u.q.h == regno) {
4005 if (value & jit_cc_a0_chg)
4010 else if ((value & jit_cc_a0_reg) && node->u.w == regno) {
4011 if (value & jit_cc_a0_chg)
4016 (value & jit_cc_a1_reg) && node->v.w == regno) {
4017 if (value & jit_cc_a1_chg)
4022 (value & jit_cc_a2_reg) && node->w.w == regno) {
4023 if (value & jit_cc_a2_chg)
4027 /* lack of extra information */
4028 if (!use && (value & (jit_cc_a0_jmp|jit_cc_a0_cnd)) &&
4029 /* In case of indirect branches, always consider
4030 * callee save registers as live. */
4031 !(jit_class(_rvs[regno].spec) & jit_class_sav))
4034 return (jit_reg_change);
4038 return (jit_reg_static);
4041 /* most of this could be done at the same time as generating jit, but
4042 * avoid complications on different cpu backends and patch spill/loads
4043 * here, by simulating jit generation */
4045 _spill_reglive_p(jit_state_t *_jit, jit_node_t *node, jit_int32_t regno)
4047 if (!jit_regset_tstbit(&_jitc->reglive, regno)) {
4048 jit_regset_setbit(&_jitc->regmask, regno);
4049 jit_update(node->next, &_jitc->reglive, &_jitc->regmask);
4050 if (!jit_regset_tstbit(&_jitc->reglive, regno) &&
4051 register_change_p(node->next, node->link, regno) != jit_reg_change)
4059 _patch_registers(jit_state_t *_jit)
4069 _jitc->function = NULL;
4071 jit_reglive_setup();
4072 for (prev = NULL, node = _jitc->head; node; node = next) {
4075 info = jit_classify(node->code);
4076 jit_regarg_set(node, info);
4078 switch (node->code) {
4080 regno = jit_regno(node->u.w);
4081 if (!spill_reglive_p(node, regno)) {
4082 /* register is not live, just remove spill/reload */
4083 jit_regarg_clr(node, info);
4084 node->link->v.w = jit_regload_delete;
4085 del_node(prev, node);
4089 /* try to find a free register of the same class */
4090 spec = jit_class(_rvs[regno].spec) & ~jit_class_arg;
4091 for (value = 0; value < _jitc->reglen; value++) {
4092 if (value != regno &&
4093 ((jit_class(_rvs[value].spec) & spec) &
4094 ~jit_class_arg) == spec &&
4095 !jit_regset_tstbit(&_jitc->regarg, value) &&
4096 !spill_reglive_p(node, value))
4099 if (value < _jitc->reglen) {
4100 jit_regarg_clr(node, info);
4101 patch_register(node->next, node->link,
4102 jit_regno_patch|node->u.w,
4103 jit_regno_patch|value);
4104 /* mark as live just in case there are nested
4105 * register patches, so that next patch will
4106 * not want to use the same register */
4107 jit_regset_setbit(&_jitc->reglive, value);
4108 /* register is not live, just remove spill/reload */
4109 node->link->v.w = jit_regload_isdead;
4110 del_node(prev, node);
4114 /* failed to find a free register */
4115 if (spec & jit_class_gpr) {
4116 if (!_jitc->function->regoff[regno])
4117 _jitc->function->regoff[regno] =
4118 jit_allocai(sizeof(jit_word_t));
4119 #if __WORDSIZE == 32
4120 node->code = jit_code_stxi_i;
4122 node->code = jit_code_stxi_l;
4126 node->code = jit_code_stxi_d;
4127 if (!_jitc->function->regoff[regno])
4128 _jitc->function->regoff[regno] =
4129 jit_allocai(sizeof(jit_float64_t));
4131 node->u.w = _jitc->function->regoff[regno];
4139 regno = jit_regno(node->u.w);
4141 if (node->v.w == jit_regload_isdead)
4142 jit_regset_clrbit(&_jitc->reglive, regno);
4143 del_node(prev, node);
4146 spec = jit_class(_rvs[regno].spec);
4147 if (spec & jit_class_gpr) {
4148 #if __WORDSIZE == 32
4149 node->code = jit_code_ldxi_i;
4151 node->code = jit_code_ldxi_l;
4155 node->code = jit_code_ldxi_d;
4158 node->w.w = _jitc->function->regoff[regno];
4161 case jit_code_prolog:
4162 _jitc->function = _jitc->functions.ptr + node->w.w;
4164 case jit_code_epilog:
4165 _jitc->function = NULL;
4171 jit_regarg_clr(node, info);
4172 /* update register live state */
4179 _patch_register(jit_state_t *_jit, jit_node_t *node, jit_node_t *link,
4180 jit_int32_t regno, jit_int32_t patch)
4184 for (; node != link; node = node->next) {
4185 value = jit_classify(node->code);
4186 if (value & jit_cc_a0_reg) {
4187 if (value & jit_cc_a0_rlh) {
4188 if (node->u.q.l == regno)
4189 node->u.q.l = patch;
4190 if (node->u.q.h == regno)
4191 node->u.q.h = patch;
4194 if (node->u.w == regno)
4198 if (value & jit_cc_a1_reg) {
4199 if (value & jit_cc_a1_rlh) {
4200 if (node->v.q.l == regno)
4201 node->v.q.l = patch;
4202 if (node->v.q.h == regno)
4203 node->v.q.h = patch;
4206 if (node->v.w == regno)
4210 if (value & jit_cc_a2_reg) {
4211 if (value & jit_cc_a2_rlh) {
4212 if (node->w.q.l == regno)
4213 node->w.q.l = patch;
4214 if (node->w.q.h == regno)
4215 node->w.q.h = patch;
4218 if (node->w.w == regno)
4225 #if __BYTE_ORDER == __LITTLE_ENDIAN
4226 # define htonr_us(r0,r1) bswapr_us(r0,r1)
4227 # define htonr_ui(r0,r1) bswapr_ui(r0,r1)
4228 # if __WORDSIZE == 64
4229 # define htonr_ul(r0,r1) bswapr_ul(r0,r1)
4232 # define htonr_us(r0,r1) extr_us(r0,r1)
4233 # if __WORDSIZE == 32
4234 # define htonr_ui(r0,r1) movr(r0,r1)
4236 # define htonr_ui(r0,r1) extr_ui(r0,r1)
4237 # define htonr_ul(r0,r1) movr(r0,r1)
4241 static maybe_unused void
4242 generic_bswapr_us(jit_state_t *_jit, jit_int32_t r0, jit_int32_t r1);
4243 static maybe_unused void
4244 generic_bswapr_ui(jit_state_t *_jit, jit_int32_t r0, jit_int32_t r1);
4245 #if __WORDSIZE == 64
4246 static maybe_unused void
4247 generic_bswapr_ul(jit_state_t *_jit, jit_int32_t r0, jit_int32_t r1);
4250 #define depi(r0, i0, i1, i2) _depi(_jit, r0, i0, i1, i2)
4251 static void _depi(jit_state_t*,jit_int32_t, jit_word_t, jit_word_t, jit_word_t);
4252 #define negi(r0, i0) _negi(_jit, r0, i0)
4253 static void _negi(jit_state_t*, jit_int32_t, jit_word_t);
4254 #define comi(r0, i0) _comi(_jit, r0, i0)
4255 static void _comi(jit_state_t*, jit_int32_t, jit_word_t);
4256 #define exti_c(r0, i0) _exti_c(_jit, r0, i0)
4257 static void _exti_c(jit_state_t*, jit_int32_t, jit_word_t);
4258 #define exti_uc(r0, i0) _exti_uc(_jit, r0, i0)
4259 static void _exti_uc(jit_state_t*, jit_int32_t, jit_word_t);
4260 #define exti_s(r0, i0) _exti_s(_jit, r0, i0)
4261 static void _exti_s(jit_state_t*, jit_int32_t, jit_word_t);
4262 #define exti_us(r0, i0) _exti_us(_jit, r0, i0)
4263 static void _exti_us(jit_state_t*, jit_int32_t, jit_word_t);
4264 #if __WORDSIZE == 64
4265 #define exti_i(r0, i0) _exti_i(_jit, r0, i0)
4266 static void _exti_i(jit_state_t*, jit_int32_t, jit_word_t);
4267 #define exti_ui(r0, i0) _exti_ui(_jit, r0, i0)
4268 static void _exti_ui(jit_state_t*, jit_int32_t, jit_word_t);
4270 #define bswapi_us(r0, i0) _bswapi_us(_jit, r0, i0)
4271 static void _bswapi_us(jit_state_t*, jit_int32_t, jit_word_t);
4272 #define bswapi_ui(r0, i0) _bswapi_ui(_jit, r0, i0)
4273 static void _bswapi_ui(jit_state_t*, jit_int32_t, jit_word_t);
4274 #if __WORDSIZE == 64
4275 # define bswapi_ul(r0, i0) _bswapi_ul(_jit, r0, i0)
4276 static void _bswapi_ul(jit_state_t*, jit_int32_t, jit_word_t);
4278 #define htoni_us(r0, i0) _htoni_us(_jit, r0, i0)
4279 static void _htoni_us(jit_state_t*, jit_int32_t, jit_word_t);
4280 #define htoni_ui(r0, i0) _htoni_ui(_jit, r0, i0)
4281 static void _htoni_ui(jit_state_t*, jit_int32_t, jit_word_t);
4282 #if __WORDSIZE == 64
4283 # define htoni_ul(r0, i0) _htoni_ul(_jit, r0, i0)
4284 static void _htoni_ul(jit_state_t*, jit_int32_t, jit_word_t);
4286 # define movi_f_w(r0, i0) _movi_f_w(_jit, r0, i0)
4287 static void _movi_f_w(jit_state_t*, jit_int32_t, jit_float32_t);
4288 #if __WORDSIZE == 32 && !(defined(__mips__) && NEW_ABI)
4289 # define movi_d_ww(r0, r1, i0) _movi_d_ww(_jit, r0, r1, i0)
4290 static void _movi_d_ww(jit_state_t*, jit_int32_t, jit_int32_t, jit_float64_t);
4292 # define movi_d_w(r0, i0) _movi_d_w(_jit, r0, i0)
4293 static void _movi_d_w(jit_state_t*, jit_int32_t, jit_float64_t);
4295 #define cloi(r0, i0) _cloi(_jit, r0, i0)
4296 static void _cloi(jit_state_t*, jit_int32_t, jit_word_t);
4297 #define clzi(r0, i0) _clzi(_jit, r0, i0)
4298 static void _clzi(jit_state_t*, jit_int32_t, jit_word_t);
4299 #define ctoi(r0, i0) _ctoi(_jit, r0, i0)
4300 static void _ctoi(jit_state_t*, jit_int32_t, jit_word_t);
4301 #define ctzi(r0, i0) _ctzi(_jit, r0, i0)
4302 static void _ctzi(jit_state_t*, jit_int32_t, jit_word_t);
4303 #define rbiti(r0, i0) _rbiti(_jit, r0, i0)
4304 static void _rbiti(jit_state_t*, jit_int32_t, jit_word_t);
4305 #define popcnti(r0, i0) _popcnti(_jit, r0, i0)
4306 static void _popcnti(jit_state_t*, jit_int32_t, jit_word_t);
4307 #define exti(r0, i0, i1, i2) _exti(_jit, r0, i0, i1, i2)
4308 static void _exti(jit_state_t*,jit_int32_t, jit_word_t, jit_word_t, jit_word_t);
4309 #define exti_u(r0, i0, i1, i2) _exti_u(_jit, r0, i0, i1, i2)
4310 static void _exti_u(jit_state_t*,
4311 jit_int32_t, jit_word_t, jit_word_t, jit_word_t);
4312 #define generic_unldr(r0, r1, i0) _generic_unldr(_jit, r0, r1, i0)
4313 static void _generic_unldr(jit_state_t*, jit_int32_t, jit_int32_t, jit_word_t);
4314 #define generic_unldi(r0, i0, i1) _generic_unldi(_jit, r0, i0, i1)
4315 static void _generic_unldi(jit_state_t*, jit_int32_t, jit_word_t, jit_word_t);
4316 #define generic_unldr_u(r0, r1, i0) _generic_unldr_u(_jit, r0, r1, i0)
4317 static void _generic_unldr_u(jit_state_t*,
4318 jit_int32_t, jit_int32_t, jit_word_t);
4319 #define generic_unldi_u(r0, i0, i1) _generic_unldi_u(_jit, r0, i0, i1)
4320 static void _generic_unldi_u(jit_state_t*, jit_int32_t, jit_word_t, jit_word_t);
4321 #define generic_unstr(r0, r1, i0) _generic_unstr(_jit, r0, r1, i0)
4322 static void _generic_unstr(jit_state_t*, jit_int32_t, jit_int32_t, jit_word_t);
4323 #define generic_unsti(i0, r0, i1) _generic_unsti(_jit, i0, r0, i1)
4324 static void _generic_unsti(jit_state_t*, jit_word_t, jit_int32_t, jit_word_t);
4325 #if !defined(__i386__) && !defined(__x86_64__)
4326 # define generic_unldr_x(r0, r1, i0) _generic_unldr_x(_jit, r0, r1, i0)
4327 static void _generic_unldr_x(jit_state_t*,
4328 jit_int32_t, jit_int32_t, jit_word_t);
4329 # define generic_unldi_x(r0, i0, i1) _generic_unldi_x(_jit, r0, i0, i1)
4330 static void _generic_unldi_x(jit_state_t*, jit_int32_t, jit_word_t, jit_word_t);
4331 # define generic_unstr_x(r0, r1, i0) _generic_unstr_x(_jit, r0, r1, i0)
4332 static void _generic_unstr_x(jit_state_t*,
4333 jit_int32_t, jit_int32_t, jit_word_t);
4334 # define generic_unsti_x(i0, r0, i1) _generic_unsti_x(_jit, i0, r0, i1)
4335 static void _generic_unsti_x(jit_state_t*, jit_word_t, jit_int32_t, jit_word_t);
4337 #define patch_alist(revert) _patch_alist(_jit, revert)
4338 static maybe_unused void _patch_alist(jit_state_t *_jit, jit_bool_t revert);
4340 #if defined(__i386__) || defined(__x86_64__)
4341 # include "jit_x86.c"
4342 #elif defined(__mips__)
4343 # include "jit_mips.c"
4344 #elif defined(__arm__)
4345 # include "jit_arm.c"
4346 #elif defined(__powerpc__)
4347 # include "jit_ppc.c"
4348 #elif defined(__sparc__)
4349 # include "jit_sparc.c"
4350 #elif defined(__ia64__)
4351 # include "jit_ia64.c"
4352 #elif defined(__hppa__)
4353 # include "jit_hppa.c"
4354 #elif defined(__aarch64__)
4355 # include "jit_aarch64.c"
4356 #elif defined(__s390__) || defined(__s390x__)
4357 # include "jit_s390.c"
4358 #elif defined(__alpha__)
4359 # include "jit_alpha.c"
4360 #elif defined(__riscv)
4361 # include "jit_riscv.c"
4362 #elif defined(__loongarch__)
4363 # include "jit_loongarch.c"
4366 static maybe_unused void
4367 generic_bswapr_us(jit_state_t *_jit, jit_int32_t r0, jit_int32_t r1)
4369 jit_int32_t reg = jit_get_reg(jit_class_gpr);
4371 rshi(rn(reg), r1, 8);
4373 andi(rn(reg), rn(reg), 0xff);
4375 orr(r0, r0, rn(reg));
4380 static maybe_unused void
4381 generic_bswapr_ui(jit_state_t *_jit, jit_int32_t r0, jit_int32_t r1)
4383 jit_int32_t reg = jit_get_reg(jit_class_gpr);
4385 rshi(rn(reg), r1, 16);
4387 bswapr_us(rn(reg), rn(reg));
4389 orr(r0, r0, rn(reg));
4394 #if __WORDSIZE == 64
4395 static maybe_unused void
4396 generic_bswapr_ul(jit_state_t *_jit, jit_int32_t r0, jit_int32_t r1)
4398 jit_int32_t reg = jit_get_reg(jit_class_gpr);
4400 rshi_u(rn(reg), r1, 32);
4402 bswapr_ui(rn(reg), rn(reg));
4404 orr(r0, r0, rn(reg));
4411 _depi(jit_state_t *_jit,
4412 jit_int32_t r0, jit_word_t i0, jit_word_t i1, jit_word_t i2)
4415 reg = jit_get_reg(jit_class_gpr);
4417 depr(r0, rn(reg), i1, i2);
4422 _negi(jit_state_t *_jit, jit_int32_t r0, jit_word_t i0)
4428 _comi(jit_state_t *_jit, jit_int32_t r0, jit_word_t i0)
4434 _exti_c(jit_state_t *_jit, jit_int32_t r0, jit_word_t i0)
4436 movi(r0, (jit_int8_t)i0);
4440 _exti_uc(jit_state_t *_jit, jit_int32_t r0, jit_word_t i0)
4442 movi(r0, (jit_uint8_t)i0);
4446 _exti_s(jit_state_t *_jit, jit_int32_t r0, jit_word_t i0)
4448 movi(r0, (jit_int16_t)i0);
4452 _exti_us(jit_state_t *_jit, jit_int32_t r0, jit_word_t i0)
4454 movi(r0, (jit_uint16_t)i0);
4457 #if __WORDSIZE == 64
4459 _exti_i(jit_state_t *_jit, jit_int32_t r0, jit_word_t i0)
4461 movi(r0, (jit_int32_t)i0);
4465 _exti_ui(jit_state_t *_jit, jit_int32_t r0, jit_word_t i0)
4467 movi(r0, (jit_uint32_t)i0);
4472 _bswapi_us(jit_state_t *_jit, jit_int32_t r0, jit_word_t i0)
4485 _bswapi_ui(jit_state_t *_jit, jit_int32_t r0, jit_word_t i0)
4499 #if __WORDSIZE == 64
4501 _bswapi_ul(jit_state_t *_jit, jit_int32_t r0, jit_word_t i0)
4521 _htoni_us(jit_state_t *_jit, jit_int32_t r0, jit_word_t i0)
4523 #if __BYTE_ORDER == __LITTLE_ENDIAN
4531 _htoni_ui(jit_state_t *_jit, jit_int32_t r0, jit_word_t i0)
4533 #if __BYTE_ORDER == __LITTLE_ENDIAN
4536 # if __WORDSIZE == 32
4544 #if __WORDSIZE == 64
4546 _htoni_ul(jit_state_t *_jit, jit_int32_t r0, jit_word_t i0)
4548 # if __BYTE_ORDER == __LITTLE_ENDIAN
4557 _movi_f_w(jit_state_t *_jit, jit_int32_t r0, jit_float32_t i0)
4564 #if defined(__ia64__)
4565 /* Should be used only in this case (with out0 == 120) */
4567 r0 = _jitc->rout + (r0 - 120);
4572 #if __WORDSIZE == 32 && !(defined(__mips__) && NEW_ABI)
4574 _movi_d_ww(jit_state_t *_jit, jit_int32_t r0, jit_int32_t r1, jit_float64_t i0)
4581 /* Mips does not change byte order of double values */
4582 # if __BYTE_ORDER == __LITTLE_ENDIAN || defined(__mips__)
4583 movi(r0, data.i[0]);
4584 movi(r1, data.i[1]);
4586 movi(r1, data.i[0]);
4587 movi(r0, data.i[1]);
4593 _movi_d_w(jit_state_t *_jit, jit_int32_t r0, jit_float64_t i0)
4600 # if defined(__ia64__)
4601 /* Should be used only in this case (with out0 == 120) */
4603 r0 = _jitc->rout + (r0 - 120);
4610 _jit_negi_f(jit_state_t *_jit, jit_fpr_t u, jit_float32_t v)
4612 jit_inc_synth_wf(negi_f, u, v);
4619 _jit_absi_f(jit_state_t *_jit, jit_fpr_t u, jit_float32_t v)
4621 jit_inc_synth_wf(absi_f, u, v);
4628 _jit_sqrti_f(jit_state_t *_jit, jit_fpr_t u, jit_float32_t v)
4630 jit_inc_synth_wf(sqrti_f, u, v);
4637 _jit_fmai_f(jit_state_t *_jit,
4638 jit_fpr_t u, jit_fpr_t v, jit_fpr_t w, jit_float32_t x)
4641 jit_inc_synth_wqf(fmai_f, u, v, w, x);
4642 if (u != v && u != w) {
4644 jit_fmar_f(u, v, w, u);
4647 y = jit_get_reg(jit_class_fpr);
4649 jit_fmar_f(u, v, w, y);
4656 _jit_fmsi_f(jit_state_t *_jit,
4657 jit_fpr_t u, jit_fpr_t v, jit_fpr_t w, jit_float32_t x)
4660 jit_inc_synth_wqf(fmai_f, u, v, w, x);
4661 if (u != v && u != w) {
4663 jit_fmsr_f(u, v, w, u);
4666 y = jit_get_reg(jit_class_fpr);
4668 jit_fmsr_f(u, v, w, y);
4675 _jit_fnmai_f(jit_state_t *_jit,
4676 jit_fpr_t u, jit_fpr_t v, jit_fpr_t w, jit_float32_t x)
4679 jit_inc_synth_wqf(fmai_f, u, v, w, x);
4680 if (u != v && u != w) {
4682 jit_fnmar_f(u, v, w, u);
4685 y = jit_get_reg(jit_class_fpr);
4687 jit_fnmar_f(u, v, w, y);
4694 _jit_fnmsi_f(jit_state_t *_jit,
4695 jit_fpr_t u, jit_fpr_t v, jit_fpr_t w, jit_float32_t x)
4698 jit_inc_synth_wqf(fmai_f, u, v, w, x);
4699 if (u != v && u != w) {
4701 jit_fnmsr_f(u, v, w, u);
4704 y = jit_get_reg(jit_class_fpr);
4706 jit_fnmsr_f(u, v, w, y);
4713 _jit_negi_d(jit_state_t *_jit, jit_fpr_t u, jit_float64_t v)
4715 jit_inc_synth_wd(negi_d, u, v);
4722 _jit_absi_d(jit_state_t *_jit, jit_fpr_t u, jit_float64_t v)
4724 jit_inc_synth_wd(absi_d, u, v);
4731 _jit_sqrti_d(jit_state_t *_jit, jit_fpr_t u, jit_float64_t v)
4733 jit_inc_synth_wd(sqrti_d, u, v);
4740 _jit_fmai_d(jit_state_t *_jit,
4741 jit_fpr_t u, jit_fpr_t v, jit_fpr_t w, jit_float64_t x)
4744 jit_inc_synth_wqd(fmai_d, u, v, w, x);
4745 if (u != v && u != w) {
4747 jit_fmar_d(u, v, w, u);
4750 y = jit_get_reg(jit_class_fpr);
4752 jit_fmar_d(u, v, w, y);
4759 _jit_fmsi_d(jit_state_t *_jit,
4760 jit_fpr_t u, jit_fpr_t v, jit_fpr_t w, jit_float64_t x)
4763 jit_inc_synth_wqd(fmai_d, u, v, w, x);
4764 if (u != v && u != w) {
4766 jit_fmsr_d(u, v, w, u);
4769 y = jit_get_reg(jit_class_fpr);
4771 jit_fmsr_d(u, v, w, y);
4778 _jit_fnmai_d(jit_state_t *_jit,
4779 jit_fpr_t u, jit_fpr_t v, jit_fpr_t w, jit_float64_t x)
4782 jit_inc_synth_wqd(fmai_d, u, v, w, x);
4783 if (u != v && u != w) {
4785 jit_fnmar_d(u, v, w, u);
4788 y = jit_get_reg(jit_class_fpr);
4790 jit_fnmar_d(u, v, w, y);
4797 _jit_fnmsi_d(jit_state_t *_jit,
4798 jit_fpr_t u, jit_fpr_t v, jit_fpr_t w, jit_float64_t x)
4801 jit_inc_synth_wqd(fmai_d, u, v, w, x);
4802 if (u != v && u != w) {
4804 jit_fnmsr_d(u, v, w, u);
4807 y = jit_get_reg(jit_class_fpr);
4809 jit_fnmsr_d(u, v, w, y);
4816 _cloi(jit_state_t *_jit, jit_int32_t r0, jit_word_t i0)
4822 _clzi(jit_state_t *_jit, jit_int32_t r0, jit_word_t i0)
4824 #if __WORDSIZE == 64 && _WIN32
4825 movi(r0, (i0) ? __builtin_clzll(i0) : __WORDSIZE);
4827 movi(r0, (i0) ? __builtin_clzl(i0) : __WORDSIZE);
4832 _ctoi(jit_state_t *_jit, jit_int32_t r0, jit_word_t i0)
4838 _ctzi(jit_state_t *_jit, jit_int32_t r0, jit_word_t i0)
4840 #if __WORDSIZE == 64 && _WIN32
4841 movi(r0, (i0) ? __builtin_ctzll(i0) : __WORDSIZE);
4843 movi(r0, (i0) ? __builtin_ctzl(i0) : __WORDSIZE);
4848 _rbiti(jit_state_t *_jit, jit_int32_t r0, jit_word_t i0)
4853 jit_uint8_t v[__WORDSIZE >> 3];
4855 static const unsigned char swap_tab[256] = {
4856 0, 128, 64, 192, 32, 160, 96, 224,
4857 16, 144, 80, 208, 48, 176, 112, 240,
4858 8, 136, 72, 200, 40, 168, 104, 232,
4859 24, 152, 88, 216 ,56, 184, 120, 248,
4860 4, 132, 68, 196, 36, 164, 100, 228,
4861 20, 148, 84, 212, 52, 180, 116, 244,
4862 12, 140, 76, 204, 44, 172, 108, 236,
4863 28, 156, 92, 220, 60, 188, 124, 252,
4864 2, 130, 66, 194, 34, 162, 98, 226,
4865 18, 146, 82, 210, 50, 178, 114, 242,
4866 10, 138, 74, 202, 42, 170, 106, 234,
4867 26, 154, 90, 218, 58, 186, 122, 250,
4868 6, 134, 70, 198, 38, 166, 102, 230,
4869 22, 150, 86, 214, 54, 182, 118, 246,
4870 14, 142, 78, 206, 46, 174, 110, 238,
4871 30, 158, 94, 222, 62, 190, 126, 254,
4872 1, 129, 65, 193, 33, 161, 97, 225,
4873 17, 145, 81, 209, 49, 177, 113, 241,
4874 9, 137, 73, 201, 41, 169, 105, 233,
4875 25, 153, 89, 217, 57, 185, 121, 249,
4876 5, 133, 69, 197, 37, 165, 101, 229,
4877 21, 149, 85, 213, 53, 181, 117, 245,
4878 13, 141, 77, 205, 45, 173, 109, 237,
4879 29, 157, 93, 221, 61, 189, 125, 253,
4880 3, 131, 67, 195, 35, 163, 99, 227,
4881 19, 147, 83, 211, 51, 179, 115, 243,
4882 11, 139, 75, 203, 43, 171, 107, 235,
4883 27, 155, 91, 219, 59, 187, 123, 251,
4884 7, 135, 71, 199, 39, 167, 103, 231,
4885 23, 151, 87, 215, 55, 183, 119, 247,
4886 15, 143, 79, 207, 47, 175, 111, 239,
4887 31, 159, 95, 223, 63, 191, 127, 255
4890 for (i = 0; i < sizeof(jit_word_t); ++i)
4891 v.v[i] = swap_tab[u.v[sizeof(jit_word_t) - i - 1]];
4896 _popcnti(jit_state_t *_jit, jit_int32_t r0, jit_word_t i0)
4898 #if __WORDSIZE == 64 && _WIN32
4899 movi(r0, (i0) ? __builtin_popcountll(i0) : __WORDSIZE);
4901 movi(r0, (i0) ? __builtin_popcountl(i0) : __WORDSIZE);
4905 static void _exti(jit_state_t *_jit,
4906 jit_int32_t r0, jit_word_t i0, jit_word_t i1, jit_word_t i2)
4908 #if __BYTE_ORDER == __BIG_ENDIAN
4909 i1 = __WORDSIZE - (i1 + i2);
4911 i0 <<= __WORDSIZE - (i1 + i2);
4912 i0 >>= __WORDSIZE - i2;
4916 static void _exti_u(jit_state_t *_jit,
4917 jit_int32_t r0, jit_word_t i0, jit_word_t i1, jit_word_t i2)
4920 #if __BYTE_ORDER == __BIG_ENDIAN
4921 i1 = __WORDSIZE - (i1 + i2);
4924 i0 >>= __WORDSIZE - i2;
4925 #if __WORDSIZE == 64 && !_WIN32
4926 i0 &= (1L << i2) - 1;
4928 i0 &= (1LL << i2) - 1;
4934 _generic_unldr(jit_state_t *_jit,
4935 jit_int32_t r0, jit_int32_t r1, jit_word_t i0)
4938 assert(i0 >= 1 && i0 <= sizeof(jit_word_t));
4939 if (i0 & (i0 - 1)) {
4940 t0 = jit_get_reg(jit_class_gpr);
4952 #if __BYTE_ORDER == __LITTLE_ENDIAN
4962 #if __WORDSIZE == 32
4971 # if __BYTE_ORDER == __LITTLE_ENDIAN
4982 # if __BYTE_ORDER == __LITTLE_ENDIAN
4993 # if __BYTE_ORDER == __LITTLE_ENDIAN
5010 if (i0 & (i0 - 1)) {
5017 _generic_unldi(jit_state_t *_jit,
5018 jit_int32_t r0, jit_word_t i0, jit_word_t i1)
5021 assert(i1 >= 1 && i1 <= sizeof(jit_word_t));
5022 if (i1 & (i1 - 1)) {
5023 t0 = jit_get_reg(jit_class_gpr);
5034 # if __BYTE_ORDER == __LITTLE_ENDIAN
5044 # if __WORDSIZE == 32
5053 # if __BYTE_ORDER == __LITTLE_ENDIAN
5064 # if __BYTE_ORDER == __LITTLE_ENDIAN
5075 # if __BYTE_ORDER == __LITTLE_ENDIAN
5092 if (i1 & (i1 - 1)) {
5099 _generic_unldr_u(jit_state_t *_jit,
5100 jit_int32_t r0, jit_int32_t r1, jit_word_t i0)
5103 assert(i0 >= 1 && i0 <= sizeof(jit_word_t));
5104 if (i0 & (i0 - 1)) {
5105 t0 = jit_get_reg(jit_class_gpr);
5117 # if __BYTE_ORDER == __LITTLE_ENDIAN
5127 # if __WORDSIZE == 32
5136 # if __BYTE_ORDER == __LITTLE_ENDIAN
5147 # if __BYTE_ORDER == __LITTLE_ENDIAN
5158 # if __BYTE_ORDER == __LITTLE_ENDIAN
5175 if (i0 & (i0 - 1)) {
5182 _generic_unldi_u(jit_state_t *_jit,
5183 jit_int32_t r0, jit_word_t i0, jit_word_t i1)
5186 assert(i1 >= 1 && i1 <= sizeof(jit_word_t));
5187 if (i1 & (i1 - 1)) {
5188 t0 = jit_get_reg(jit_class_gpr);
5199 # if __BYTE_ORDER == __LITTLE_ENDIAN
5209 # if __WORDSIZE == 32
5218 # if __BYTE_ORDER == __LITTLE_ENDIAN
5229 # if __BYTE_ORDER == __LITTLE_ENDIAN
5240 # if __BYTE_ORDER == __LITTLE_ENDIAN
5257 if (i1 & (i1 - 1)) {
5264 _generic_unstr(jit_state_t *_jit, jit_int32_t r0, jit_int32_t r1, jit_word_t i0)
5267 assert(i0 > 0 && i0 <= sizeof(jit_word_t));
5268 if (i0 & (i0 - 1)) {
5269 t0 = jit_get_reg(jit_class_gpr);
5280 # if __BYTE_ORDER == __LITTLE_ENDIAN
5290 # if __WORDSIZE == 32
5299 # if __BYTE_ORDER == __LITTLE_ENDIAN
5310 # if __BYTE_ORDER == __LITTLE_ENDIAN
5321 # if __BYTE_ORDER == __LITTLE_ENDIAN
5345 _generic_unsti(jit_state_t *_jit, jit_word_t i0, jit_int32_t r0, jit_word_t i1)
5348 assert(i1 > 0 && i1 <= sizeof(jit_word_t));
5349 if (i1 & (i1 - 1)) {
5350 t0 = jit_get_reg(jit_class_gpr);
5361 # if __BYTE_ORDER == __LITTLE_ENDIAN
5371 # if __WORDSIZE == 32
5380 # if __BYTE_ORDER == __LITTLE_ENDIAN
5391 # if __BYTE_ORDER == __LITTLE_ENDIAN
5402 # if __BYTE_ORDER == __LITTLE_ENDIAN
5425 #if !defined(__i386__) && !defined(__x86_64__) && !defined(__arm__)
5427 _generic_unldr_x(jit_state_t *_jit,
5428 jit_int32_t r0, jit_int32_t r1, jit_word_t i0)
5430 assert(i0 == 4 || i0 == 8);
5438 _generic_unldi_x(jit_state_t *_jit,
5439 jit_int32_t r0, jit_word_t i0, jit_word_t i1)
5441 assert(i1 == 4 || i1 == 8);
5449 _generic_unstr_x(jit_state_t *_jit,
5450 jit_int32_t r0, jit_int32_t r1, jit_word_t i0)
5452 assert(i0 == 4 || i0 == 8);
5460 _generic_unsti_x(jit_state_t *_jit,
5461 jit_word_t i0, jit_int32_t r0, jit_word_t i1)
5463 assert(i1 == 4 || i1 == 8);
5471 #if defined(stack_framesize)
5472 static maybe_unused void
5473 _patch_alist(jit_state_t *_jit, jit_bool_t revert)
5477 diff = jit_diffsize();
5481 for (node = _jitc->function->alist; node; node = node->link) {
5482 switch (node->code) {
5483 case jit_code_ldxi_c: case jit_code_ldxi_uc:
5484 case jit_code_ldxi_s: case jit_code_ldxi_us:
5485 case jit_code_ldxi_i:
5486 #if __WORDSIZE == 64
5487 case jit_code_ldxi_ui: case jit_code_ldxi_l:
5489 case jit_code_ldxi_f: case jit_code_ldxi_d:
5492 case jit_code_stxi_c: case jit_code_stxi_s:
5493 case jit_code_stxi_i:
5494 #if __WORDSIZE == 64
5495 case jit_code_stxi_l:
5497 case jit_code_stxi_f: case jit_code_stxi_d: