2 * Copyright (C) 2012-2019 Free Software Foundation, Inc.
4 * This file is part of GNU lightning.
6 * GNU lightning is free software; you can redistribute it and/or modify it
7 * under the terms of the GNU Lesser General Public License as published
8 * by the Free Software Foundation; either version 3, or (at your option)
11 * GNU lightning is distributed in the hope that it will be useful, but
12 * WITHOUT ANY WARRANTY; without even the implied warranty of MERCHANTABILITY
13 * or FITNESS FOR A PARTICULAR PURPOSE. See the GNU Lesser General Public
14 * License for more details.
17 * Paulo Cesar Pereira de Andrade
20 #include <lightning.h>
21 #include <lightning/jit_private.h>
23 # include <sys/mman.h>
30 # define MAP_ANON MAP_ANONYMOUS
31 # ifndef MAP_ANONYMOUS
32 # define MAP_ANONYMOUS 0
36 #define jit_regload_reload 0 /* convert to reload */
37 #define jit_regload_delete 1 /* just remove node */
38 #define jit_regload_isdead 2 /* delete and unset live bit */
43 static jit_word_t hash_data(const void*, jit_word_t);
45 #define new_pool() _new_pool(_jit)
46 static void _new_pool(jit_state_t*);
48 #define new_node(u) _new_node(_jit, u)
49 static jit_node_t *_new_node(jit_state_t*, jit_code_t);
51 #define link_node(u) _link_node(_jit, u)
52 static inline jit_node_t *_link_node(jit_state_t*, jit_node_t*);
54 #define del_node(u, v) _del_node(_jit, u, v)
55 static inline void _del_node(jit_state_t*, jit_node_t*, jit_node_t*);
57 #define free_node(u) _free_node(_jit, u)
58 static inline void _free_node(jit_state_t*, jit_node_t*);
60 #define del_label(u, v) _del_label(_jit, u, v)
61 static void _del_label(jit_state_t*, jit_node_t*, jit_node_t*);
63 #define jit_dataset() _jit_dataset(_jit)
65 _jit_dataset(jit_state_t *_jit);
67 #define jit_setup(block) _jit_setup(_jit, block)
69 _jit_setup(jit_state_t *_jit, jit_block_t *block);
71 #define jit_follow(block, todo) _jit_follow(_jit, block, todo)
73 _jit_follow(jit_state_t *_jit, jit_block_t *block, jit_bool_t *todo);
75 #define jit_update(node, live, mask) _jit_update(_jit, node, live, mask)
77 _jit_update(jit_state_t *_jit, jit_node_t *node,
78 jit_regset_t *live, jit_regset_t *mask);
80 #define thread_jumps() _thread_jumps(_jit)
82 _thread_jumps(jit_state_t *_jit);
84 #define sequential_labels() _sequential_labels(_jit)
86 _sequential_labels(jit_state_t *_jit);
88 #define split_branches() _split_branches(_jit)
90 _split_branches(jit_state_t *_jit);
92 #define shortcut_jump(prev, node) _shortcut_jump(_jit, prev, node)
94 _shortcut_jump(jit_state_t *_jit, jit_node_t *prev, jit_node_t *node);
96 #define redundant_jump(prev, node) _redundant_jump(_jit, prev, node)
98 _redundant_jump(jit_state_t *_jit, jit_node_t *prev, jit_node_t *node);
101 reverse_jump_code(jit_code_t code);
103 #define reverse_jump(prev, node) _reverse_jump(_jit, prev, node)
105 _reverse_jump(jit_state_t *_jit, jit_node_t *prev, jit_node_t *node);
107 #define redundant_store(node, jump) _redundant_store(_jit, node, jump)
109 _redundant_store(jit_state_t *_jit, jit_node_t *node, jit_bool_t jump);
111 #define simplify_movr(p, n, k, s) _simplify_movr(_jit, p, n, k, s)
113 _simplify_movr(jit_state_t *_jit, jit_node_t *prev, jit_node_t *node,
114 jit_int32_t kind, jit_int32_t size);
116 #define simplify_movi(p, n, k, s) _simplify_movi(_jit, p, n, k, s)
118 _simplify_movi(jit_state_t *_jit, jit_node_t *prev, jit_node_t *node,
119 jit_int32_t kind, jit_int32_t size);
121 #define simplify_ldxi(prev, node) _simplify_ldxi(_jit, prev, node)
123 _simplify_ldxi(jit_state_t *_jit, jit_node_t *prev, jit_node_t *node);
125 #define simplify_stxi(prev, node) _simplify_stxi(_jit, prev, node)
127 _simplify_stxi(jit_state_t *_jit, jit_node_t *prev, jit_node_t *node);
129 #define simplify_spill(node, regno) _simplify_spill(_jit, node, regno)
131 _simplify_spill(jit_state_t *_jit, jit_node_t *node, jit_int32_t regno);
133 #define simplify() _simplify(_jit)
135 _simplify(jit_state_t *_jit);
137 #define jit_reg_undef -1
138 #define jit_reg_static 0
139 #define jit_reg_change 1
140 #define register_change_p(n, l, r) _register_change_p(_jit, n, l, r)
142 _register_change_p(jit_state_t *_jit, jit_node_t *node, jit_node_t *link,
145 #define spill_reglive_p(node, regno) _spill_reglive_p(_jit, node, regno)
147 _spill_reglive_p(jit_state_t *_jit, jit_node_t *node, jit_int32_t regno);
149 #define patch_registers() _patch_registers(_jit)
151 _patch_registers(jit_state_t *_jit);
153 #define patch_register(n,l,r,p) _patch_register(_jit,n,l,r,p)
155 _patch_register(jit_state_t *jit, jit_node_t *node, jit_node_t *link,
156 jit_int32_t regno, jit_int32_t patch);
169 init_jit(const char *progname)
172 jit_init_debug(progname);
184 _jit_get_reg(jit_state_t *_jit, jit_int32_t regspec)
189 spec = regspec & ~(jit_class_chk|jit_class_nospill);
190 if (spec & jit_class_named) {
191 regno = jit_regno(spec);
192 if (jit_regset_tstbit(&_jitc->regsav, regno))
193 /* fail if register is spilled */
195 if (jit_regset_tstbit(&_jitc->regarg, regno))
196 /* fail if register is an argument to current instruction */
198 if (jit_regset_tstbit(&_jitc->reglive, regno)) {
199 if (regspec & jit_class_nospill)
200 /* fail if register is live and should not spill/reload */
204 jit_regset_setbit(&_jitc->regarg, regno);
208 assert(jit_class(spec) != 0);
211 /* search for a free register matching spec */
212 for (regno = 0; regno < _jitc->reglen; regno++) {
213 if ((jit_class(_rvs[regno].spec) & spec) == spec &&
214 !jit_regset_tstbit(&_jitc->regarg, regno) &&
215 !jit_regset_tstbit(&_jitc->reglive, regno))
219 /* search for a register matching spec that is not an argument
220 * for the current instruction */
221 for (regno = 0; regno < _jitc->reglen; regno++) {
222 if ((jit_class(_rvs[regno].spec) & spec) == spec &&
223 !jit_regset_tstbit(&_jitc->regsav, regno) &&
224 !jit_regset_tstbit(&_jitc->regarg, regno) &&
225 !(regspec & jit_class_nospill)) {
227 assert(_jitc->function != NULL);
228 if (spec & jit_class_gpr) {
229 if (!_jitc->function->regoff[regno]) {
230 _jitc->function->regoff[regno] =
231 jit_allocai(sizeof(jit_word_t));
235 /* emit_stxi must not need temporary registers */
236 assert(!_jitc->getreg);
239 emit_stxi(_jitc->function->regoff[regno], JIT_FP, regno);
245 if (!_jitc->function->regoff[regno]) {
246 _jitc->function->regoff[regno] =
247 jit_allocai(sizeof(jit_float64_t));
251 /* emit_stxi must not need temporary registers */
252 assert(!_jitc->getreg);
255 emit_stxi_d(_jitc->function->regoff[regno], JIT_FP, regno);
260 jit_regset_setbit(&_jitc->regsav, regno);
262 jit_regset_setbit(&_jitc->regarg, regno);
263 if (jit_class(_rvs[regno].spec) & jit_class_sav) {
264 /* if will modify callee save registers without a
265 * function prolog, better patch this assertion */
266 assert(_jitc->function != NULL);
267 if (!jit_regset_tstbit(&_jitc->function->regset, regno)) {
268 jit_regset_setbit(&_jitc->function->regset, regno);
277 /* nospill hint only valid during emit" */
278 assert(!(regspec & jit_class_nospill));
279 for (regno = 0; regno < _jitc->reglen; regno++) {
280 if ((jit_class(_rvs[regno].spec) & spec) == spec &&
281 !jit_regset_tstbit(&_jitc->regsav, regno) &&
282 !jit_regset_tstbit(&_jitc->regarg, regno)) {
283 jit_regset_setbit(&_jitc->regarg, regno);
284 jit_regset_setbit(&_jitc->regsav, regno);
286 return (jit_regno_patch|regno);
291 /* Out of hardware registers */
293 assert(regspec & jit_class_chk);
298 _jit_unget_reg(jit_state_t *_jit, jit_int32_t regno)
300 regno = jit_regno(regno);
301 if (jit_regset_tstbit(&_jitc->regsav, regno)) {
304 /* emit_ldxi must not need a temporary register */
305 assert(!_jitc->getreg);
308 if (jit_class(_rvs[regno].spec) & jit_class_gpr)
309 emit_ldxi(regno, JIT_FP, _jitc->function->regoff[regno]);
311 emit_ldxi_d(regno, JIT_FP, _jitc->function->regoff[regno]);
313 /* emit_ldxi must not need a temporary register */
319 jit_regset_clrbit(&_jitc->regsav, regno);
321 #if defined(jit_carry)
322 assert((regno == jit_carry /*&& _NOREG != jit_carry*/) ||
323 jit_regset_tstbit(&_jitc->regarg, regno) != 0);
325 assert(jit_regset_tstbit(&_jitc->regarg, regno) != 0);
327 jit_regset_clrbit(&_jitc->regarg, regno);
331 _jit_callee_save_p(jit_state_t *_jit, jit_int32_t regno)
333 assert(regno >= 0 && regno < JIT_NOREG);
334 return (!!(_rvs[regno].spec & jit_class_sav));
338 _jit_pointer_p(jit_state_t *_jit, jit_pointer_t address)
340 return ((jit_uint8_t *)address >= _jit->code.ptr &&
341 (jit_word_t)address < _jit->pc.w);
346 jit_regset_com(jit_regset_t *u, jit_regset_t *v)
348 u->rl = ~v->rl; u->rh = ~v->rh;
349 u->fl = ~v->fl; u->fh = ~v->fh;
353 jit_regset_and(jit_regset_t *u, jit_regset_t *v, jit_regset_t *w)
355 u->rl = v->rl & w->rl; u->rh = v->rh & w->rh;
356 u->fl = v->fl & w->fl; u->fh = v->fh & w->fh;
360 jit_regset_ior(jit_regset_t *u, jit_regset_t *v, jit_regset_t *w)
362 u->rl = v->rl | w->rl; u->rh = v->rh | w->rh;
363 u->fl = v->fl | w->fl; u->fh = v->fh | w->fh;
367 jit_regset_xor(jit_regset_t *u, jit_regset_t *v, jit_regset_t *w)
369 u->rl = v->rl ^ w->rl; u->rh = v->rh ^ w->rh;
370 u->fl = v->fl ^ w->fl; u->fh = v->fh ^ w->fh;
374 jit_regset_set(jit_regset_t *u, jit_regset_t *v)
376 u->rl = v->rl; u->rh = v->rh;
377 u->fl = v->fl; u->fh = v->fh;
381 jit_regset_set_mask(jit_regset_t *u, jit_int32_t v)
383 jit_bool_t w = !!(v & (v - 1));
385 assert(v >= 0 && v <= 256);
387 u->rl = u->rh = u->fl = u->fh = -1LL;
389 u->rl = w ? (1LL << v) - 1 : -1LL;
390 u->rh = u->fl = u->fh = 0;
394 u->rh = w ? (1LL << (v - 64)) - 1 : -1LL;
398 u->rl = u->rh = -1LL;
399 u->fl = w ? (1LL << (v - 128)) - 1 : -1LL;
403 u->rl = u->rh = u->fl = -1LL;
404 u->fh = w ? (1LL << (v - 128)) - 1 : -1LL;
409 jit_regset_cmp_ui(jit_regset_t *u, jit_word_t v)
411 return !((u->rl == v && u->rh == 0 && u->fl == 0 && u->fh == 0));
415 jit_regset_set_ui(jit_regset_t *u, jit_word_t v)
418 u->rh = u->fl = u->fh = 0;
422 jit_regset_set_p(jit_regset_t *u)
424 return (u->rl || u->rh || u->fl || u->fh);
428 jit_regset_clrbit(jit_regset_t *set, jit_int32_t bit)
430 assert(bit >= 0 && bit <= 255);
432 set->rl &= ~(1LL << bit);
434 set->rh &= ~(1LL << (bit - 64));
436 set->fl &= ~(1LL << (bit - 128));
438 set->fh &= ~(1LL << (bit - 192));
442 jit_regset_setbit(jit_regset_t *set, jit_int32_t bit)
444 assert(bit >= 0 && bit <= 255);
446 set->rl |= 1LL << bit;
448 set->rh |= 1LL << (bit - 64);
450 set->fl |= 1LL << (bit - 128);
452 set->fh |= 1LL << (bit - 192);
456 jit_regset_tstbit(jit_regset_t *set, jit_int32_t bit)
458 assert(bit >= 0 && bit <= 255);
460 return (!!(set->rl & (1LL << bit)));
462 return (!!(set->rh & (1LL << (bit - 64))));
464 return (!!(set->fl & (1LL << (bit - 128))));
465 return (!!(set->fh & (1LL << (bit - 192))));
469 jit_regset_scan1(jit_regset_t *set, jit_int32_t offset)
471 assert(offset >= 0 && offset <= 255);
472 for (; offset < 64; offset++) {
473 if (set->rl & (1LL << offset))
476 for (; offset < 128; offset++) {
477 if (set->rh & (1LL << (offset - 64)))
480 for (; offset < 192; offset++) {
481 if (set->fl & (1LL << (offset - 128)))
484 for (; offset < 256; offset++) {
485 if (set->fh & (1LL << (offset - 192)))
491 #elif __sparc__ && __WORDSIZE == 64
493 jit_regset_com(jit_regset_t *u, jit_regset_t *v)
495 u->rl = ~v->rl; u->rh = ~v->rh;
499 jit_regset_and(jit_regset_t *u, jit_regset_t *v, jit_regset_t *w)
501 u->rl = v->rl & w->rl; u->rh = v->rh & w->rh;
505 jit_regset_ior(jit_regset_t *u, jit_regset_t *v, jit_regset_t *w)
507 u->rl = v->rl | w->rl; u->rh = v->rh | w->rh;
511 jit_regset_xor(jit_regset_t *u, jit_regset_t *v, jit_regset_t *w)
513 u->rl = v->rl ^ w->rl; u->rh = v->rh ^ w->rh;
517 jit_regset_set(jit_regset_t *u, jit_regset_t *v)
519 u->rl = v->rl; u->rh = v->rh;
523 jit_regset_set_mask(jit_regset_t *u, jit_int32_t v)
525 jit_bool_t w = !!(v & (v - 1));
527 assert(v >= 0 && v <= 128);
529 u->rl = u->rh = -1LL;
531 u->rl = w ? (1LL << v) - 1 : -1LL;
536 u->rh = w ? (1LL << (v - 64)) - 1 : -1LL;
541 jit_regset_cmp_ui(jit_regset_t *u, jit_word_t v)
543 return !((u->rl == v && u->rh == 0));
547 jit_regset_set_ui(jit_regset_t *u, jit_word_t v)
554 jit_regset_set_p(jit_regset_t *u)
556 return (u->rl || u->rh);
560 jit_regset_clrbit(jit_regset_t *set, jit_int32_t bit)
562 assert(bit >= 0 && bit <= 128);
564 set->rl &= ~(1LL << bit);
566 set->rh &= ~(1LL << (bit - 64));
570 jit_regset_setbit(jit_regset_t *set, jit_int32_t bit)
572 assert(bit >= 0 && bit <= 127);
574 set->rl |= 1LL << bit;
576 set->rh |= 1LL << (bit - 64);
580 jit_regset_tstbit(jit_regset_t *set, jit_int32_t bit)
582 assert(bit >= 0 && bit <= 127);
584 return (!!(set->rl & (1LL << bit)));
586 return (!!(set->rh & (1LL << (bit - 64))));
590 jit_regset_scan1(jit_regset_t *set, jit_int32_t offset)
592 assert(offset >= 0 && offset <= 127);
593 for (; offset < 64; offset++) {
594 if (set->rl & (1LL << offset))
597 for (; offset < 128; offset++) {
598 if (set->rh & (1LL << (offset - 64)))
606 jit_regset_scan1(jit_regset_t *set, jit_int32_t offset)
609 assert(offset >= 0 && offset <= 63);
610 if ((mask = *set >> offset)) {
623 _jit_save(jit_state_t *_jit, jit_int32_t reg)
625 reg = jit_regno(reg);
626 assert(!_jitc->realize);
627 _jitc->spill[reg] = jit_new_node_w(jit_code_save, reg);
631 _jit_load(jit_state_t *_jit, jit_int32_t reg)
635 reg = jit_regno(reg);
636 assert(!_jitc->realize);
637 assert(_jitc->spill[reg] != NULL);
638 node = jit_new_node_w(jit_code_load, reg);
639 /* create a path to flag the save/load is not required */
640 node->link = _jitc->spill[reg];
641 node->link->link = node;
642 _jitc->spill[reg] = NULL;
646 hash_data(const void *data, jit_word_t length)
648 const jit_uint8_t *ptr;
650 for (i = key = 0, ptr = data; i < length; i++)
651 key = (key << (key & 1)) ^ ptr[i];
656 _jit_address(jit_state_t *_jit, jit_node_t *node)
659 assert(node != NULL &&
660 /* If a node type that is documented to be a fixed marker */
661 (node->code == jit_code_note || node->code == jit_code_name ||
662 /* If another special fixed marker, returned by jit_indirect() */
663 (node->code == jit_code_label && (node->flag & jit_flag_use) != 0)));
664 return ((jit_pointer_t)node->u.w);
668 _jit_data(jit_state_t *_jit, const void *data,
669 jit_word_t length, jit_int32_t align)
674 assert(!_jitc->realize);
676 /* Ensure there is space even if asking for a duplicate */
677 if (((_jitc->data.offset + 7) & -8) + length > _jit->data.length) {
680 size = (_jit->data.length + length + 4096) & - 4095;
681 assert(size >= _jit->data.length);
682 if (_jitc->data.ptr == NULL)
683 jit_alloc((jit_pointer_t *)&_jitc->data.ptr, size);
685 jit_realloc((jit_pointer_t *)&_jitc->data.ptr,
686 _jit->data.length, size);
687 _jit->data.length = size;
689 if (_jitc->data.table == NULL)
690 jit_alloc((jit_pointer_t *)&_jitc->data.table,
691 (_jitc->data.size = 16) * sizeof(jit_node_t*));
693 key = hash_data(data, length) & (_jitc->data.size - 1);
694 node = _jitc->data.table[key];
695 for (; node; node = node->next) {
696 if (node->v.w == length &&
697 memcmp(_jitc->data.ptr + node->u.w, data, length) == 0)
702 node = jit_new_node_no_link(jit_code_data);
709 _jitc->data.offset = (_jitc->data.offset + 1) & -2;
712 _jitc->data.offset = (_jitc->data.offset + 3) & -4;
715 _jitc->data.offset = (_jitc->data.offset + 7) & -8;
718 node->u.w = _jitc->data.offset;
720 jit_memcpy(_jitc->data.ptr + _jitc->data.offset, data, length);
721 _jitc->data.offset += length;
723 node->next = _jitc->data.table[key];
724 _jitc->data.table[key] = node;
727 /* Rehash if more than 75% used table */
728 if (_jitc->data.count >
729 (_jitc->data.size >> 1) + (_jitc->data.size >> 2) &&
730 (_jitc->data.size << 1) > _jitc->data.size) {
736 jit_alloc((jit_pointer_t *)&hash,
737 (_jitc->data.size << 1) * sizeof(jit_node_t*));
738 for (i = 0; i < _jitc->data.size; i++) {
739 temp = _jitc->data.table[i];
740 for (; temp; temp = next) {
742 key = hash_data(_jitc->data.ptr + temp->u.w, temp->v.w) &
743 ((_jitc->data.size << 1) - 1);
744 temp->next = hash[key];
748 jit_free((jit_pointer_t *)&_jitc->data.table);
749 _jitc->data.table = hash;
750 _jitc->data.size <<= 1;
758 _new_pool(jit_state_t *_jit)
763 if (_jitc->pool.offset >= _jitc->pool.length) {
766 length = _jitc->pool.length + 16;
767 jit_realloc((jit_pointer_t *)&_jitc->pool.ptr,
768 _jitc->pool.length * sizeof(jit_node_t *),
769 length * sizeof(jit_node_t *));
770 _jitc->pool.length = length;
772 jit_alloc((jit_pointer_t *)(_jitc->pool.ptr + _jitc->pool.offset),
773 sizeof(jit_node_t) * 1024);
774 list = _jitc->pool.ptr[_jitc->pool.offset];
775 for (offset = 1; offset < 1024; offset++, list++)
776 list->next = list + 1;
777 list->next = _jitc->list;
778 _jitc->list = _jitc->pool.ptr[_jitc->pool.offset];
779 ++_jitc->pool.offset;
783 _new_node(jit_state_t *_jit, jit_code_t code)
787 if (_jitc->list == NULL)
790 _jitc->list = node->next;
792 node->flag |= jit_flag_synth;
799 static inline jit_node_t *
800 _link_node(jit_state_t *_jit, jit_node_t *node)
803 _jitc->tail->next = node;
806 return (_jitc->tail = node);
810 _del_node(jit_state_t *_jit, jit_node_t *prev, jit_node_t *node)
813 assert(prev == _jitc->head);
814 _jitc->head = node->next;
817 prev->next = node->next;
818 memset(node, 0, sizeof(jit_node_t));
819 node->next = _jitc->list;
824 _free_node(jit_state_t *_jit, jit_node_t *node)
826 memset(node, 0, sizeof(jit_node_t));
827 node->next = _jitc->list;
832 _del_label(jit_state_t *_jit, jit_node_t *prev, jit_node_t *node)
836 /* only allow call to del_label on linked labels */
837 block = _jitc->blocks.ptr + node->v.w;
838 assert(block->label == node);
840 /* del_label() should only be called when optimizing.
841 * This will leave an empty block index */
842 jit_regset_del(&block->reglive);
843 jit_regset_del(&block->regmask);
846 /* redundant, should be already true */
847 assert(node->link == NULL);
848 del_node(prev, node);
856 jit_alloc((jit_pointer_t *)&_jit, sizeof(jit_state_t));
857 jit_alloc((jit_pointer_t *)&_jitc, sizeof(jit_compiler_t));
858 jit_regset_new(&_jitc->regarg);
859 jit_regset_new(&_jitc->regsav);
860 jit_regset_new(&_jitc->reglive);
861 jit_regset_new(&_jitc->regmask);
865 jit_alloc((jit_pointer_t *)&_jitc->spill,
866 _jitc->reglen * sizeof(jit_node_t*));
867 jit_alloc((jit_pointer_t *)&_jitc->gen,
868 _jitc->reglen * sizeof(jit_int32_t));
869 jit_alloc((jit_pointer_t *)&_jitc->values,
870 _jitc->reglen * sizeof(jit_value_t));
872 jit_alloc((jit_pointer_t *)&_jitc->patches.ptr,
873 (_jitc->patches.length = 1024) * sizeof(jit_patch_t));
874 jit_alloc((jit_pointer_t *)&_jitc->functions.ptr,
875 (_jitc->functions.length = 16) * sizeof(jit_function_t));
876 jit_alloc((jit_pointer_t *)&_jitc->pool.ptr,
877 (_jitc->pool.length = 16) * sizeof(jit_node_t*));
878 jit_alloc((jit_pointer_t *)&_jitc->blocks.ptr,
879 (_jitc->blocks.length = 16) * sizeof(jit_block_t));
880 #if __arm__ && DISASSEMBLER
881 jit_alloc((jit_pointer_t *)&_jitc->data_info.ptr,
882 (_jitc->data_info.length = 1024) * sizeof(jit_data_info_t));
885 /* allocate at most one extra note in case jit_name() is
886 * never called, or called after adding at least one note */
887 _jit->note.length = 1;
888 _jitc->note.size = sizeof(jit_note_t);
894 _jit_clear_state(jit_state_t *_jit)
896 #if DEVEL_DISASSEMBLER
897 # define jit_really_clear_state() _jit_really_clear_state(_jit)
900 void _jit_really_clear_state(jit_state_t *_jit)
904 jit_function_t *function;
906 /* release memory not required at jit execution time and set
907 * pointers to NULL to explicitly know they are released */
908 _jitc->head = _jitc->tail = NULL;
910 jit_free((jit_pointer_t *)&_jitc->data.table);
911 _jitc->data.size = _jitc->data.count = 0;
913 jit_free((jit_pointer_t *)&_jitc->spill);
914 jit_free((jit_pointer_t *)&_jitc->gen);
915 jit_free((jit_pointer_t *)&_jitc->values);
917 jit_free((jit_pointer_t *)&_jitc->blocks.ptr);
919 jit_free((jit_pointer_t *)&_jitc->patches.ptr);
920 _jitc->patches.offset = _jitc->patches.length = 0;
922 for (offset = 0; offset < _jitc->functions.offset; offset++) {
923 function = _jitc->functions.ptr + offset;
924 jit_free((jit_pointer_t *)&function->regoff);
926 jit_free((jit_pointer_t *)&_jitc->functions.ptr);
927 _jitc->functions.offset = _jitc->functions.length = 0;
928 _jitc->function = NULL;
930 for (offset = 0; offset < _jitc->pool.offset; offset++)
931 jit_free((jit_pointer_t *)(_jitc->pool.ptr + offset));
932 jit_free((jit_pointer_t *)&_jitc->pool.ptr);
933 _jitc->pool.offset = _jitc->pool.length = 0;
936 _jitc->note.head = _jitc->note.tail =
937 _jitc->note.name = _jitc->note.note = NULL;
938 _jitc->note.base = NULL;
940 #if __arm__ && DISASSEMBLER
941 jit_free((jit_pointer_t *)&_jitc->data_info.ptr);
944 #if (__powerpc__ && _CALL_AIXDESC) || __ia64__
945 jit_free((jit_pointer_t *)&_jitc->prolog.ptr);
949 jit_regset_del(&_jitc->regs);
952 jit_free((jit_pointer_t *)&_jitc);
956 _jit_destroy_state(jit_state_t *_jit)
958 #if DEVEL_DISASSEMBLER
959 jit_really_clear_state();
962 if (!_jit->user_code)
963 munmap(_jit->code.ptr, _jit->code.length);
964 if (!_jit->user_data)
965 munmap(_jit->data.ptr, _jit->data.length);
967 jit_free((jit_pointer_t *)&_jit);
971 _jit_synth_inc(jit_state_t *_jit)
973 assert(_jitc->synth < 8);
978 _jit_new_node(jit_state_t *_jit, jit_code_t code)
980 assert(!_jitc->realize);
981 return (link_node(new_node(code)));
985 _jit_new_node_no_link(jit_state_t *_jit, jit_code_t code)
987 assert(!_jitc->realize);
988 return (new_node(code));
992 _jit_link_node(jit_state_t *_jit, jit_node_t *node)
994 assert(!_jitc->realize);
999 _jit_synth_dec(jit_state_t *_jit)
1001 assert(_jitc->synth > 0);
1006 _jit_new_node_w(jit_state_t *_jit, jit_code_t code,
1009 jit_node_t *node = new_node(code);
1010 assert(!_jitc->realize);
1012 return (link_node(node));
1016 _jit_new_node_f(jit_state_t *_jit, jit_code_t code,
1019 jit_node_t *node = new_node(code);
1020 assert(!_jitc->realize);
1022 return (link_node(node));
1026 _jit_new_node_d(jit_state_t *_jit, jit_code_t code,
1029 jit_node_t *node = new_node(code);
1030 assert(!_jitc->realize);
1032 return (link_node(node));
1036 _jit_new_node_p(jit_state_t *_jit, jit_code_t code,
1039 jit_node_t *node = new_node(code);
1040 assert(!_jitc->realize);
1042 return (link_node(node));
1046 _jit_new_node_ww(jit_state_t *_jit, jit_code_t code,
1047 jit_word_t u, jit_word_t v)
1049 jit_node_t *node = new_node(code);
1050 assert(!_jitc->realize);
1053 return (link_node(node));
1057 _jit_new_node_wp(jit_state_t *_jit, jit_code_t code,
1058 jit_word_t u, jit_pointer_t v)
1060 return (jit_new_node_ww(code, u, (jit_word_t)v));
1064 _jit_new_node_fp(jit_state_t *_jit, jit_code_t code,
1065 jit_float32_t u, jit_pointer_t v)
1067 jit_node_t *node = new_node(code);
1068 assert(!_jitc->realize);
1070 node->v.w = (jit_word_t)v;
1071 return (link_node(node));
1075 _jit_new_node_dp(jit_state_t *_jit, jit_code_t code,
1076 jit_float64_t u, jit_pointer_t v)
1078 jit_node_t *node = new_node(code);
1079 assert(!_jitc->realize);
1081 node->v.w = (jit_word_t)v;
1082 return (link_node(node));
1086 _jit_new_node_pw(jit_state_t *_jit, jit_code_t code,
1087 jit_pointer_t u, jit_word_t v)
1089 return (jit_new_node_ww(code, (jit_word_t)u, v));
1093 _jit_new_node_wf(jit_state_t *_jit, jit_code_t code,
1094 jit_word_t u, jit_float32_t v)
1096 jit_node_t *node = new_node(code);
1097 assert(!_jitc->realize);
1100 return (link_node(node));
1104 _jit_new_node_wd(jit_state_t *_jit, jit_code_t code,
1105 jit_word_t u, jit_float64_t v)
1107 jit_node_t *node = new_node(code);
1108 assert(!_jitc->realize);
1111 return (link_node(node));
1115 _jit_new_node_www(jit_state_t *_jit, jit_code_t code,
1116 jit_word_t u, jit_word_t v, jit_word_t w)
1118 jit_node_t *node = new_node(code);
1119 assert(!_jitc->realize);
1123 return (link_node(node));
1127 _jit_new_node_qww(jit_state_t *_jit, jit_code_t code,
1128 jit_int32_t l, jit_int32_t h,
1129 jit_word_t v, jit_word_t w)
1131 jit_node_t *node = new_node(code);
1132 assert(!_jitc->realize);
1138 return (link_node(node));
1142 _jit_new_node_wwf(jit_state_t *_jit, jit_code_t code,
1143 jit_word_t u, jit_word_t v, jit_float32_t w)
1145 jit_node_t *node = new_node(code);
1146 assert(!_jitc->realize);
1150 return (link_node(node));
1154 _jit_new_node_wwd(jit_state_t *_jit, jit_code_t code,
1155 jit_word_t u, jit_word_t v, jit_float64_t w)
1157 jit_node_t *node = new_node(code);
1158 assert(!_jitc->realize);
1162 return (link_node(node));
1166 _jit_new_node_pww(jit_state_t *_jit, jit_code_t code,
1167 jit_pointer_t u, jit_word_t v, jit_word_t w)
1169 jit_node_t *node = new_node(code);
1170 assert(!_jitc->realize);
1174 return (link_node(node));
1178 _jit_new_node_pwf(jit_state_t *_jit, jit_code_t code,
1179 jit_pointer_t u, jit_word_t v, jit_float32_t w)
1181 jit_node_t *node = new_node(code);
1182 assert(!_jitc->realize);
1186 return (link_node(node));
1190 _jit_new_node_pwd(jit_state_t *_jit, jit_code_t code,
1191 jit_pointer_t u, jit_word_t v, jit_float64_t w)
1193 jit_node_t *node = new_node(code);
1194 assert(!_jitc->realize);
1198 return (link_node(node));
1202 _jit_label(jit_state_t *_jit)
1206 if (!(node = _jitc->tail) || node->code != jit_code_label) {
1207 node = jit_forward();
1215 _jit_forward(jit_state_t *_jit)
1217 return (jit_new_node_no_link(jit_code_label));
1221 _jit_indirect(jit_state_t *_jit)
1226 node->flag |= jit_flag_use;
1232 _jit_link(jit_state_t *_jit, jit_node_t *node)
1236 assert((node->code == jit_code_label ||
1237 node->code == jit_code_prolog ||
1238 node->code == jit_code_epilog) && !node->next);
1239 jit_link_node(node);
1240 if (_jitc->blocks.offset >= _jitc->blocks.length) {
1243 length = _jitc->blocks.length + 16;
1244 jit_realloc((jit_pointer_t *)&_jitc->blocks.ptr,
1245 _jitc->blocks.length * sizeof(jit_block_t),
1246 length * sizeof(jit_block_t));
1247 _jitc->blocks.length = length;
1249 block = _jitc->blocks.ptr + _jitc->blocks.offset;
1250 block->label = node;
1251 node->v.w = _jitc->blocks.offset;
1252 jit_regset_new(&block->reglive);
1253 jit_regset_new(&block->regmask);
1254 ++_jitc->blocks.offset;
1258 _jit_forward_p(jit_state_t *_jit, jit_node_t *node)
1260 return (node->code == jit_code_label && !node->next && node != _jitc->tail);
1264 _jit_indirect_p(jit_state_t *_jit, jit_node_t *node)
1266 return (node->code == jit_code_label && !!(node->flag & jit_flag_use));
1270 _jit_target_p(jit_state_t *_jit, jit_node_t *node)
1272 return (node->code == jit_code_label && !!node->link);
1276 _jit_prepare(jit_state_t *_jit)
1278 assert(_jitc->function != NULL);
1279 _jitc->function->call.call = jit_call_default;
1280 _jitc->function->call.argi =
1281 _jitc->function->call.argf =
1282 _jitc->function->call.size = 0;
1283 _jitc->prepare = jit_new_node(jit_code_prepare);
1287 _jit_patch(jit_state_t* _jit, jit_node_t *instr)
1291 if (!(label = _jitc->tail) || label->code != jit_code_label)
1292 label = jit_label();
1293 jit_patch_at(instr, label);
1297 _jit_classify(jit_state_t *_jit, jit_code_t code)
1302 case jit_code_data: case jit_code_save: case jit_code_load:
1303 case jit_code_name: case jit_code_label: case jit_code_note:
1304 case jit_code_prolog: case jit_code_ellipsis: case jit_code_va_push:
1305 case jit_code_epilog: case jit_code_ret: case jit_code_prepare:
1308 case jit_code_live: case jit_code_va_end:
1309 case jit_code_retr: case jit_code_retr_f: case jit_code_retr_d:
1310 case jit_code_pushargr: case jit_code_pushargr_f:
1311 case jit_code_pushargr_d:
1312 case jit_code_finishr: /* synthesized will set jit_cc_a0_jmp */
1313 mask = jit_cc_a0_reg;
1315 case jit_code_align: case jit_code_reti: case jit_code_pushargi:
1316 case jit_code_finishi: /* synthesized will set jit_cc_a0_jmp */
1317 mask = jit_cc_a0_int;
1319 case jit_code_reti_f: case jit_code_pushargi_f:
1320 mask = jit_cc_a0_flt;
1322 case jit_code_reti_d: case jit_code_pushargi_d:
1323 mask = jit_cc_a0_dbl;
1325 case jit_code_allocai:
1326 mask = jit_cc_a0_int|jit_cc_a1_int;
1328 case jit_code_arg: case jit_code_arg_f: case jit_code_arg_d:
1329 mask = jit_cc_a0_int|jit_cc_a0_arg;
1331 case jit_code_calli: case jit_code_jmpi:
1332 mask = jit_cc_a0_jmp;
1334 case jit_code_callr: case jit_code_jmpr:
1335 mask = jit_cc_a0_reg|jit_cc_a0_jmp;
1337 case jit_code_retval_c: case jit_code_retval_uc:
1338 case jit_code_retval_s: case jit_code_retval_us:
1339 case jit_code_retval_i: case jit_code_retval_ui:
1340 case jit_code_retval_l:
1341 case jit_code_retval_f: case jit_code_retval_d:
1342 case jit_code_va_start:
1343 mask = jit_cc_a0_reg|jit_cc_a0_chg;
1345 case jit_code_getarg_c: case jit_code_getarg_uc:
1346 case jit_code_getarg_s: case jit_code_getarg_us:
1347 case jit_code_getarg_i: case jit_code_getarg_ui:
1348 case jit_code_getarg_l:
1349 case jit_code_getarg_f: case jit_code_getarg_d:
1350 mask = jit_cc_a0_reg|jit_cc_a0_chg|jit_cc_a1_arg;
1352 case jit_code_putargr: case jit_code_putargr_f:
1353 case jit_code_putargr_d:
1354 mask = jit_cc_a0_reg|jit_cc_a1_arg;
1356 case jit_code_putargi:
1357 mask = jit_cc_a0_int|jit_cc_a1_arg;
1359 case jit_code_putargi_f:
1360 mask = jit_cc_a0_flt|jit_cc_a1_arg;
1362 case jit_code_putargi_d:
1363 mask = jit_cc_a0_dbl|jit_cc_a1_arg;
1365 case jit_code_movi: case jit_code_ldi_c: case jit_code_ldi_uc:
1366 case jit_code_ldi_s: case jit_code_ldi_us: case jit_code_ldi_i:
1367 case jit_code_ldi_ui: case jit_code_ldi_l: case jit_code_ldi_f:
1368 case jit_code_ldi_d:
1369 mask = jit_cc_a0_reg|jit_cc_a0_chg|jit_cc_a1_int;
1371 case jit_code_movi_f: case jit_code_movi_f_w:
1372 mask = jit_cc_a0_reg|jit_cc_a0_chg|jit_cc_a1_flt;
1374 case jit_code_movi_d: case jit_code_movi_d_w:
1375 mask = jit_cc_a0_reg|jit_cc_a0_chg|jit_cc_a1_dbl;
1377 case jit_code_movi_d_ww:
1378 mask = jit_cc_a0_reg|jit_cc_a0_chg|jit_cc_a1_reg|jit_cc_a1_chg|
1381 case jit_code_negr: case jit_code_comr: case jit_code_movr:
1382 case jit_code_extr_c: case jit_code_extr_uc: case jit_code_extr_s:
1383 case jit_code_extr_us: case jit_code_extr_i: case jit_code_extr_ui:
1384 case jit_code_truncr_f_i: case jit_code_truncr_f_l:
1385 case jit_code_truncr_d_i: case jit_code_truncr_d_l:
1386 case jit_code_htonr_us: case jit_code_htonr_ui: case jit_code_htonr_ul:
1387 case jit_code_bswapr_us: case jit_code_bswapr_ui: case jit_code_bswapr_ul:
1388 case jit_code_ldr_c: case jit_code_ldr_uc:
1389 case jit_code_ldr_s: case jit_code_ldr_us: case jit_code_ldr_i:
1390 case jit_code_ldr_ui: case jit_code_ldr_l: case jit_code_negr_f:
1391 case jit_code_absr_f: case jit_code_sqrtr_f: case jit_code_movr_f:
1392 case jit_code_extr_f: case jit_code_extr_d_f: case jit_code_ldr_f:
1393 case jit_code_negr_d: case jit_code_absr_d: case jit_code_sqrtr_d:
1394 case jit_code_movr_d: case jit_code_extr_d: case jit_code_extr_f_d:
1395 case jit_code_ldr_d:
1396 case jit_code_movr_w_f: case jit_code_movr_f_w:
1397 case jit_code_movr_w_d: case jit_code_movr_d_w:
1398 case jit_code_va_arg: case jit_code_va_arg_d:
1399 mask = jit_cc_a0_reg|jit_cc_a0_chg|jit_cc_a1_reg;
1401 case jit_code_movr_d_ww:
1402 mask = jit_cc_a0_reg|jit_cc_a0_chg|jit_cc_a1_reg|jit_cc_a1_chg|
1405 case jit_code_addi: case jit_code_addxi: case jit_code_addci:
1406 case jit_code_subi: case jit_code_subxi: case jit_code_subci:
1408 case jit_code_muli: case jit_code_divi: case jit_code_divi_u:
1409 case jit_code_remi: case jit_code_remi_u: case jit_code_andi:
1410 case jit_code_ori: case jit_code_xori: case jit_code_lshi:
1411 case jit_code_rshi: case jit_code_rshi_u: case jit_code_lti:
1412 case jit_code_lti_u: case jit_code_lei: case jit_code_lei_u:
1413 case jit_code_eqi: case jit_code_gei: case jit_code_gei_u:
1414 case jit_code_gti: case jit_code_gti_u: case jit_code_nei:
1415 case jit_code_ldxi_c: case jit_code_ldxi_uc: case jit_code_ldxi_s:
1416 case jit_code_ldxi_us: case jit_code_ldxi_i: case jit_code_ldxi_ui:
1417 case jit_code_ldxi_l: case jit_code_ldxi_f: case jit_code_ldxi_d:
1418 mask = jit_cc_a0_reg|jit_cc_a0_chg|jit_cc_a1_reg|jit_cc_a2_int;
1420 case jit_code_qmuli: case jit_code_qmuli_u:
1421 case jit_code_qdivi: case jit_code_qdivi_u:
1422 mask = jit_cc_a0_reg|jit_cc_a0_rlh|jit_cc_a0_chg|
1423 jit_cc_a1_reg|jit_cc_a2_int;
1425 case jit_code_addi_f: case jit_code_subi_f: case jit_code_rsbi_f:
1426 case jit_code_muli_f: case jit_code_divi_f: case jit_code_lti_f:
1427 case jit_code_lei_f: case jit_code_eqi_f: case jit_code_gei_f:
1428 case jit_code_gti_f: case jit_code_nei_f: case jit_code_unlti_f:
1429 case jit_code_unlei_f: case jit_code_uneqi_f: case jit_code_ungei_f:
1430 case jit_code_ungti_f: case jit_code_ltgti_f: case jit_code_ordi_f:
1431 case jit_code_unordi_f:
1432 mask = jit_cc_a0_reg|jit_cc_a0_chg|jit_cc_a1_reg|jit_cc_a2_flt;
1434 case jit_code_addi_d: case jit_code_subi_d: case jit_code_rsbi_d:
1435 case jit_code_muli_d: case jit_code_divi_d: case jit_code_lti_d:
1436 case jit_code_lei_d: case jit_code_eqi_d: case jit_code_gei_d:
1437 case jit_code_gti_d: case jit_code_nei_d: case jit_code_unlti_d:
1438 case jit_code_unlei_d: case jit_code_uneqi_d: case jit_code_ungei_d:
1439 case jit_code_ungti_d: case jit_code_ltgti_d: case jit_code_ordi_d:
1440 case jit_code_unordi_d:
1441 mask = jit_cc_a0_reg|jit_cc_a0_chg|jit_cc_a1_reg|jit_cc_a2_dbl;
1443 case jit_code_addr: case jit_code_addxr: case jit_code_addcr:
1444 case jit_code_subr: case jit_code_subxr: case jit_code_subcr:
1445 case jit_code_mulr: case jit_code_divr: case jit_code_divr_u:
1446 case jit_code_remr: case jit_code_remr_u: case jit_code_andr:
1447 case jit_code_orr: case jit_code_xorr: case jit_code_lshr:
1448 case jit_code_rshr: case jit_code_rshr_u: case jit_code_ltr:
1449 case jit_code_ltr_u: case jit_code_ler: case jit_code_ler_u:
1450 case jit_code_eqr: case jit_code_ger: case jit_code_ger_u:
1451 case jit_code_gtr: case jit_code_gtr_u: case jit_code_ner:
1452 case jit_code_ldxr_c: case jit_code_ldxr_uc: case jit_code_ldxr_s:
1453 case jit_code_ldxr_us: case jit_code_ldxr_i: case jit_code_ldxr_ui:
1454 case jit_code_ldxr_l: case jit_code_addr_f: case jit_code_subr_f:
1455 case jit_code_mulr_f: case jit_code_divr_f: case jit_code_ltr_f:
1456 case jit_code_ler_f: case jit_code_eqr_f: case jit_code_ger_f:
1457 case jit_code_gtr_f: case jit_code_ner_f: case jit_code_unltr_f:
1458 case jit_code_unler_f: case jit_code_uneqr_f: case jit_code_unger_f:
1459 case jit_code_ungtr_f: case jit_code_ltgtr_f: case jit_code_ordr_f:
1460 case jit_code_unordr_f: case jit_code_ldxr_f: case jit_code_addr_d:
1461 case jit_code_subr_d: case jit_code_mulr_d: case jit_code_divr_d:
1462 case jit_code_ltr_d: case jit_code_ler_d: case jit_code_eqr_d:
1463 case jit_code_ger_d: case jit_code_gtr_d: case jit_code_ner_d:
1464 case jit_code_unltr_d: case jit_code_unler_d: case jit_code_uneqr_d:
1465 case jit_code_unger_d: case jit_code_ungtr_d: case jit_code_ltgtr_d:
1466 case jit_code_ordr_d: case jit_code_unordr_d: case jit_code_ldxr_d:
1467 case jit_code_movr_ww_d:
1468 mask = jit_cc_a0_reg|jit_cc_a0_chg|jit_cc_a1_reg|jit_cc_a2_reg;
1470 case jit_code_qmulr: case jit_code_qmulr_u:
1471 case jit_code_qdivr: case jit_code_qdivr_u:
1472 mask = jit_cc_a0_reg|jit_cc_a0_rlh|jit_cc_a0_chg|
1473 jit_cc_a1_reg|jit_cc_a2_reg;
1475 case jit_code_sti_c: case jit_code_sti_s: case jit_code_sti_i:
1476 case jit_code_sti_l: case jit_code_sti_f: case jit_code_sti_d:
1477 mask = jit_cc_a0_int|jit_cc_a1_reg;
1479 case jit_code_blti: case jit_code_blti_u: case jit_code_blei:
1480 case jit_code_blei_u: case jit_code_beqi: case jit_code_bgei:
1481 case jit_code_bgei_u: case jit_code_bgti: case jit_code_bgti_u:
1482 case jit_code_bnei: case jit_code_bmsi: case jit_code_bmci:
1483 mask = jit_cc_a0_jmp|jit_cc_a1_reg|jit_cc_a2_int;
1485 case jit_code_blti_f: case jit_code_blei_f: case jit_code_beqi_f:
1486 case jit_code_bgei_f: case jit_code_bgti_f: case jit_code_bnei_f:
1487 case jit_code_bunlti_f: case jit_code_bunlei_f: case jit_code_buneqi_f:
1488 case jit_code_bungei_f: case jit_code_bungti_f: case jit_code_bltgti_f:
1489 case jit_code_bordi_f: case jit_code_bunordi_f:
1490 mask = jit_cc_a0_jmp|jit_cc_a1_reg|jit_cc_a2_flt;
1492 case jit_code_blti_d: case jit_code_blei_d: case jit_code_beqi_d:
1493 case jit_code_bgei_d: case jit_code_bgti_d: case jit_code_bnei_d:
1494 case jit_code_bunlti_d: case jit_code_bunlei_d: case jit_code_buneqi_d:
1495 case jit_code_bungei_d: case jit_code_bungti_d: case jit_code_bltgti_d:
1496 case jit_code_bordi_d: case jit_code_bunordi_d:
1497 mask = jit_cc_a0_jmp|jit_cc_a1_reg|jit_cc_a2_dbl;
1499 case jit_code_allocar: /* synthesized instructions make it
1500 * equivalent to jit_cc_a0_chg */
1501 case jit_code_str_c: case jit_code_str_s: case jit_code_str_i:
1502 case jit_code_str_l: case jit_code_str_f: case jit_code_str_d:
1503 mask = jit_cc_a0_reg|jit_cc_a1_reg;
1505 case jit_code_stxi_c: case jit_code_stxi_s: case jit_code_stxi_i:
1506 case jit_code_stxi_l: case jit_code_stxi_f: case jit_code_stxi_d:
1507 mask = jit_cc_a0_int|jit_cc_a1_reg|jit_cc_a2_reg;
1509 case jit_code_bltr: case jit_code_bltr_u: case jit_code_bler:
1510 case jit_code_bler_u: case jit_code_beqr: case jit_code_bger:
1511 case jit_code_bger_u: case jit_code_bgtr: case jit_code_bgtr_u:
1512 case jit_code_bner: case jit_code_bmsr: case jit_code_bmcr:
1513 case jit_code_bltr_f: case jit_code_bler_f: case jit_code_beqr_f:
1514 case jit_code_bger_f: case jit_code_bgtr_f: case jit_code_bner_f:
1515 case jit_code_bunltr_f: case jit_code_bunler_f: case jit_code_buneqr_f:
1516 case jit_code_bunger_f: case jit_code_bungtr_f: case jit_code_bltgtr_f:
1517 case jit_code_bordr_f: case jit_code_bunordr_f:case jit_code_bltr_d:
1518 case jit_code_bler_d: case jit_code_beqr_d: case jit_code_bger_d:
1519 case jit_code_bgtr_d: case jit_code_bner_d: case jit_code_bunltr_d:
1520 case jit_code_bunler_d: case jit_code_buneqr_d: case jit_code_bunger_d:
1521 case jit_code_bungtr_d: case jit_code_bltgtr_d: case jit_code_bordr_d:
1522 case jit_code_bunordr_d:
1523 mask = jit_cc_a0_jmp|jit_cc_a1_reg|jit_cc_a2_reg;
1525 case jit_code_boaddi: case jit_code_boaddi_u: case jit_code_bxaddi:
1526 case jit_code_bxaddi_u: case jit_code_bosubi: case jit_code_bosubi_u:
1527 case jit_code_bxsubi: case jit_code_bxsubi_u:
1528 mask = jit_cc_a0_jmp|jit_cc_a1_reg|jit_cc_a1_chg|jit_cc_a2_int;
1530 case jit_code_stxr_c: case jit_code_stxr_s: case jit_code_stxr_i:
1531 case jit_code_stxr_l: case jit_code_stxr_f: case jit_code_stxr_d:
1532 mask = jit_cc_a0_reg|jit_cc_a1_reg|jit_cc_a2_reg;
1534 case jit_code_boaddr: case jit_code_boaddr_u: case jit_code_bxaddr:
1535 case jit_code_bxaddr_u: case jit_code_bosubr: case jit_code_bosubr_u:
1536 case jit_code_bxsubr: case jit_code_bxsubr_u:
1537 mask = jit_cc_a0_jmp|jit_cc_a1_reg|jit_cc_a1_chg|jit_cc_a2_reg;
1539 case jit_code_movnr: case jit_code_movzr:
1540 mask = jit_cc_a0_reg|jit_cc_a0_cnd|jit_cc_a1_reg|jit_cc_a2_reg;
1550 _jit_patch_abs(jit_state_t *_jit, jit_node_t *instr, jit_pointer_t address)
1554 switch (instr->code) {
1555 case jit_code_movi: case jit_code_ldi_c: case jit_code_ldi_uc:
1556 case jit_code_ldi_s: case jit_code_ldi_us: case jit_code_ldi_i:
1557 case jit_code_ldi_ui: case jit_code_ldi_l: case jit_code_ldi_f:
1558 case jit_code_ldi_d:
1559 instr->v.p = address;
1561 case jit_code_sti_c: case jit_code_sti_s: case jit_code_sti_i:
1562 case jit_code_sti_l: case jit_code_sti_f: case jit_code_sti_d:
1563 instr->u.p = address;
1566 mask = jit_classify(instr->code);
1567 assert((mask & (jit_cc_a0_reg|jit_cc_a0_jmp)) == jit_cc_a0_jmp);
1568 instr->u.p = address;
1573 _jit_patch_at(jit_state_t *_jit, jit_node_t *instr, jit_node_t *label)
1577 assert(!(instr->flag & jit_flag_node));
1578 instr->flag |= jit_flag_node;
1579 switch (instr->code) {
1581 assert(label->code == jit_code_label ||
1582 label->code == jit_code_data);
1584 if (label->code == jit_code_data)
1585 instr->flag |= jit_flag_data;
1588 assert(label->code == jit_code_label ||
1589 label->code == jit_code_epilog);
1593 mask = jit_classify(instr->code);
1594 assert((mask & (jit_cc_a0_reg|jit_cc_a0_jmp)) == jit_cc_a0_jmp);
1595 assert(label->code == jit_code_label);
1599 /* link field is used as list of nodes associated with a given label */
1600 instr->link = label->link;
1601 label->link = instr;
1605 _jit_optimize(jit_state_t *_jit)
1614 _jitc->function = NULL;
1617 sequential_labels();
1620 /* create initial mapping of live register values
1621 * at the start of a basic block */
1622 for (offset = 0; offset < _jitc->blocks.offset; offset++) {
1623 block = _jitc->blocks.ptr + offset;
1626 if (block->label->code != jit_code_epilog)
1630 /* set live state of registers not referenced in a block, but
1631 * referenced in a jump target or normal flow */
1634 for (offset = 0; offset < _jitc->blocks.offset; offset++) {
1635 block = _jitc->blocks.ptr + offset;
1638 if (block->label->code != jit_code_epilog)
1639 jit_follow(block, &todo);
1646 /* figure out labels that are only reached with a jump
1647 * and is required to do a simple redundant_store removal
1648 * on jit_beqi below */
1650 for (node = _jitc->head; node; node = node->next) {
1651 switch (node->code) {
1652 case jit_code_label:
1654 node->flag |= jit_flag_head;
1656 case jit_code_jmpi: case jit_code_jmpr:
1657 case jit_code_epilog:
1660 case jit_code_data: case jit_code_note:
1668 for (node = _jitc->head; node; node = node->next) {
1669 mask = jit_classify(node->code);
1670 if (mask & jit_cc_a0_reg)
1671 node->u.w &= ~jit_regno_patch;
1672 if (mask & jit_cc_a1_reg)
1673 node->v.w &= ~jit_regno_patch;
1674 if (mask & jit_cc_a2_reg)
1675 node->w.w &= ~jit_regno_patch;
1676 switch (node->code) {
1677 case jit_code_prolog:
1678 _jitc->function = _jitc->functions.ptr + node->w.w;
1680 case jit_code_epilog:
1681 _jitc->function = NULL;
1684 redundant_store(node, 1);
1687 redundant_store(node, 0);
1691 if (mask & jit_cc_a0_flt) {
1692 node->u.p = jit_data(&node->u.f, sizeof(jit_float32_t), 4);
1693 node->flag |= jit_flag_node | jit_flag_data;
1695 else if (mask & jit_cc_a0_dbl) {
1696 node->u.p = jit_data(&node->u.d, sizeof(jit_float64_t), 8);
1697 node->flag |= jit_flag_node | jit_flag_data;
1699 else if (mask & jit_cc_a1_flt) {
1700 node->v.p = jit_data(&node->v.f, sizeof(jit_float32_t), 4);
1701 node->flag |= jit_flag_node | jit_flag_data;
1703 else if (mask & jit_cc_a1_dbl) {
1704 node->v.p = jit_data(&node->v.d, sizeof(jit_float64_t), 8);
1705 node->flag |= jit_flag_node | jit_flag_data;
1707 else if (mask & jit_cc_a2_flt) {
1708 node->w.p = jit_data(&node->w.f, sizeof(jit_float32_t), 4);
1709 node->flag |= jit_flag_node | jit_flag_data;
1711 else if (mask & jit_cc_a2_dbl) {
1712 node->w.p = jit_data(&node->w.d, sizeof(jit_float64_t), 8);
1713 node->flag |= jit_flag_node | jit_flag_data;
1716 if (_jitc->function) {
1717 if ((mask & (jit_cc_a0_reg|jit_cc_a0_chg)) ==
1718 (jit_cc_a0_reg|jit_cc_a0_chg)) {
1719 if (mask & jit_cc_a0_rlh) {
1720 jit_regset_setbit(&_jitc->function->regset,
1721 jit_regno(node->u.q.l));
1722 jit_regset_setbit(&_jitc->function->regset,
1723 jit_regno(node->u.q.h));
1726 jit_regset_setbit(&_jitc->function->regset,
1727 jit_regno(node->u.w));
1729 if ((mask & (jit_cc_a1_reg|jit_cc_a1_chg)) ==
1730 (jit_cc_a1_reg|jit_cc_a1_chg))
1731 jit_regset_setbit(&_jitc->function->regset,
1732 jit_regno(node->v.w));
1733 if ((mask & (jit_cc_a2_reg|jit_cc_a2_chg)) ==
1734 (jit_cc_a2_reg|jit_cc_a2_chg))
1735 jit_regset_setbit(&_jitc->function->regset,
1736 jit_regno(node->w.w));
1744 _jit_reglive(jit_state_t *_jit, jit_node_t *node)
1750 switch (node->code) {
1751 case jit_code_label: case jit_code_prolog: case jit_code_epilog:
1752 block = _jitc->blocks.ptr + node->v.w;
1753 jit_regset_set(&_jitc->reglive, &block->reglive);
1755 case jit_code_callr:
1756 value = jit_regno(node->u.w);
1757 if (!(node->u.w & jit_regno_patch)) {
1758 jit_regset_setbit(&_jitc->reglive, value);
1760 case jit_code_calli:
1761 for (value = 0; value < _jitc->reglen; value++) {
1762 spec = jit_class(_rvs[value].spec);
1763 if ((spec & jit_class_arg) && jit_regarg_p(node, value))
1764 jit_regset_setbit(&_jitc->reglive, value);
1765 else if (!(spec & jit_class_sav))
1766 jit_regset_clrbit(&_jitc->reglive, value);
1770 value = jit_classify(node->code);
1771 if (value & jit_cc_a0_reg) {
1772 if (value & jit_cc_a0_rlh) {
1773 if (!(node->u.q.l & jit_regno_patch)) {
1774 if (value & jit_cc_a0_chg) {
1775 jit_regset_clrbit(&_jitc->reglive, node->u.q.l);
1776 jit_regset_setbit(&_jitc->regmask, node->u.q.l);
1779 jit_regset_setbit(&_jitc->reglive, node->u.q.l);
1781 if (!(node->u.q.h & jit_regno_patch)) {
1782 if (value & jit_cc_a0_chg) {
1783 jit_regset_clrbit(&_jitc->reglive, node->u.q.h);
1784 jit_regset_setbit(&_jitc->regmask, node->u.q.h);
1787 jit_regset_setbit(&_jitc->reglive, node->u.q.h);
1791 if (!(node->u.w & jit_regno_patch)) {
1792 if (value & jit_cc_a0_chg) {
1793 jit_regset_clrbit(&_jitc->reglive, node->u.w);
1794 jit_regset_setbit(&_jitc->regmask, node->u.w);
1797 jit_regset_setbit(&_jitc->reglive, node->u.w);
1801 if ((value & jit_cc_a1_reg) && !(node->v.w & jit_regno_patch)) {
1802 if (value & jit_cc_a1_chg) {
1803 jit_regset_clrbit(&_jitc->reglive, node->v.w);
1804 jit_regset_setbit(&_jitc->regmask, node->v.w);
1807 jit_regset_setbit(&_jitc->reglive, node->v.w);
1809 if ((value & jit_cc_a2_reg) && !(node->w.w & jit_regno_patch)) {
1810 if (value & jit_cc_a2_chg) {
1811 jit_regset_clrbit(&_jitc->reglive, node->w.w);
1812 jit_regset_setbit(&_jitc->regmask, node->w.w);
1815 jit_regset_setbit(&_jitc->reglive, node->w.w);
1817 if (jit_regset_set_p(&_jitc->regmask)) {
1818 jit_update(node->next, &_jitc->reglive, &_jitc->regmask);
1819 if (jit_regset_set_p(&_jitc->regmask)) {
1820 /* any unresolved live state is considered as live */
1821 jit_regset_ior(&_jitc->reglive,
1822 &_jitc->reglive, &_jitc->regmask);
1823 jit_regset_set_ui(&_jitc->regmask, 0);
1831 _jit_regarg_set(jit_state_t *_jit, jit_node_t *node, jit_int32_t value)
1836 if (value & jit_cc_a0_reg) {
1837 if (value & jit_cc_a0_rlh) {
1838 jit_regset_setbit(&_jitc->regarg, jit_regno(node->u.q.l));
1839 jit_regset_setbit(&_jitc->regarg, jit_regno(node->u.q.h));
1842 jit_regset_setbit(&_jitc->regarg, jit_regno(node->u.w));
1844 if (value & jit_cc_a1_reg)
1845 jit_regset_setbit(&_jitc->regarg, jit_regno(node->v.w));
1846 if (value & jit_cc_a2_reg)
1847 jit_regset_setbit(&_jitc->regarg, jit_regno(node->w.w));
1851 _jit_regarg_clr(jit_state_t *_jit, jit_node_t *node, jit_int32_t value)
1854 jit_size_collect(node);
1856 if (value & jit_cc_a0_reg) {
1857 if (value & jit_cc_a0_rlh) {
1858 jit_regset_clrbit(&_jitc->regarg, jit_regno(node->u.q.l));
1859 jit_regset_clrbit(&_jitc->regarg, jit_regno(node->u.q.h));
1862 jit_regset_clrbit(&_jitc->regarg, jit_regno(node->u.w));
1864 if (value & jit_cc_a1_reg)
1865 jit_regset_clrbit(&_jitc->regarg, jit_regno(node->v.w));
1866 if (value & jit_cc_a2_reg)
1867 jit_regset_clrbit(&_jitc->regarg, jit_regno(node->w.w));
1871 _jit_realize(jit_state_t *_jit)
1873 assert(!_jitc->realize);
1874 if (_jitc->function)
1879 /* ensure it is aligned */
1880 _jitc->data.offset = (_jitc->data.offset + 7) & -8;
1883 /* Heuristic to guess code buffer size */
1885 _jit->code.length = _jitc->pool.length * 1024 * _jitc->mult;
1887 _jit->code.length = jit_get_size();
1892 _jit_dataset(jit_state_t *_jit)
1901 assert(!_jitc->dataset);
1903 assert(_jit->user_data);
1905 if (!_jit->user_data) {
1907 /* create read only data buffer */
1908 _jit->data.length = (_jitc->data.offset +
1909 /* reserve space for annotations */
1910 _jitc->note.size + 4095) & -4096;
1912 mmap_fd = open("/dev/zero", O_RDWR);
1914 _jit->data.ptr = mmap(NULL, _jit->data.length,
1915 PROT_READ | PROT_WRITE,
1916 MAP_PRIVATE | MAP_ANON, mmap_fd, 0);
1917 assert(_jit->data.ptr != MAP_FAILED);
1922 #endif /* !HAVE_MMAP */
1924 if (!_jitc->no_data)
1925 jit_memcpy(_jit->data.ptr, _jitc->data.ptr, _jitc->data.offset);
1927 if (_jitc->no_note) {
1928 /* Space for one note is always allocated, so revert it here
1929 * if after jit_new_state was called, it is also requested to
1930 * not generate annotation information */
1931 _jit->note.length = 0;
1932 _jitc->note.size = 0;
1935 _jitc->note.base = _jit->data.ptr;
1936 if (!_jitc->no_data)
1937 _jitc->note.base += _jitc->data.offset;
1938 memset(_jitc->note.base, 0, _jitc->note.size);
1941 if (_jit->user_data)
1942 /* Need the temporary hashed data until jit_emit is finished */
1943 ptr = _jitc->no_data ? _jitc->data.ptr : _jit->data.ptr;
1945 ptr = _jit->data.ptr;
1946 /* Temporary hashed data no longer required */
1947 jit_free((jit_pointer_t *)&_jitc->data.ptr);
1950 for (offset = 0; offset < _jitc->data.size; offset++) {
1951 for (node = _jitc->data.table[offset]; node; node = node->next) {
1952 node->flag |= jit_flag_patch;
1953 node->u.w = (jit_word_t)(ptr + node->u.w);
1961 _jit_get_code(jit_state_t *_jit, jit_word_t *length)
1963 assert(_jitc->realize);
1966 /* If code already generated, return exact size of code */
1967 *length = _jit->pc.uc - _jit->code.ptr;
1969 /* Else return current size of the code buffer */
1970 *length = _jit->code.length;
1973 return (_jit->code.ptr);
1977 _jit_set_code(jit_state_t *_jit, jit_pointer_t ptr, jit_word_t length)
1979 assert(_jitc->realize);
1980 _jit->code.ptr = ptr;
1981 _jit->code.length = length;
1982 _jit->user_code = 1;
1986 _jit_get_data(jit_state_t *_jit, jit_word_t *data_size, jit_word_t *note_size)
1988 assert(_jitc->realize);
1990 *data_size = _jitc->data.offset;
1992 *note_size = _jitc->note.size;
1993 return (_jit->data.ptr);
1997 _jit_set_data(jit_state_t *_jit, jit_pointer_t ptr,
1998 jit_word_t length, jit_word_t flags)
2000 assert(_jitc->realize);
2001 if (flags & JIT_DISABLE_DATA)
2004 assert(length >= _jitc->data.offset);
2005 if (flags & JIT_DISABLE_NOTE)
2008 if (flags & JIT_DISABLE_DATA)
2009 assert(length >= _jitc->note.size);
2011 assert(length >= _jitc->data.offset + _jitc->note.size);
2013 _jit->data.ptr = ptr;
2014 _jit->data.length = length;
2015 _jit->user_data = 1;
2019 _jit_emit(jit_state_t *_jit)
2029 if (!_jitc->realize)
2032 if (!_jitc->dataset)
2038 assert(_jit->user_code);
2040 if (!_jit->user_code) {
2042 mmap_fd = open("/dev/zero", O_RDWR);
2044 _jit->code.ptr = mmap(NULL, _jit->code.length,
2045 PROT_EXEC | PROT_READ | PROT_WRITE,
2046 MAP_PRIVATE | MAP_ANON, mmap_fd, 0);
2047 assert(_jit->code.ptr != MAP_FAILED);
2049 #endif /* !HAVE_MMAP */
2050 _jitc->code.end = _jit->code.ptr + _jit->code.length -
2051 jit_get_max_instr();
2052 _jit->pc.uc = _jit->code.ptr;
2055 if ((code = emit_code()) == NULL) {
2056 _jitc->patches.offset = 0;
2057 for (node = _jitc->head; node; node = node->next) {
2059 (node->code == jit_code_label ||
2060 node->code == jit_code_epilog))
2061 node->flag &= ~jit_flag_patch;
2064 assert(_jit->user_code);
2066 if (_jit->user_code)
2070 length = _jitc->pool.length * 1024 * _jitc->mult;
2072 /* Should only happen on very special cases */
2073 length = _jit->code.length + 4096;
2077 munmap(_jit->code.ptr, _jit->code.length);
2082 _jit->code.ptr = mremap(_jit->code.ptr, _jit->code.length,
2083 _jit->code.ptr, length, 0);
2085 _jit->code.ptr = mremap(_jit->code.ptr, _jit->code.length,
2086 length, MREMAP_MAYMOVE, NULL);
2089 _jit->code.ptr = mmap(NULL, length,
2090 PROT_EXEC | PROT_READ | PROT_WRITE,
2091 MAP_PRIVATE | MAP_ANON, mmap_fd, 0);
2094 assert(_jit->code.ptr != MAP_FAILED);
2095 _jit->code.length = length;
2096 _jitc->code.end = _jit->code.ptr + _jit->code.length -
2097 jit_get_max_instr();
2098 _jit->pc.uc = _jit->code.ptr;
2099 #endif /* !HAVE_MMAP */
2106 if (!_jit->user_code)
2111 if (!_jitc->no_note)
2114 if (_jit->user_data)
2115 jit_free((jit_pointer_t *)&_jitc->data.ptr);
2118 result = mprotect(_jit->data.ptr, _jit->data.length, PROT_READ);
2119 assert(result == 0);
2121 if (!_jit->user_code) {
2122 result = mprotect(_jit->code.ptr, _jit->code.length,
2123 PROT_READ | PROT_EXEC);
2124 assert(result == 0);
2126 #endif /* HAVE_MMAP */
2128 return (_jit->code.ptr);
2134 _jit_frame(jit_state_t *_jit, jit_int32_t frame)
2136 jit_trampoline(frame, 1);
2140 _jit_tramp(jit_state_t *_jit, jit_int32_t frame)
2142 jit_trampoline(frame, 0);
2146 _jit_trampoline(jit_state_t *_jit, jit_int32_t frame, jit_bool_t prolog)
2150 /* Must be called after prolog, actually, just to simplify
2151 * tests and know there is a current function and that
2152 * _jitc->function->self.aoff is at the before any alloca value */
2153 assert(_jitc->tail && _jitc->tail->code == jit_code_prolog);
2155 /* + 24 for 3 possible spilled temporaries (that could be a double) */
2157 #if defined(__hppa__)
2158 frame += _jitc->function->self.aoff;
2160 frame -= _jitc->function->self.aoff;
2162 _jitc->function->frame = frame;
2164 _jitc->function->define_frame = 1;
2166 _jitc->function->assume_frame = 1;
2167 for (regno = 0; regno < _jitc->reglen; regno++)
2168 if (jit_class(_rvs[regno].spec) & jit_class_sav)
2169 jit_regset_setbit(&_jitc->function->regset, regno);
2172 /* Compute initial reglive and regmask set values of a basic block.
2173 * reglive is the set of known live registers
2174 * regmask is the set of registers not referenced in the block
2175 * Registers in regmask might be live.
2178 _jit_setup(jit_state_t *_jit, jit_block_t *block)
2182 unsigned long value;
2184 jit_regset_set_mask(&block->regmask, _jitc->reglen);
2185 for (value = 0; value < _jitc->reglen; ++value)
2186 if (!(jit_class(_rvs[value].spec) & (jit_class_gpr|jit_class_fpr)))
2187 jit_regset_clrbit(&block->regmask, value);
2189 for (node = block->label->next; node; node = node->next) {
2190 switch (node->code) {
2191 case jit_code_label: case jit_code_prolog:
2192 case jit_code_epilog:
2195 /* Check argument registers in reverse order to properly
2196 * handle registers that are both, argument and result */
2197 value = jit_classify(node->code);
2198 if ((value & jit_cc_a2_reg) &&
2199 !(node->w.w & jit_regno_patch) &&
2200 jit_regset_tstbit(&block->regmask, node->w.w)) {
2201 live = !(value & jit_cc_a2_chg);
2202 jit_regset_clrbit(&block->regmask, node->w.w);
2204 jit_regset_setbit(&block->reglive, node->w.w);
2206 if ((value & jit_cc_a1_reg) &&
2207 !(node->v.w & jit_regno_patch) &&
2208 jit_regset_tstbit(&block->regmask, node->v.w)) {
2209 live = !(value & jit_cc_a1_chg);
2210 jit_regset_clrbit(&block->regmask, node->v.w);
2212 jit_regset_setbit(&block->reglive, node->v.w);
2214 if (value & jit_cc_a0_reg) {
2215 live = !(value & jit_cc_a0_chg);
2216 if (value & jit_cc_a0_rlh) {
2217 if (!(node->u.q.l & jit_regno_patch) &&
2218 jit_regset_tstbit(&block->regmask, node->u.q.l)) {
2219 jit_regset_clrbit(&block->regmask, node->u.q.l);
2221 jit_regset_setbit(&block->reglive, node->u.q.l);
2223 if (!(node->u.q.h & jit_regno_patch) &&
2224 jit_regset_tstbit(&block->regmask, node->u.q.h)) {
2225 jit_regset_clrbit(&block->regmask, node->u.q.h);
2227 jit_regset_setbit(&block->reglive, node->u.q.h);
2231 if (!(node->u.w & jit_regno_patch) &&
2232 jit_regset_tstbit(&block->regmask, node->u.w)) {
2233 jit_regset_clrbit(&block->regmask, node->u.w);
2235 jit_regset_setbit(&block->reglive, node->u.w);
2244 /* Update regmask and reglive of blocks at entry point of branch targets
2245 * or normal flow that have a live register not used in this block.
2248 _jit_follow(jit_state_t *_jit, jit_block_t *block, jit_bool_t *todo)
2254 unsigned long value;
2256 jit_regset_t reglive;
2257 jit_regset_t regmask;
2258 jit_regset_t regtemp;
2260 jit_regset_set(®live, &block->reglive);
2261 jit_regset_set(®mask, &block->regmask);
2262 for (node = block->label->next; node; node = node->next) {
2263 switch (node->code) {
2264 case jit_code_label:
2265 /* Do not consider jmpi and jmpr cannot jump to the
2266 * next instruction. */
2267 next = _jitc->blocks.ptr + node->v.w;
2268 /* Set of live registers in next block that are at unknown
2269 * state in this block. */
2270 jit_regset_and(®temp, ®mask, &next->reglive);
2271 if (jit_regset_set_p(®temp)) {
2272 /* Add live state of next block to current block. */
2273 jit_regset_ior(&block->reglive, &block->reglive, ®temp);
2274 /* Remove from unknown state bitmask. */
2275 jit_regset_com(®temp, ®temp);
2276 jit_regset_and(&block->regmask, &block->regmask, ®temp);
2279 case jit_code_prolog:
2280 case jit_code_epilog:
2282 case jit_code_callr:
2283 value = jit_regno(node->u.w);
2284 if (!(node->u.w & jit_regno_patch)) {
2285 if (jit_regset_tstbit(®mask, value)) {
2286 jit_regset_clrbit(®mask, value);
2287 jit_regset_setbit(®live, value);
2290 case jit_code_calli:
2291 for (value = 0; value < _jitc->reglen; ++value) {
2292 value = jit_regset_scan1(®mask, value);
2293 if (value >= _jitc->reglen)
2295 spec = jit_class(_rvs[value].spec);
2296 if (!(spec & jit_class_sav))
2297 jit_regset_clrbit(®mask, value);
2298 if ((spec & jit_class_arg) && jit_regarg_p(node, value))
2299 jit_regset_setbit(®live, value);
2303 value = jit_classify(node->code);
2304 if (value & jit_cc_a2_reg) {
2305 if (!(node->w.w & jit_regno_patch)) {
2306 if (jit_regset_tstbit(®mask, node->w.w)) {
2307 jit_regset_clrbit(®mask, node->w.w);
2308 if (!(value & jit_cc_a2_chg))
2309 jit_regset_setbit(®live, node->w.w);
2313 if (value & jit_cc_a1_reg) {
2314 if (!(node->v.w & jit_regno_patch)) {
2315 if (jit_regset_tstbit(®mask, node->v.w)) {
2316 jit_regset_clrbit(®mask, node->v.w);
2317 if (!(value & jit_cc_a1_chg))
2318 jit_regset_setbit(®live, node->v.w);
2322 if (value & jit_cc_a0_reg) {
2323 if (value & jit_cc_a0_rlh) {
2324 if (!(node->u.q.l & jit_regno_patch)) {
2325 if (jit_regset_tstbit(®mask, node->u.q.l)) {
2326 jit_regset_clrbit(®mask, node->u.q.l);
2327 if (!(value & jit_cc_a0_chg))
2328 jit_regset_setbit(®live, node->u.q.l);
2331 if (!(node->u.q.h & jit_regno_patch)) {
2332 if (jit_regset_tstbit(®mask, node->u.q.h)) {
2333 jit_regset_clrbit(®mask, node->u.q.h);
2334 if (!(value & jit_cc_a0_chg))
2335 jit_regset_setbit(®live, node->u.q.h);
2340 if (!(node->u.w & jit_regno_patch)) {
2341 if (jit_regset_tstbit(®mask, node->u.w)) {
2342 jit_regset_clrbit(®mask, node->u.w);
2343 if (!(value & jit_cc_a0_chg))
2344 jit_regset_setbit(®live, node->u.w);
2349 if (value & jit_cc_a0_jmp) {
2350 if (node->flag & jit_flag_node) {
2352 /* Do not consider jmpi and jmpr cannot jump to the
2353 * next instruction. */
2354 next = _jitc->blocks.ptr + label->v.w;
2355 jit_regset_and(®temp, ®mask, &next->reglive);
2356 if (jit_regset_set_p(®temp)) {
2357 /* Add live state. */
2358 jit_regset_ior(&block->reglive,
2359 &block->reglive, ®temp);
2360 /* Remove from unknown state bitmask. */
2361 jit_regset_com(®temp, ®temp);
2362 jit_regset_and(&block->regmask,
2363 &block->regmask, ®temp);
2368 /* Jump to unknown location.
2369 * This is a pitfall of the implementation.
2370 * Only jmpi to not a jit code should reach here,
2371 * or a jmpr of a computed address.
2372 * Because the implementation needs jit_class_nospill
2373 * registers, must treat jmpr as a function call. This
2374 * means that only JIT_Vn registers can be trusted on
2377 for (regno = 0; regno < _jitc->reglen; regno++) {
2378 spec = jit_class(_rvs[regno].spec);
2379 if (jit_regset_tstbit(®mask, regno) &&
2380 (spec & (jit_class_gpr|jit_class_fpr)) &&
2381 !(spec & jit_class_sav))
2382 jit_regset_clrbit(®mask, regno);
2384 /* Assume non callee save registers are live due
2385 * to jump to unknown location. */
2386 /* Treat all callee save as live. */
2387 jit_regset_ior(®live, ®live, ®mask);
2388 /* Treat anything else as dead. */
2389 jit_regset_set_ui(®mask, 0);
2397 /* Follow code generation up to finding a label or end of code.
2398 * When finding a label, update the set of live registers.
2399 * On branches, update based on taken branch or normal flow.
2402 _jit_update(jit_state_t *_jit, jit_node_t *node,
2403 jit_regset_t *live, jit_regset_t *mask)
2407 unsigned long value;
2410 jit_regset_t regtemp;
2412 for (; node; node = node->next) {
2413 if (jit_regset_set_p(mask) == 0)
2415 switch (node->code) {
2416 case jit_code_label:
2417 block = _jitc->blocks.ptr + node->v.w;
2418 jit_regset_and(®temp, mask, &block->reglive);
2419 if (jit_regset_set_p(®temp)) {
2420 /* Add live state. */
2421 jit_regset_ior(live, live, ®temp);
2422 /* Remove from unknown state bitmask. */
2423 jit_regset_com(®temp, ®temp);
2424 jit_regset_and(mask, mask, ®temp);
2427 case jit_code_prolog:
2428 jit_regset_set_ui(mask, 0);
2430 case jit_code_epilog:
2431 jit_regset_set_ui(mask, 0);
2433 case jit_code_callr:
2434 value = jit_regno(node->u.w);
2435 if (!(node->u.w & jit_regno_patch)) {
2436 if (jit_regset_tstbit(mask, value)) {
2437 jit_regset_clrbit(mask, value);
2438 jit_regset_setbit(live, value);
2441 case jit_code_calli:
2442 for (value = 0; value < _jitc->reglen; ++value) {
2443 value = jit_regset_scan1(mask, value);
2444 if (value >= _jitc->reglen)
2446 spec = jit_class(_rvs[value].spec);
2447 if (!(spec & jit_class_sav))
2448 jit_regset_clrbit(mask, value);
2449 if ((spec & jit_class_arg) && jit_regarg_p(node, value))
2450 jit_regset_setbit(live, value);
2454 value = jit_classify(node->code);
2455 if (value & jit_cc_a2_reg) {
2456 if (!(node->w.w & jit_regno_patch)) {
2457 if (jit_regset_tstbit(mask, node->w.w)) {
2458 jit_regset_clrbit(mask, node->w.w);
2459 if (!(value & jit_cc_a2_chg))
2460 jit_regset_setbit(live, node->w.w);
2464 if (value & jit_cc_a1_reg) {
2465 if (!(node->v.w & jit_regno_patch)) {
2466 if (jit_regset_tstbit(mask, node->v.w)) {
2467 jit_regset_clrbit(mask, node->v.w);
2468 if (!(value & jit_cc_a1_chg))
2469 jit_regset_setbit(live, node->v.w);
2473 if (value & jit_cc_a0_reg) {
2474 if (value & jit_cc_a0_rlh) {
2475 if (!(node->u.q.l & jit_regno_patch)) {
2476 if (jit_regset_tstbit(mask, node->u.q.l)) {
2477 jit_regset_clrbit(mask, node->u.q.l);
2478 if (!(value & jit_cc_a0_chg))
2479 jit_regset_setbit(live, node->u.q.l);
2482 if (!(node->u.q.h & jit_regno_patch)) {
2483 if (jit_regset_tstbit(mask, node->u.q.h)) {
2484 jit_regset_clrbit(mask, node->u.q.h);
2485 if (!(value & jit_cc_a0_chg))
2486 jit_regset_setbit(live, node->u.q.h);
2491 if (!(node->u.w & jit_regno_patch)) {
2492 if (jit_regset_tstbit(mask, node->u.w)) {
2493 jit_regset_clrbit(mask, node->u.w);
2494 if (!(value & jit_cc_a0_chg))
2495 jit_regset_setbit(live, node->u.w);
2500 if (value & jit_cc_a0_jmp) {
2501 if (node->flag & jit_flag_node) {
2503 /* Do not consider jmpi and jmpr cannot jump to the
2504 * next instruction. */
2505 block = _jitc->blocks.ptr + label->v.w;
2506 jit_regset_and(®temp, mask, &block->reglive);
2507 if (jit_regset_set_p(®temp)) {
2508 /* Add live state. */
2509 jit_regset_ior(live, live, ®temp);
2510 /* Remove from unknown state bitmask. */
2511 jit_regset_com(®temp, ®temp);
2512 jit_regset_and(mask, mask, ®temp);
2516 /* Jump to unknown location.
2517 * This is a pitfall of the implementation.
2518 * Only jmpi to not a jit code should reach here,
2519 * or a jmpr of a computed address.
2520 * Because the implementation needs jit_class_nospill
2521 * registers, must treat jmpr as a function call. This
2522 * means that only JIT_Vn registers can be trusted on
2525 for (regno = 0; regno < _jitc->reglen; regno++) {
2526 spec = jit_class(_rvs[regno].spec);
2527 if (jit_regset_tstbit(mask, regno) &&
2528 (spec & (jit_class_gpr|jit_class_fpr)) &&
2529 !(spec & jit_class_sav))
2530 jit_regset_clrbit(mask, regno);
2532 /* Assume non callee save registers are live due
2533 * to jump to unknown location. */
2534 /* Treat all callee save as live. */
2535 jit_regset_ior(live, live, mask);
2536 /* Treat anything else as dead. */
2537 jit_regset_set_ui(mask, 0);
2546 _thread_jumps(jit_state_t *_jit)
2553 for (prev = node = _jitc->head; node;) {
2555 switch (node->code) {
2557 if (redundant_jump(prev, node)) {
2561 if (shortcut_jump(prev, node))
2565 case jit_code_callr: case jit_code_calli:
2566 /* non optimizable jump like code */
2569 mask = jit_classify(node->code);
2570 if (mask & jit_cc_a0_jmp) {
2571 if (reverse_jump(prev, node) ||
2572 shortcut_jump(prev, node))
2583 _sequential_labels(jit_state_t *_jit)
2591 for (prev = node = _jitc->head; node; node = next) {
2593 if (node->code == jit_code_label) {
2596 del_label(prev, node);
2599 if (prev != node && prev->code == jit_code_label) {
2600 if ((jump = node->link)) {
2601 for (; jump; jump = link) {
2604 jump->link = prev->link;
2609 del_label(prev, node);
2613 if (next && next->code == jit_code_label && !next->flag) {
2614 if ((jump = next->link)) {
2615 for (; jump; jump = link) {
2618 jump->link = node->link;
2623 del_label(node, next);
2633 _split_branches(jit_state_t *_jit)
2640 for (node = _jitc->head; node; node = next) {
2641 if ((next = node->next)) {
2642 if (next->code == jit_code_label ||
2643 next->code == jit_code_prolog ||
2644 next->code == jit_code_epilog)
2646 /* split block on branches */
2647 if (jit_classify(node->code) & jit_cc_a0_jmp) {
2648 label = new_node(jit_code_label);
2651 if (_jitc->blocks.offset >= _jitc->blocks.length) {
2654 length = _jitc->blocks.length + 16;
2655 jit_realloc((jit_pointer_t *)&_jitc->blocks.ptr,
2656 _jitc->blocks.length * sizeof(jit_block_t),
2657 length * sizeof(jit_block_t));
2658 _jitc->blocks.length = length;
2660 block = _jitc->blocks.ptr + _jitc->blocks.offset;
2661 block->label = label;
2662 label->v.w = _jitc->blocks.offset;
2663 jit_regset_new(&block->reglive);
2664 jit_regset_new(&block->regmask);
2665 ++_jitc->blocks.offset;
2672 _shortcut_jump(jit_state_t *_jit, jit_node_t *prev, jit_node_t *node)
2679 if (!(node->flag & jit_flag_node))
2681 assert(node->code != jit_code_jmpr);
2682 cond = node->code != jit_code_jmpi;
2684 for (next = jump->next; next; next = next->next) {
2685 switch (next->code) {
2687 if (!(next->flag & jit_flag_node))
2689 if (jump->link == node)
2690 jump->link = node->link;
2692 for (temp = jump->link;
2695 assert(temp != NULL);
2696 temp->link = node->link;
2700 node->link = jump->link;
2706 node->code = jit_code_jmpr;
2707 node->u.w = next->u.w;
2709 node->flag &= ~jit_flag_node;
2711 case jit_code_note: case jit_code_label:
2721 _redundant_jump(jit_state_t *_jit, jit_node_t *prev, jit_node_t *node)
2723 jit_node_t *local_prev;
2724 jit_node_t *local_next;
2726 if (!(node->flag & jit_flag_node))
2728 for (local_prev = node, local_next = node->next;
2730 local_prev = local_next, local_next = local_next->next) {
2732 switch (local_next->code) {
2733 case jit_code_label: case jit_code_epilog:
2734 if (node->u.n == local_next) {
2735 if (local_next->link == node)
2736 local_next->link = node->link;
2738 for (local_prev = local_next->link;
2739 local_prev->link != node;
2740 local_prev = local_prev->link)
2741 assert(local_prev != NULL);
2742 local_prev->link = node->link;
2744 del_node(prev, node);
2748 case jit_code_name: case jit_code_note:
2749 case jit_code_align:
2759 reverse_jump_code(jit_code_t code)
2762 case jit_code_bltr: return (jit_code_bger);
2763 case jit_code_blti: return (jit_code_bgei);
2764 case jit_code_bltr_u: return (jit_code_bger_u);
2765 case jit_code_blti_u: return (jit_code_bgei_u);
2766 case jit_code_bler: return (jit_code_bgtr);
2767 case jit_code_blei: return (jit_code_bgti);
2768 case jit_code_bler_u: return (jit_code_bgtr_u);
2769 case jit_code_blei_u: return (jit_code_bgti_u);
2770 case jit_code_beqr: return (jit_code_bner);
2771 case jit_code_beqi: return (jit_code_bnei);
2772 case jit_code_bger: return (jit_code_bltr);
2773 case jit_code_bgei: return (jit_code_blti);
2774 case jit_code_bger_u: return (jit_code_bltr_u);
2775 case jit_code_bgei_u: return (jit_code_blti_u);
2776 case jit_code_bgtr: return (jit_code_bler);
2777 case jit_code_bgti: return (jit_code_blei);
2778 case jit_code_bgtr_u: return (jit_code_bler_u);
2779 case jit_code_bgti_u: return (jit_code_blei_u);
2780 case jit_code_bner: return (jit_code_beqr);
2781 case jit_code_bnei: return (jit_code_beqi);
2782 case jit_code_bmsr: return (jit_code_bmcr);
2783 case jit_code_bmsi: return (jit_code_bmci);
2784 case jit_code_bmcr: return (jit_code_bmsr);
2785 case jit_code_bmci: return (jit_code_bmsi);
2786 case jit_code_bltr_f: return (jit_code_bunger_f);
2787 case jit_code_blti_f: return (jit_code_bungei_f);
2788 case jit_code_bler_f: return (jit_code_bungtr_f);
2789 case jit_code_blei_f: return (jit_code_bungti_f);
2791 case jit_code_beqr_f: return (jit_code_bner_f);
2792 case jit_code_beqi_f: return (jit_code_bnei_f);
2794 case jit_code_bger_f: return (jit_code_bunltr_f);
2795 case jit_code_bgei_f: return (jit_code_bunlti_f);
2796 case jit_code_bgtr_f: return (jit_code_bunler_f);
2797 case jit_code_bgti_f: return (jit_code_bunlei_f);
2799 case jit_code_bner_f: return (jit_code_beqr_f);
2800 case jit_code_bnei_f: return (jit_code_beqr_f);
2802 case jit_code_bunltr_f: return (jit_code_bger_f);
2803 case jit_code_bunlti_f: return (jit_code_bgei_f);
2804 case jit_code_bunler_f: return (jit_code_bgtr_f);
2805 case jit_code_bunlei_f: return (jit_code_bgti_f);
2807 case jit_code_buneqr_f: return (jit_code_bltgtr_f);
2808 case jit_code_buneqi_f: return (jit_code_bltgti_f);
2810 case jit_code_bunger_f: return (jit_code_bltr_f);
2811 case jit_code_bungei_f: return (jit_code_blti_f);
2812 case jit_code_bungtr_f: return (jit_code_bler_f);
2813 case jit_code_bungti_f: return (jit_code_blei_f);
2815 case jit_code_bltgtr_f: return (jit_code_buneqr_f);
2816 case jit_code_bltgti_f: return (jit_code_buneqi_f);
2818 case jit_code_bordr_f: return (jit_code_bunordr_f);
2819 case jit_code_bordi_f: return (jit_code_bunordi_f);
2820 case jit_code_bunordr_f:return (jit_code_bordr_f);
2821 case jit_code_bunordi_f:return (jit_code_bordi_f);
2822 case jit_code_bltr_d: return (jit_code_bunger_d);
2823 case jit_code_blti_d: return (jit_code_bungei_d);
2824 case jit_code_bler_d: return (jit_code_bungtr_d);
2825 case jit_code_blei_d: return (jit_code_bungti_d);
2827 case jit_code_beqr_d: return (jit_code_bner_d);
2828 case jit_code_beqi_d: return (jit_code_bnei_d);
2830 case jit_code_bger_d: return (jit_code_bunltr_d);
2831 case jit_code_bgei_d: return (jit_code_bunlti_d);
2832 case jit_code_bgtr_d: return (jit_code_bunler_d);
2833 case jit_code_bgti_d: return (jit_code_bunlei_d);
2835 case jit_code_bner_d: return (jit_code_beqr_d);
2836 case jit_code_bnei_d: return (jit_code_beqi_d);
2838 case jit_code_bunltr_d: return (jit_code_bger_d);
2839 case jit_code_bunlti_d: return (jit_code_bgei_d);
2840 case jit_code_bunler_d: return (jit_code_bgtr_d);
2841 case jit_code_bunlei_d: return (jit_code_bgti_d);
2843 case jit_code_buneqr_d: return (jit_code_bltgtr_d);
2844 case jit_code_buneqi_d: return (jit_code_bltgti_d);
2846 case jit_code_bunger_d: return (jit_code_bltr_d);
2847 case jit_code_bungei_d: return (jit_code_blti_d);
2848 case jit_code_bungtr_d: return (jit_code_bler_d);
2849 case jit_code_bungti_d: return (jit_code_blei_d);
2851 case jit_code_bltgtr_d: return (jit_code_buneqr_d);
2852 case jit_code_bltgti_d: return (jit_code_buneqi_d);
2854 case jit_code_bordr_d: return (jit_code_bunordr_d);
2855 case jit_code_bordi_d: return (jit_code_bunordi_d);
2856 case jit_code_bunordr_d:return (jit_code_bordr_d);
2857 case jit_code_bunordi_d:return (jit_code_bordi_d);
2858 case jit_code_boaddr: return (jit_code_bxaddr);
2859 case jit_code_boaddi: return (jit_code_bxaddi);
2860 case jit_code_boaddr_u: return (jit_code_bxaddr_u);
2861 case jit_code_boaddi_u: return (jit_code_bxaddi_u);
2862 case jit_code_bxaddr: return (jit_code_boaddr);
2863 case jit_code_bxaddi: return (jit_code_boaddi);
2864 case jit_code_bxaddr_u: return (jit_code_boaddr_u);
2865 case jit_code_bxaddi_u: return (jit_code_boaddi_u);
2866 case jit_code_bosubr: return (jit_code_bxsubr);
2867 case jit_code_bosubi: return (jit_code_bxsubi);
2868 case jit_code_bosubr_u: return (jit_code_bxsubr_u);
2869 case jit_code_bosubi_u: return (jit_code_bxsubi_u);
2870 case jit_code_bxsubr: return (jit_code_bosubr);
2871 case jit_code_bxsubi: return (jit_code_bosubi);
2872 case jit_code_bxsubr_u: return (jit_code_bosubr_u);
2873 case jit_code_bxsubi_u: return (jit_code_bosubi_u);
2874 default: abort(); /* invalid jump code */
2879 * change common pattern:
2880 * <cond_jump L0> <jump L1> <label L0>
2882 * <reverse_cond_jump L1>
2885 _reverse_jump(jit_state_t *_jit, jit_node_t *prev, jit_node_t *node)
2887 jit_node_t *local_prev;
2888 jit_node_t *local_next;
2889 jit_node_t *local_jump;
2891 if (!(node->flag & jit_flag_node))
2893 /* =><cond_jump L0> <jump L1> <label L0> */
2894 local_next = node->next;
2895 if (local_next->code != jit_code_jmpi ||
2896 !(local_next->flag & jit_flag_node))
2898 /* <cond_jump L0> =><jump L1> <label L0> */
2900 local_jump = local_next->u.n;
2901 for (local_prev = local_next, local_next = local_next->next;
2903 local_prev = local_next, local_next = local_next->next) {
2904 switch (local_next->code) {
2905 case jit_code_label: case jit_code_epilog:
2906 if (node->u.n == local_next) {
2907 if (local_next->link == node)
2908 local_next->link = node->link;
2910 for (local_prev = local_next->link;
2911 local_prev->link != node;
2912 local_prev = local_prev->link)
2913 assert(local_prev != NULL);
2914 local_prev->link = node->link;
2916 del_node(node, node->next);
2917 node->code = reverse_jump_code(node->code);
2918 node->u.n = local_jump;
2919 node->link = local_jump->link;
2920 local_jump->link = node;
2934 _redundant_store(jit_state_t *_jit, jit_node_t *node, jit_bool_t jump)
2944 if (prev->code == jit_code_epilog)
2946 assert(prev->code == jit_code_label);
2947 if ((prev->flag & jit_flag_head) || node->link || prev->link != node)
2948 /* multiple sources */
2950 /* if there are sequential labels it will return below */
2955 regno = jit_regno(node->v.w);
2956 for (iter = prev->next; iter; prev = iter, iter = iter->next) {
2957 switch (iter->code) {
2958 case jit_code_label: case jit_code_prolog:
2959 case jit_code_epilog:
2962 if (regno == jit_regno(iter->u.w)) {
2963 if (iter->flag || iter->v.w != word)
2965 del_node(prev, iter);
2970 spec = jit_classify(iter->code);
2971 if (spec & jit_cc_a0_jmp)
2973 if ((spec & (jit_cc_a0_reg|jit_cc_a0_chg)) ==
2974 (jit_cc_a0_reg|jit_cc_a0_chg)) {
2975 if (spec & jit_cc_a0_rlh) {
2976 if (regno == jit_regno(iter->u.q.l) ||
2977 regno == jit_regno(iter->u.q.h))
2981 if (regno == jit_regno(iter->u.w))
2985 if ((spec & (jit_cc_a1_reg|jit_cc_a1_chg)) ==
2986 (jit_cc_a1_reg|jit_cc_a1_chg)) {
2987 if (regno == jit_regno(iter->v.w))
2990 if ((spec & (jit_cc_a2_reg|jit_cc_a2_chg)) ==
2991 (jit_cc_a2_reg|jit_cc_a2_chg)) {
2992 if (regno == jit_regno(iter->w.w))
3001 _simplify_movr(jit_state_t *_jit, jit_node_t *prev, jit_node_t *node,
3002 jit_int32_t kind, jit_int32_t size)
3008 regno = jit_regno(node->u.w);
3009 right = jit_regno(node->v.w);
3010 value = _jitc->values + regno;
3011 if ((value->kind == jit_kind_register &&
3012 jit_regno(value->base.q.l) == right &&
3013 value->base.q.h == _jitc->gen[right]) ||
3014 (value->kind == kind && _jitc->values[right].kind == kind &&
3015 memcmp(&value->base.w, &_jitc->values[right].base.w, size) == 0)) {
3016 del_node(prev, node);
3019 if (_jitc->values[right].kind == jit_kind_word)
3020 jit_memcpy(value, _jitc->values + right, sizeof(jit_value_t));
3022 value->kind = jit_kind_register;
3023 value->base.q.l = right;
3024 value->base.q.h = _jitc->gen[right];
3026 ++_jitc->gen[regno];
3032 _simplify_movi(jit_state_t *_jit, jit_node_t *prev, jit_node_t *node,
3033 jit_int32_t kind, jit_int32_t size)
3040 regno = jit_regno(node->u.w);
3041 value = _jitc->values + regno;
3042 if (node->flag & jit_flag_node) {
3043 /* set to undefined if value will be patched */
3045 ++_jitc->gen[regno];
3048 if (value->kind == kind) {
3049 if (memcmp(&node->v.w, &value->base.w, size) == 0) {
3050 del_node(prev, node);
3053 spec = jit_class(_rvs[regno].spec);
3054 if (kind == jit_kind_word)
3055 spec &= jit_class_gpr;
3057 spec &= (jit_class_xpr | jit_class_fpr);
3058 for (offset = 0; offset < _jitc->reglen; offset++) {
3059 if (_jitc->values[offset].kind == kind &&
3060 memcmp(&node->v.w, &_jitc->values[offset].base.w, size) == 0 &&
3061 (jit_class(_rvs[offset].spec) & spec) == spec) {
3062 if (kind == jit_kind_word)
3063 node->code = jit_code_movr;
3064 else if (kind == jit_kind_float32)
3065 node->code = jit_code_movr_f;
3067 node->code = jit_code_movr_d;
3069 jit_memcpy(value, _jitc->values + offset, sizeof(jit_value_t));
3070 ++_jitc->gen[regno];
3076 jit_memcpy(&value->base.w, &node->v.w, size);
3077 ++_jitc->gen[regno];
3082 /* simple/safe redundandy test not checking if another register
3083 * holds the same value
3086 _simplify_ldxi(jit_state_t *_jit, jit_node_t *prev, jit_node_t *node)
3092 regno = jit_regno(node->u.w);
3093 right = jit_regno(node->v.w);
3094 value = _jitc->values + regno;
3095 if (regno != right &&
3096 value->kind == jit_kind_code && value->code == node->code &&
3097 value->base.q.l == right && value->base.q.h == _jitc->gen[right] &&
3098 node->w.w == value->disp.w) {
3099 del_node(prev, node);
3102 value->kind = jit_kind_code;
3103 value->code = node->code;
3104 value->base.q.l = right;
3105 value->base.q.h = _jitc->gen[right];
3106 value->disp.w = node->w.w;
3107 ++_jitc->gen[regno];
3113 _simplify_stxi(jit_state_t *_jit, jit_node_t *prev, jit_node_t *node)
3120 regno = jit_regno(node->w.w);
3121 right = jit_regno(node->v.w);
3122 value = _jitc->values + regno;
3124 /* check for redundant store after load */
3125 if (regno != right &&
3126 value->kind == jit_kind_code && value->code == node->code &&
3127 value->base.q.l == right && value->base.q.h == _jitc->gen[right] &&
3128 node->u.w == value->disp.w) {
3129 del_node(prev, node);
3133 /* assume anything can alias, and invalidate tracked values */
3134 for (offset = 0; offset < _jitc->reglen; offset++) {
3135 if (_jitc->values[offset].kind == jit_kind_code) {
3136 _jitc->values[offset].kind = 0;
3137 ++_jitc->gen[offset];
3141 /* no multiple information, so, if set to a constant,
3142 * prefer to keep that information */
3143 if (value->kind == 0) {
3144 value->kind = jit_kind_code;
3145 switch (node->code) {
3146 /* no information about signed/unsigned either */
3147 case jit_code_stxi_c: value->code = jit_code_ldxi_c; break;
3148 case jit_code_stxi_s: value->code = jit_code_ldxi_s; break;
3149 case jit_code_stxi_i: value->code = jit_code_ldxi_i; break;
3150 case jit_code_stxi_l: value->code = jit_code_ldxi_l; break;
3151 case jit_code_stxi_f: value->code = jit_code_ldxi_f; break;
3152 case jit_code_stxi_d: value->code = jit_code_ldxi_d; break;
3155 value->kind = jit_kind_code;
3156 value->base.q.l = right;
3157 value->base.q.h = _jitc->gen[right];
3158 value->disp.w = node->u.w;
3164 /* usually there should be only one store in the
3165 * jit_get_reg/jit_unget_reg, but properly handle
3166 * multiple ones by moving the save node */
3168 _simplify_spill(jit_state_t *_jit, jit_node_t *node, jit_int32_t regno)
3173 if ((temp = _jitc->spill[regno]) && (save = temp->next) != node) {
3174 temp->next = save->next;
3175 save->next = node->next;
3177 _jitc->spill[regno] = node;
3181 /* checks for simple cases where a register is set more than
3182 * once to the same value, and is a common pattern of calls
3183 * to jit_pushargi and jit_pushargr
3186 _simplify(jit_state_t *_jit)
3194 for (prev = NULL, node = _jitc->head; node; prev = node, node = next) {
3196 switch (node->code) {
3197 case jit_code_label: case jit_code_prolog:
3198 case jit_code_callr: case jit_code_calli:
3200 memset(_jitc->gen, 0, sizeof(jit_int32_t) * _jitc->reglen);
3201 memset(_jitc->values, 0, sizeof(jit_value_t) * _jitc->reglen);
3204 _jitc->spill[jit_regno(node->u.w)] = prev;
3207 regno = jit_regno(node->u.w);
3208 if (register_change_p(node->link->next, node, regno) !=
3210 /* spill not required due to optimizing common
3211 * redundancy case of calling jit_get_reg/jit_unget_reg
3212 * and then setting the register to the value it is
3213 * already holding */
3214 patch_register(node->link->next, node,
3215 jit_regno_patch|regno, regno);
3216 del_node(_jitc->spill[regno], node->link);
3217 del_node(prev, node);
3220 _jitc->spill[regno] = NULL;
3223 regno = jit_regno(node->u.w);
3224 if (simplify_movr(prev, node,
3225 jit_kind_word, sizeof(jit_word_t)))
3226 simplify_spill(node = prev, regno);
3229 regno = jit_regno(node->u.w);
3230 if (simplify_movi(prev, node,
3231 jit_kind_word, sizeof(jit_word_t)))
3232 simplify_spill(node = prev, regno);
3234 case jit_code_movr_f:
3235 regno = jit_regno(node->u.w);
3236 if (simplify_movr(prev, node,
3237 jit_kind_float32, sizeof(jit_float32_t)))
3238 simplify_spill(node = prev, regno);
3240 case jit_code_movi_f:
3241 regno = jit_regno(node->u.w);
3242 if (simplify_movi(prev, node,
3243 jit_kind_float32, sizeof(jit_float32_t)))
3244 simplify_spill(node = prev, regno);
3246 case jit_code_movr_d:
3247 regno = jit_regno(node->u.w);
3248 if (simplify_movr(prev, node,
3249 jit_kind_float64, sizeof(jit_float64_t)))
3250 simplify_spill(node = prev, regno);
3252 case jit_code_movi_d:
3253 regno = jit_regno(node->u.w);
3254 if (simplify_movi(prev, node,
3255 jit_kind_float64, sizeof(jit_float64_t)))
3256 simplify_spill(node = prev, regno);
3258 case jit_code_ldxi_c: case jit_code_ldxi_uc:
3259 case jit_code_ldxi_s: case jit_code_ldxi_us:
3260 case jit_code_ldxi_i: case jit_code_ldxi_ui:
3261 case jit_code_ldxi_l:
3262 case jit_code_ldxi_f: case jit_code_ldxi_d:
3263 regno = jit_regno(node->u.w);
3264 if (simplify_ldxi(prev, node))
3265 simplify_spill(node = prev, regno);
3267 case jit_code_stxi_c: case jit_code_stxi_s:
3268 case jit_code_stxi_i: case jit_code_stxi_l:
3269 case jit_code_stxi_f: case jit_code_stxi_d:
3270 regno = jit_regno(node->u.w);
3271 if (simplify_stxi(prev, node))
3272 simplify_spill(node = prev, regno);
3275 info = jit_classify(node->code);
3276 if (info & jit_cc_a0_jmp)
3277 /* labels are not implicitly added when not taking
3278 * a conditional branch */
3280 if (info & jit_cc_a0_chg) {
3281 if (info & jit_cc_a0_rlh) {
3282 regno = jit_regno(node->u.q.l);
3283 _jitc->values[regno].kind = 0;
3284 ++_jitc->gen[regno];
3285 regno = jit_regno(node->u.q.h);
3286 _jitc->values[regno].kind = 0;
3287 ++_jitc->gen[regno];
3290 regno = jit_regno(node->u.w);
3291 _jitc->values[regno].kind = 0;
3292 ++_jitc->gen[regno];
3295 if (info & jit_cc_a1_chg) {
3296 regno = jit_regno(node->v.w);
3297 _jitc->values[regno].kind = 0;
3298 ++_jitc->gen[regno];
3300 if (info & jit_cc_a2_chg) {
3301 regno = jit_regno(node->w.w);
3302 _jitc->values[regno].kind = 0;
3303 ++_jitc->gen[regno];
3311 _register_change_p(jit_state_t *_jit, jit_node_t *node, jit_node_t *link,
3316 for (; node != link; node = node->next) {
3317 switch (node->code) {
3318 case jit_code_label: case jit_code_prolog:
3319 /* lack of extra information so cannot say it is undefined */
3320 return (jit_reg_change);
3321 case jit_code_callr: case jit_code_calli:
3322 if (!(jit_class(_rvs[regno].spec) & jit_class_sav))
3323 return (jit_reg_undef);
3326 value = jit_classify(node->code);
3327 /* lack of extra information */
3328 if (value & (jit_cc_a0_jmp|jit_cc_a0_cnd))
3329 return (jit_reg_change);
3330 else if ((value & (jit_cc_a0_reg|jit_cc_a0_chg)) ==
3331 (jit_cc_a0_reg|jit_cc_a0_chg) &&
3332 (((value & jit_cc_a0_rlh) &&
3333 (node->u.q.l == regno || node->u.q.h == regno)) ||
3334 (!(value & jit_cc_a0_rlh) &&
3335 node->u.w == regno)))
3336 return (jit_reg_change);
3337 else if ((value & jit_cc_a1_reg) && node->v.w == regno &&
3338 (value & jit_cc_a1_chg))
3339 return (jit_reg_change);
3340 else if ((value & jit_cc_a2_reg) && node->w.w == regno &&
3341 (value & jit_cc_a2_chg))
3342 return (jit_reg_change);
3346 return (jit_reg_static);
3349 /* most of this could be done at the same time as generating jit, but
3350 * avoid complications on different cpu backends and patch spill/loads
3351 * here, by simulating jit generation */
3353 _spill_reglive_p(jit_state_t *_jit, jit_node_t *node, jit_int32_t regno)
3355 if (!jit_regset_tstbit(&_jitc->reglive, regno)) {
3356 jit_regset_setbit(&_jitc->regmask, regno);
3357 jit_update(node->next, &_jitc->reglive, &_jitc->regmask);
3358 if (!jit_regset_tstbit(&_jitc->reglive, regno) &&
3359 register_change_p(node->next, node->link, regno) != jit_reg_change)
3367 _patch_registers(jit_state_t *_jit)
3377 _jitc->function = NULL;
3379 jit_reglive_setup();
3380 for (prev = NULL, node = _jitc->head; node; node = next) {
3383 info = jit_classify(node->code);
3384 jit_regarg_set(node, info);
3386 switch (node->code) {
3388 regno = jit_regno(node->u.w);
3389 if (!spill_reglive_p(node, regno)) {
3390 /* register is not live, just remove spill/reload */
3391 jit_regarg_clr(node, info);
3392 node->link->v.w = jit_regload_delete;
3393 del_node(prev, node);
3397 /* try to find a free register of the same class */
3398 spec = jit_class(_rvs[regno].spec) & ~jit_class_arg;
3399 for (value = 0; value < _jitc->reglen; value++) {
3400 if (value != regno &&
3401 ((jit_class(_rvs[value].spec) & spec) &
3402 ~jit_class_arg) == spec &&
3403 !jit_regset_tstbit(&_jitc->regarg, value) &&
3404 !spill_reglive_p(node, value))
3407 if (value < _jitc->reglen) {
3408 jit_regarg_clr(node, info);
3409 patch_register(node->next, node->link,
3410 jit_regno_patch|node->u.w,
3411 jit_regno_patch|value);
3412 /* mark as live just in case there are nested
3413 * register patches, so that next patch will
3414 * not want to use the same register */
3415 jit_regset_setbit(&_jitc->reglive, value);
3416 /* register is not live, just remove spill/reload */
3417 node->link->v.w = jit_regload_isdead;
3418 del_node(prev, node);
3422 /* failed to find a free register */
3423 if (spec & jit_class_gpr) {
3424 if (!_jitc->function->regoff[regno])
3425 _jitc->function->regoff[regno] =
3426 jit_allocai(sizeof(jit_word_t));
3427 #if __WORDSIZE == 32
3428 node->code = jit_code_stxi_i;
3430 node->code = jit_code_stxi_l;
3434 node->code = jit_code_stxi_d;
3435 if (!_jitc->function->regoff[regno])
3436 _jitc->function->regoff[regno] =
3437 jit_allocai(sizeof(jit_float64_t));
3439 node->u.w = _jitc->function->regoff[regno];
3447 regno = jit_regno(node->u.w);
3449 if (node->v.w == jit_regload_isdead)
3450 jit_regset_clrbit(&_jitc->reglive, regno);
3451 del_node(prev, node);
3454 spec = jit_class(_rvs[regno].spec);
3455 if (spec & jit_class_gpr) {
3456 #if __WORDSIZE == 32
3457 node->code = jit_code_ldxi_i;
3459 node->code = jit_code_ldxi_l;
3463 node->code = jit_code_ldxi_d;
3466 node->w.w = _jitc->function->regoff[regno];
3469 case jit_code_prolog:
3470 _jitc->function = _jitc->functions.ptr + node->w.w;
3472 case jit_code_epilog:
3473 _jitc->function = NULL;
3479 jit_regarg_clr(node, info);
3480 /* update register live state */
3487 _patch_register(jit_state_t *_jit, jit_node_t *node, jit_node_t *link,
3488 jit_int32_t regno, jit_int32_t patch)
3492 for (; node != link; node = node->next) {
3493 value = jit_classify(node->code);
3494 if (value & jit_cc_a0_reg) {
3495 if (value & jit_cc_a0_rlh) {
3496 if (node->u.q.l == regno)
3497 node->u.q.l = patch;
3498 if (node->u.q.h == regno)
3499 node->u.q.h = patch;
3502 if (node->u.w == regno)
3506 if ((value & jit_cc_a1_reg) && node->v.w == regno)
3508 if ((value & jit_cc_a2_reg) && node->w.w == regno)
3513 #if __BYTE_ORDER == __LITTLE_ENDIAN
3514 # define htonr_us(r0,r1) bswapr_us(r0,r1)
3515 # define htonr_ui(r0,r1) bswapr_ui(r0,r1)
3516 # if __WORDSIZE == 64
3517 # define htonr_ul(r0,r1) bswapr_ul(r0,r1)
3520 # define htonr_us(r0,r1) extr_us(r0,r1)
3521 # if __WORDSIZE == 32
3522 # define htonr_ui(r0,r1) movr(r0,r1)
3524 # define htonr_ui(r0,r1) extr_ui(r0,r1)
3525 # define htonr_ul(r0,r1) movr(r0,r1)
3529 static maybe_unused void
3530 generic_bswapr_us(jit_state_t *_jit, jit_int32_t r0, jit_int32_t r1);
3531 static maybe_unused void
3532 generic_bswapr_ui(jit_state_t *_jit, jit_int32_t r0, jit_int32_t r1);
3533 #if __WORDSIZE == 64
3534 static maybe_unused void
3535 generic_bswapr_ul(jit_state_t *_jit, jit_int32_t r0, jit_int32_t r1);
3538 #if defined(__i386__) || defined(__x86_64__)
3539 # include "jit_x86.c"
3540 #elif defined(__mips__)
3541 # include "jit_mips.c"
3542 #elif defined(__arm__)
3543 # include "jit_arm.c"
3544 #elif defined(__powerpc__)
3545 # include "jit_ppc.c"
3546 #elif defined(__sparc__)
3547 # include "jit_sparc.c"
3548 #elif defined(__ia64__)
3549 # include "jit_ia64.c"
3550 #elif defined(__hppa__)
3551 # include "jit_hppa.c"
3552 #elif defined(__aarch64__)
3553 # include "jit_aarch64.c"
3554 #elif defined(__s390__) || defined(__s390x__)
3555 # include "jit_s390.c"
3556 #elif defined(__alpha__)
3557 # include "jit_alpha.c"
3558 #elif defined(__riscv)
3559 # include "jit_riscv.c"
3562 static maybe_unused void
3563 generic_bswapr_us(jit_state_t *_jit, jit_int32_t r0, jit_int32_t r1)
3565 jit_int32_t reg = jit_get_reg(jit_class_gpr);
3567 rshi(rn(reg), r1, 8);
3569 andi(rn(reg), rn(reg), 0xff);
3571 orr(r0, r0, rn(reg));
3576 static maybe_unused void
3577 generic_bswapr_ui(jit_state_t *_jit, jit_int32_t r0, jit_int32_t r1)
3579 jit_int32_t reg = jit_get_reg(jit_class_gpr);
3581 rshi(rn(reg), r1, 16);
3583 bswapr_us(rn(reg), rn(reg));
3585 orr(r0, r0, rn(reg));
3590 #if __WORDSIZE == 64
3591 static maybe_unused void
3592 generic_bswapr_ul(jit_state_t *_jit, jit_int32_t r0, jit_int32_t r1)
3594 jit_int32_t reg = jit_get_reg(jit_class_gpr);
3596 rshi_u(rn(reg), r1, 32);
3598 bswapr_ui(rn(reg), rn(reg));
3600 orr(r0, r0, rn(reg));