2 * Copyright (C) 2012-2019 Free Software Foundation, Inc.
4 * This file is part of GNU lightning.
6 * GNU lightning is free software; you can redistribute it and/or modify it
7 * under the terms of the GNU Lesser General Public License as published
8 * by the Free Software Foundation; either version 3, or (at your option)
11 * GNU lightning is distributed in the hope that it will be useful, but
12 * WITHOUT ANY WARRANTY; without even the implied warranty of MERCHANTABILITY
13 * or FITNESS FOR A PARTICULAR PURPOSE. See the GNU Lesser General Public
14 * License for more details.
17 * Paulo Cesar Pereira de Andrade
20 #include <lightning.h>
21 #include <lightning/jit_private.h>
28 # define MAP_ANON MAP_ANONYMOUS
29 # ifndef MAP_ANONYMOUS
30 # define MAP_ANONYMOUS 0
34 #define jit_regload_reload 0 /* convert to reload */
35 #define jit_regload_delete 1 /* just remove node */
36 #define jit_regload_isdead 2 /* delete and unset live bit */
41 static jit_word_t hash_data(const void*, jit_word_t);
43 #define new_pool() _new_pool(_jit)
44 static void _new_pool(jit_state_t*);
46 #define new_node(u) _new_node(_jit, u)
47 static jit_node_t *_new_node(jit_state_t*, jit_code_t);
49 #define link_node(u) _link_node(_jit, u)
50 static inline jit_node_t *_link_node(jit_state_t*, jit_node_t*);
52 #define del_node(u, v) _del_node(_jit, u, v)
53 static inline void _del_node(jit_state_t*, jit_node_t*, jit_node_t*);
55 #define free_node(u) _free_node(_jit, u)
56 static inline void _free_node(jit_state_t*, jit_node_t*);
58 #define del_label(u, v) _del_label(_jit, u, v)
59 static void _del_label(jit_state_t*, jit_node_t*, jit_node_t*);
61 #define jit_dataset() _jit_dataset(_jit)
63 _jit_dataset(jit_state_t *_jit);
65 #define jit_setup(block) _jit_setup(_jit, block)
67 _jit_setup(jit_state_t *_jit, jit_block_t *block);
69 #define jit_follow(block, todo) _jit_follow(_jit, block, todo)
71 _jit_follow(jit_state_t *_jit, jit_block_t *block, jit_bool_t *todo);
73 #define jit_update(node, live, mask) _jit_update(_jit, node, live, mask)
75 _jit_update(jit_state_t *_jit, jit_node_t *node,
76 jit_regset_t *live, jit_regset_t *mask);
78 #define thread_jumps() _thread_jumps(_jit)
80 _thread_jumps(jit_state_t *_jit);
82 #define sequential_labels() _sequential_labels(_jit)
84 _sequential_labels(jit_state_t *_jit);
86 #define split_branches() _split_branches(_jit)
88 _split_branches(jit_state_t *_jit);
90 #define shortcut_jump(prev, node) _shortcut_jump(_jit, prev, node)
92 _shortcut_jump(jit_state_t *_jit, jit_node_t *prev, jit_node_t *node);
94 #define redundant_jump(prev, node) _redundant_jump(_jit, prev, node)
96 _redundant_jump(jit_state_t *_jit, jit_node_t *prev, jit_node_t *node);
99 reverse_jump_code(jit_code_t code);
101 #define reverse_jump(prev, node) _reverse_jump(_jit, prev, node)
103 _reverse_jump(jit_state_t *_jit, jit_node_t *prev, jit_node_t *node);
105 #define redundant_store(node, jump) _redundant_store(_jit, node, jump)
107 _redundant_store(jit_state_t *_jit, jit_node_t *node, jit_bool_t jump);
109 #define simplify_movr(p, n, k, s) _simplify_movr(_jit, p, n, k, s)
111 _simplify_movr(jit_state_t *_jit, jit_node_t *prev, jit_node_t *node,
112 jit_int32_t kind, jit_int32_t size);
114 #define simplify_movi(p, n, k, s) _simplify_movi(_jit, p, n, k, s)
116 _simplify_movi(jit_state_t *_jit, jit_node_t *prev, jit_node_t *node,
117 jit_int32_t kind, jit_int32_t size);
119 #define simplify_ldxi(prev, node) _simplify_ldxi(_jit, prev, node)
121 _simplify_ldxi(jit_state_t *_jit, jit_node_t *prev, jit_node_t *node);
123 #define simplify_stxi(prev, node) _simplify_stxi(_jit, prev, node)
125 _simplify_stxi(jit_state_t *_jit, jit_node_t *prev, jit_node_t *node);
127 #define simplify_spill(node, regno) _simplify_spill(_jit, node, regno)
129 _simplify_spill(jit_state_t *_jit, jit_node_t *node, jit_int32_t regno);
131 #define simplify() _simplify(_jit)
133 _simplify(jit_state_t *_jit);
135 #define jit_reg_undef -1
136 #define jit_reg_static 0
137 #define jit_reg_change 1
138 #define register_change_p(n, l, r) _register_change_p(_jit, n, l, r)
140 _register_change_p(jit_state_t *_jit, jit_node_t *node, jit_node_t *link,
143 #define spill_reglive_p(node, regno) _spill_reglive_p(_jit, node, regno)
145 _spill_reglive_p(jit_state_t *_jit, jit_node_t *node, jit_int32_t regno);
147 #define patch_registers() _patch_registers(_jit)
149 _patch_registers(jit_state_t *_jit);
151 #define patch_register(n,l,r,p) _patch_register(_jit,n,l,r,p)
153 _patch_register(jit_state_t *jit, jit_node_t *node, jit_node_t *link,
154 jit_int32_t regno, jit_int32_t patch);
167 init_jit(const char *progname)
170 jit_init_debug(progname);
182 _jit_get_reg(jit_state_t *_jit, jit_int32_t regspec)
187 spec = regspec & ~(jit_class_chk|jit_class_nospill);
188 if (spec & jit_class_named) {
189 regno = jit_regno(spec);
190 if (jit_regset_tstbit(&_jitc->regsav, regno))
191 /* fail if register is spilled */
193 if (jit_regset_tstbit(&_jitc->regarg, regno))
194 /* fail if register is an argument to current instruction */
196 if (jit_regset_tstbit(&_jitc->reglive, regno)) {
197 if (regspec & jit_class_nospill)
198 /* fail if register is live and should not spill/reload */
202 jit_regset_setbit(&_jitc->regarg, regno);
206 assert(jit_class(spec) != 0);
209 /* search for a free register matching spec */
210 for (regno = 0; regno < _jitc->reglen; regno++) {
211 if ((jit_class(_rvs[regno].spec) & spec) == spec &&
212 !jit_regset_tstbit(&_jitc->regarg, regno) &&
213 !jit_regset_tstbit(&_jitc->reglive, regno))
217 /* search for a register matching spec that is not an argument
218 * for the current instruction */
219 for (regno = 0; regno < _jitc->reglen; regno++) {
220 if ((jit_class(_rvs[regno].spec) & spec) == spec &&
221 !jit_regset_tstbit(&_jitc->regsav, regno) &&
222 !jit_regset_tstbit(&_jitc->regarg, regno) &&
223 !(regspec & jit_class_nospill)) {
225 assert(_jitc->function != NULL);
226 if (spec & jit_class_gpr) {
227 if (!_jitc->function->regoff[regno]) {
228 _jitc->function->regoff[regno] =
229 jit_allocai(sizeof(jit_word_t));
233 /* emit_stxi must not need temporary registers */
234 assert(!_jitc->getreg);
237 emit_stxi(_jitc->function->regoff[regno], JIT_FP, regno);
243 if (!_jitc->function->regoff[regno]) {
244 _jitc->function->regoff[regno] =
245 jit_allocai(sizeof(jit_float64_t));
249 /* emit_stxi must not need temporary registers */
250 assert(!_jitc->getreg);
253 emit_stxi_d(_jitc->function->regoff[regno], JIT_FP, regno);
258 jit_regset_setbit(&_jitc->regsav, regno);
260 jit_regset_setbit(&_jitc->regarg, regno);
261 if (jit_class(_rvs[regno].spec) & jit_class_sav) {
262 /* if will modify callee save registers without a
263 * function prolog, better patch this assertion */
264 assert(_jitc->function != NULL);
265 if (!jit_regset_tstbit(&_jitc->function->regset, regno)) {
266 jit_regset_setbit(&_jitc->function->regset, regno);
275 /* nospill hint only valid during emit" */
276 assert(!(regspec & jit_class_nospill));
277 for (regno = 0; regno < _jitc->reglen; regno++) {
278 if ((jit_class(_rvs[regno].spec) & spec) == spec &&
279 !jit_regset_tstbit(&_jitc->regsav, regno) &&
280 !jit_regset_tstbit(&_jitc->regarg, regno)) {
281 jit_regset_setbit(&_jitc->regarg, regno);
282 jit_regset_setbit(&_jitc->regsav, regno);
284 return (jit_regno_patch|regno);
289 /* Out of hardware registers */
291 assert(regspec & jit_class_chk);
296 _jit_unget_reg(jit_state_t *_jit, jit_int32_t regno)
298 regno = jit_regno(regno);
299 if (jit_regset_tstbit(&_jitc->regsav, regno)) {
302 /* emit_ldxi must not need a temporary register */
303 assert(!_jitc->getreg);
306 if (jit_class(_rvs[regno].spec) & jit_class_gpr)
307 emit_ldxi(regno, JIT_FP, _jitc->function->regoff[regno]);
309 emit_ldxi_d(regno, JIT_FP, _jitc->function->regoff[regno]);
311 /* emit_ldxi must not need a temporary register */
317 jit_regset_clrbit(&_jitc->regsav, regno);
319 #if defined(jit_carry)
320 assert((regno == jit_carry /*&& _NOREG != jit_carry*/) ||
321 jit_regset_tstbit(&_jitc->regarg, regno) != 0);
323 assert(jit_regset_tstbit(&_jitc->regarg, regno) != 0);
325 jit_regset_clrbit(&_jitc->regarg, regno);
329 _jit_callee_save_p(jit_state_t *_jit, jit_int32_t regno)
331 assert(regno >= 0 && regno < JIT_NOREG);
332 return (!!(_rvs[regno].spec & jit_class_sav));
336 _jit_pointer_p(jit_state_t *_jit, jit_pointer_t address)
338 return ((jit_uint8_t *)address >= _jit->code.ptr &&
339 (jit_word_t)address < _jit->pc.w);
344 jit_regset_com(jit_regset_t *u, jit_regset_t *v)
346 u->rl = ~v->rl; u->rh = ~v->rh;
347 u->fl = ~v->fl; u->fh = ~v->fh;
351 jit_regset_and(jit_regset_t *u, jit_regset_t *v, jit_regset_t *w)
353 u->rl = v->rl & w->rl; u->rh = v->rh & w->rh;
354 u->fl = v->fl & w->fl; u->fh = v->fh & w->fh;
358 jit_regset_ior(jit_regset_t *u, jit_regset_t *v, jit_regset_t *w)
360 u->rl = v->rl | w->rl; u->rh = v->rh | w->rh;
361 u->fl = v->fl | w->fl; u->fh = v->fh | w->fh;
365 jit_regset_xor(jit_regset_t *u, jit_regset_t *v, jit_regset_t *w)
367 u->rl = v->rl ^ w->rl; u->rh = v->rh ^ w->rh;
368 u->fl = v->fl ^ w->fl; u->fh = v->fh ^ w->fh;
372 jit_regset_set(jit_regset_t *u, jit_regset_t *v)
374 u->rl = v->rl; u->rh = v->rh;
375 u->fl = v->fl; u->fh = v->fh;
379 jit_regset_set_mask(jit_regset_t *u, jit_int32_t v)
381 jit_bool_t w = !!(v & (v - 1));
383 assert(v >= 0 && v <= 256);
385 u->rl = u->rh = u->fl = u->fh = -1LL;
387 u->rl = w ? (1LL << v) - 1 : -1LL;
388 u->rh = u->fl = u->fh = 0;
392 u->rh = w ? (1LL << (v - 64)) - 1 : -1LL;
396 u->rl = u->rh = -1LL;
397 u->fl = w ? (1LL << (v - 128)) - 1 : -1LL;
401 u->rl = u->rh = u->fl = -1LL;
402 u->fh = w ? (1LL << (v - 128)) - 1 : -1LL;
407 jit_regset_cmp_ui(jit_regset_t *u, jit_word_t v)
409 return !((u->rl == v && u->rh == 0 && u->fl == 0 && u->fh == 0));
413 jit_regset_set_ui(jit_regset_t *u, jit_word_t v)
416 u->rh = u->fl = u->fh = 0;
420 jit_regset_set_p(jit_regset_t *u)
422 return (u->rl || u->rh || u->fl || u->fh);
426 jit_regset_clrbit(jit_regset_t *set, jit_int32_t bit)
428 assert(bit >= 0 && bit <= 255);
430 set->rl &= ~(1LL << bit);
432 set->rh &= ~(1LL << (bit - 64));
434 set->fl &= ~(1LL << (bit - 128));
436 set->fh &= ~(1LL << (bit - 192));
440 jit_regset_setbit(jit_regset_t *set, jit_int32_t bit)
442 assert(bit >= 0 && bit <= 255);
444 set->rl |= 1LL << bit;
446 set->rh |= 1LL << (bit - 64);
448 set->fl |= 1LL << (bit - 128);
450 set->fh |= 1LL << (bit - 192);
454 jit_regset_tstbit(jit_regset_t *set, jit_int32_t bit)
456 assert(bit >= 0 && bit <= 255);
458 return (!!(set->rl & (1LL << bit)));
460 return (!!(set->rh & (1LL << (bit - 64))));
462 return (!!(set->fl & (1LL << (bit - 128))));
463 return (!!(set->fh & (1LL << (bit - 192))));
467 jit_regset_scan1(jit_regset_t *set, jit_int32_t offset)
469 assert(offset >= 0 && offset <= 255);
470 for (; offset < 64; offset++) {
471 if (set->rl & (1LL << offset))
474 for (; offset < 128; offset++) {
475 if (set->rh & (1LL << (offset - 64)))
478 for (; offset < 192; offset++) {
479 if (set->fl & (1LL << (offset - 128)))
482 for (; offset < 256; offset++) {
483 if (set->fh & (1LL << (offset - 192)))
489 #elif __sparc__ && __WORDSIZE == 64
491 jit_regset_com(jit_regset_t *u, jit_regset_t *v)
493 u->rl = ~v->rl; u->rh = ~v->rh;
497 jit_regset_and(jit_regset_t *u, jit_regset_t *v, jit_regset_t *w)
499 u->rl = v->rl & w->rl; u->rh = v->rh & w->rh;
503 jit_regset_ior(jit_regset_t *u, jit_regset_t *v, jit_regset_t *w)
505 u->rl = v->rl | w->rl; u->rh = v->rh | w->rh;
509 jit_regset_xor(jit_regset_t *u, jit_regset_t *v, jit_regset_t *w)
511 u->rl = v->rl ^ w->rl; u->rh = v->rh ^ w->rh;
515 jit_regset_set(jit_regset_t *u, jit_regset_t *v)
517 u->rl = v->rl; u->rh = v->rh;
521 jit_regset_set_mask(jit_regset_t *u, jit_int32_t v)
523 jit_bool_t w = !!(v & (v - 1));
525 assert(v >= 0 && v <= 128);
527 u->rl = u->rh = -1LL;
529 u->rl = w ? (1LL << v) - 1 : -1LL;
534 u->rh = w ? (1LL << (v - 64)) - 1 : -1LL;
539 jit_regset_cmp_ui(jit_regset_t *u, jit_word_t v)
541 return !((u->rl == v && u->rh == 0));
545 jit_regset_set_ui(jit_regset_t *u, jit_word_t v)
552 jit_regset_set_p(jit_regset_t *u)
554 return (u->rl || u->rh);
558 jit_regset_clrbit(jit_regset_t *set, jit_int32_t bit)
560 assert(bit >= 0 && bit <= 128);
562 set->rl &= ~(1LL << bit);
564 set->rh &= ~(1LL << (bit - 64));
568 jit_regset_setbit(jit_regset_t *set, jit_int32_t bit)
570 assert(bit >= 0 && bit <= 127);
572 set->rl |= 1LL << bit;
574 set->rh |= 1LL << (bit - 64);
578 jit_regset_tstbit(jit_regset_t *set, jit_int32_t bit)
580 assert(bit >= 0 && bit <= 127);
582 return (!!(set->rl & (1LL << bit)));
584 return (!!(set->rh & (1LL << (bit - 64))));
588 jit_regset_scan1(jit_regset_t *set, jit_int32_t offset)
590 assert(offset >= 0 && offset <= 127);
591 for (; offset < 64; offset++) {
592 if (set->rl & (1LL << offset))
595 for (; offset < 128; offset++) {
596 if (set->rh & (1LL << (offset - 64)))
604 jit_regset_scan1(jit_regset_t *set, jit_int32_t offset)
607 assert(offset >= 0 && offset <= 63);
608 if ((mask = *set >> offset)) {
621 _jit_save(jit_state_t *_jit, jit_int32_t reg)
623 reg = jit_regno(reg);
624 assert(!_jitc->realize);
625 _jitc->spill[reg] = jit_new_node_w(jit_code_save, reg);
629 _jit_load(jit_state_t *_jit, jit_int32_t reg)
633 reg = jit_regno(reg);
634 assert(!_jitc->realize);
635 assert(_jitc->spill[reg] != NULL);
636 node = jit_new_node_w(jit_code_load, reg);
637 /* create a path to flag the save/load is not required */
638 node->link = _jitc->spill[reg];
639 node->link->link = node;
640 _jitc->spill[reg] = NULL;
644 hash_data(const void *data, jit_word_t length)
646 const jit_uint8_t *ptr;
648 for (i = key = 0, ptr = data; i < length; i++)
649 key = (key << (key & 1)) ^ ptr[i];
654 _jit_address(jit_state_t *_jit, jit_node_t *node)
657 assert(node != NULL &&
658 /* If a node type that is documented to be a fixed marker */
659 (node->code == jit_code_note || node->code == jit_code_name ||
660 /* If another special fixed marker, returned by jit_indirect() */
661 (node->code == jit_code_label && (node->flag & jit_flag_use) != 0)));
662 return ((jit_pointer_t)node->u.w);
666 _jit_data(jit_state_t *_jit, const void *data,
667 jit_word_t length, jit_int32_t align)
672 assert(!_jitc->realize);
674 /* Ensure there is space even if asking for a duplicate */
675 if (((_jitc->data.offset + 7) & -8) + length > _jit->data.length) {
678 size = (_jit->data.length + length + 4096) & - 4095;
679 assert(size >= _jit->data.length);
680 if (_jitc->data.ptr == NULL)
681 jit_alloc((jit_pointer_t *)&_jitc->data.ptr, size);
683 jit_realloc((jit_pointer_t *)&_jitc->data.ptr,
684 _jit->data.length, size);
685 _jit->data.length = size;
687 if (_jitc->data.table == NULL)
688 jit_alloc((jit_pointer_t *)&_jitc->data.table,
689 (_jitc->data.size = 16) * sizeof(jit_node_t*));
691 key = hash_data(data, length) & (_jitc->data.size - 1);
692 node = _jitc->data.table[key];
693 for (; node; node = node->next) {
694 if (node->v.w == length &&
695 memcmp(_jitc->data.ptr + node->u.w, data, length) == 0)
700 node = jit_new_node_no_link(jit_code_data);
707 _jitc->data.offset = (_jitc->data.offset + 1) & -2;
710 _jitc->data.offset = (_jitc->data.offset + 3) & -4;
713 _jitc->data.offset = (_jitc->data.offset + 7) & -8;
716 node->u.w = _jitc->data.offset;
718 jit_memcpy(_jitc->data.ptr + _jitc->data.offset, data, length);
719 _jitc->data.offset += length;
721 node->next = _jitc->data.table[key];
722 _jitc->data.table[key] = node;
725 /* Rehash if more than 75% used table */
726 if (_jitc->data.count >
727 (_jitc->data.size >> 1) + (_jitc->data.size >> 2) &&
728 (_jitc->data.size << 1) > _jitc->data.size) {
734 jit_alloc((jit_pointer_t *)&hash,
735 (_jitc->data.size << 1) * sizeof(jit_node_t*));
736 for (i = 0; i < _jitc->data.size; i++) {
737 temp = _jitc->data.table[i];
738 for (; temp; temp = next) {
740 key = hash_data(_jitc->data.ptr + temp->u.w, temp->v.w) &
741 ((_jitc->data.size << 1) - 1);
742 temp->next = hash[key];
746 jit_free((jit_pointer_t *)&_jitc->data.table);
747 _jitc->data.table = hash;
748 _jitc->data.size <<= 1;
756 _new_pool(jit_state_t *_jit)
761 if (_jitc->pool.offset >= _jitc->pool.length) {
764 length = _jitc->pool.length + 16;
765 jit_realloc((jit_pointer_t *)&_jitc->pool.ptr,
766 _jitc->pool.length * sizeof(jit_node_t *),
767 length * sizeof(jit_node_t *));
768 _jitc->pool.length = length;
770 jit_alloc((jit_pointer_t *)(_jitc->pool.ptr + _jitc->pool.offset),
771 sizeof(jit_node_t) * 1024);
772 list = _jitc->pool.ptr[_jitc->pool.offset];
773 for (offset = 1; offset < 1024; offset++, list++)
774 list->next = list + 1;
775 list->next = _jitc->list;
776 _jitc->list = _jitc->pool.ptr[_jitc->pool.offset];
777 ++_jitc->pool.offset;
781 _new_node(jit_state_t *_jit, jit_code_t code)
785 if (_jitc->list == NULL)
788 _jitc->list = node->next;
790 node->flag |= jit_flag_synth;
797 static inline jit_node_t *
798 _link_node(jit_state_t *_jit, jit_node_t *node)
801 _jitc->tail->next = node;
804 return (_jitc->tail = node);
808 _del_node(jit_state_t *_jit, jit_node_t *prev, jit_node_t *node)
811 assert(prev == _jitc->head);
812 _jitc->head = node->next;
815 prev->next = node->next;
816 memset(node, 0, sizeof(jit_node_t));
817 node->next = _jitc->list;
822 _free_node(jit_state_t *_jit, jit_node_t *node)
824 memset(node, 0, sizeof(jit_node_t));
825 node->next = _jitc->list;
830 _del_label(jit_state_t *_jit, jit_node_t *prev, jit_node_t *node)
834 /* only allow call to del_label on linked labels */
835 block = _jitc->blocks.ptr + node->v.w;
836 assert(block->label == node);
838 /* del_label() should only be called when optimizing.
839 * This will leave an empty block index */
840 jit_regset_del(&block->reglive);
841 jit_regset_del(&block->regmask);
844 /* redundant, should be already true */
845 assert(node->link == NULL);
846 del_node(prev, node);
854 jit_alloc((jit_pointer_t *)&_jit, sizeof(jit_state_t));
855 jit_alloc((jit_pointer_t *)&_jitc, sizeof(jit_compiler_t));
856 jit_regset_new(&_jitc->regarg);
857 jit_regset_new(&_jitc->regsav);
858 jit_regset_new(&_jitc->reglive);
859 jit_regset_new(&_jitc->regmask);
863 jit_alloc((jit_pointer_t *)&_jitc->spill,
864 _jitc->reglen * sizeof(jit_node_t*));
865 jit_alloc((jit_pointer_t *)&_jitc->gen,
866 _jitc->reglen * sizeof(jit_int32_t));
867 jit_alloc((jit_pointer_t *)&_jitc->values,
868 _jitc->reglen * sizeof(jit_value_t));
870 jit_alloc((jit_pointer_t *)&_jitc->patches.ptr,
871 (_jitc->patches.length = 1024) * sizeof(jit_patch_t));
872 jit_alloc((jit_pointer_t *)&_jitc->functions.ptr,
873 (_jitc->functions.length = 16) * sizeof(jit_function_t));
874 jit_alloc((jit_pointer_t *)&_jitc->pool.ptr,
875 (_jitc->pool.length = 16) * sizeof(jit_node_t*));
876 jit_alloc((jit_pointer_t *)&_jitc->blocks.ptr,
877 (_jitc->blocks.length = 16) * sizeof(jit_block_t));
878 #if __arm__ && DISASSEMBLER
879 jit_alloc((jit_pointer_t *)&_jitc->data_info.ptr,
880 (_jitc->data_info.length = 1024) * sizeof(jit_data_info_t));
883 /* allocate at most one extra note in case jit_name() is
884 * never called, or called after adding at least one note */
885 _jit->note.length = 1;
886 _jitc->note.size = sizeof(jit_note_t);
892 _jit_clear_state(jit_state_t *_jit)
894 #if DEVEL_DISASSEMBLER
895 # define jit_really_clear_state() _jit_really_clear_state(_jit)
898 void _jit_really_clear_state(jit_state_t *_jit)
902 jit_function_t *function;
904 /* release memory not required at jit execution time and set
905 * pointers to NULL to explicitly know they are released */
906 _jitc->head = _jitc->tail = NULL;
908 jit_free((jit_pointer_t *)&_jitc->data.table);
909 _jitc->data.size = _jitc->data.count = 0;
911 jit_free((jit_pointer_t *)&_jitc->spill);
912 jit_free((jit_pointer_t *)&_jitc->gen);
913 jit_free((jit_pointer_t *)&_jitc->values);
915 jit_free((jit_pointer_t *)&_jitc->blocks.ptr);
917 jit_free((jit_pointer_t *)&_jitc->patches.ptr);
918 _jitc->patches.offset = _jitc->patches.length = 0;
920 for (offset = 0; offset < _jitc->functions.offset; offset++) {
921 function = _jitc->functions.ptr + offset;
922 jit_free((jit_pointer_t *)&function->regoff);
924 jit_free((jit_pointer_t *)&_jitc->functions.ptr);
925 _jitc->functions.offset = _jitc->functions.length = 0;
926 _jitc->function = NULL;
928 for (offset = 0; offset < _jitc->pool.offset; offset++)
929 jit_free((jit_pointer_t *)(_jitc->pool.ptr + offset));
930 jit_free((jit_pointer_t *)&_jitc->pool.ptr);
931 _jitc->pool.offset = _jitc->pool.length = 0;
934 _jitc->note.head = _jitc->note.tail =
935 _jitc->note.name = _jitc->note.note = NULL;
936 _jitc->note.base = NULL;
938 #if __arm__ && DISASSEMBLER
939 jit_free((jit_pointer_t *)&_jitc->data_info.ptr);
942 #if (__powerpc__ && _CALL_AIXDESC) || __ia64__
943 jit_free((jit_pointer_t *)&_jitc->prolog.ptr);
947 jit_regset_del(&_jitc->regs);
950 jit_free((jit_pointer_t *)&_jitc);
954 _jit_destroy_state(jit_state_t *_jit)
956 #if DEVEL_DISASSEMBLER
957 jit_really_clear_state();
959 if (!_jit->user_code)
960 munmap(_jit->code.ptr, _jit->code.length);
961 if (!_jit->user_data)
962 munmap(_jit->data.ptr, _jit->data.length);
963 jit_free((jit_pointer_t *)&_jit);
967 _jit_synth_inc(jit_state_t *_jit)
969 assert(_jitc->synth < 8);
974 _jit_new_node(jit_state_t *_jit, jit_code_t code)
976 assert(!_jitc->realize);
977 return (link_node(new_node(code)));
981 _jit_new_node_no_link(jit_state_t *_jit, jit_code_t code)
983 assert(!_jitc->realize);
984 return (new_node(code));
988 _jit_link_node(jit_state_t *_jit, jit_node_t *node)
990 assert(!_jitc->realize);
995 _jit_synth_dec(jit_state_t *_jit)
997 assert(_jitc->synth > 0);
1002 _jit_new_node_w(jit_state_t *_jit, jit_code_t code,
1005 jit_node_t *node = new_node(code);
1006 assert(!_jitc->realize);
1008 return (link_node(node));
1012 _jit_new_node_f(jit_state_t *_jit, jit_code_t code,
1015 jit_node_t *node = new_node(code);
1016 assert(!_jitc->realize);
1018 return (link_node(node));
1022 _jit_new_node_d(jit_state_t *_jit, jit_code_t code,
1025 jit_node_t *node = new_node(code);
1026 assert(!_jitc->realize);
1028 return (link_node(node));
1032 _jit_new_node_p(jit_state_t *_jit, jit_code_t code,
1035 jit_node_t *node = new_node(code);
1036 assert(!_jitc->realize);
1038 return (link_node(node));
1042 _jit_new_node_ww(jit_state_t *_jit, jit_code_t code,
1043 jit_word_t u, jit_word_t v)
1045 jit_node_t *node = new_node(code);
1046 assert(!_jitc->realize);
1049 return (link_node(node));
1053 _jit_new_node_wp(jit_state_t *_jit, jit_code_t code,
1054 jit_word_t u, jit_pointer_t v)
1056 return (jit_new_node_ww(code, u, (jit_word_t)v));
1060 _jit_new_node_fp(jit_state_t *_jit, jit_code_t code,
1061 jit_float32_t u, jit_pointer_t v)
1063 jit_node_t *node = new_node(code);
1064 assert(!_jitc->realize);
1066 node->v.w = (jit_word_t)v;
1067 return (link_node(node));
1071 _jit_new_node_dp(jit_state_t *_jit, jit_code_t code,
1072 jit_float64_t u, jit_pointer_t v)
1074 jit_node_t *node = new_node(code);
1075 assert(!_jitc->realize);
1077 node->v.w = (jit_word_t)v;
1078 return (link_node(node));
1082 _jit_new_node_pw(jit_state_t *_jit, jit_code_t code,
1083 jit_pointer_t u, jit_word_t v)
1085 return (jit_new_node_ww(code, (jit_word_t)u, v));
1089 _jit_new_node_wf(jit_state_t *_jit, jit_code_t code,
1090 jit_word_t u, jit_float32_t v)
1092 jit_node_t *node = new_node(code);
1093 assert(!_jitc->realize);
1096 return (link_node(node));
1100 _jit_new_node_wd(jit_state_t *_jit, jit_code_t code,
1101 jit_word_t u, jit_float64_t v)
1103 jit_node_t *node = new_node(code);
1104 assert(!_jitc->realize);
1107 return (link_node(node));
1111 _jit_new_node_www(jit_state_t *_jit, jit_code_t code,
1112 jit_word_t u, jit_word_t v, jit_word_t w)
1114 jit_node_t *node = new_node(code);
1115 assert(!_jitc->realize);
1119 return (link_node(node));
1123 _jit_new_node_qww(jit_state_t *_jit, jit_code_t code,
1124 jit_int32_t l, jit_int32_t h,
1125 jit_word_t v, jit_word_t w)
1127 jit_node_t *node = new_node(code);
1128 assert(!_jitc->realize);
1134 return (link_node(node));
1138 _jit_new_node_wwf(jit_state_t *_jit, jit_code_t code,
1139 jit_word_t u, jit_word_t v, jit_float32_t w)
1141 jit_node_t *node = new_node(code);
1142 assert(!_jitc->realize);
1146 return (link_node(node));
1150 _jit_new_node_wwd(jit_state_t *_jit, jit_code_t code,
1151 jit_word_t u, jit_word_t v, jit_float64_t w)
1153 jit_node_t *node = new_node(code);
1154 assert(!_jitc->realize);
1158 return (link_node(node));
1162 _jit_new_node_pww(jit_state_t *_jit, jit_code_t code,
1163 jit_pointer_t u, jit_word_t v, jit_word_t w)
1165 jit_node_t *node = new_node(code);
1166 assert(!_jitc->realize);
1170 return (link_node(node));
1174 _jit_new_node_pwf(jit_state_t *_jit, jit_code_t code,
1175 jit_pointer_t u, jit_word_t v, jit_float32_t w)
1177 jit_node_t *node = new_node(code);
1178 assert(!_jitc->realize);
1182 return (link_node(node));
1186 _jit_new_node_pwd(jit_state_t *_jit, jit_code_t code,
1187 jit_pointer_t u, jit_word_t v, jit_float64_t w)
1189 jit_node_t *node = new_node(code);
1190 assert(!_jitc->realize);
1194 return (link_node(node));
1198 _jit_label(jit_state_t *_jit)
1202 if (!(node = _jitc->tail) || node->code != jit_code_label) {
1203 node = jit_forward();
1211 _jit_forward(jit_state_t *_jit)
1213 return (jit_new_node_no_link(jit_code_label));
1217 _jit_indirect(jit_state_t *_jit)
1222 node->flag |= jit_flag_use;
1228 _jit_link(jit_state_t *_jit, jit_node_t *node)
1232 assert((node->code == jit_code_label ||
1233 node->code == jit_code_prolog ||
1234 node->code == jit_code_epilog) && !node->next);
1235 jit_link_node(node);
1236 if (_jitc->blocks.offset >= _jitc->blocks.length) {
1239 length = _jitc->blocks.length + 16;
1240 jit_realloc((jit_pointer_t *)&_jitc->blocks.ptr,
1241 _jitc->blocks.length * sizeof(jit_block_t),
1242 length * sizeof(jit_block_t));
1243 _jitc->blocks.length = length;
1245 block = _jitc->blocks.ptr + _jitc->blocks.offset;
1246 block->label = node;
1247 node->v.w = _jitc->blocks.offset;
1248 jit_regset_new(&block->reglive);
1249 jit_regset_new(&block->regmask);
1250 ++_jitc->blocks.offset;
1254 _jit_forward_p(jit_state_t *_jit, jit_node_t *node)
1256 return (node->code == jit_code_label && !node->next && node != _jitc->tail);
1260 _jit_indirect_p(jit_state_t *_jit, jit_node_t *node)
1262 return (node->code == jit_code_label && !!(node->flag & jit_flag_use));
1266 _jit_target_p(jit_state_t *_jit, jit_node_t *node)
1268 return (node->code == jit_code_label && !!node->link);
1272 _jit_prepare(jit_state_t *_jit)
1274 assert(_jitc->function != NULL);
1275 _jitc->function->call.call = jit_call_default;
1276 _jitc->function->call.argi =
1277 _jitc->function->call.argf =
1278 _jitc->function->call.size = 0;
1279 _jitc->prepare = jit_new_node(jit_code_prepare);
1283 _jit_patch(jit_state_t* _jit, jit_node_t *instr)
1287 if (!(label = _jitc->tail) || label->code != jit_code_label)
1288 label = jit_label();
1289 jit_patch_at(instr, label);
1293 _jit_classify(jit_state_t *_jit, jit_code_t code)
1298 case jit_code_data: case jit_code_save: case jit_code_load:
1299 case jit_code_name: case jit_code_label: case jit_code_note:
1300 case jit_code_prolog: case jit_code_ellipsis: case jit_code_va_push:
1301 case jit_code_epilog: case jit_code_ret: case jit_code_prepare:
1304 case jit_code_live: case jit_code_va_end:
1305 case jit_code_retr: case jit_code_retr_f: case jit_code_retr_d:
1306 case jit_code_pushargr: case jit_code_pushargr_f:
1307 case jit_code_pushargr_d:
1308 case jit_code_finishr: /* synthesized will set jit_cc_a0_jmp */
1309 mask = jit_cc_a0_reg;
1311 case jit_code_align: case jit_code_reti: case jit_code_pushargi:
1312 case jit_code_finishi: /* synthesized will set jit_cc_a0_jmp */
1313 mask = jit_cc_a0_int;
1315 case jit_code_reti_f: case jit_code_pushargi_f:
1316 mask = jit_cc_a0_flt;
1318 case jit_code_reti_d: case jit_code_pushargi_d:
1319 mask = jit_cc_a0_dbl;
1321 case jit_code_allocai:
1322 mask = jit_cc_a0_int|jit_cc_a1_int;
1324 case jit_code_arg: case jit_code_arg_f: case jit_code_arg_d:
1325 mask = jit_cc_a0_int|jit_cc_a0_arg;
1327 case jit_code_calli: case jit_code_jmpi:
1328 mask = jit_cc_a0_jmp;
1330 case jit_code_callr: case jit_code_jmpr:
1331 mask = jit_cc_a0_reg|jit_cc_a0_jmp;
1333 case jit_code_retval_c: case jit_code_retval_uc:
1334 case jit_code_retval_s: case jit_code_retval_us:
1335 case jit_code_retval_i: case jit_code_retval_ui:
1336 case jit_code_retval_l:
1337 case jit_code_retval_f: case jit_code_retval_d:
1338 case jit_code_va_start:
1339 mask = jit_cc_a0_reg|jit_cc_a0_chg;
1341 case jit_code_getarg_c: case jit_code_getarg_uc:
1342 case jit_code_getarg_s: case jit_code_getarg_us:
1343 case jit_code_getarg_i: case jit_code_getarg_ui:
1344 case jit_code_getarg_l:
1345 case jit_code_getarg_f: case jit_code_getarg_d:
1346 mask = jit_cc_a0_reg|jit_cc_a0_chg|jit_cc_a1_arg;
1348 case jit_code_putargr: case jit_code_putargr_f:
1349 case jit_code_putargr_d:
1350 mask = jit_cc_a0_reg|jit_cc_a1_arg;
1352 case jit_code_putargi:
1353 mask = jit_cc_a0_int|jit_cc_a1_arg;
1355 case jit_code_putargi_f:
1356 mask = jit_cc_a0_flt|jit_cc_a1_arg;
1358 case jit_code_putargi_d:
1359 mask = jit_cc_a0_dbl|jit_cc_a1_arg;
1361 case jit_code_movi: case jit_code_ldi_c: case jit_code_ldi_uc:
1362 case jit_code_ldi_s: case jit_code_ldi_us: case jit_code_ldi_i:
1363 case jit_code_ldi_ui: case jit_code_ldi_l: case jit_code_ldi_f:
1364 case jit_code_ldi_d:
1365 mask = jit_cc_a0_reg|jit_cc_a0_chg|jit_cc_a1_int;
1367 case jit_code_movi_f: case jit_code_movi_f_w:
1368 mask = jit_cc_a0_reg|jit_cc_a0_chg|jit_cc_a1_flt;
1370 case jit_code_movi_d: case jit_code_movi_d_w:
1371 mask = jit_cc_a0_reg|jit_cc_a0_chg|jit_cc_a1_dbl;
1373 case jit_code_movi_d_ww:
1374 mask = jit_cc_a0_reg|jit_cc_a0_chg|jit_cc_a1_reg|jit_cc_a1_chg|
1377 case jit_code_negr: case jit_code_comr: case jit_code_movr:
1378 case jit_code_extr_c: case jit_code_extr_uc: case jit_code_extr_s:
1379 case jit_code_extr_us: case jit_code_extr_i: case jit_code_extr_ui:
1380 case jit_code_truncr_f_i: case jit_code_truncr_f_l:
1381 case jit_code_truncr_d_i: case jit_code_truncr_d_l:
1382 case jit_code_htonr_us: case jit_code_htonr_ui: case jit_code_htonr_ul:
1383 case jit_code_ldr_c: case jit_code_ldr_uc:
1384 case jit_code_ldr_s: case jit_code_ldr_us: case jit_code_ldr_i:
1385 case jit_code_ldr_ui: case jit_code_ldr_l: case jit_code_negr_f:
1386 case jit_code_absr_f: case jit_code_sqrtr_f: case jit_code_movr_f:
1387 case jit_code_extr_f: case jit_code_extr_d_f: case jit_code_ldr_f:
1388 case jit_code_negr_d: case jit_code_absr_d: case jit_code_sqrtr_d:
1389 case jit_code_movr_d: case jit_code_extr_d: case jit_code_extr_f_d:
1390 case jit_code_ldr_d:
1391 case jit_code_movr_w_f: case jit_code_movr_f_w:
1392 case jit_code_movr_w_d: case jit_code_movr_d_w:
1393 case jit_code_va_arg: case jit_code_va_arg_d:
1394 mask = jit_cc_a0_reg|jit_cc_a0_chg|jit_cc_a1_reg;
1396 case jit_code_movr_d_ww:
1397 mask = jit_cc_a0_reg|jit_cc_a0_chg|jit_cc_a1_reg|jit_cc_a1_chg|
1400 case jit_code_addi: case jit_code_addxi: case jit_code_addci:
1401 case jit_code_subi: case jit_code_subxi: case jit_code_subci:
1403 case jit_code_muli: case jit_code_divi: case jit_code_divi_u:
1404 case jit_code_remi: case jit_code_remi_u: case jit_code_andi:
1405 case jit_code_ori: case jit_code_xori: case jit_code_lshi:
1406 case jit_code_rshi: case jit_code_rshi_u: case jit_code_lti:
1407 case jit_code_lti_u: case jit_code_lei: case jit_code_lei_u:
1408 case jit_code_eqi: case jit_code_gei: case jit_code_gei_u:
1409 case jit_code_gti: case jit_code_gti_u: case jit_code_nei:
1410 case jit_code_ldxi_c: case jit_code_ldxi_uc: case jit_code_ldxi_s:
1411 case jit_code_ldxi_us: case jit_code_ldxi_i: case jit_code_ldxi_ui:
1412 case jit_code_ldxi_l: case jit_code_ldxi_f: case jit_code_ldxi_d:
1413 mask = jit_cc_a0_reg|jit_cc_a0_chg|jit_cc_a1_reg|jit_cc_a2_int;
1415 case jit_code_qmuli: case jit_code_qmuli_u:
1416 case jit_code_qdivi: case jit_code_qdivi_u:
1417 mask = jit_cc_a0_reg|jit_cc_a0_rlh|jit_cc_a0_chg|
1418 jit_cc_a1_reg|jit_cc_a2_int;
1420 case jit_code_addi_f: case jit_code_subi_f: case jit_code_rsbi_f:
1421 case jit_code_muli_f: case jit_code_divi_f: case jit_code_lti_f:
1422 case jit_code_lei_f: case jit_code_eqi_f: case jit_code_gei_f:
1423 case jit_code_gti_f: case jit_code_nei_f: case jit_code_unlti_f:
1424 case jit_code_unlei_f: case jit_code_uneqi_f: case jit_code_ungei_f:
1425 case jit_code_ungti_f: case jit_code_ltgti_f: case jit_code_ordi_f:
1426 case jit_code_unordi_f:
1427 mask = jit_cc_a0_reg|jit_cc_a0_chg|jit_cc_a1_reg|jit_cc_a2_flt;
1429 case jit_code_addi_d: case jit_code_subi_d: case jit_code_rsbi_d:
1430 case jit_code_muli_d: case jit_code_divi_d: case jit_code_lti_d:
1431 case jit_code_lei_d: case jit_code_eqi_d: case jit_code_gei_d:
1432 case jit_code_gti_d: case jit_code_nei_d: case jit_code_unlti_d:
1433 case jit_code_unlei_d: case jit_code_uneqi_d: case jit_code_ungei_d:
1434 case jit_code_ungti_d: case jit_code_ltgti_d: case jit_code_ordi_d:
1435 case jit_code_unordi_d:
1436 mask = jit_cc_a0_reg|jit_cc_a0_chg|jit_cc_a1_reg|jit_cc_a2_dbl;
1438 case jit_code_addr: case jit_code_addxr: case jit_code_addcr:
1439 case jit_code_subr: case jit_code_subxr: case jit_code_subcr:
1440 case jit_code_mulr: case jit_code_divr: case jit_code_divr_u:
1441 case jit_code_remr: case jit_code_remr_u: case jit_code_andr:
1442 case jit_code_orr: case jit_code_xorr: case jit_code_lshr:
1443 case jit_code_rshr: case jit_code_rshr_u: case jit_code_ltr:
1444 case jit_code_ltr_u: case jit_code_ler: case jit_code_ler_u:
1445 case jit_code_eqr: case jit_code_ger: case jit_code_ger_u:
1446 case jit_code_gtr: case jit_code_gtr_u: case jit_code_ner:
1447 case jit_code_ldxr_c: case jit_code_ldxr_uc: case jit_code_ldxr_s:
1448 case jit_code_ldxr_us: case jit_code_ldxr_i: case jit_code_ldxr_ui:
1449 case jit_code_ldxr_l: case jit_code_addr_f: case jit_code_subr_f:
1450 case jit_code_mulr_f: case jit_code_divr_f: case jit_code_ltr_f:
1451 case jit_code_ler_f: case jit_code_eqr_f: case jit_code_ger_f:
1452 case jit_code_gtr_f: case jit_code_ner_f: case jit_code_unltr_f:
1453 case jit_code_unler_f: case jit_code_uneqr_f: case jit_code_unger_f:
1454 case jit_code_ungtr_f: case jit_code_ltgtr_f: case jit_code_ordr_f:
1455 case jit_code_unordr_f: case jit_code_ldxr_f: case jit_code_addr_d:
1456 case jit_code_subr_d: case jit_code_mulr_d: case jit_code_divr_d:
1457 case jit_code_ltr_d: case jit_code_ler_d: case jit_code_eqr_d:
1458 case jit_code_ger_d: case jit_code_gtr_d: case jit_code_ner_d:
1459 case jit_code_unltr_d: case jit_code_unler_d: case jit_code_uneqr_d:
1460 case jit_code_unger_d: case jit_code_ungtr_d: case jit_code_ltgtr_d:
1461 case jit_code_ordr_d: case jit_code_unordr_d: case jit_code_ldxr_d:
1462 case jit_code_movr_ww_d:
1463 mask = jit_cc_a0_reg|jit_cc_a0_chg|jit_cc_a1_reg|jit_cc_a2_reg;
1465 case jit_code_qmulr: case jit_code_qmulr_u:
1466 case jit_code_qdivr: case jit_code_qdivr_u:
1467 mask = jit_cc_a0_reg|jit_cc_a0_rlh|jit_cc_a0_chg|
1468 jit_cc_a1_reg|jit_cc_a2_reg;
1470 case jit_code_sti_c: case jit_code_sti_s: case jit_code_sti_i:
1471 case jit_code_sti_l: case jit_code_sti_f: case jit_code_sti_d:
1472 mask = jit_cc_a0_int|jit_cc_a1_reg;
1474 case jit_code_blti: case jit_code_blti_u: case jit_code_blei:
1475 case jit_code_blei_u: case jit_code_beqi: case jit_code_bgei:
1476 case jit_code_bgei_u: case jit_code_bgti: case jit_code_bgti_u:
1477 case jit_code_bnei: case jit_code_bmsi: case jit_code_bmci:
1478 mask = jit_cc_a0_jmp|jit_cc_a1_reg|jit_cc_a2_int;
1480 case jit_code_blti_f: case jit_code_blei_f: case jit_code_beqi_f:
1481 case jit_code_bgei_f: case jit_code_bgti_f: case jit_code_bnei_f:
1482 case jit_code_bunlti_f: case jit_code_bunlei_f: case jit_code_buneqi_f:
1483 case jit_code_bungei_f: case jit_code_bungti_f: case jit_code_bltgti_f:
1484 case jit_code_bordi_f: case jit_code_bunordi_f:
1485 mask = jit_cc_a0_jmp|jit_cc_a1_reg|jit_cc_a2_flt;
1487 case jit_code_blti_d: case jit_code_blei_d: case jit_code_beqi_d:
1488 case jit_code_bgei_d: case jit_code_bgti_d: case jit_code_bnei_d:
1489 case jit_code_bunlti_d: case jit_code_bunlei_d: case jit_code_buneqi_d:
1490 case jit_code_bungei_d: case jit_code_bungti_d: case jit_code_bltgti_d:
1491 case jit_code_bordi_d: case jit_code_bunordi_d:
1492 mask = jit_cc_a0_jmp|jit_cc_a1_reg|jit_cc_a2_dbl;
1494 case jit_code_allocar: /* synthesized instructions make it
1495 * equivalent to jit_cc_a0_chg */
1496 case jit_code_str_c: case jit_code_str_s: case jit_code_str_i:
1497 case jit_code_str_l: case jit_code_str_f: case jit_code_str_d:
1498 mask = jit_cc_a0_reg|jit_cc_a1_reg;
1500 case jit_code_stxi_c: case jit_code_stxi_s: case jit_code_stxi_i:
1501 case jit_code_stxi_l: case jit_code_stxi_f: case jit_code_stxi_d:
1502 mask = jit_cc_a0_int|jit_cc_a1_reg|jit_cc_a2_reg;
1504 case jit_code_bltr: case jit_code_bltr_u: case jit_code_bler:
1505 case jit_code_bler_u: case jit_code_beqr: case jit_code_bger:
1506 case jit_code_bger_u: case jit_code_bgtr: case jit_code_bgtr_u:
1507 case jit_code_bner: case jit_code_bmsr: case jit_code_bmcr:
1508 case jit_code_bltr_f: case jit_code_bler_f: case jit_code_beqr_f:
1509 case jit_code_bger_f: case jit_code_bgtr_f: case jit_code_bner_f:
1510 case jit_code_bunltr_f: case jit_code_bunler_f: case jit_code_buneqr_f:
1511 case jit_code_bunger_f: case jit_code_bungtr_f: case jit_code_bltgtr_f:
1512 case jit_code_bordr_f: case jit_code_bunordr_f:case jit_code_bltr_d:
1513 case jit_code_bler_d: case jit_code_beqr_d: case jit_code_bger_d:
1514 case jit_code_bgtr_d: case jit_code_bner_d: case jit_code_bunltr_d:
1515 case jit_code_bunler_d: case jit_code_buneqr_d: case jit_code_bunger_d:
1516 case jit_code_bungtr_d: case jit_code_bltgtr_d: case jit_code_bordr_d:
1517 case jit_code_bunordr_d:
1518 mask = jit_cc_a0_jmp|jit_cc_a1_reg|jit_cc_a2_reg;
1520 case jit_code_boaddi: case jit_code_boaddi_u: case jit_code_bxaddi:
1521 case jit_code_bxaddi_u: case jit_code_bosubi: case jit_code_bosubi_u:
1522 case jit_code_bxsubi: case jit_code_bxsubi_u:
1523 mask = jit_cc_a0_jmp|jit_cc_a1_reg|jit_cc_a1_chg|jit_cc_a2_int;
1525 case jit_code_stxr_c: case jit_code_stxr_s: case jit_code_stxr_i:
1526 case jit_code_stxr_l: case jit_code_stxr_f: case jit_code_stxr_d:
1527 mask = jit_cc_a0_reg|jit_cc_a1_reg|jit_cc_a2_reg;
1529 case jit_code_boaddr: case jit_code_boaddr_u: case jit_code_bxaddr:
1530 case jit_code_bxaddr_u: case jit_code_bosubr: case jit_code_bosubr_u:
1531 case jit_code_bxsubr: case jit_code_bxsubr_u:
1532 mask = jit_cc_a0_jmp|jit_cc_a1_reg|jit_cc_a1_chg|jit_cc_a2_reg;
1542 _jit_patch_abs(jit_state_t *_jit, jit_node_t *instr, jit_pointer_t address)
1546 switch (instr->code) {
1547 case jit_code_movi: case jit_code_ldi_c: case jit_code_ldi_uc:
1548 case jit_code_ldi_s: case jit_code_ldi_us: case jit_code_ldi_i:
1549 case jit_code_ldi_ui: case jit_code_ldi_l: case jit_code_ldi_f:
1550 case jit_code_ldi_d:
1551 instr->v.p = address;
1553 case jit_code_sti_c: case jit_code_sti_s: case jit_code_sti_i:
1554 case jit_code_sti_l: case jit_code_sti_f: case jit_code_sti_d:
1555 instr->u.p = address;
1558 mask = jit_classify(instr->code);
1559 assert((mask & (jit_cc_a0_reg|jit_cc_a0_jmp)) == jit_cc_a0_jmp);
1560 instr->u.p = address;
1565 _jit_patch_at(jit_state_t *_jit, jit_node_t *instr, jit_node_t *label)
1569 assert(!(instr->flag & jit_flag_node));
1570 instr->flag |= jit_flag_node;
1571 switch (instr->code) {
1573 assert(label->code == jit_code_label ||
1574 label->code == jit_code_data);
1576 if (label->code == jit_code_data)
1577 instr->flag |= jit_flag_data;
1580 assert(label->code == jit_code_label ||
1581 label->code == jit_code_epilog);
1585 mask = jit_classify(instr->code);
1586 assert((mask & (jit_cc_a0_reg|jit_cc_a0_jmp)) == jit_cc_a0_jmp);
1587 assert(label->code == jit_code_label);
1591 /* link field is used as list of nodes associated with a given label */
1592 instr->link = label->link;
1593 label->link = instr;
1597 _jit_optimize(jit_state_t *_jit)
1606 _jitc->function = NULL;
1609 sequential_labels();
1612 /* create initial mapping of live register values
1613 * at the start of a basic block */
1614 for (offset = 0; offset < _jitc->blocks.offset; offset++) {
1615 block = _jitc->blocks.ptr + offset;
1618 if (block->label->code != jit_code_epilog)
1622 /* set live state of registers not referenced in a block, but
1623 * referenced in a jump target or normal flow */
1626 for (offset = 0; offset < _jitc->blocks.offset; offset++) {
1627 block = _jitc->blocks.ptr + offset;
1630 if (block->label->code != jit_code_epilog)
1631 jit_follow(block, &todo);
1638 /* figure out labels that are only reached with a jump
1639 * and is required to do a simple redundant_store removal
1640 * on jit_beqi below */
1642 for (node = _jitc->head; node; node = node->next) {
1643 switch (node->code) {
1644 case jit_code_label:
1646 node->flag |= jit_flag_head;
1648 case jit_code_jmpi: case jit_code_jmpr:
1649 case jit_code_epilog:
1652 case jit_code_data: case jit_code_note:
1660 for (node = _jitc->head; node; node = node->next) {
1661 mask = jit_classify(node->code);
1662 if (mask & jit_cc_a0_reg)
1663 node->u.w &= ~jit_regno_patch;
1664 if (mask & jit_cc_a1_reg)
1665 node->v.w &= ~jit_regno_patch;
1666 if (mask & jit_cc_a2_reg)
1667 node->w.w &= ~jit_regno_patch;
1668 switch (node->code) {
1669 case jit_code_prolog:
1670 _jitc->function = _jitc->functions.ptr + node->w.w;
1672 case jit_code_epilog:
1673 _jitc->function = NULL;
1676 redundant_store(node, 1);
1679 redundant_store(node, 0);
1683 if (mask & jit_cc_a0_flt) {
1684 node->u.p = jit_data(&node->u.f, sizeof(jit_float32_t), 4);
1685 node->flag |= jit_flag_node | jit_flag_data;
1687 else if (mask & jit_cc_a0_dbl) {
1688 node->u.p = jit_data(&node->u.d, sizeof(jit_float64_t), 8);
1689 node->flag |= jit_flag_node | jit_flag_data;
1691 else if (mask & jit_cc_a1_flt) {
1692 node->v.p = jit_data(&node->v.f, sizeof(jit_float32_t), 4);
1693 node->flag |= jit_flag_node | jit_flag_data;
1695 else if (mask & jit_cc_a1_dbl) {
1696 node->v.p = jit_data(&node->v.d, sizeof(jit_float64_t), 8);
1697 node->flag |= jit_flag_node | jit_flag_data;
1699 else if (mask & jit_cc_a2_flt) {
1700 node->w.p = jit_data(&node->w.f, sizeof(jit_float32_t), 4);
1701 node->flag |= jit_flag_node | jit_flag_data;
1703 else if (mask & jit_cc_a2_dbl) {
1704 node->w.p = jit_data(&node->w.d, sizeof(jit_float64_t), 8);
1705 node->flag |= jit_flag_node | jit_flag_data;
1708 if (_jitc->function) {
1709 if ((mask & (jit_cc_a0_reg|jit_cc_a0_chg)) ==
1710 (jit_cc_a0_reg|jit_cc_a0_chg)) {
1711 if (mask & jit_cc_a0_rlh) {
1712 jit_regset_setbit(&_jitc->function->regset,
1713 jit_regno(node->u.q.l));
1714 jit_regset_setbit(&_jitc->function->regset,
1715 jit_regno(node->u.q.h));
1718 jit_regset_setbit(&_jitc->function->regset,
1719 jit_regno(node->u.w));
1721 if ((mask & (jit_cc_a1_reg|jit_cc_a1_chg)) ==
1722 (jit_cc_a1_reg|jit_cc_a1_chg))
1723 jit_regset_setbit(&_jitc->function->regset,
1724 jit_regno(node->v.w));
1725 if ((mask & (jit_cc_a2_reg|jit_cc_a2_chg)) ==
1726 (jit_cc_a2_reg|jit_cc_a2_chg))
1727 jit_regset_setbit(&_jitc->function->regset,
1728 jit_regno(node->w.w));
1736 _jit_reglive(jit_state_t *_jit, jit_node_t *node)
1742 switch (node->code) {
1743 case jit_code_label: case jit_code_prolog: case jit_code_epilog:
1744 block = _jitc->blocks.ptr + node->v.w;
1745 jit_regset_set(&_jitc->reglive, &block->reglive);
1747 case jit_code_callr:
1748 value = jit_regno(node->u.w);
1749 if (!(node->u.w & jit_regno_patch)) {
1750 jit_regset_setbit(&_jitc->reglive, value);
1752 case jit_code_calli:
1753 for (value = 0; value < _jitc->reglen; value++) {
1754 spec = jit_class(_rvs[value].spec);
1755 if ((spec & jit_class_arg) && jit_regarg_p(node, value))
1756 jit_regset_setbit(&_jitc->reglive, value);
1757 else if (!(spec & jit_class_sav))
1758 jit_regset_clrbit(&_jitc->reglive, value);
1762 value = jit_classify(node->code);
1763 if (value & jit_cc_a0_reg) {
1764 if (value & jit_cc_a0_rlh) {
1765 if (!(node->u.q.l & jit_regno_patch)) {
1766 if (value & jit_cc_a0_chg) {
1767 jit_regset_clrbit(&_jitc->reglive, node->u.q.l);
1768 jit_regset_setbit(&_jitc->regmask, node->u.q.l);
1771 jit_regset_setbit(&_jitc->reglive, node->u.q.l);
1773 if (!(node->u.q.h & jit_regno_patch)) {
1774 if (value & jit_cc_a0_chg) {
1775 jit_regset_clrbit(&_jitc->reglive, node->u.q.h);
1776 jit_regset_setbit(&_jitc->regmask, node->u.q.h);
1779 jit_regset_setbit(&_jitc->reglive, node->u.q.h);
1783 if (!(node->u.w & jit_regno_patch)) {
1784 if (value & jit_cc_a0_chg) {
1785 jit_regset_clrbit(&_jitc->reglive, node->u.w);
1786 jit_regset_setbit(&_jitc->regmask, node->u.w);
1789 jit_regset_setbit(&_jitc->reglive, node->u.w);
1793 if ((value & jit_cc_a1_reg) && !(node->v.w & jit_regno_patch)) {
1794 if (value & jit_cc_a1_chg) {
1795 jit_regset_clrbit(&_jitc->reglive, node->v.w);
1796 jit_regset_setbit(&_jitc->regmask, node->v.w);
1799 jit_regset_setbit(&_jitc->reglive, node->v.w);
1801 if ((value & jit_cc_a2_reg) && !(node->w.w & jit_regno_patch)) {
1802 if (value & jit_cc_a2_chg) {
1803 jit_regset_clrbit(&_jitc->reglive, node->w.w);
1804 jit_regset_setbit(&_jitc->regmask, node->w.w);
1807 jit_regset_setbit(&_jitc->reglive, node->w.w);
1809 if (jit_regset_set_p(&_jitc->regmask)) {
1810 jit_update(node->next, &_jitc->reglive, &_jitc->regmask);
1811 if (jit_regset_set_p(&_jitc->regmask)) {
1812 /* any unresolved live state is considered as live */
1813 jit_regset_ior(&_jitc->reglive,
1814 &_jitc->reglive, &_jitc->regmask);
1815 jit_regset_set_ui(&_jitc->regmask, 0);
1823 _jit_regarg_set(jit_state_t *_jit, jit_node_t *node, jit_int32_t value)
1828 if (value & jit_cc_a0_reg) {
1829 if (value & jit_cc_a0_rlh) {
1830 jit_regset_setbit(&_jitc->regarg, jit_regno(node->u.q.l));
1831 jit_regset_setbit(&_jitc->regarg, jit_regno(node->u.q.h));
1834 jit_regset_setbit(&_jitc->regarg, jit_regno(node->u.w));
1836 if (value & jit_cc_a1_reg)
1837 jit_regset_setbit(&_jitc->regarg, jit_regno(node->v.w));
1838 if (value & jit_cc_a2_reg)
1839 jit_regset_setbit(&_jitc->regarg, jit_regno(node->w.w));
1843 _jit_regarg_clr(jit_state_t *_jit, jit_node_t *node, jit_int32_t value)
1846 jit_size_collect(node);
1848 if (value & jit_cc_a0_reg) {
1849 if (value & jit_cc_a0_rlh) {
1850 jit_regset_clrbit(&_jitc->regarg, jit_regno(node->u.q.l));
1851 jit_regset_clrbit(&_jitc->regarg, jit_regno(node->u.q.h));
1854 jit_regset_clrbit(&_jitc->regarg, jit_regno(node->u.w));
1856 if (value & jit_cc_a1_reg)
1857 jit_regset_clrbit(&_jitc->regarg, jit_regno(node->v.w));
1858 if (value & jit_cc_a2_reg)
1859 jit_regset_clrbit(&_jitc->regarg, jit_regno(node->w.w));
1863 _jit_realize(jit_state_t *_jit)
1865 assert(!_jitc->realize);
1866 if (_jitc->function)
1871 /* ensure it is aligned */
1872 _jitc->data.offset = (_jitc->data.offset + 7) & -8;
1875 /* Heuristic to guess code buffer size */
1877 _jit->code.length = _jitc->pool.length * 1024 * _jitc->mult;
1879 _jit->code.length = jit_get_size();
1884 _jit_dataset(jit_state_t *_jit)
1893 assert(!_jitc->dataset);
1894 if (!_jit->user_data) {
1896 /* create read only data buffer */
1897 _jit->data.length = (_jitc->data.offset +
1898 /* reserve space for annotations */
1899 _jitc->note.size + 4095) & -4096;
1901 mmap_fd = open("/dev/zero", O_RDWR);
1903 _jit->data.ptr = mmap(NULL, _jit->data.length,
1904 PROT_READ | PROT_WRITE,
1905 MAP_PRIVATE | MAP_ANON, mmap_fd, 0);
1906 assert(_jit->data.ptr != MAP_FAILED);
1912 if (!_jitc->no_data)
1913 jit_memcpy(_jit->data.ptr, _jitc->data.ptr, _jitc->data.offset);
1915 if (_jitc->no_note) {
1916 /* Space for one note is always allocated, so revert it here
1917 * if after jit_new_state was called, it is also requested to
1918 * not generate annotation information */
1919 _jit->note.length = 0;
1920 _jitc->note.size = 0;
1923 _jitc->note.base = _jit->data.ptr;
1924 if (!_jitc->no_data)
1925 _jitc->note.base += _jitc->data.offset;
1926 memset(_jitc->note.base, 0, _jitc->note.size);
1929 if (_jit->user_data)
1930 /* Need the temporary hashed data until jit_emit is finished */
1931 ptr = _jitc->no_data ? _jitc->data.ptr : _jit->data.ptr;
1933 ptr = _jit->data.ptr;
1934 /* Temporary hashed data no longer required */
1935 jit_free((jit_pointer_t *)&_jitc->data.ptr);
1938 for (offset = 0; offset < _jitc->data.size; offset++) {
1939 for (node = _jitc->data.table[offset]; node; node = node->next) {
1940 node->flag |= jit_flag_patch;
1941 node->u.w = (jit_word_t)(ptr + node->u.w);
1949 _jit_get_code(jit_state_t *_jit, jit_word_t *length)
1951 assert(_jitc->realize);
1954 /* If code already generated, return exact size of code */
1955 *length = _jit->pc.uc - _jit->code.ptr;
1957 /* Else return current size of the code buffer */
1958 *length = _jit->code.length;
1961 return (_jit->code.ptr);
1965 _jit_set_code(jit_state_t *_jit, jit_pointer_t ptr, jit_word_t length)
1967 assert(_jitc->realize);
1968 _jit->code.ptr = ptr;
1969 _jit->code.length = length;
1970 _jit->user_code = 1;
1974 _jit_get_data(jit_state_t *_jit, jit_word_t *data_size, jit_word_t *note_size)
1976 assert(_jitc->realize);
1978 *data_size = _jitc->data.offset;
1980 *note_size = _jitc->note.size;
1981 return (_jit->data.ptr);
1985 _jit_set_data(jit_state_t *_jit, jit_pointer_t ptr,
1986 jit_word_t length, jit_word_t flags)
1988 assert(_jitc->realize);
1989 if (flags & JIT_DISABLE_DATA)
1992 assert(length >= _jitc->data.offset);
1993 if (flags & JIT_DISABLE_NOTE)
1996 if (flags & JIT_DISABLE_DATA)
1997 assert(length >= _jitc->note.size);
1999 assert(length >= _jitc->data.offset + _jitc->note.size);
2001 _jit->data.ptr = ptr;
2002 _jit->data.length = length;
2003 _jit->user_data = 1;
2007 _jit_emit(jit_state_t *_jit)
2017 if (!_jitc->realize)
2020 if (!_jitc->dataset)
2025 if (!_jit->user_code) {
2027 mmap_fd = open("/dev/zero", O_RDWR);
2029 _jit->code.ptr = mmap(NULL, _jit->code.length,
2030 PROT_EXEC | PROT_READ | PROT_WRITE,
2031 MAP_PRIVATE | MAP_ANON, mmap_fd, 0);
2032 assert(_jit->code.ptr != MAP_FAILED);
2034 _jitc->code.end = _jit->code.ptr + _jit->code.length -
2035 jit_get_max_instr();
2036 _jit->pc.uc = _jit->code.ptr;
2039 if ((code = emit_code()) == NULL) {
2040 _jitc->patches.offset = 0;
2041 for (node = _jitc->head; node; node = node->next) {
2043 (node->code == jit_code_label ||
2044 node->code == jit_code_epilog))
2045 node->flag &= ~jit_flag_patch;
2047 if (_jit->user_code)
2051 length = _jitc->pool.length * 1024 * _jitc->mult;
2053 /* Should only happen on very special cases */
2054 length = _jit->code.length + 4096;
2058 munmap(_jit->code.ptr, _jit->code.length);
2063 _jit->code.ptr = mremap(_jit->code.ptr, _jit->code.length,
2064 _jit->code.ptr, length, 0);
2066 _jit->code.ptr = mremap(_jit->code.ptr, _jit->code.length,
2067 length, MREMAP_MAYMOVE, NULL);
2070 _jit->code.ptr = mmap(NULL, length,
2071 PROT_EXEC | PROT_READ | PROT_WRITE,
2072 MAP_PRIVATE | MAP_ANON, mmap_fd, 0);
2075 assert(_jit->code.ptr != MAP_FAILED);
2076 _jit->code.length = length;
2077 _jitc->code.end = _jit->code.ptr + _jit->code.length -
2078 jit_get_max_instr();
2079 _jit->pc.uc = _jit->code.ptr;
2086 if (!_jit->user_code)
2091 if (!_jitc->no_note)
2094 if (_jit->user_data)
2095 jit_free((jit_pointer_t *)&_jitc->data.ptr);
2097 result = mprotect(_jit->data.ptr, _jit->data.length, PROT_READ);
2098 assert(result == 0);
2100 if (!_jit->user_code) {
2101 result = mprotect(_jit->code.ptr, _jit->code.length,
2102 PROT_READ | PROT_EXEC);
2103 assert(result == 0);
2106 return (_jit->code.ptr);
2112 _jit_frame(jit_state_t *_jit, jit_int32_t frame)
2114 jit_trampoline(frame, 1);
2118 _jit_tramp(jit_state_t *_jit, jit_int32_t frame)
2120 jit_trampoline(frame, 0);
2124 _jit_trampoline(jit_state_t *_jit, jit_int32_t frame, jit_bool_t prolog)
2128 /* Must be called after prolog, actually, just to simplify
2129 * tests and know there is a current function and that
2130 * _jitc->function->self.aoff is at the before any alloca value */
2131 assert(_jitc->tail && _jitc->tail->code == jit_code_prolog);
2133 /* + 24 for 3 possible spilled temporaries (that could be a double) */
2135 #if defined(__hppa__)
2136 frame += _jitc->function->self.aoff;
2138 frame -= _jitc->function->self.aoff;
2140 _jitc->function->frame = frame;
2142 _jitc->function->define_frame = 1;
2144 _jitc->function->assume_frame = 1;
2145 for (regno = 0; regno < _jitc->reglen; regno++)
2146 if (jit_class(_rvs[regno].spec) & jit_class_sav)
2147 jit_regset_setbit(&_jitc->function->regset, regno);
2150 /* Compute initial reglive and regmask set values of a basic block.
2151 * reglive is the set of known live registers
2152 * regmask is the set of registers not referenced in the block
2153 * Registers in regmask might be live.
2156 _jit_setup(jit_state_t *_jit, jit_block_t *block)
2160 unsigned long value;
2162 jit_regset_set_mask(&block->regmask, _jitc->reglen);
2163 for (value = 0; value < _jitc->reglen; ++value)
2164 if (!(jit_class(_rvs[value].spec) & (jit_class_gpr|jit_class_fpr)))
2165 jit_regset_clrbit(&block->regmask, value);
2167 for (node = block->label->next; node; node = node->next) {
2168 switch (node->code) {
2169 case jit_code_label: case jit_code_prolog:
2170 case jit_code_epilog:
2173 /* Check argument registers in reverse order to properly
2174 * handle registers that are both, argument and result */
2175 value = jit_classify(node->code);
2176 if ((value & jit_cc_a2_reg) &&
2177 !(node->w.w & jit_regno_patch) &&
2178 jit_regset_tstbit(&block->regmask, node->w.w)) {
2179 live = !(value & jit_cc_a2_chg);
2180 jit_regset_clrbit(&block->regmask, node->w.w);
2182 jit_regset_setbit(&block->reglive, node->w.w);
2184 if ((value & jit_cc_a1_reg) &&
2185 !(node->v.w & jit_regno_patch) &&
2186 jit_regset_tstbit(&block->regmask, node->v.w)) {
2187 live = !(value & jit_cc_a1_chg);
2188 jit_regset_clrbit(&block->regmask, node->v.w);
2190 jit_regset_setbit(&block->reglive, node->v.w);
2192 if (value & jit_cc_a0_reg) {
2193 live = !(value & jit_cc_a0_chg);
2194 if (value & jit_cc_a0_rlh) {
2195 if (!(node->u.q.l & jit_regno_patch) &&
2196 jit_regset_tstbit(&block->regmask, node->u.q.l)) {
2197 jit_regset_clrbit(&block->regmask, node->u.q.l);
2199 jit_regset_setbit(&block->reglive, node->u.q.l);
2201 if (!(node->u.q.h & jit_regno_patch) &&
2202 jit_regset_tstbit(&block->regmask, node->u.q.h)) {
2203 jit_regset_clrbit(&block->regmask, node->u.q.h);
2205 jit_regset_setbit(&block->reglive, node->u.q.h);
2209 if (!(node->u.w & jit_regno_patch) &&
2210 jit_regset_tstbit(&block->regmask, node->u.w)) {
2211 jit_regset_clrbit(&block->regmask, node->u.w);
2213 jit_regset_setbit(&block->reglive, node->u.w);
2222 /* Update regmask and reglive of blocks at entry point of branch targets
2223 * or normal flow that have a live register not used in this block.
2226 _jit_follow(jit_state_t *_jit, jit_block_t *block, jit_bool_t *todo)
2232 unsigned long value;
2234 jit_regset_t reglive;
2235 jit_regset_t regmask;
2236 jit_regset_t regtemp;
2238 jit_regset_set(®live, &block->reglive);
2239 jit_regset_set(®mask, &block->regmask);
2240 for (node = block->label->next; node; node = node->next) {
2241 switch (node->code) {
2242 case jit_code_label:
2243 /* Do not consider jmpi and jmpr cannot jump to the
2244 * next instruction. */
2245 next = _jitc->blocks.ptr + node->v.w;
2246 /* Set of live registers in next block that are at unknown
2247 * state in this block. */
2248 jit_regset_and(®temp, ®mask, &next->reglive);
2249 if (jit_regset_set_p(®temp)) {
2250 /* Add live state of next block to current block. */
2251 jit_regset_ior(&block->reglive, &block->reglive, ®temp);
2252 /* Remove from unknown state bitmask. */
2253 jit_regset_com(®temp, ®temp);
2254 jit_regset_and(&block->regmask, &block->regmask, ®temp);
2257 case jit_code_prolog:
2258 case jit_code_epilog:
2260 case jit_code_callr:
2261 value = jit_regno(node->u.w);
2262 if (!(node->u.w & jit_regno_patch)) {
2263 if (jit_regset_tstbit(®mask, value)) {
2264 jit_regset_clrbit(®mask, value);
2265 jit_regset_setbit(®live, value);
2268 case jit_code_calli:
2269 for (value = 0; value < _jitc->reglen; ++value) {
2270 value = jit_regset_scan1(®mask, value);
2271 if (value >= _jitc->reglen)
2273 spec = jit_class(_rvs[value].spec);
2274 if (!(spec & jit_class_sav))
2275 jit_regset_clrbit(®mask, value);
2276 if ((spec & jit_class_arg) && jit_regarg_p(node, value))
2277 jit_regset_setbit(®live, value);
2281 value = jit_classify(node->code);
2282 if (value & jit_cc_a2_reg) {
2283 if (!(node->w.w & jit_regno_patch)) {
2284 if (jit_regset_tstbit(®mask, node->w.w)) {
2285 jit_regset_clrbit(®mask, node->w.w);
2286 if (!(value & jit_cc_a2_chg))
2287 jit_regset_setbit(®live, node->w.w);
2291 if (value & jit_cc_a1_reg) {
2292 if (!(node->v.w & jit_regno_patch)) {
2293 if (jit_regset_tstbit(®mask, node->v.w)) {
2294 jit_regset_clrbit(®mask, node->v.w);
2295 if (!(value & jit_cc_a1_chg))
2296 jit_regset_setbit(®live, node->v.w);
2300 if (value & jit_cc_a0_reg) {
2301 if (value & jit_cc_a0_rlh) {
2302 if (!(node->u.q.l & jit_regno_patch)) {
2303 if (jit_regset_tstbit(®mask, node->u.q.l)) {
2304 jit_regset_clrbit(®mask, node->u.q.l);
2305 if (!(value & jit_cc_a0_chg))
2306 jit_regset_setbit(®live, node->u.q.l);
2309 if (!(node->u.q.h & jit_regno_patch)) {
2310 if (jit_regset_tstbit(®mask, node->u.q.h)) {
2311 jit_regset_clrbit(®mask, node->u.q.h);
2312 if (!(value & jit_cc_a0_chg))
2313 jit_regset_setbit(®live, node->u.q.h);
2318 if (!(node->u.w & jit_regno_patch)) {
2319 if (jit_regset_tstbit(®mask, node->u.w)) {
2320 jit_regset_clrbit(®mask, node->u.w);
2321 if (!(value & jit_cc_a0_chg))
2322 jit_regset_setbit(®live, node->u.w);
2327 if (value & jit_cc_a0_jmp) {
2328 if (node->flag & jit_flag_node) {
2330 /* Do not consider jmpi and jmpr cannot jump to the
2331 * next instruction. */
2332 next = _jitc->blocks.ptr + label->v.w;
2333 jit_regset_and(®temp, ®mask, &next->reglive);
2334 if (jit_regset_set_p(®temp)) {
2335 /* Add live state. */
2336 jit_regset_ior(&block->reglive,
2337 &block->reglive, ®temp);
2338 /* Remove from unknown state bitmask. */
2339 jit_regset_com(®temp, ®temp);
2340 jit_regset_and(&block->regmask,
2341 &block->regmask, ®temp);
2346 /* Jump to unknown location.
2347 * This is a pitfall of the implementation.
2348 * Only jmpi to not a jit code should reach here,
2349 * or a jmpr of a computed address.
2350 * Because the implementation needs jit_class_nospill
2351 * registers, must treat jmpr as a function call. This
2352 * means that only JIT_Vn registers can be trusted on
2355 for (regno = 0; regno < _jitc->reglen; regno++) {
2356 spec = jit_class(_rvs[regno].spec);
2357 if (jit_regset_tstbit(®mask, regno) &&
2358 (spec & (jit_class_gpr|jit_class_fpr)) &&
2359 !(spec & jit_class_sav))
2360 jit_regset_clrbit(®mask, regno);
2362 /* Assume non callee save registers are live due
2363 * to jump to unknown location. */
2364 /* Treat all callee save as live. */
2365 jit_regset_ior(®live, ®live, ®mask);
2366 /* Treat anything else as dead. */
2367 jit_regset_set_ui(®mask, 0);
2375 /* Follow code generation up to finding a label or end of code.
2376 * When finding a label, update the set of live registers.
2377 * On branches, update based on taken branch or normal flow.
2380 _jit_update(jit_state_t *_jit, jit_node_t *node,
2381 jit_regset_t *live, jit_regset_t *mask)
2385 unsigned long value;
2388 jit_regset_t regtemp;
2390 for (; node; node = node->next) {
2391 if (jit_regset_set_p(mask) == 0)
2393 switch (node->code) {
2394 case jit_code_label:
2395 block = _jitc->blocks.ptr + node->v.w;
2396 jit_regset_and(®temp, mask, &block->reglive);
2397 if (jit_regset_set_p(®temp)) {
2398 /* Add live state. */
2399 jit_regset_ior(live, live, ®temp);
2400 /* Remove from unknown state bitmask. */
2401 jit_regset_com(®temp, ®temp);
2402 jit_regset_and(mask, mask, ®temp);
2405 case jit_code_prolog:
2406 jit_regset_set_ui(mask, 0);
2408 case jit_code_epilog:
2409 jit_regset_set_ui(mask, 0);
2411 case jit_code_callr:
2412 value = jit_regno(node->u.w);
2413 if (!(node->u.w & jit_regno_patch)) {
2414 if (jit_regset_tstbit(mask, value)) {
2415 jit_regset_clrbit(mask, value);
2416 jit_regset_setbit(live, value);
2419 case jit_code_calli:
2420 for (value = 0; value < _jitc->reglen; ++value) {
2421 value = jit_regset_scan1(mask, value);
2422 if (value >= _jitc->reglen)
2424 spec = jit_class(_rvs[value].spec);
2425 if (!(spec & jit_class_sav))
2426 jit_regset_clrbit(mask, value);
2427 if ((spec & jit_class_arg) && jit_regarg_p(node, value))
2428 jit_regset_setbit(live, value);
2432 value = jit_classify(node->code);
2433 if (value & jit_cc_a2_reg) {
2434 if (!(node->w.w & jit_regno_patch)) {
2435 if (jit_regset_tstbit(mask, node->w.w)) {
2436 jit_regset_clrbit(mask, node->w.w);
2437 if (!(value & jit_cc_a2_chg))
2438 jit_regset_setbit(live, node->w.w);
2442 if (value & jit_cc_a1_reg) {
2443 if (!(node->v.w & jit_regno_patch)) {
2444 if (jit_regset_tstbit(mask, node->v.w)) {
2445 jit_regset_clrbit(mask, node->v.w);
2446 if (!(value & jit_cc_a1_chg))
2447 jit_regset_setbit(live, node->v.w);
2451 if (value & jit_cc_a0_reg) {
2452 if (value & jit_cc_a0_rlh) {
2453 if (!(node->u.q.l & jit_regno_patch)) {
2454 if (jit_regset_tstbit(mask, node->u.q.l)) {
2455 jit_regset_clrbit(mask, node->u.q.l);
2456 if (!(value & jit_cc_a0_chg))
2457 jit_regset_setbit(live, node->u.q.l);
2460 if (!(node->u.q.h & jit_regno_patch)) {
2461 if (jit_regset_tstbit(mask, node->u.q.h)) {
2462 jit_regset_clrbit(mask, node->u.q.h);
2463 if (!(value & jit_cc_a0_chg))
2464 jit_regset_setbit(live, node->u.q.h);
2469 if (!(node->u.w & jit_regno_patch)) {
2470 if (jit_regset_tstbit(mask, node->u.w)) {
2471 jit_regset_clrbit(mask, node->u.w);
2472 if (!(value & jit_cc_a0_chg))
2473 jit_regset_setbit(live, node->u.w);
2478 if (value & jit_cc_a0_jmp) {
2479 if (node->flag & jit_flag_node) {
2481 /* Do not consider jmpi and jmpr cannot jump to the
2482 * next instruction. */
2483 block = _jitc->blocks.ptr + label->v.w;
2484 jit_regset_and(®temp, mask, &block->reglive);
2485 if (jit_regset_set_p(®temp)) {
2486 /* Add live state. */
2487 jit_regset_ior(live, live, ®temp);
2488 /* Remove from unknown state bitmask. */
2489 jit_regset_com(®temp, ®temp);
2490 jit_regset_and(mask, mask, ®temp);
2494 /* Jump to unknown location.
2495 * This is a pitfall of the implementation.
2496 * Only jmpi to not a jit code should reach here,
2497 * or a jmpr of a computed address.
2498 * Because the implementation needs jit_class_nospill
2499 * registers, must treat jmpr as a function call. This
2500 * means that only JIT_Vn registers can be trusted on
2503 for (regno = 0; regno < _jitc->reglen; regno++) {
2504 spec = jit_class(_rvs[regno].spec);
2505 if (jit_regset_tstbit(mask, regno) &&
2506 (spec & (jit_class_gpr|jit_class_fpr)) &&
2507 !(spec & jit_class_sav))
2508 jit_regset_clrbit(mask, regno);
2510 /* Assume non callee save registers are live due
2511 * to jump to unknown location. */
2512 /* Treat all callee save as live. */
2513 jit_regset_ior(live, live, mask);
2514 /* Treat anything else as dead. */
2515 jit_regset_set_ui(mask, 0);
2524 _thread_jumps(jit_state_t *_jit)
2531 for (prev = node = _jitc->head; node;) {
2533 switch (node->code) {
2535 if (redundant_jump(prev, node)) {
2539 if (shortcut_jump(prev, node))
2543 case jit_code_callr: case jit_code_calli:
2544 /* non optimizable jump like code */
2547 mask = jit_classify(node->code);
2548 if (mask & jit_cc_a0_jmp) {
2549 if (reverse_jump(prev, node) ||
2550 shortcut_jump(prev, node))
2561 _sequential_labels(jit_state_t *_jit)
2569 for (prev = node = _jitc->head; node; node = next) {
2571 if (node->code == jit_code_label) {
2574 del_label(prev, node);
2577 if (prev != node && prev->code == jit_code_label) {
2578 if ((jump = node->link)) {
2579 for (; jump; jump = link) {
2582 jump->link = prev->link;
2587 del_label(prev, node);
2591 if (next && next->code == jit_code_label && !next->flag) {
2592 if ((jump = next->link)) {
2593 for (; jump; jump = link) {
2596 jump->link = node->link;
2601 del_label(node, next);
2611 _split_branches(jit_state_t *_jit)
2618 for (node = _jitc->head; node; node = next) {
2619 if ((next = node->next)) {
2620 if (next->code == jit_code_label ||
2621 next->code == jit_code_prolog ||
2622 next->code == jit_code_epilog)
2624 /* split block on branches */
2625 if (jit_classify(node->code) & jit_cc_a0_jmp) {
2626 label = new_node(jit_code_label);
2629 if (_jitc->blocks.offset >= _jitc->blocks.length) {
2632 length = _jitc->blocks.length + 16;
2633 jit_realloc((jit_pointer_t *)&_jitc->blocks.ptr,
2634 _jitc->blocks.length * sizeof(jit_block_t),
2635 length * sizeof(jit_block_t));
2636 _jitc->blocks.length = length;
2638 block = _jitc->blocks.ptr + _jitc->blocks.offset;
2639 block->label = label;
2640 label->v.w = _jitc->blocks.offset;
2641 jit_regset_new(&block->reglive);
2642 jit_regset_new(&block->regmask);
2643 ++_jitc->blocks.offset;
2650 _shortcut_jump(jit_state_t *_jit, jit_node_t *prev, jit_node_t *node)
2657 if (!(node->flag & jit_flag_node))
2659 assert(node->code != jit_code_jmpr);
2660 cond = node->code != jit_code_jmpi;
2662 for (next = jump->next; next; next = next->next) {
2663 switch (next->code) {
2665 if (!(next->flag & jit_flag_node))
2667 if (jump->link == node)
2668 jump->link = node->link;
2670 for (temp = jump->link;
2673 assert(temp != NULL);
2674 temp->link = node->link;
2678 node->link = jump->link;
2684 node->code = jit_code_jmpr;
2685 node->u.w = next->u.w;
2687 node->flag &= ~jit_flag_node;
2689 case jit_code_note: case jit_code_label:
2699 _redundant_jump(jit_state_t *_jit, jit_node_t *prev, jit_node_t *node)
2701 jit_node_t *local_prev;
2702 jit_node_t *local_next;
2704 if (!(node->flag & jit_flag_node))
2706 for (local_prev = node, local_next = node->next;
2708 local_prev = local_next, local_next = local_next->next) {
2710 switch (local_next->code) {
2711 case jit_code_label: case jit_code_epilog:
2712 if (node->u.n == local_next) {
2713 if (local_next->link == node)
2714 local_next->link = node->link;
2716 for (local_prev = local_next->link;
2717 local_prev->link != node;
2718 local_prev = local_prev->link)
2719 assert(local_prev != NULL);
2720 local_prev->link = node->link;
2722 del_node(prev, node);
2726 case jit_code_name: case jit_code_note:
2727 case jit_code_align:
2737 reverse_jump_code(jit_code_t code)
2740 case jit_code_bltr: return (jit_code_bger);
2741 case jit_code_blti: return (jit_code_bgei);
2742 case jit_code_bltr_u: return (jit_code_bger_u);
2743 case jit_code_blti_u: return (jit_code_bgei_u);
2744 case jit_code_bler: return (jit_code_bgtr);
2745 case jit_code_blei: return (jit_code_bgti);
2746 case jit_code_bler_u: return (jit_code_bgtr_u);
2747 case jit_code_blei_u: return (jit_code_bgti_u);
2748 case jit_code_beqr: return (jit_code_bner);
2749 case jit_code_beqi: return (jit_code_bnei);
2750 case jit_code_bger: return (jit_code_bltr);
2751 case jit_code_bgei: return (jit_code_blti);
2752 case jit_code_bger_u: return (jit_code_bltr_u);
2753 case jit_code_bgei_u: return (jit_code_blti_u);
2754 case jit_code_bgtr: return (jit_code_bler);
2755 case jit_code_bgti: return (jit_code_blei);
2756 case jit_code_bgtr_u: return (jit_code_bler_u);
2757 case jit_code_bgti_u: return (jit_code_blei_u);
2758 case jit_code_bner: return (jit_code_beqr);
2759 case jit_code_bnei: return (jit_code_beqi);
2760 case jit_code_bmsr: return (jit_code_bmcr);
2761 case jit_code_bmsi: return (jit_code_bmci);
2762 case jit_code_bmcr: return (jit_code_bmsr);
2763 case jit_code_bmci: return (jit_code_bmsi);
2764 case jit_code_bltr_f: return (jit_code_bunger_f);
2765 case jit_code_blti_f: return (jit_code_bungei_f);
2766 case jit_code_bler_f: return (jit_code_bungtr_f);
2767 case jit_code_blei_f: return (jit_code_bungti_f);
2769 case jit_code_beqr_f: return (jit_code_bner_f);
2770 case jit_code_beqi_f: return (jit_code_bnei_f);
2772 case jit_code_bger_f: return (jit_code_bunltr_f);
2773 case jit_code_bgei_f: return (jit_code_bunlti_f);
2774 case jit_code_bgtr_f: return (jit_code_bunler_f);
2775 case jit_code_bgti_f: return (jit_code_bunlei_f);
2777 case jit_code_bner_f: return (jit_code_beqr_f);
2778 case jit_code_bnei_f: return (jit_code_beqr_f);
2780 case jit_code_bunltr_f: return (jit_code_bger_f);
2781 case jit_code_bunlti_f: return (jit_code_bgei_f);
2782 case jit_code_bunler_f: return (jit_code_bgtr_f);
2783 case jit_code_bunlei_f: return (jit_code_bgti_f);
2785 case jit_code_buneqr_f: return (jit_code_bltgtr_f);
2786 case jit_code_buneqi_f: return (jit_code_bltgti_f);
2788 case jit_code_bunger_f: return (jit_code_bltr_f);
2789 case jit_code_bungei_f: return (jit_code_blti_f);
2790 case jit_code_bungtr_f: return (jit_code_bler_f);
2791 case jit_code_bungti_f: return (jit_code_blei_f);
2793 case jit_code_bltgtr_f: return (jit_code_buneqr_f);
2794 case jit_code_bltgti_f: return (jit_code_buneqi_f);
2796 case jit_code_bordr_f: return (jit_code_bunordr_f);
2797 case jit_code_bordi_f: return (jit_code_bunordi_f);
2798 case jit_code_bunordr_f:return (jit_code_bordr_f);
2799 case jit_code_bunordi_f:return (jit_code_bordi_f);
2800 case jit_code_bltr_d: return (jit_code_bunger_d);
2801 case jit_code_blti_d: return (jit_code_bungei_d);
2802 case jit_code_bler_d: return (jit_code_bungtr_d);
2803 case jit_code_blei_d: return (jit_code_bungti_d);
2805 case jit_code_beqr_d: return (jit_code_bner_d);
2806 case jit_code_beqi_d: return (jit_code_bnei_d);
2808 case jit_code_bger_d: return (jit_code_bunltr_d);
2809 case jit_code_bgei_d: return (jit_code_bunlti_d);
2810 case jit_code_bgtr_d: return (jit_code_bunler_d);
2811 case jit_code_bgti_d: return (jit_code_bunlei_d);
2813 case jit_code_bner_d: return (jit_code_beqr_d);
2814 case jit_code_bnei_d: return (jit_code_beqi_d);
2816 case jit_code_bunltr_d: return (jit_code_bger_d);
2817 case jit_code_bunlti_d: return (jit_code_bgei_d);
2818 case jit_code_bunler_d: return (jit_code_bgtr_d);
2819 case jit_code_bunlei_d: return (jit_code_bgti_d);
2821 case jit_code_buneqr_d: return (jit_code_bltgtr_d);
2822 case jit_code_buneqi_d: return (jit_code_bltgti_d);
2824 case jit_code_bunger_d: return (jit_code_bltr_d);
2825 case jit_code_bungei_d: return (jit_code_blti_d);
2826 case jit_code_bungtr_d: return (jit_code_bler_d);
2827 case jit_code_bungti_d: return (jit_code_blei_d);
2829 case jit_code_bltgtr_d: return (jit_code_buneqr_d);
2830 case jit_code_bltgti_d: return (jit_code_buneqi_d);
2832 case jit_code_bordr_d: return (jit_code_bunordr_d);
2833 case jit_code_bordi_d: return (jit_code_bunordi_d);
2834 case jit_code_bunordr_d:return (jit_code_bordr_d);
2835 case jit_code_bunordi_d:return (jit_code_bordi_d);
2836 case jit_code_boaddr: return (jit_code_bxaddr);
2837 case jit_code_boaddi: return (jit_code_bxaddi);
2838 case jit_code_boaddr_u: return (jit_code_bxaddr_u);
2839 case jit_code_boaddi_u: return (jit_code_bxaddi_u);
2840 case jit_code_bxaddr: return (jit_code_boaddr);
2841 case jit_code_bxaddi: return (jit_code_boaddi);
2842 case jit_code_bxaddr_u: return (jit_code_boaddr_u);
2843 case jit_code_bxaddi_u: return (jit_code_boaddi_u);
2844 case jit_code_bosubr: return (jit_code_bxsubr);
2845 case jit_code_bosubi: return (jit_code_bxsubi);
2846 case jit_code_bosubr_u: return (jit_code_bxsubr_u);
2847 case jit_code_bosubi_u: return (jit_code_bxsubi_u);
2848 case jit_code_bxsubr: return (jit_code_bosubr);
2849 case jit_code_bxsubi: return (jit_code_bosubi);
2850 case jit_code_bxsubr_u: return (jit_code_bosubr_u);
2851 case jit_code_bxsubi_u: return (jit_code_bosubi_u);
2852 default: abort(); /* invalid jump code */
2857 * change common pattern:
2858 * <cond_jump L0> <jump L1> <label L0>
2860 * <reverse_cond_jump L1>
2863 _reverse_jump(jit_state_t *_jit, jit_node_t *prev, jit_node_t *node)
2865 jit_node_t *local_prev;
2866 jit_node_t *local_next;
2867 jit_node_t *local_jump;
2869 if (!(node->flag & jit_flag_node))
2871 /* =><cond_jump L0> <jump L1> <label L0> */
2872 local_next = node->next;
2873 if (local_next->code != jit_code_jmpi ||
2874 !(local_next->flag & jit_flag_node))
2876 /* <cond_jump L0> =><jump L1> <label L0> */
2878 local_jump = local_next->u.n;
2879 for (local_prev = local_next, local_next = local_next->next;
2881 local_prev = local_next, local_next = local_next->next) {
2882 switch (local_next->code) {
2883 case jit_code_label: case jit_code_epilog:
2884 if (node->u.n == local_next) {
2885 if (local_next->link == node)
2886 local_next->link = node->link;
2888 for (local_prev = local_next->link;
2889 local_prev->link != node;
2890 local_prev = local_prev->link)
2891 assert(local_prev != NULL);
2892 local_prev->link = node->link;
2894 del_node(node, node->next);
2895 node->code = reverse_jump_code(node->code);
2896 node->u.n = local_jump;
2897 node->link = local_jump->link;
2898 local_jump->link = node;
2912 _redundant_store(jit_state_t *_jit, jit_node_t *node, jit_bool_t jump)
2922 if (prev->code == jit_code_epilog)
2924 assert(prev->code == jit_code_label);
2925 if ((prev->flag & jit_flag_head) || node->link || prev->link != node)
2926 /* multiple sources */
2928 /* if there are sequential labels it will return below */
2933 regno = jit_regno(node->v.w);
2934 for (iter = prev->next; iter; prev = iter, iter = iter->next) {
2935 switch (iter->code) {
2936 case jit_code_label: case jit_code_prolog:
2937 case jit_code_epilog:
2940 if (regno == jit_regno(iter->u.w)) {
2941 if (iter->flag || iter->v.w != word)
2943 del_node(prev, iter);
2948 spec = jit_classify(iter->code);
2949 if (spec & jit_cc_a0_jmp)
2951 if ((spec & (jit_cc_a0_reg|jit_cc_a0_chg)) ==
2952 (jit_cc_a0_reg|jit_cc_a0_chg)) {
2953 if (spec & jit_cc_a0_rlh) {
2954 if (regno == jit_regno(iter->u.q.l) ||
2955 regno == jit_regno(iter->u.q.h))
2959 if (regno == jit_regno(iter->u.w))
2963 if ((spec & (jit_cc_a1_reg|jit_cc_a1_chg)) ==
2964 (jit_cc_a1_reg|jit_cc_a1_chg)) {
2965 if (regno == jit_regno(iter->v.w))
2968 if ((spec & (jit_cc_a2_reg|jit_cc_a2_chg)) ==
2969 (jit_cc_a2_reg|jit_cc_a2_chg)) {
2970 if (regno == jit_regno(iter->w.w))
2979 _simplify_movr(jit_state_t *_jit, jit_node_t *prev, jit_node_t *node,
2980 jit_int32_t kind, jit_int32_t size)
2986 regno = jit_regno(node->u.w);
2987 right = jit_regno(node->v.w);
2988 value = _jitc->values + regno;
2989 if ((value->kind == jit_kind_register &&
2990 jit_regno(value->base.q.l) == right &&
2991 value->base.q.h == _jitc->gen[right]) ||
2992 (value->kind == kind && _jitc->values[right].kind == kind &&
2993 memcmp(&value->base.w, &_jitc->values[right].base.w, size) == 0)) {
2994 del_node(prev, node);
2997 if (_jitc->values[right].kind == jit_kind_word)
2998 jit_memcpy(value, _jitc->values + right, sizeof(jit_value_t));
3000 value->kind = jit_kind_register;
3001 value->base.q.l = right;
3002 value->base.q.h = _jitc->gen[right];
3004 ++_jitc->gen[regno];
3010 _simplify_movi(jit_state_t *_jit, jit_node_t *prev, jit_node_t *node,
3011 jit_int32_t kind, jit_int32_t size)
3018 regno = jit_regno(node->u.w);
3019 value = _jitc->values + regno;
3020 if (node->flag & jit_flag_node) {
3021 /* set to undefined if value will be patched */
3023 ++_jitc->gen[regno];
3026 if (value->kind == kind) {
3027 if (memcmp(&node->v.w, &value->base.w, size) == 0) {
3028 del_node(prev, node);
3031 spec = jit_class(_rvs[regno].spec);
3032 if (kind == jit_kind_word)
3033 spec &= jit_class_gpr;
3035 spec &= (jit_class_xpr | jit_class_fpr);
3036 for (offset = 0; offset < _jitc->reglen; offset++) {
3037 if (_jitc->values[offset].kind == kind &&
3038 memcmp(&node->v.w, &_jitc->values[offset].base.w, size) == 0 &&
3039 (jit_class(_rvs[offset].spec) & spec) == spec) {
3040 if (kind == jit_kind_word)
3041 node->code = jit_code_movr;
3042 else if (kind == jit_kind_float32)
3043 node->code = jit_code_movr_f;
3045 node->code = jit_code_movr_d;
3047 jit_memcpy(value, _jitc->values + offset, sizeof(jit_value_t));
3048 ++_jitc->gen[regno];
3054 jit_memcpy(&value->base.w, &node->v.w, size);
3055 ++_jitc->gen[regno];
3060 /* simple/safe redundandy test not checking if another register
3061 * holds the same value
3064 _simplify_ldxi(jit_state_t *_jit, jit_node_t *prev, jit_node_t *node)
3070 regno = jit_regno(node->u.w);
3071 right = jit_regno(node->v.w);
3072 value = _jitc->values + regno;
3073 if (regno != right &&
3074 value->kind == jit_kind_code && value->code == node->code &&
3075 value->base.q.l == right && value->base.q.h == _jitc->gen[right] &&
3076 node->w.w == value->disp.w) {
3077 del_node(prev, node);
3080 value->kind = jit_kind_code;
3081 value->code = node->code;
3082 value->base.q.l = right;
3083 value->base.q.h = _jitc->gen[right];
3084 value->disp.w = node->w.w;
3085 ++_jitc->gen[regno];
3091 _simplify_stxi(jit_state_t *_jit, jit_node_t *prev, jit_node_t *node)
3098 regno = jit_regno(node->w.w);
3099 right = jit_regno(node->v.w);
3100 value = _jitc->values + regno;
3102 /* check for redundant store after load */
3103 if (regno != right &&
3104 value->kind == jit_kind_code && value->code == node->code &&
3105 value->base.q.l == right && value->base.q.h == _jitc->gen[right] &&
3106 node->u.w == value->disp.w) {
3107 del_node(prev, node);
3111 /* assume anything can alias, and invalidate tracked values */
3112 for (offset = 0; offset < _jitc->reglen; offset++) {
3113 if (_jitc->values[offset].kind == jit_kind_code) {
3114 _jitc->values[offset].kind = 0;
3115 ++_jitc->gen[offset];
3119 /* no multiple information, so, if set to a constant,
3120 * prefer to keep that information */
3121 if (value->kind == 0) {
3122 value->kind = jit_kind_code;
3123 switch (node->code) {
3124 /* no information about signed/unsigned either */
3125 case jit_code_stxi_c: value->code = jit_code_ldxi_c; break;
3126 case jit_code_stxi_s: value->code = jit_code_ldxi_s; break;
3127 case jit_code_stxi_i: value->code = jit_code_ldxi_i; break;
3128 case jit_code_stxi_l: value->code = jit_code_ldxi_l; break;
3129 case jit_code_stxi_f: value->code = jit_code_ldxi_f; break;
3130 case jit_code_stxi_d: value->code = jit_code_ldxi_d; break;
3133 value->kind = jit_kind_code;
3134 value->base.q.l = right;
3135 value->base.q.h = _jitc->gen[right];
3136 value->disp.w = node->u.w;
3142 /* usually there should be only one store in the
3143 * jit_get_reg/jit_unget_reg, but properly handle
3144 * multiple ones by moving the save node */
3146 _simplify_spill(jit_state_t *_jit, jit_node_t *node, jit_int32_t regno)
3151 if ((temp = _jitc->spill[regno]) && (save = temp->next) != node) {
3152 temp->next = save->next;
3153 save->next = node->next;
3155 _jitc->spill[regno] = node;
3159 /* checks for simple cases where a register is set more than
3160 * once to the same value, and is a common pattern of calls
3161 * to jit_pushargi and jit_pushargr
3164 _simplify(jit_state_t *_jit)
3172 for (prev = NULL, node = _jitc->head; node; prev = node, node = next) {
3174 switch (node->code) {
3175 case jit_code_label: case jit_code_prolog:
3176 case jit_code_callr: case jit_code_calli:
3178 memset(_jitc->gen, 0, sizeof(jit_int32_t) * _jitc->reglen);
3179 memset(_jitc->values, 0, sizeof(jit_value_t) * _jitc->reglen);
3182 _jitc->spill[jit_regno(node->u.w)] = prev;
3185 regno = jit_regno(node->u.w);
3186 if (register_change_p(node->link->next, node, regno) !=
3188 /* spill not required due to optimizing common
3189 * redundancy case of calling jit_get_reg/jit_unget_reg
3190 * and then setting the register to the value it is
3191 * already holding */
3192 patch_register(node->link->next, node,
3193 jit_regno_patch|regno, regno);
3194 del_node(_jitc->spill[regno], node->link);
3195 del_node(prev, node);
3198 _jitc->spill[regno] = NULL;
3201 regno = jit_regno(node->u.w);
3202 if (simplify_movr(prev, node,
3203 jit_kind_word, sizeof(jit_word_t)))
3204 simplify_spill(node = prev, regno);
3207 regno = jit_regno(node->u.w);
3208 if (simplify_movi(prev, node,
3209 jit_kind_word, sizeof(jit_word_t)))
3210 simplify_spill(node = prev, regno);
3212 case jit_code_movr_f:
3213 regno = jit_regno(node->u.w);
3214 if (simplify_movr(prev, node,
3215 jit_kind_float32, sizeof(jit_float32_t)))
3216 simplify_spill(node = prev, regno);
3218 case jit_code_movi_f:
3219 regno = jit_regno(node->u.w);
3220 if (simplify_movi(prev, node,
3221 jit_kind_float32, sizeof(jit_float32_t)))
3222 simplify_spill(node = prev, regno);
3224 case jit_code_movr_d:
3225 regno = jit_regno(node->u.w);
3226 if (simplify_movr(prev, node,
3227 jit_kind_float64, sizeof(jit_float64_t)))
3228 simplify_spill(node = prev, regno);
3230 case jit_code_movi_d:
3231 regno = jit_regno(node->u.w);
3232 if (simplify_movi(prev, node,
3233 jit_kind_float64, sizeof(jit_float64_t)))
3234 simplify_spill(node = prev, regno);
3236 case jit_code_ldxi_c: case jit_code_ldxi_uc:
3237 case jit_code_ldxi_s: case jit_code_ldxi_us:
3238 case jit_code_ldxi_i: case jit_code_ldxi_ui:
3239 case jit_code_ldxi_l:
3240 case jit_code_ldxi_f: case jit_code_ldxi_d:
3241 regno = jit_regno(node->u.w);
3242 if (simplify_ldxi(prev, node))
3243 simplify_spill(node = prev, regno);
3245 case jit_code_stxi_c: case jit_code_stxi_s:
3246 case jit_code_stxi_i: case jit_code_stxi_l:
3247 case jit_code_stxi_f: case jit_code_stxi_d:
3248 regno = jit_regno(node->u.w);
3249 if (simplify_stxi(prev, node))
3250 simplify_spill(node = prev, regno);
3253 info = jit_classify(node->code);
3254 if (info & jit_cc_a0_jmp)
3255 /* labels are not implicitly added when not taking
3256 * a conditional branch */
3258 if (info & jit_cc_a0_chg) {
3259 if (info & jit_cc_a0_rlh) {
3260 regno = jit_regno(node->u.q.l);
3261 _jitc->values[regno].kind = 0;
3262 ++_jitc->gen[regno];
3263 regno = jit_regno(node->u.q.h);
3264 _jitc->values[regno].kind = 0;
3265 ++_jitc->gen[regno];
3268 regno = jit_regno(node->u.w);
3269 _jitc->values[regno].kind = 0;
3270 ++_jitc->gen[regno];
3273 if (info & jit_cc_a1_chg) {
3274 regno = jit_regno(node->v.w);
3275 _jitc->values[regno].kind = 0;
3276 ++_jitc->gen[regno];
3278 if (info & jit_cc_a2_chg) {
3279 regno = jit_regno(node->w.w);
3280 _jitc->values[regno].kind = 0;
3281 ++_jitc->gen[regno];
3289 _register_change_p(jit_state_t *_jit, jit_node_t *node, jit_node_t *link,
3294 for (; node != link; node = node->next) {
3295 switch (node->code) {
3296 case jit_code_label: case jit_code_prolog:
3297 /* lack of extra information so cannot say it is undefined */
3298 return (jit_reg_change);
3299 case jit_code_callr: case jit_code_calli:
3300 if (!(jit_class(_rvs[regno].spec) & jit_class_sav))
3301 return (jit_reg_undef);
3304 value = jit_classify(node->code);
3305 /* lack of extra information */
3306 if (value & jit_cc_a0_jmp)
3307 return (jit_reg_change);
3308 else if ((value & (jit_cc_a0_reg|jit_cc_a0_chg)) ==
3309 (jit_cc_a0_reg|jit_cc_a0_chg) &&
3310 (((value & jit_cc_a0_rlh) &&
3311 (node->u.q.l == regno || node->u.q.h == regno)) ||
3312 (!(value & jit_cc_a0_rlh) &&
3313 node->u.w == regno)))
3314 return (jit_reg_change);
3315 else if ((value & jit_cc_a1_reg) && node->v.w == regno &&
3316 (value & jit_cc_a1_chg))
3317 return (jit_reg_change);
3318 else if ((value & jit_cc_a2_reg) && node->w.w == regno &&
3319 (value & jit_cc_a2_chg))
3320 return (jit_reg_change);
3324 return (jit_reg_static);
3327 /* most of this could be done at the same time as generating jit, but
3328 * avoid complications on different cpu backends and patch spill/loads
3329 * here, by simulating jit generation */
3331 _spill_reglive_p(jit_state_t *_jit, jit_node_t *node, jit_int32_t regno)
3333 if (!jit_regset_tstbit(&_jitc->reglive, regno)) {
3334 jit_regset_setbit(&_jitc->regmask, regno);
3335 jit_update(node->next, &_jitc->reglive, &_jitc->regmask);
3336 if (!jit_regset_tstbit(&_jitc->reglive, regno) &&
3337 register_change_p(node->next, node->link, regno) != jit_reg_change)
3345 _patch_registers(jit_state_t *_jit)
3355 _jitc->function = NULL;
3357 jit_reglive_setup();
3358 for (prev = NULL, node = _jitc->head; node; node = next) {
3361 info = jit_classify(node->code);
3362 jit_regarg_set(node, info);
3364 switch (node->code) {
3366 regno = jit_regno(node->u.w);
3367 if (!spill_reglive_p(node, regno)) {
3368 /* register is not live, just remove spill/reload */
3369 jit_regarg_clr(node, info);
3370 node->link->v.w = jit_regload_delete;
3371 del_node(prev, node);
3375 /* try to find a free register of the same class */
3376 spec = jit_class(_rvs[regno].spec) & ~jit_class_arg;
3377 for (value = 0; value < _jitc->reglen; value++) {
3378 if (value != regno &&
3379 ((jit_class(_rvs[value].spec) & spec) &
3380 ~jit_class_arg) == spec &&
3381 !jit_regset_tstbit(&_jitc->regarg, value) &&
3382 !spill_reglive_p(node, value))
3385 if (value < _jitc->reglen) {
3386 jit_regarg_clr(node, info);
3387 patch_register(node->next, node->link,
3388 jit_regno_patch|node->u.w,
3389 jit_regno_patch|value);
3390 /* mark as live just in case there are nested
3391 * register patches, so that next patch will
3392 * not want to use the same register */
3393 jit_regset_setbit(&_jitc->reglive, value);
3394 /* register is not live, just remove spill/reload */
3395 node->link->v.w = jit_regload_isdead;
3396 del_node(prev, node);
3400 /* failed to find a free register */
3401 if (spec & jit_class_gpr) {
3402 if (!_jitc->function->regoff[regno])
3403 _jitc->function->regoff[regno] =
3404 jit_allocai(sizeof(jit_word_t));
3405 #if __WORDSIZE == 32
3406 node->code = jit_code_stxi_i;
3408 node->code = jit_code_stxi_l;
3412 node->code = jit_code_stxi_d;
3413 if (!_jitc->function->regoff[regno])
3414 _jitc->function->regoff[regno] =
3415 jit_allocai(sizeof(jit_float64_t));
3417 node->u.w = _jitc->function->regoff[regno];
3425 regno = jit_regno(node->u.w);
3427 if (node->v.w == jit_regload_isdead)
3428 jit_regset_clrbit(&_jitc->reglive, regno);
3429 del_node(prev, node);
3432 spec = jit_class(_rvs[regno].spec);
3433 if (spec & jit_class_gpr) {
3434 #if __WORDSIZE == 32
3435 node->code = jit_code_ldxi_i;
3437 node->code = jit_code_ldxi_l;
3441 node->code = jit_code_ldxi_d;
3444 node->w.w = _jitc->function->regoff[regno];
3447 case jit_code_prolog:
3448 _jitc->function = _jitc->functions.ptr + node->w.w;
3450 case jit_code_epilog:
3451 _jitc->function = NULL;
3457 jit_regarg_clr(node, info);
3458 /* update register live state */
3465 _patch_register(jit_state_t *_jit, jit_node_t *node, jit_node_t *link,
3466 jit_int32_t regno, jit_int32_t patch)
3470 for (; node != link; node = node->next) {
3471 value = jit_classify(node->code);
3472 if (value & jit_cc_a0_reg) {
3473 if (value & jit_cc_a0_rlh) {
3474 if (node->u.q.l == regno)
3475 node->u.q.l = patch;
3476 if (node->u.q.h == regno)
3477 node->u.q.h = patch;
3480 if (node->u.w == regno)
3484 if ((value & jit_cc_a1_reg) && node->v.w == regno)
3486 if ((value & jit_cc_a2_reg) && node->w.w == regno)
3491 #if defined(__i386__) || defined(__x86_64__)
3492 # include "jit_x86.c"
3493 #elif defined(__mips__)
3494 # include "jit_mips.c"
3495 #elif defined(__arm__)
3496 # include "jit_arm.c"
3497 #elif defined(__powerpc__)
3498 # include "jit_ppc.c"
3499 #elif defined(__sparc__)
3500 # include "jit_sparc.c"
3501 #elif defined(__ia64__)
3502 # include "jit_ia64.c"
3503 #elif defined(__hppa__)
3504 # include "jit_hppa.c"
3505 #elif defined(__aarch64__)
3506 # include "jit_aarch64.c"
3507 #elif defined(__s390__) || defined(__s390x__)
3508 # include "jit_s390.c"
3509 #elif defined(__alpha__)
3510 # include "jit_alpha.c"
3511 #elif defined(__riscv)
3512 # include "jit_riscv.c"