2 * Copyright (C) 2012-2023 Free Software Foundation, Inc.
4 * This file is part of GNU lightning.
6 * GNU lightning is free software; you can redistribute it and/or modify it
7 * under the terms of the GNU Lesser General Public License as published
8 * by the Free Software Foundation; either version 3, or (at your option)
11 * GNU lightning is distributed in the hope that it will be useful, but
12 * WITHOUT ANY WARRANTY; without even the implied warranty of MERCHANTABILITY
13 * or FITNESS FOR A PARTICULAR PURPOSE. See the GNU Lesser General Public
14 * License for more details.
17 * Paulo Cesar Pereira de Andrade
20 #if defined(__linux__)
21 # include <sys/cachectl.h>
25 /* callee save + variadic arguments
26 * align16(ra+fp+s[0-7]++f20+f22+f24+f26+f28+f30) + align16(a[0-7]) */
27 # define stack_framesize (128 + 64)
30 * align16(ra+fp+s[0-7]+f16+f18+f20+f22+f24+f26+f28+f30) */
31 # define stack_framesize 128
35 # define NUM_WORD_ARGS 8
37 # define STACK_SHIFT 3
39 # define NUM_WORD_ARGS 4
41 # define STACK_SHIFT 2
43 #if NEW_ABI && __BYTE_ORDER == __BIG_ENDIAN && __WORDSIZE == 32
44 # define WORD_ADJUST 4
46 # define WORD_ADJUST 0
48 #define jit_arg_reg_p(i) ((i) >= 0 && (i) < NUM_WORD_ARGS)
49 #if __BYTE_ORDER == __LITTLE_ENDIAN
54 # define C_DISP STACK_SLOT - sizeof(jit_int8_t)
55 # define S_DISP STACK_SLOT - sizeof(jit_int16_t)
56 # define I_DISP STACK_SLOT - sizeof(jit_int32_t)
62 typedef struct jit_pointer_t jit_va_list_t;
67 #define jit_make_arg(node,code) _jit_make_arg(_jit,node,code)
68 static jit_node_t *_jit_make_arg(jit_state_t*,jit_node_t*,jit_code_t);
69 #define jit_make_arg_f(node) _jit_make_arg_f(_jit,node)
70 static jit_node_t *_jit_make_arg_f(jit_state_t*,jit_node_t*);
71 #define jit_make_arg_d(node) _jit_make_arg_d(_jit,node)
72 static jit_node_t *_jit_make_arg_d(jit_state_t*,jit_node_t*);
73 #define compute_framesize() _compute_framesize(_jit)
74 static void _compute_framesize(jit_state_t*);
75 #define patch(instr, node) _patch(_jit, instr, node)
76 static void _patch(jit_state_t*,jit_word_t,jit_node_t*);
79 # include "jit_rewind.c"
80 # include "jit_mips-cpu.c"
81 # include "jit_mips-fpu.c"
82 # include "jit_fallback.c"
89 jit_register_t _rvs[] = {
90 { rc(gpr) | 0x01, "at" },
91 { rc(gpr) | 0x02, "v0" },
92 { rc(gpr) | 0x03, "v1" },
94 { rc(gpr) | 0x08, "t0" },
95 { rc(gpr) | 0x09, "t1" },
96 { rc(gpr) | 0x0a, "t2" },
97 { rc(gpr) | 0x0b, "t3" },
99 { rc(gpr) | 0x0c, "t4" },
100 { rc(gpr) | 0x0d, "t5" },
101 { rc(gpr) | 0x0e, "t6" },
102 { rc(gpr) | 0x0f, "t7" },
103 { rc(gpr) | 0x18, "t8" },
104 { rc(gpr) | 0x19, "t9" },
105 { rc(sav) | rc(gpr) | 0x10, "s0" },
106 { rc(sav) | rc(gpr) | 0x11, "s1" },
107 { rc(sav) | rc(gpr) | 0x12, "s2" },
108 { rc(sav) | rc(gpr) | 0x13, "s3" },
109 { rc(sav) | rc(gpr) | 0x14, "s4" },
110 { rc(sav) | rc(gpr) | 0x15, "s5" },
111 { rc(sav) | rc(gpr) | 0x16, "s6" },
112 { rc(sav) | rc(gpr) | 0x17, "s7" },
116 { rc(sav) | 0x1f, "ra" },
117 { rc(sav) | 0x1c, "gp" },
118 { rc(sav) | 0x1d, "sp" },
119 { rc(sav) | 0x1e, "fp" },
121 { rc(gpr) | 0x0b, "a7" },
122 { rc(gpr) | 0x0a, "a6" },
123 { rc(gpr) | 0x09, "a5" },
124 { rc(gpr) | 0x08, "a4" },
126 { rc(arg) | rc(gpr) | 0x07, "a3" },
127 { rc(arg) | rc(gpr) | 0x06, "a2" },
128 { rc(arg) | rc(gpr) | 0x05, "a1" },
129 { rc(arg) | rc(gpr) | 0x04, "a0" },
130 { rc(fpr) | 0x00, "$f0" },
131 { rc(fpr) | 0x02, "$f2" },
132 { rc(fpr) | 0x04, "$f4" },
133 { rc(fpr) | 0x06, "$f6" },
134 { rc(fpr) | 0x08, "$f8" },
135 { rc(fpr) | 0x0a, "$f10" },
137 { rc(sav) | rc(fpr) | 0x10, "$f16" },
138 { rc(sav) | rc(fpr) | 0x12, "$f18" },
140 { rc(sav) | rc(fpr) | 0x14, "$f20" },
141 { rc(sav) | rc(fpr) | 0x16, "$f22" },
142 { rc(sav) | rc(fpr) | 0x18, "$f24" },
143 { rc(sav) | rc(fpr) | 0x1a, "$f26" },
144 { rc(sav) | rc(fpr) | 0x1c, "$f28" },
145 { rc(sav) | rc(fpr) | 0x1e, "$f30" },
147 { rc(arg) | rc(fpr) | 0x13, "$f19" },
148 { rc(arg) | rc(fpr) | 0x12, "$f18" },
149 { rc(arg) | rc(fpr) | 0x11, "$f17" },
150 { rc(arg) | rc(fpr) | 0x10, "$f16" },
151 { rc(arg) | rc(fpr) | 0x0f, "$f15" },
152 { rc(arg) | rc(fpr) | 0x0e, "$f14" },
153 { rc(arg) | rc(fpr) | 0x0d, "$f13" },
154 { rc(arg) | rc(fpr) | 0x0c, "$f12" },
156 { rc(arg) | rc(fpr) | 0x0e, "$f14" },
157 { rc(arg) | rc(fpr) | 0x0c, "$f12" },
159 { _NOREG, "<none>" },
162 static jit_int32_t iregs[] = {
163 _S0, _S1, _S2, _S3, _S4, _S5, _S6, _S7
166 static jit_int32_t fregs[] = {
170 _F20, _F22, _F24, _F26, _F28, _F30
179 #if defined(__linux__)
184 if ((fp = fopen("/proc/cpuinfo", "r")) != NULL) {
185 while (fgets(buf, sizeof(buf), fp)) {
186 if (strncmp(buf, "isa : ", 8) == 0) {
187 if ((ptr = strstr(buf + 9, "mips64r")))
188 jit_cpu.release = strtoul(ptr + 7, NULL, 10);
196 if (!jit_cpu.release)
197 jit_cpu.release = __mips_isa_rev;
198 #elif defined _MIPS_ARCH
199 if (!jit_cpu.release)
200 jit_cpu.release = strtoul(&_MIPS_ARCH[4], NULL, 10);
201 #elif defined(__mips) && __mips < 6
202 if (!jit_cpu.release)
203 jit_cpu.release = __mips;
208 _jit_init(jit_state_t *_jit)
210 _jitc->reglen = jit_size(_rvs) - 1;
212 * o reserve a register for carry (overkill)
213 * o use MTLO/MFLO (performance hit)
214 * So, keep a register allocated after setting carry, and implicitly
215 * deallocate it if it can no longer be tracked
221 _jit_prolog(jit_state_t *_jit)
227 assert(jit_regset_cmp_ui(&_jitc->regarg, 0) == 0);
228 jit_regset_set_ui(&_jitc->regsav, 0);
229 offset = _jitc->functions.offset;
230 if (offset >= _jitc->functions.length) {
231 jit_realloc((jit_pointer_t *)&_jitc->functions.ptr,
232 _jitc->functions.length * sizeof(jit_function_t),
233 (_jitc->functions.length + 16) * sizeof(jit_function_t));
234 _jitc->functions.length += 16;
236 _jitc->function = _jitc->functions.ptr + _jitc->functions.offset++;
237 _jitc->function->self.size = stack_framesize;
238 _jitc->function->self.argi = _jitc->function->self.argf =
239 _jitc->function->self.aoff = _jitc->function->self.alen = 0;
240 _jitc->function->self.call = jit_call_default;
241 jit_alloc((jit_pointer_t *)&_jitc->function->regoff,
242 _jitc->reglen * sizeof(jit_int32_t));
244 /* _no_link here does not mean the jit_link() call can be removed
246 * _jitc->function->prolog = jit_new_node(jit_code_prolog);
248 _jitc->function->prolog = jit_new_node_no_link(jit_code_prolog);
249 jit_link(_jitc->function->prolog);
250 _jitc->function->prolog->w.w = offset;
251 _jitc->function->epilog = jit_new_node_no_link(jit_code_epilog);
253 * v: offset in blocks vector
254 * w: offset in functions vector
256 _jitc->function->epilog->w.w = offset;
258 jit_regset_new(&_jitc->function->regset);
262 _jit_allocai(jit_state_t *_jit, jit_int32_t length)
264 assert(_jitc->function);
267 case 0: case 1: break;
268 case 2: _jitc->function->self.aoff &= -2; break;
269 case 3: case 4: _jitc->function->self.aoff &= -4; break;
270 default: _jitc->function->self.aoff &= -8; break;
272 _jitc->function->self.aoff -= length;
273 if (!_jitc->realize) {
274 jit_inc_synth_ww(allocai, _jitc->function->self.aoff, length);
277 return (_jitc->function->self.aoff);
281 _jit_allocar(jit_state_t *_jit, jit_int32_t u, jit_int32_t v)
284 assert(_jitc->function);
285 jit_inc_synth_ww(allocar, u, v);
286 if (!_jitc->function->allocar) {
287 _jitc->function->aoffoff = jit_allocai(sizeof(jit_int32_t));
288 _jitc->function->allocar = 1;
290 reg = jit_get_reg(jit_class_gpr);
292 jit_andi(reg, reg, -8);
293 jit_ldxi_i(u, JIT_FP, _jitc->function->aoffoff);
295 jit_addr(JIT_SP, JIT_SP, reg);
296 jit_stxi_i(_jitc->function->aoffoff, JIT_FP, u);
302 _jit_ret(jit_state_t *_jit)
305 assert(_jitc->function);
309 jit_patch_at(instr, _jitc->function->epilog);
314 _jit_retr(jit_state_t *_jit, jit_int32_t u, jit_code_t code)
316 jit_code_inc_synth_w(code, u);
317 jit_movr(JIT_RET, u);
323 _jit_reti(jit_state_t *_jit, jit_word_t u, jit_code_t code)
325 jit_code_inc_synth_w(code, u);
326 jit_movi(JIT_RET, u);
332 _jit_retr_f(jit_state_t *_jit, jit_int32_t u)
334 jit_inc_synth_w(retr_f, u);
336 jit_movr_f(JIT_FRET, u);
344 _jit_reti_f(jit_state_t *_jit, jit_float32_t u)
346 jit_inc_synth_f(reti_f, u);
347 jit_movi_f(JIT_FRET, u);
353 _jit_retr_d(jit_state_t *_jit, jit_int32_t u)
355 jit_inc_synth_w(retr_d, u);
357 jit_movr_d(JIT_FRET, u);
365 _jit_reti_d(jit_state_t *_jit, jit_float64_t u)
367 jit_inc_synth_d(reti_d, u);
368 jit_movi_d(JIT_FRET, u);
374 _jit_epilog(jit_state_t *_jit)
376 assert(_jitc->function);
377 assert(_jitc->function->epilog->next == NULL);
378 jit_link(_jitc->function->epilog);
379 _jitc->function = NULL;
383 _jit_arg_register_p(jit_state_t *_jit, jit_node_t *u)
385 if (u->code >= jit_code_arg_c && u->code <= jit_code_arg)
386 return (jit_arg_reg_p(u->u.w));
387 assert(u->code == jit_code_arg_f || u->code == jit_code_arg_d);
389 return (jit_arg_reg_p(u->u.w) || jit_arg_reg_p(u->u.w - 8));
396 _jit_make_arg(jit_state_t *_jit, jit_node_t *node, jit_code_t code)
400 if (jit_arg_reg_p(_jitc->function->self.argi))
401 offset = _jitc->function->self.argi++;
403 offset = _jitc->function->self.size;
404 _jitc->function->self.size += STACK_SLOT;
407 offset = (_jitc->function->self.size - stack_framesize) >> STACK_SHIFT;
408 ++_jitc->function->self.argi;
410 offset = _jitc->function->self.size;
411 _jitc->function->self.size += STACK_SLOT;
413 if (node == (jit_node_t *)0)
414 node = jit_new_node(code);
418 node->v.w = ++_jitc->function->self.argn;
424 _jit_make_arg_f(jit_state_t *_jit, jit_node_t *node)
428 if (jit_arg_reg_p(_jitc->function->self.argi)) {
429 offset = _jitc->function->self.argi++;
430 if (_jitc->function->self.call & jit_call_varargs)
434 offset = _jitc->function->self.size;
435 _jitc->function->self.size += STACK_SLOT;
438 offset = (_jitc->function->self.size - stack_framesize) >> STACK_SHIFT;
439 if (offset < NUM_WORD_ARGS) {
440 if (!_jitc->function->self.argi &&
441 !(_jitc->function->self.call & jit_call_varargs)) {
446 _jitc->function->self.argi = 1;
448 /* Use as flag to rewind in case of varargs function */
449 ++_jitc->function->self.argf;
453 offset = _jitc->function->self.size;
454 _jitc->function->self.size += STACK_SLOT;
456 if (node == (jit_node_t *)0)
457 node = jit_new_node(jit_code_arg_f);
461 node->v.w = ++_jitc->function->self.argn;
467 _jit_make_arg_d(jit_state_t *_jit, jit_node_t *node)
471 if (jit_arg_reg_p(_jitc->function->self.argi)) {
472 offset = _jitc->function->self.argi++;
473 if (_jitc->function->self.call & jit_call_varargs)
477 offset = _jitc->function->self.size;
478 _jitc->function->self.size += STACK_SLOT;
481 if (_jitc->function->self.size & 7) {
482 _jitc->function->self.size += 4;
483 _jitc->function->self.argi = 1;
485 offset = (_jitc->function->self.size - stack_framesize) >> STACK_SHIFT;
486 if (offset < NUM_WORD_ARGS) {
487 if (!_jitc->function->self.argi &&
488 !(_jitc->function->self.call & jit_call_varargs)) {
490 /* Use as flag to rewind in case of varargs function */
491 ++_jitc->function->self.argf;
495 offset = _jitc->function->self.size;
496 _jitc->function->self.size += sizeof(jit_float64_t);
498 if (node == (jit_node_t *)0)
499 node = jit_new_node(jit_code_arg_d);
503 node->v.w = ++_jitc->function->self.argn;
509 _jit_ellipsis(jit_state_t *_jit)
511 if (_jitc->prepare) {
512 assert(!(_jitc->function->call.call & jit_call_varargs));
513 _jitc->function->call.call |= jit_call_varargs;
515 if (_jitc->function->call.argf)
520 assert(!(_jitc->function->self.call & jit_call_varargs));
522 if (jit_arg_reg_p(_jitc->function->self.argi))
524 /* Do not set during possible rewind. */
525 _jitc->function->self.call |= jit_call_varargs;
527 _jitc->function->self.call |= jit_call_varargs;
528 if (_jitc->function->self.argf)
531 _jitc->function->vagp = _jitc->function->self.argi;
533 jit_inc_synth(ellipsis);
543 _jit_va_push(jit_state_t *_jit, jit_int32_t u)
545 jit_inc_synth_w(va_push, u);
551 _jit_arg(jit_state_t *_jit, jit_code_t code)
553 assert(_jitc->function);
554 assert(!(_jitc->function->self.call & jit_call_varargs));
555 #if STRONG_TYPE_CHECKING
556 assert(code >= jit_code_arg_c && code <= jit_code_arg);
558 return (jit_make_arg((jit_node_t*)0, code));
562 _jit_arg_f(jit_state_t *_jit)
564 assert(_jitc->function);
565 return (jit_make_arg_f((jit_node_t*)0));
569 _jit_arg_d(jit_state_t *_jit)
571 assert(_jitc->function);
572 return (jit_make_arg_d((jit_node_t*)0));
576 _jit_getarg_c(jit_state_t *_jit, jit_int32_t u, jit_node_t *v)
578 assert_arg_type(v->code, jit_code_arg_c);
579 jit_inc_synth_wp(getarg_c, u, v);
580 if (jit_arg_reg_p(v->u.w))
581 jit_extr_c(u, _A0 - v->u.w);
583 jit_node_t *node = jit_ldxi_c(u, _FP, v->u.w + C_DISP);
584 jit_link_alist(node);
591 _jit_getarg_uc(jit_state_t *_jit, jit_int32_t u, jit_node_t *v)
593 assert_arg_type(v->code, jit_code_arg_c);
594 jit_inc_synth_wp(getarg_uc, u, v);
595 if (jit_arg_reg_p(v->u.w))
596 jit_extr_uc(u, _A0 - v->u.w);
598 jit_node_t *node = jit_ldxi_uc(u, _FP, v->u.w + C_DISP);
599 jit_link_alist(node);
606 _jit_getarg_s(jit_state_t *_jit, jit_int32_t u, jit_node_t *v)
608 assert_arg_type(v->code, jit_code_arg_s);
609 jit_inc_synth_wp(getarg_s, u, v);
610 if (jit_arg_reg_p(v->u.w))
611 jit_extr_s(u, _A0 - v->u.w);
613 jit_node_t *node = jit_ldxi_s(u, _FP, v->u.w + S_DISP);
614 jit_link_alist(node);
621 _jit_getarg_us(jit_state_t *_jit, jit_int32_t u, jit_node_t *v)
623 assert_arg_type(v->code, jit_code_arg_s);
624 jit_inc_synth_wp(getarg_us, u, v);
625 if (jit_arg_reg_p(v->u.w))
626 jit_extr_us(u, _A0 - v->u.w);
628 jit_node_t *node = jit_ldxi_us(u, _FP, v->u.w + S_DISP);
629 jit_link_alist(node);
636 _jit_getarg_i(jit_state_t *_jit, jit_int32_t u, jit_node_t *v)
638 assert_arg_type(v->code, jit_code_arg_i);
639 jit_inc_synth_wp(getarg_i, u, v);
640 if (jit_arg_reg_p(v->u.w)) {
642 jit_extr_i(u, _A0 - v->u.w);
644 jit_movr(u, _A0 - v->u.w);
648 jit_node_t *node = jit_ldxi_i(u, _FP, v->u.w + I_DISP);
649 jit_link_alist(node);
657 _jit_getarg_ui(jit_state_t *_jit, jit_int32_t u, jit_node_t *v)
659 assert_arg_type(v->code, jit_code_arg_i);
660 jit_inc_synth_wp(getarg_ui, u, v);
661 if (jit_arg_reg_p(v->u.w))
662 jit_extr_ui(u, _A0 - v->u.w);
664 jit_node_t *node = jit_ldxi_ui(u, _FP, v->u.w + I_DISP);
665 jit_link_alist(node);
672 _jit_getarg_l(jit_state_t *_jit, jit_int32_t u, jit_node_t *v)
674 assert_arg_type(v->code, jit_code_arg_l);
675 jit_inc_synth_wp(getarg_l, u, v);
676 if (jit_arg_reg_p(v->u.w))
677 jit_movr(u, _A0 - v->u.w);
679 jit_node_t *node = jit_ldxi_l(u, _FP, v->u.w);
680 jit_link_alist(node);
688 _jit_putargr(jit_state_t *_jit, jit_int32_t u, jit_node_t *v, jit_code_t code)
690 assert_putarg_type(code, v->code);
691 jit_code_inc_synth_wp(code, u, v);
692 if (jit_arg_reg_p(v->u.w))
693 jit_movr(_A0 - v->u.w, u);
695 jit_node_t *node = jit_stxi(v->u.w + WORD_ADJUST, _FP, u);
696 jit_link_alist(node);
703 _jit_putargi(jit_state_t *_jit, jit_word_t u, jit_node_t *v, jit_code_t code)
706 assert_putarg_type(code, v->code);
707 jit_code_inc_synth_wp(code, u, v);
708 if (jit_arg_reg_p(v->u.w))
709 jit_movi(_A0 - v->u.w, u);
712 regno = jit_get_reg(jit_class_gpr);
714 node = jit_stxi(v->u.w + WORD_ADJUST, _FP, regno);
715 jit_link_alist(node);
717 jit_unget_reg(regno);
723 _jit_getarg_f(jit_state_t *_jit, jit_int32_t u, jit_node_t *v)
725 assert(v->code == jit_code_arg_f);
726 jit_inc_synth_wp(getarg_f, u, v);
728 if (jit_arg_reg_p(v->u.w))
729 jit_movr_f(u, _F12 - v->u.w);
730 else if (jit_arg_reg_p(v->u.w - 8))
731 jit_movr_w_f(u, _A0 - (v->u.w - 8));
734 jit_movr_w_f(u, _A0 - v->u.w);
736 jit_movr_f(u, _F12 - ((v->u.w - 4) >> 1));
739 jit_node_t *node = jit_ldxi_f(u, _FP, v->u.w);
740 jit_link_alist(node);
747 _jit_putargr_f(jit_state_t *_jit, jit_int32_t u, jit_node_t *v)
749 assert(v->code == jit_code_arg_f);
750 jit_inc_synth_wp(putargr_f, u, v);
752 if (jit_arg_reg_p(v->u.w))
753 jit_movr_f(_F12 - v->u.w, u);
754 else if (jit_arg_reg_p(v->u.w - 8))
755 jit_movr_f_w(_A0 - (v->u.w - 8), u);
758 jit_movr_f_w(_A0 - v->u.w, u);
760 jit_movr_f(_F12 - ((v->u.w - 4) >> 1), u);
763 jit_node_t *node = jit_stxi_f(v->u.w, _FP, u);
764 jit_link_alist(node);
771 _jit_putargi_f(jit_state_t *_jit, jit_float32_t u, jit_node_t *v)
774 assert(v->code == jit_code_arg_f);
775 jit_inc_synth_fp(putargi_f, u, v);
777 if (jit_arg_reg_p(v->u.w))
778 jit_movi_f(_F12 - v->u.w, u);
779 else if (jit_arg_reg_p(v->u.w - 8))
780 jit_movi_f_w(_A0 - (v->u.w - 8), u);
783 regno = jit_get_reg(jit_class_fpr);
784 jit_movi_f(regno, u);
785 jit_movr_f_w(_A0 - ((v->u.w - 4) >> 1), regno);
786 jit_unget_reg(regno);
789 jit_movi_f(_F12 - ((v->u.w - 4) >> 1), u);
793 regno = jit_get_reg(jit_class_fpr);
794 jit_movi_f(regno, u);
795 node = jit_stxi_f(v->u.w, _FP, regno);
796 jit_link_alist(node);
798 jit_unget_reg(regno);
804 _jit_getarg_d(jit_state_t *_jit, jit_int32_t u, jit_node_t *v)
806 assert(v->code == jit_code_arg_d);
807 jit_inc_synth_wp(getarg_d, u, v);
809 if (jit_arg_reg_p(v->u.w))
810 jit_movr_d(u, _F12 - v->u.w);
811 else if (jit_arg_reg_p(v->u.w - 8))
812 jit_movr_d_w(_A0 - (v->u.w - 8), u);
815 jit_movr_ww_d(u, _A0 - v->u.w, _A0 - (v->u.w + 1));
817 jit_movr_d(u, _F12 - ((v->u.w - 4) >> 1));
820 jit_node_t *node = jit_ldxi_d(u, _FP, v->u.w);
821 jit_link_alist(node);
828 _jit_putargr_d(jit_state_t *_jit, jit_int32_t u, jit_node_t *v)
830 assert(v->code == jit_code_arg_d);
831 jit_inc_synth_wp(putargr_d, u, v);
833 if (jit_arg_reg_p(v->u.w))
834 jit_movr_d(_F12 - v->u.w, u);
835 else if (jit_arg_reg_p(v->u.w - 8))
836 jit_movr_d_w(_A0 - (v->u.w - 8), u);
839 jit_movr_d_ww(_A0 - v->u.w, _A0 - (v->u.w + 1), u);
841 jit_movr_d(_F12 - ((v->u.w - 4) >> 1), u);
844 jit_node_t *node = jit_stxi_d(v->u.w, _FP, u);
845 jit_link_alist(node);
852 _jit_putargi_d(jit_state_t *_jit, jit_float64_t u, jit_node_t *v)
855 assert(v->code == jit_code_arg_d);
856 jit_inc_synth_dp(putargi_d, u, v);
858 if (jit_arg_reg_p(v->u.w))
859 jit_movi_d(_F12 - v->u.w, u);
860 else if (jit_arg_reg_p(v->u.w - 8))
861 jit_movi_d_w(_A0 - (v->u.w - 8), u);
864 regno = jit_get_reg(jit_class_fpr);
865 jit_movi_d(regno, u);
866 jit_movr_d_ww(_A0 - v->u.w, _A0 - (v->u.w + 1), regno);
867 jit_unget_reg(regno);
870 jit_movi_d(_F12 - ((v->u.w - 4) >> 1), u);
874 regno = jit_get_reg(jit_class_fpr);
875 jit_movi_d(regno, u);
876 node = jit_stxi_d(v->u.w, _FP, regno);
877 jit_link_alist(node);
879 jit_unget_reg(regno);
885 _jit_pushargr(jit_state_t *_jit, jit_int32_t u, jit_code_t code)
887 jit_code_inc_synth_w(code, u);
890 assert(_jitc->function);
891 if (jit_arg_reg_p(_jitc->function->call.argi)) {
892 jit_movr(_A0 - _jitc->function->call.argi, u);
893 ++_jitc->function->call.argi;
897 jit_stxi(_jitc->function->call.size + WORD_ADJUST, JIT_SP, u);
898 _jitc->function->call.size += STACK_SLOT;
902 assert(_jitc->function);
903 offset = _jitc->function->call.size >> STACK_SHIFT;
904 ++_jitc->function->call.argi;
905 if (jit_arg_reg_p(offset))
906 jit_movr(_A0 - offset, u);
909 jit_stxi(_jitc->function->call.size, JIT_SP, u);
911 _jitc->function->call.size += STACK_SLOT;
917 _jit_pushargi(jit_state_t *_jit, jit_word_t u, jit_code_t code)
923 assert(_jitc->function);
924 jit_code_inc_synth_w(code, u);
927 if (jit_arg_reg_p(_jitc->function->call.argi)) {
928 jit_movi(_A0 - _jitc->function->call.argi, u);
929 ++_jitc->function->call.argi;
933 regno = jit_get_reg(jit_class_gpr);
935 jit_stxi(_jitc->function->call.size + WORD_ADJUST, JIT_SP, regno);
936 _jitc->function->call.size += STACK_SLOT;
937 jit_unget_reg(regno);
940 offset = _jitc->function->call.size >> STACK_SHIFT;
941 ++_jitc->function->call.argi;
942 if (jit_arg_reg_p(offset))
943 jit_movi(_A0 - offset, u);
946 regno = jit_get_reg(jit_class_gpr);
948 jit_stxi(_jitc->function->call.size, JIT_SP, regno);
949 jit_unget_reg(regno);
951 _jitc->function->call.size += STACK_SLOT;
957 _jit_pushargr_f(jit_state_t *_jit, jit_int32_t u)
962 assert(_jitc->function);
963 jit_inc_synth_w(pushargr_f, u);
966 if (jit_arg_reg_p(_jitc->function->call.argi)) {
967 if (!(_jitc->function->call.call & jit_call_varargs))
968 jit_movr_f(_F12 - _jitc->function->call.argi, u);
970 jit_movr_f_w(_A0 - _jitc->function->call.argi, u);
971 ++_jitc->function->call.argi;
975 jit_stxi_f(_jitc->function->call.size, JIT_SP, u);
976 _jitc->function->call.size += STACK_SLOT;
979 offset = _jitc->function->call.size >> STACK_SHIFT;
980 if (offset < 2 && !_jitc->function->call.argi &&
981 !(_jitc->function->call.call & jit_call_varargs)) {
982 ++_jitc->function->call.argf;
983 jit_movr_f(_F12 - offset, u);
985 else if (offset < 4) {
986 ++_jitc->function->call.argi;
987 jit_movr_f_w(_A0 - offset, u);
991 jit_stxi_f(_jitc->function->call.size, JIT_SP, u);
993 _jitc->function->call.size += STACK_SLOT;
999 _jit_pushargi_f(jit_state_t *_jit, jit_float32_t u)
1005 assert(_jitc->function);
1006 jit_inc_synth_f(pushargi_f, u);
1009 if (jit_arg_reg_p(_jitc->function->call.argi)) {
1010 if (!(_jitc->function->call.call & jit_call_varargs))
1011 jit_movi_f(_F12 - _jitc->function->call.argi, u);
1013 jit_movi_f_w(_A0 - _jitc->function->call.argi, u);
1014 ++_jitc->function->call.argi;
1018 regno = jit_get_reg(jit_class_fpr);
1019 jit_movi_f(regno, u);
1020 jit_stxi_f(_jitc->function->call.size, JIT_SP, regno);
1021 _jitc->function->call.size += STACK_SLOT;
1022 jit_unget_reg(regno);
1025 offset = _jitc->function->call.size >> STACK_SHIFT;
1026 if (offset < 2 && !_jitc->function->call.argi &&
1027 !(_jitc->function->call.call & jit_call_varargs)) {
1028 ++_jitc->function->call.argf;
1029 jit_movi_f(_F12 - offset, u);
1031 else if (offset < 4) {
1032 ++_jitc->function->call.argi;
1033 jit_movi_f_w(_A0 - offset, u);
1037 regno = jit_get_reg(jit_class_fpr);
1038 jit_movi_f(regno, u);
1039 jit_stxi_f(_jitc->function->call.size, JIT_SP, regno);
1040 jit_unget_reg(regno);
1042 _jitc->function->call.size += STACK_SLOT;
1048 _jit_pushargr_d(jit_state_t *_jit, jit_int32_t u)
1054 assert(_jitc->function);
1055 jit_inc_synth_w(pushargr_d, u);
1058 if (jit_arg_reg_p(_jitc->function->call.argi)) {
1059 if (!(_jitc->function->call.call & jit_call_varargs))
1060 jit_movr_d(_F12 - _jitc->function->call.argi, u);
1062 jit_movr_d_w(_A0 - _jitc->function->call.argi, u);
1063 ++_jitc->function->call.argi;
1067 jit_stxi_d(_jitc->function->call.size, JIT_SP, u);
1068 _jitc->function->call.size += STACK_SLOT;
1071 adjust = !!_jitc->function->call.argi;
1072 if (_jitc->function->call.size & 7) {
1073 _jitc->function->call.size += 4;
1076 offset = _jitc->function->call.size >> STACK_SHIFT;
1078 if (adjust || (_jitc->function->call.call & jit_call_varargs)) {
1079 jit_movr_d_ww(_A0 - offset, _A0 - (offset + 1), u);
1080 _jitc->function->call.argi += 2;
1083 jit_movr_d(_F12 - (offset >> 1), u);
1084 ++_jitc->function->call.argf;
1089 jit_stxi_d(_jitc->function->call.size, JIT_SP, u);
1091 _jitc->function->call.size += sizeof(jit_float64_t);
1097 _jit_pushargi_d(jit_state_t *_jit, jit_float64_t u)
1104 assert(_jitc->function);
1105 jit_inc_synth_d(pushargi_d, u);
1108 if (jit_arg_reg_p(_jitc->function->call.argi)) {
1109 if (!(_jitc->function->call.call & jit_call_varargs))
1110 jit_movi_d(_F12 - _jitc->function->call.argi, u);
1112 jit_movi_d_w(_A0 - _jitc->function->call.argi, u);
1113 ++_jitc->function->call.argi;
1117 regno = jit_get_reg(jit_class_fpr);
1118 jit_movi_d(regno, u);
1119 jit_stxi_d(_jitc->function->call.size, JIT_SP, regno);
1120 _jitc->function->call.size += STACK_SLOT;
1121 jit_unget_reg(regno);
1124 adjust = !!_jitc->function->call.argi;
1125 if (_jitc->function->call.size & 7) {
1126 _jitc->function->call.size += 4;
1129 offset = _jitc->function->call.size >> STACK_SHIFT;
1131 if (adjust || (_jitc->function->call.call & jit_call_varargs)) {
1132 jit_movi_d_ww(_A0 - offset, _A0 - (offset + 1), u);
1133 _jitc->function->call.argi += 2;
1136 jit_movi_d(_F12 - (offset >> 1), u);
1137 ++_jitc->function->call.argf;
1142 regno = jit_get_reg(jit_class_fpr);
1143 jit_movi_d(regno, u);
1144 jit_stxi_d(_jitc->function->call.size, JIT_SP, regno);
1145 jit_unget_reg(regno);
1147 _jitc->function->call.size += sizeof(jit_float64_t);
1153 _jit_regarg_p(jit_state_t *_jit, jit_node_t *node, jit_int32_t regno)
1157 spec = jit_class(_rvs[regno].spec);
1158 if (spec & jit_class_arg) {
1159 if (spec & jit_class_gpr) {
1160 regno = _A0 - regno;
1161 if (regno >= 0 && regno < node->v.w)
1164 else if (spec & jit_class_fpr) {
1165 regno = _F12 - regno;
1166 if (regno >= 0 && regno < node->w.w)
1175 _jit_finishr(jit_state_t *_jit, jit_int32_t r0)
1178 assert(_jitc->function);
1180 jit_inc_synth_w(finishr, r0);
1181 if (_jitc->function->self.alen < _jitc->function->call.size)
1182 _jitc->function->self.alen = _jitc->function->call.size;
1183 call = jit_callr(r0);
1184 call->v.w = _jitc->function->self.argi;
1186 call->w.w = call->v.w;
1188 call->w.w = _jitc->function->self.argf;
1190 _jitc->function->call.argi = _jitc->function->call.argf =
1191 _jitc->function->call.size = 0;
1197 _jit_finishi(jit_state_t *_jit, jit_pointer_t i0)
1200 assert(_jitc->function);
1202 jit_inc_synth_w(finishi, (jit_word_t)i0);
1203 if (_jitc->function->self.alen < _jitc->function->call.size)
1204 _jitc->function->self.alen = _jitc->function->call.size;
1205 call = jit_calli(i0);
1206 call->v.w = _jitc->function->call.argi;
1208 call->w.w = call->v.w;
1210 call->w.w = _jitc->function->call.argf;
1212 _jitc->function->call.argi = _jitc->function->call.argf =
1213 _jitc->function->call.size = 0;
1220 _jit_retval_c(jit_state_t *_jit, jit_int32_t r0)
1222 jit_extr_c(r0, JIT_RET);
1226 _jit_retval_uc(jit_state_t *_jit, jit_int32_t r0)
1228 jit_extr_uc(r0, JIT_RET);
1232 _jit_retval_s(jit_state_t *_jit, jit_int32_t r0)
1234 jit_extr_s(r0, JIT_RET);
1238 _jit_retval_us(jit_state_t *_jit, jit_int32_t r0)
1240 jit_extr_us(r0, JIT_RET);
1244 _jit_retval_i(jit_state_t *_jit, jit_int32_t r0)
1246 #if __WORDSIZE == 32
1248 jit_movr(r0, JIT_RET);
1250 jit_extr_i(r0, JIT_RET);
1254 #if __WORDSIZE == 64
1256 _jit_retval_ui(jit_state_t *_jit, jit_int32_t r0)
1258 jit_extr_ui(r0, JIT_RET);
1262 _jit_retval_l(jit_state_t *_jit, jit_int32_t r0)
1265 jit_movr(r0, JIT_RET);
1270 _jit_retval_f(jit_state_t *_jit, jit_int32_t r0)
1273 jit_movr_f(r0, JIT_FRET);
1277 _jit_retval_d(jit_state_t *_jit, jit_int32_t r0)
1280 jit_movr_d(r0, JIT_FRET);
1284 _emit_code(jit_state_t *_jit)
1295 jit_function_t func;
1296 #if DEVEL_DISASSEMBLER
1299 jit_int32_t patch_offset;
1301 #if DEVEL_DISASSEMBLER
1305 _jitc->function = NULL;
1307 jit_reglive_setup();
1311 undo.patch_offset = 0;
1312 #define case_rr(name, type) \
1313 case jit_code_##name##r##type: \
1314 name##r##type(rn(node->u.w), rn(node->v.w)); \
1316 #define case_rw(name, type) \
1317 case jit_code_##name##i##type: \
1318 name##i##type(rn(node->u.w), node->v.w); \
1320 #define case_wr(name, type) \
1321 case jit_code_##name##i##type: \
1322 name##i##type(node->u.w, rn(node->v.w)); \
1324 #define case_rrr(name, type) \
1325 case jit_code_##name##r##type: \
1326 name##r##type(rn(node->u.w), \
1327 rn(node->v.w), rn(node->w.w)); \
1329 #define case_rrw(name, type) \
1330 case jit_code_##name##i##type: \
1331 name##i##type(rn(node->u.w), rn(node->v.w), node->w.w); \
1333 #define case_rrrr(name, type) \
1334 case jit_code_##name##r##type: \
1335 name##r##type(rn(node->u.q.l), rn(node->u.q.h), \
1336 rn(node->v.w), rn(node->w.w)); \
1338 #define case_rrrw(name, type) \
1339 case jit_code_##name##i##type: \
1340 name##i##type(rn(node->u.q.l), rn(node->u.q.h), \
1341 rn(node->v.w), node->w.w); \
1343 #define case_rrf(name, type, size) \
1344 case jit_code_##name##i##type: \
1345 assert(node->flag & jit_flag_data); \
1346 name##i##type(rn(node->u.w), rn(node->v.w), \
1347 (jit_float##size##_t *)node->w.n->u.w); \
1349 #define case_wrr(name, type) \
1350 case jit_code_##name##i##type: \
1351 name##i##type(node->u.w, rn(node->v.w), rn(node->w.w)); \
1353 #define case_brr(name, type) \
1354 case jit_code_##name##r##type: \
1356 assert(temp->code == jit_code_label || \
1357 temp->code == jit_code_epilog); \
1358 if (temp->flag & jit_flag_patch) \
1359 name##r##type(temp->u.w, rn(node->v.w), \
1362 word = name##r##type(_jit->pc.w, \
1363 rn(node->v.w), rn(node->w.w)); \
1364 patch(word, node); \
1367 #define case_brw(name, type) \
1368 case jit_code_##name##i##type: \
1370 assert(temp->code == jit_code_label || \
1371 temp->code == jit_code_epilog); \
1372 if (temp->flag & jit_flag_patch) \
1373 name##i##type(temp->u.w, \
1374 rn(node->v.w), node->w.w); \
1376 word = name##i##type(_jit->pc.w, \
1377 rn(node->v.w), node->w.w); \
1378 patch(word, node); \
1381 #define case_brf(name, type, size) \
1382 case jit_code_##name##i##type: \
1384 assert(temp->code == jit_code_label || \
1385 temp->code == jit_code_epilog); \
1386 if (temp->flag & jit_flag_patch) \
1387 name##i##type(temp->u.w, rn(node->v.w), \
1388 (jit_float##size##_t *)node->w.n->u.w); \
1390 word = name##i##type(_jit->pc.w, rn(node->v.w), \
1391 (jit_float##size##_t *)node->w.n->u.w); \
1392 patch(word, node); \
1395 #if DEVEL_DISASSEMBLER
1398 for (node = _jitc->head; node; node = node->next) {
1399 if (_jit->pc.uc >= _jitc->code.end)
1402 #if DEVEL_DISASSEMBLER
1403 node->offset = (jit_uword_t)_jit->pc.w - (jit_uword_t)prevw;
1406 value = jit_classify(node->code);
1410 jit_regarg_set(node, value);
1411 switch (node->code) {
1412 case jit_code_align:
1413 /* Must align to a power of two */
1414 assert(!(node->u.w & (node->u.w - 1)));
1416 if ((word = _jit->pc.w & (node->u.w - 1)))
1417 nop(node->u.w - word);
1422 nop((node->u.w + 3) & ~3);
1425 case jit_code_note: case jit_code_name:
1427 node->u.w = _jit->pc.w;
1429 case jit_code_label:
1431 /* remember label is defined */
1432 node->flag |= jit_flag_patch;
1433 node->u.w = _jit->pc.w;
1452 case_rrrr(qmul, _u);
1453 case_rrrw(qmul, _u);
1460 case_rrrr(qdiv, _u);
1461 case_rrrw(qdiv, _u);
1478 case_rr(trunc, _f_i);
1479 case_rr(trunc, _d_i);
1480 #if __WORDSIZE == 64
1481 case_rr(trunc, _f_l);
1482 case_rr(trunc, _d_l);
1494 #if __WORDSIZE == 64
1510 #if __WORDSIZE == 64
1522 #if __WORDSIZE == 64
1532 #if __WORDSIZE == 64
1538 #if __WORDSIZE == 64
1541 case_rr(bswap, _us);
1542 case_rr(bswap, _ui);
1543 #if __WORDSIZE == 64
1544 case_rr(bswap, _ul);
1550 #if __WORDSIZE == 64
1555 casr(rn(node->u.w), rn(node->v.w),
1556 rn(node->w.q.l), rn(node->w.q.h));
1559 casi(rn(node->u.w), node->v.w,
1560 rn(node->w.q.l), rn(node->w.q.h));
1566 if (node->flag & jit_flag_node) {
1568 if (temp->code == jit_code_data ||
1569 (temp->code == jit_code_label &&
1570 (temp->flag & jit_flag_patch)))
1571 movi(rn(node->u.w), temp->u.w);
1573 assert(temp->code == jit_code_label ||
1574 temp->code == jit_code_epilog);
1575 word = movi_p(rn(node->u.w), node->v.w);
1580 movi(rn(node->u.w), node->v.w);
1630 case_brr(boadd, _u);
1631 case_brw(boadd, _u);
1634 case_brr(bxadd, _u);
1635 case_brw(bxadd, _u);
1638 case_brr(bosub, _u);
1639 case_brw(bosub, _u);
1642 case_brr(bxsub, _u);
1643 case_brw(bxsub, _u);
1649 case_rrf(add, _f, 32);
1651 case_rrf(sub, _f, 32);
1652 case_rrf(rsb, _f, 32);
1654 case_rrf(mul, _f, 32);
1656 case_rrf(div, _f, 32);
1670 case jit_code_movi_f:
1671 assert(node->flag & jit_flag_data);
1672 movi_f(rn(node->u.w), (jit_float32_t *)node->v.n->u.w);
1676 case_rrf(lt, _f, 32);
1678 case_rrf(le, _f, 32);
1680 case_rrf(eq, _f, 32);
1682 case_rrf(ge, _f, 32);
1684 case_rrf(gt, _f, 32);
1686 case_rrf(ne, _f, 32);
1688 case_rrf(unlt, _f, 32);
1690 case_rrf(unle, _f, 32);
1692 case_rrf(uneq, _f, 32);
1694 case_rrf(unge, _f, 32);
1696 case_rrf(ungt, _f, 32);
1698 case_rrf(ltgt, _f, 32);
1700 case_rrf(ord, _f, 32);
1701 case_rrr(unord, _f);
1702 case_rrf(unord, _f, 32);
1704 case_brf(blt, _f, 32);
1706 case_brf(ble, _f, 32);
1708 case_brf(beq, _f, 32);
1710 case_brf(bge, _f, 32);
1712 case_brf(bgt, _f, 32);
1714 case_brf(bne, _f, 32);
1715 case_brr(bunlt, _f);
1716 case_brf(bunlt, _f, 32);
1717 case_brr(bunle, _f);
1718 case_brf(bunle, _f, 32);
1719 case_brr(buneq, _f);
1720 case_brf(buneq, _f, 32);
1721 case_brr(bunge, _f);
1722 case_brf(bunge, _f, 32);
1723 case_brr(bungt, _f);
1724 case_brf(bungt, _f, 32);
1725 case_brr(bltgt, _f);
1726 case_brf(bltgt, _f, 32);
1728 case_brf(bord, _f, 32);
1729 case_brr(bunord, _f);
1730 case_brf(bunord, _f, 32);
1732 case_rrf(add, _d, 64);
1734 case_rrf(sub, _d, 64);
1735 case_rrf(rsb, _d, 64);
1737 case_rrf(mul, _d, 64);
1739 case_rrf(div, _d, 64);
1753 case jit_code_movi_d:
1754 assert(node->flag & jit_flag_data);
1755 movi_d(rn(node->u.w), (jit_float64_t *)node->v.n->u.w);
1759 case_rrf(lt, _d, 64);
1761 case_rrf(le, _d, 64);
1763 case_rrf(eq, _d, 64);
1765 case_rrf(ge, _d, 64);
1767 case_rrf(gt, _d, 64);
1769 case_rrf(ne, _d, 64);
1771 case_rrf(unlt, _d, 64);
1773 case_rrf(unle, _d, 64);
1775 case_rrf(uneq, _d, 64);
1777 case_rrf(unge, _d, 64);
1779 case_rrf(ungt, _d, 64);
1781 case_rrf(ltgt, _d, 64);
1783 case_rrf(ord, _d, 64);
1784 case_rrr(unord, _d);
1785 case_rrf(unord, _d, 64);
1787 case_brf(blt, _d, 64);
1789 case_brf(ble, _d, 64);
1791 case_brf(beq, _d, 64);
1793 case_brf(bge, _d, 64);
1795 case_brf(bgt, _d, 64);
1797 case_brf(bne, _d, 64);
1798 case_brr(bunlt, _d);
1799 case_brf(bunlt, _d, 64);
1800 case_brr(bunle, _d);
1801 case_brf(bunle, _d, 64);
1802 case_brr(buneq, _d);
1803 case_brf(buneq, _d, 64);
1804 case_brr(bunge, _d);
1805 case_brf(bunge, _d, 64);
1806 case_brr(bungt, _d);
1807 case_brf(bungt, _d, 64);
1808 case_brr(bltgt, _d);
1809 case_brf(bltgt, _d, 64);
1811 case_brf(bord, _d, 64);
1812 case_brr(bunord, _d);
1813 case_brf(bunord, _d, 64);
1816 jmpr(rn(node->u.w));
1819 if (node->flag & jit_flag_node) {
1821 assert(temp->code == jit_code_label ||
1822 temp->code == jit_code_epilog);
1823 if (temp->flag & jit_flag_patch)
1826 word = _jit->code.length -
1827 (_jit->pc.uc - _jit->code.ptr);
1828 if (jit_mips2_p() && can_relative_jump_p(word))
1829 word = jmpi(_jit->pc.w, 1);
1831 word = jmpi_p(_jit->pc.w);
1840 case jit_code_callr:
1842 callr(rn(node->u.w));
1844 case jit_code_calli:
1845 if (node->flag & jit_flag_node) {
1847 assert(temp->code == jit_code_label ||
1848 temp->code == jit_code_epilog);
1849 if (temp->flag & jit_flag_patch)
1850 calli(temp->u.w, 0);
1852 word = _jit->code.length -
1853 (_jit->pc.uc - _jit->code.ptr);
1854 if (jit_mips2_p() && can_relative_jump_p(word))
1855 word = calli(_jit->pc.w, 1);
1857 word = calli_p(_jit->pc.w);
1863 calli(node->u.w, 0);
1866 case jit_code_prolog:
1868 _jitc->function = _jitc->functions.ptr + node->w.w;
1870 undo.word = _jit->pc.w;
1871 memcpy(&undo.func, _jitc->function, sizeof(undo.func));
1872 #if DEVEL_DISASSEMBLER
1875 undo.patch_offset = _jitc->patches.offset;
1878 compute_framesize();
1882 case jit_code_epilog:
1883 assert(_jitc->function == _jitc->functions.ptr + node->w.w);
1885 for (temp = undo.node->next;
1886 temp != node; temp = temp->next) {
1887 if (temp->code == jit_code_label ||
1888 temp->code == jit_code_epilog)
1889 temp->flag &= ~jit_flag_patch;
1891 temp->flag &= ~jit_flag_patch;
1893 _jit->pc.w = undo.word;
1894 /* undo.func.self.aoff and undo.func.regset should not
1895 * be undone, as they will be further updated, and are
1896 * the reason of the undo. */
1897 undo.func.self.aoff = _jitc->function->frame +
1898 _jitc->function->self.aoff;
1899 undo.func.need_frame = _jitc->function->need_frame;
1900 jit_regset_set(&undo.func.regset, &_jitc->function->regset);
1901 /* allocar information also does not need to be undone */
1902 undo.func.aoffoff = _jitc->function->aoffoff;
1903 undo.func.allocar = _jitc->function->allocar;
1904 /* this will be recomputed but undo anyway to have it
1905 * better self documented.*/
1906 undo.func.need_stack = _jitc->function->need_stack;
1907 memcpy(_jitc->function, &undo.func, sizeof(undo.func));
1908 #if DEVEL_DISASSEMBLER
1911 _jitc->patches.offset = undo.patch_offset;
1913 goto restart_function;
1915 /* remember label is defined */
1917 node->flag |= jit_flag_patch;
1918 node->u.w = _jit->pc.w;
1920 _jitc->function = NULL;
1923 case jit_code_movr_w_f:
1924 movr_w_f(rn(node->u.w), rn(node->v.w));
1927 case jit_code_movr_f_w:
1928 movr_f_w(rn(node->u.w), rn(node->v.w));
1930 case jit_code_movi_f_w:
1931 assert(node->flag & jit_flag_data);
1932 movi_f_w(rn(node->u.w), (jit_float32_t *)node->v.n->u.w);
1935 case jit_code_movr_d_w:
1936 movr_d_w(rn(node->u.w), rn(node->v.w));
1938 case jit_code_movi_d_w:
1939 assert(node->flag & jit_flag_data);
1940 movi_d_w(rn(node->u.w), (jit_float64_t *)node->v.n->u.w);
1943 case jit_code_movr_ww_d:
1944 movr_ww_d(rn(node->u.w), rn(node->v.w), rn(node->w.w));
1946 case jit_code_movr_d_ww:
1947 movr_d_ww(rn(node->u.w), rn(node->v.w), rn(node->w.w));
1949 case jit_code_movi_d_ww:
1950 assert(node->flag & jit_flag_data);
1951 movi_d_ww(rn(node->u.w), rn(node->v.w),
1952 (jit_float64_t *)node->w.n->u.w);
1955 case jit_code_va_start:
1956 vastart(rn(node->u.w));
1958 case jit_code_va_arg:
1959 vaarg(rn(node->u.w), rn(node->v.w));
1961 case jit_code_va_arg_d:
1962 vaarg_d(rn(node->u.w), rn(node->v.w));
1964 case jit_code_live: case jit_code_ellipsis:
1965 case jit_code_va_push:
1966 case jit_code_allocai: case jit_code_allocar:
1967 case jit_code_arg_c: case jit_code_arg_s:
1968 case jit_code_arg_i:
1969 # if __WORDSIZE == 64
1970 case jit_code_arg_l:
1972 case jit_code_arg_f: case jit_code_arg_d:
1973 case jit_code_va_end:
1975 case jit_code_retr_c: case jit_code_reti_c:
1976 case jit_code_retr_uc: case jit_code_reti_uc:
1977 case jit_code_retr_s: case jit_code_reti_s:
1978 case jit_code_retr_us: case jit_code_reti_us:
1979 case jit_code_retr_i: case jit_code_reti_i:
1980 #if __WORDSIZE == 64
1981 case jit_code_retr_ui: case jit_code_reti_ui:
1982 case jit_code_retr_l: case jit_code_reti_l:
1984 case jit_code_retr_f: case jit_code_reti_f:
1985 case jit_code_retr_d: case jit_code_reti_d:
1986 case jit_code_getarg_c: case jit_code_getarg_uc:
1987 case jit_code_getarg_s: case jit_code_getarg_us:
1988 case jit_code_getarg_i:
1989 #if __WORDSIZE == 64
1990 case jit_code_getarg_ui: case jit_code_getarg_l:
1992 case jit_code_getarg_f: case jit_code_getarg_d:
1993 case jit_code_putargr_c: case jit_code_putargi_c:
1994 case jit_code_putargr_uc: case jit_code_putargi_uc:
1995 case jit_code_putargr_s: case jit_code_putargi_s:
1996 case jit_code_putargr_us: case jit_code_putargi_us:
1997 case jit_code_putargr_i: case jit_code_putargi_i:
1998 #if __WORDSIZE == 64
1999 case jit_code_putargr_ui: case jit_code_putargi_ui:
2000 case jit_code_putargr_l: case jit_code_putargi_l:
2002 case jit_code_putargr_f: case jit_code_putargi_f:
2003 case jit_code_putargr_d: case jit_code_putargi_d:
2004 case jit_code_pushargr_c: case jit_code_pushargi_c:
2005 case jit_code_pushargr_uc: case jit_code_pushargi_uc:
2006 case jit_code_pushargr_s: case jit_code_pushargi_s:
2007 case jit_code_pushargr_us: case jit_code_pushargi_us:
2008 case jit_code_pushargr_i: case jit_code_pushargi_i:
2009 #if __WORDSIZE == 64
2010 case jit_code_pushargr_ui: case jit_code_pushargi_ui:
2011 case jit_code_pushargr_l: case jit_code_pushargi_l:
2013 case jit_code_pushargr_f: case jit_code_pushargi_f:
2014 case jit_code_pushargr_d: case jit_code_pushargi_d:
2015 case jit_code_retval_c: case jit_code_retval_uc:
2016 case jit_code_retval_s: case jit_code_retval_us:
2017 case jit_code_retval_i:
2018 #if __WORDSIZE == 64
2019 case jit_code_retval_ui: case jit_code_retval_l:
2021 case jit_code_retval_f: case jit_code_retval_d:
2022 case jit_code_prepare:
2023 case jit_code_finishr: case jit_code_finishi:
2028 if (jit_carry != _NOREG) {
2029 switch (node->code) {
2031 case jit_code_addcr: case jit_code_addci:
2032 case jit_code_addxr: case jit_code_addxi:
2033 case jit_code_subcr: case jit_code_subci:
2034 case jit_code_subxr: case jit_code_subxi:
2037 jit_unget_reg(jit_carry);
2045 jit_regarg_clr(node, value);
2046 assert(_jitc->regarg == 0 ||
2047 (jit_carry != _NOREG && _jitc->regarg == (1 << jit_carry)));
2048 assert(_jitc->synth == 0);
2049 /* update register live state */
2064 for (offset = 0; offset < _jitc->patches.offset; offset++) {
2065 node = _jitc->patches.ptr[offset].node;
2066 word = node->code == jit_code_movi ? node->v.n->u.w : node->u.n->u.w;
2067 patch_at(_jitc->patches.ptr[offset].inst, word);
2070 jit_flush(_jit->code.ptr, _jit->pc.uc);
2072 return (_jit->code.ptr);
2076 # include "jit_rewind.c"
2077 # include "jit_mips-cpu.c"
2078 # include "jit_mips-fpu.c"
2079 # include "jit_fallback.c"
2083 jit_flush(void *fptr, void *tptr)
2085 #if defined(__linux__)
2088 s = sysconf(_SC_PAGE_SIZE);
2089 f = (jit_word_t)fptr & -s;
2090 t = (((jit_word_t)tptr) + s - 1) & -s;
2091 _flush_cache((void *)f, t - f, ICACHE);
2096 _emit_ldxi(jit_state_t *_jit, jit_int32_t r0, jit_int32_t r1, jit_word_t i0)
2098 ldxi(rn(r0), rn(r1), i0);
2102 _emit_stxi(jit_state_t *_jit, jit_word_t i0, jit_int32_t r0, jit_int32_t r1)
2104 stxi(i0, rn(r0), rn(r1));
2108 _emit_ldxi_d(jit_state_t *_jit, jit_int32_t r0, jit_int32_t r1, jit_word_t i0)
2110 ldxi_d(rn(r0), rn(r1), i0);
2114 _emit_stxi_d(jit_state_t *_jit, jit_word_t i0, jit_int32_t r0, jit_int32_t r1)
2116 stxi_d(i0, rn(r0), rn(r1));
2120 _compute_framesize(jit_state_t *_jit)
2123 _jitc->framesize = STACK_SLOT << 1; /* ra+fp */
2124 for (reg = 0; reg < jit_size(iregs); reg++)
2125 if (jit_regset_tstbit(&_jitc->function->regset, iregs[reg]))
2126 _jitc->framesize += STACK_SLOT;
2128 for (reg = 0; reg < jit_size(fregs); reg++)
2129 if (jit_regset_tstbit(&_jitc->function->regset, fregs[reg]))
2130 _jitc->framesize += sizeof(jit_float64_t);
2133 /* Space to store variadic arguments */
2134 if (_jitc->function->self.call & jit_call_varargs)
2135 _jitc->framesize += (NUM_WORD_ARGS - _jitc->function->vagp) * STACK_SLOT;
2138 /* Make sure functions called have a 16 byte aligned stack */
2139 _jitc->framesize = (_jitc->framesize + 15) & -16;
2143 _patch(jit_state_t *_jit, jit_word_t instr, jit_node_t *node)
2147 assert(node->flag & jit_flag_node);
2148 if (node->code == jit_code_movi)
2149 flag = node->v.n->flag;
2151 flag = node->u.n->flag;
2152 assert(!(flag & jit_flag_patch));
2153 if (_jitc->patches.offset >= _jitc->patches.length) {
2154 jit_realloc((jit_pointer_t *)&_jitc->patches.ptr,
2155 _jitc->patches.length * sizeof(jit_patch_t),
2156 (_jitc->patches.length + 1024) * sizeof(jit_patch_t));
2157 _jitc->patches.length += 1024;
2159 _jitc->patches.ptr[_jitc->patches.offset].inst = instr;
2160 _jitc->patches.ptr[_jitc->patches.offset].node = node;
2161 ++_jitc->patches.offset;