2 * Copyright (C) 2012-2022 Free Software Foundation, Inc.
4 * This file is part of GNU lightning.
6 * GNU lightning is free software; you can redistribute it and/or modify it
7 * under the terms of the GNU Lesser General Public License as published
8 * by the Free Software Foundation; either version 3, or (at your option)
11 * GNU lightning is distributed in the hope that it will be useful, but
12 * WITHOUT ANY WARRANTY; without even the implied warranty of MERCHANTABILITY
13 * or FITNESS FOR A PARTICULAR PURPOSE. See the GNU Lesser General Public
14 * License for more details.
17 * Paulo Cesar Pereira de Andrade
20 #if defined(__linux__)
21 # include <sys/cachectl.h>
25 # define NUM_WORD_ARGS 8
27 # define STACK_SHIFT 3
29 # define NUM_WORD_ARGS 4
31 # define STACK_SHIFT 2
33 #if NEW_ABI && __BYTE_ORDER == __BIG_ENDIAN && __WORDSIZE == 32
34 # define WORD_ADJUST 4
36 # define WORD_ADJUST 0
38 #define jit_arg_reg_p(i) ((i) >= 0 && (i) < NUM_WORD_ARGS)
39 #if __BYTE_ORDER == __LITTLE_ENDIAN
44 # define C_DISP STACK_SLOT - sizeof(jit_int8_t)
45 # define S_DISP STACK_SLOT - sizeof(jit_int16_t)
46 # define I_DISP STACK_SLOT - sizeof(jit_int32_t)
52 typedef struct jit_pointer_t jit_va_list_t;
57 #define jit_make_arg(node) _jit_make_arg(_jit,node)
58 static jit_node_t *_jit_make_arg(jit_state_t*,jit_node_t*);
59 #define jit_make_arg_f(node) _jit_make_arg_f(_jit,node)
60 static jit_node_t *_jit_make_arg_f(jit_state_t*,jit_node_t*);
61 #define jit_make_arg_d(node) _jit_make_arg_d(_jit,node)
62 static jit_node_t *_jit_make_arg_d(jit_state_t*,jit_node_t*);
63 #define patch(instr, node) _patch(_jit, instr, node)
64 static void _patch(jit_state_t*,jit_word_t,jit_node_t*);
67 # include "jit_rewind.c"
68 # include "jit_mips-cpu.c"
69 # include "jit_mips-fpu.c"
75 jit_register_t _rvs[] = {
76 { rc(gpr) | 0x01, "at" },
77 { rc(gpr) | 0x02, "v0" },
78 { rc(gpr) | 0x03, "v1" },
80 { rc(gpr) | 0x08, "t0" },
81 { rc(gpr) | 0x09, "t1" },
82 { rc(gpr) | 0x0a, "t2" },
83 { rc(gpr) | 0x0b, "t3" },
85 { rc(gpr) | 0x0c, "t4" },
86 { rc(gpr) | 0x0d, "t5" },
87 { rc(gpr) | 0x0e, "t6" },
88 { rc(gpr) | 0x0f, "t7" },
89 { rc(gpr) | 0x18, "t8" },
90 { rc(gpr) | 0x19, "t9" },
91 { rc(sav) | rc(gpr) | 0x10, "s0" },
92 { rc(sav) | rc(gpr) | 0x11, "s1" },
93 { rc(sav) | rc(gpr) | 0x12, "s2" },
94 { rc(sav) | rc(gpr) | 0x13, "s3" },
95 { rc(sav) | rc(gpr) | 0x14, "s4" },
96 { rc(sav) | rc(gpr) | 0x15, "s5" },
97 { rc(sav) | rc(gpr) | 0x16, "s6" },
98 { rc(sav) | rc(gpr) | 0x17, "s7" },
102 { rc(sav) | 0x1f, "ra" },
103 { rc(sav) | 0x1c, "gp" },
104 { rc(sav) | 0x1d, "sp" },
105 { rc(sav) | 0x1e, "fp" },
107 { rc(gpr) | 0x0b, "a7" },
108 { rc(gpr) | 0x0a, "a6" },
109 { rc(gpr) | 0x09, "a5" },
110 { rc(gpr) | 0x08, "a4" },
112 { rc(arg) | rc(gpr) | 0x07, "a3" },
113 { rc(arg) | rc(gpr) | 0x06, "a2" },
114 { rc(arg) | rc(gpr) | 0x05, "a1" },
115 { rc(arg) | rc(gpr) | 0x04, "a0" },
116 { rc(fpr) | 0x00, "$f0" },
117 { rc(fpr) | 0x02, "$f2" },
118 { rc(fpr) | 0x04, "$f4" },
119 { rc(fpr) | 0x06, "$f6" },
120 { rc(fpr) | 0x08, "$f8" },
121 { rc(fpr) | 0x0a, "$f10" },
123 { rc(sav) | rc(fpr) | 0x10, "$f16" },
124 { rc(sav) | rc(fpr) | 0x12, "$f18" },
126 { rc(sav) | rc(fpr) | 0x14, "$f20" },
127 { rc(sav) | rc(fpr) | 0x16, "$f22" },
128 { rc(sav) | rc(fpr) | 0x18, "$f24" },
129 { rc(sav) | rc(fpr) | 0x1a, "$f26" },
130 { rc(sav) | rc(fpr) | 0x1c, "$f28" },
131 { rc(sav) | rc(fpr) | 0x1e, "$f30" },
133 { rc(arg) | rc(fpr) | 0x13, "$f19" },
134 { rc(arg) | rc(fpr) | 0x12, "$f18" },
135 { rc(arg) | rc(fpr) | 0x11, "$f17" },
136 { rc(arg) | rc(fpr) | 0x10, "$f16" },
137 { rc(arg) | rc(fpr) | 0x0f, "$f15" },
138 { rc(arg) | rc(fpr) | 0x0e, "$f14" },
139 { rc(arg) | rc(fpr) | 0x0d, "$f13" },
140 { rc(arg) | rc(fpr) | 0x0c, "$f12" },
142 { rc(arg) | rc(fpr) | 0x0e, "$f14" },
143 { rc(arg) | rc(fpr) | 0x0c, "$f12" },
145 { _NOREG, "<none>" },
157 _jit_init(jit_state_t *_jit)
159 _jitc->reglen = jit_size(_rvs) - 1;
161 * o reserve a register for carry (overkill)
162 * o use MTLO/MFLO (performance hit)
163 * So, keep a register allocated after setting carry, and implicitly
164 * deallocate it if it can no longer be tracked
170 _jit_prolog(jit_state_t *_jit)
176 assert(jit_regset_cmp_ui(&_jitc->regarg, 0) == 0);
177 jit_regset_set_ui(&_jitc->regsav, 0);
178 offset = _jitc->functions.offset;
179 if (offset >= _jitc->functions.length) {
180 jit_realloc((jit_pointer_t *)&_jitc->functions.ptr,
181 _jitc->functions.length * sizeof(jit_function_t),
182 (_jitc->functions.length + 16) * sizeof(jit_function_t));
183 _jitc->functions.length += 16;
185 _jitc->function = _jitc->functions.ptr + _jitc->functions.offset++;
186 _jitc->function->self.size = stack_framesize;
187 _jitc->function->self.argi = _jitc->function->self.argf =
188 _jitc->function->self.aoff = _jitc->function->self.alen = 0;
189 _jitc->function->self.call = jit_call_default;
190 jit_alloc((jit_pointer_t *)&_jitc->function->regoff,
191 _jitc->reglen * sizeof(jit_int32_t));
193 /* _no_link here does not mean the jit_link() call can be removed
195 * _jitc->function->prolog = jit_new_node(jit_code_prolog);
197 _jitc->function->prolog = jit_new_node_no_link(jit_code_prolog);
198 jit_link(_jitc->function->prolog);
199 _jitc->function->prolog->w.w = offset;
200 _jitc->function->epilog = jit_new_node_no_link(jit_code_epilog);
202 * v: offset in blocks vector
203 * w: offset in functions vector
205 _jitc->function->epilog->w.w = offset;
207 jit_regset_new(&_jitc->function->regset);
211 _jit_allocai(jit_state_t *_jit, jit_int32_t length)
213 assert(_jitc->function);
215 case 0: case 1: break;
216 case 2: _jitc->function->self.aoff &= -2; break;
217 case 3: case 4: _jitc->function->self.aoff &= -4; break;
218 default: _jitc->function->self.aoff &= -8; break;
220 _jitc->function->self.aoff -= length;
221 if (!_jitc->realize) {
222 jit_inc_synth_ww(allocai, _jitc->function->self.aoff, length);
225 return (_jitc->function->self.aoff);
229 _jit_allocar(jit_state_t *_jit, jit_int32_t u, jit_int32_t v)
232 assert(_jitc->function);
233 jit_inc_synth_ww(allocar, u, v);
234 if (!_jitc->function->allocar) {
235 _jitc->function->aoffoff = jit_allocai(sizeof(jit_int32_t));
236 _jitc->function->allocar = 1;
238 reg = jit_get_reg(jit_class_gpr);
240 jit_andi(reg, reg, -8);
241 jit_ldxi_i(u, JIT_FP, _jitc->function->aoffoff);
243 jit_addr(JIT_SP, JIT_SP, reg);
244 jit_stxi_i(_jitc->function->aoffoff, JIT_FP, u);
250 _jit_ret(jit_state_t *_jit)
253 assert(_jitc->function);
257 jit_patch_at(instr, _jitc->function->epilog);
262 _jit_retr(jit_state_t *_jit, jit_int32_t u)
264 jit_inc_synth_w(retr, u);
266 jit_movr(JIT_RET, u);
273 _jit_reti(jit_state_t *_jit, jit_word_t u)
275 jit_inc_synth_w(reti, u);
276 jit_movi(JIT_RET, u);
282 _jit_retr_f(jit_state_t *_jit, jit_int32_t u)
284 jit_inc_synth_w(retr_f, u);
286 jit_movr_f(JIT_FRET, u);
294 _jit_reti_f(jit_state_t *_jit, jit_float32_t u)
296 jit_inc_synth_f(reti_f, u);
297 jit_movi_f(JIT_FRET, u);
303 _jit_retr_d(jit_state_t *_jit, jit_int32_t u)
305 jit_inc_synth_w(retr_d, u);
307 jit_movr_d(JIT_FRET, u);
315 _jit_reti_d(jit_state_t *_jit, jit_float64_t u)
317 jit_inc_synth_d(reti_d, u);
318 jit_movi_d(JIT_FRET, u);
324 _jit_epilog(jit_state_t *_jit)
326 assert(_jitc->function);
327 assert(_jitc->function->epilog->next == NULL);
328 jit_link(_jitc->function->epilog);
329 _jitc->function = NULL;
333 _jit_arg_register_p(jit_state_t *_jit, jit_node_t *u)
335 if (u->code == jit_code_arg)
336 return (jit_arg_reg_p(u->u.w));
337 assert(u->code == jit_code_arg_f || u->code == jit_code_arg_d);
339 return (jit_arg_reg_p(u->u.w));
346 _jit_make_arg(jit_state_t *_jit, jit_node_t *node)
350 if (jit_arg_reg_p(_jitc->function->self.argi))
351 offset = _jitc->function->self.argi++;
353 offset = _jitc->function->self.size;
354 _jitc->function->self.size += STACK_SLOT;
357 offset = (_jitc->function->self.size - stack_framesize) >> STACK_SHIFT;
358 _jitc->function->self.argi = 1;
360 offset = _jitc->function->self.size;
361 _jitc->function->self.size += STACK_SLOT;
363 if (node == (jit_node_t *)0)
364 node = jit_new_node(jit_code_arg);
368 node->v.w = ++_jitc->function->self.argn;
374 _jit_make_arg_f(jit_state_t *_jit, jit_node_t *node)
378 if (jit_arg_reg_p(_jitc->function->self.argi)) {
379 offset = _jitc->function->self.argi++;
380 if (_jitc->function->self.call & jit_call_varargs)
384 offset = _jitc->function->self.size;
385 _jitc->function->self.size += STACK_SLOT;
388 offset = (_jitc->function->self.size - stack_framesize) >> STACK_SHIFT;
389 if (offset < NUM_WORD_ARGS) {
390 if (!_jitc->function->self.argi &&
391 !(_jitc->function->self.call & jit_call_varargs)) {
396 _jitc->function->self.argi = 1;
398 /* Use as flag to rewind in case of varargs function */
399 ++_jitc->function->self.argf;
403 offset = _jitc->function->self.size;
404 _jitc->function->self.size += STACK_SLOT;
406 if (node == (jit_node_t *)0)
407 node = jit_new_node(jit_code_arg_f);
411 node->v.w = ++_jitc->function->self.argn;
417 _jit_make_arg_d(jit_state_t *_jit, jit_node_t *node)
421 if (jit_arg_reg_p(_jitc->function->self.argi)) {
422 offset = _jitc->function->self.argi++;
423 if (_jitc->function->self.call & jit_call_varargs)
427 offset = _jitc->function->self.size;
428 _jitc->function->self.size += STACK_SLOT;
431 if (_jitc->function->self.size & 7) {
432 _jitc->function->self.size += 4;
433 _jitc->function->self.argi = 1;
435 offset = (_jitc->function->self.size - stack_framesize) >> STACK_SHIFT;
436 if (offset < NUM_WORD_ARGS) {
437 if (!_jitc->function->self.argi &&
438 !(_jitc->function->self.call & jit_call_varargs)) {
440 /* Use as flag to rewind in case of varargs function */
441 ++_jitc->function->self.argf;
445 offset = _jitc->function->self.size;
446 _jitc->function->self.size += sizeof(jit_float64_t);
448 if (node == (jit_node_t *)0)
449 node = jit_new_node(jit_code_arg_d);
453 node->v.w = ++_jitc->function->self.argn;
459 _jit_ellipsis(jit_state_t *_jit)
461 if (_jitc->prepare) {
462 assert(!(_jitc->function->call.call & jit_call_varargs));
463 _jitc->function->call.call |= jit_call_varargs;
465 if (_jitc->function->call.argf)
470 assert(!(_jitc->function->self.call & jit_call_varargs));
472 /* If varargs start in a register, allocate extra 64 bytes. */
473 if (jit_arg_reg_p(_jitc->function->self.argi))
475 /* Do not set during possible rewind. */
476 _jitc->function->self.call |= jit_call_varargs;
478 _jitc->function->self.call |= jit_call_varargs;
479 if (_jitc->function->self.argf)
482 _jitc->function->vagp = _jitc->function->self.argi;
484 jit_inc_synth(ellipsis);
493 _jit_va_push(jit_state_t *_jit, jit_int32_t u)
495 jit_inc_synth_w(va_push, u);
501 _jit_arg(jit_state_t *_jit)
503 assert(_jitc->function);
504 return (jit_make_arg((jit_node_t*)0));
508 _jit_arg_f(jit_state_t *_jit)
510 assert(_jitc->function);
511 return (jit_make_arg_f((jit_node_t*)0));
515 _jit_arg_d(jit_state_t *_jit)
517 assert(_jitc->function);
518 return (jit_make_arg_d((jit_node_t*)0));
522 _jit_getarg_c(jit_state_t *_jit, jit_int32_t u, jit_node_t *v)
524 assert(v->code == jit_code_arg);
525 jit_inc_synth_wp(getarg_c, u, v);
526 if (jit_arg_reg_p(v->u.w))
527 jit_extr_c(u, _A0 - v->u.w);
529 jit_ldxi_c(u, _FP, v->u.w + C_DISP);
534 _jit_getarg_uc(jit_state_t *_jit, jit_int32_t u, jit_node_t *v)
536 assert(v->code == jit_code_arg);
537 jit_inc_synth_wp(getarg_uc, u, v);
538 if (jit_arg_reg_p(v->u.w))
539 jit_extr_uc(u, _A0 - v->u.w);
541 jit_ldxi_uc(u, _FP, v->u.w + C_DISP);
546 _jit_getarg_s(jit_state_t *_jit, jit_int32_t u, jit_node_t *v)
548 assert(v->code == jit_code_arg);
549 jit_inc_synth_wp(getarg_s, u, v);
550 if (jit_arg_reg_p(v->u.w))
551 jit_extr_s(u, _A0 - v->u.w);
553 jit_ldxi_s(u, _FP, v->u.w + S_DISP);
558 _jit_getarg_us(jit_state_t *_jit, jit_int32_t u, jit_node_t *v)
560 assert(v->code == jit_code_arg);
561 jit_inc_synth_wp(getarg_us, u, v);
562 if (jit_arg_reg_p(v->u.w))
563 jit_extr_us(u, _A0 - v->u.w);
565 jit_ldxi_us(u, _FP, v->u.w + S_DISP);
570 _jit_getarg_i(jit_state_t *_jit, jit_int32_t u, jit_node_t *v)
572 assert(v->code == jit_code_arg);
573 jit_inc_synth_wp(getarg_i, u, v);
574 if (jit_arg_reg_p(v->u.w)) {
576 jit_extr_i(u, _A0 - v->u.w);
578 jit_movr(u, _A0 - v->u.w);
582 jit_ldxi_i(u, _FP, v->u.w + I_DISP);
588 _jit_getarg_ui(jit_state_t *_jit, jit_int32_t u, jit_node_t *v)
590 assert(v->code == jit_code_arg);
591 jit_inc_synth_wp(getarg_ui, u, v);
592 if (jit_arg_reg_p(v->u.w))
593 jit_extr_ui(u, _A0 - v->u.w);
595 jit_ldxi_ui(u, _FP, v->u.w + I_DISP);
600 _jit_getarg_l(jit_state_t *_jit, jit_int32_t u, jit_node_t *v)
602 assert(v->code == jit_code_arg);
603 jit_inc_synth_wp(getarg_l, u, v);
604 if (jit_arg_reg_p(v->u.w))
605 jit_movr(u, _A0 - v->u.w);
607 jit_ldxi_l(u, _FP, v->u.w);
613 _jit_putargr(jit_state_t *_jit, jit_int32_t u, jit_node_t *v)
615 jit_inc_synth_wp(putargr, u, v);
616 assert(v->code == jit_code_arg);
617 if (jit_arg_reg_p(v->u.w))
618 jit_movr(_A0 - v->u.w, u);
620 jit_stxi(v->u.w + WORD_ADJUST, _FP, u);
625 _jit_putargi(jit_state_t *_jit, jit_word_t u, jit_node_t *v)
628 assert(v->code == jit_code_arg);
629 jit_inc_synth_wp(putargi, u, v);
630 if (jit_arg_reg_p(v->u.w))
631 jit_movi(_A0 - v->u.w, u);
633 regno = jit_get_reg(jit_class_gpr);
635 jit_stxi(v->u.w + WORD_ADJUST, _FP, regno);
636 jit_unget_reg(regno);
642 _jit_getarg_f(jit_state_t *_jit, jit_int32_t u, jit_node_t *v)
644 assert(v->code == jit_code_arg_f);
645 jit_inc_synth_wp(getarg_f, u, v);
647 if (jit_arg_reg_p(v->u.w))
648 jit_movr_f(u, _F12 - v->u.w);
649 else if (jit_arg_reg_p(v->u.w - 8))
650 jit_movr_w_f(u, _A0 - v->u.w - 8);
653 jit_movr_w_f(u, _A0 - v->u.w);
655 jit_movr_f(u, _F12 - ((v->u.w - 4) >> 1));
658 jit_ldxi_f(u, _FP, v->u.w);
663 _jit_putargr_f(jit_state_t *_jit, jit_int32_t u, jit_node_t *v)
665 assert(v->code == jit_code_arg_f);
666 jit_inc_synth_wp(putargr_f, u, v);
668 if (jit_arg_reg_p(v->u.w))
669 jit_movr_f(_F12 - v->u.w, u);
670 else if (jit_arg_reg_p(v->u.w - 8))
671 jit_movr_f_w(_A0 - v->u.w - 8, u);
674 jit_movr_f_w(_A0 - v->u.w, u);
676 jit_movr_f(_F12 - ((v->u.w - 4) >> 1), u);
679 jit_stxi_f(v->u.w, _FP, u);
684 _jit_putargi_f(jit_state_t *_jit, jit_float32_t u, jit_node_t *v)
687 assert(v->code == jit_code_arg_f);
688 jit_inc_synth_fp(putargi_f, u, v);
690 if (jit_arg_reg_p(v->u.w))
691 jit_movi_f(_F12 - v->u.w, u);
692 else if (jit_arg_reg_p(v->u.w - 8)) {
693 regno = jit_get_reg(jit_class_fpr);
694 jit_movi_f(regno, u);
695 jit_movr_f_w(_A0 - v->u.w - 8, u);
696 jit_unget_reg(regno);
700 regno = jit_get_reg(jit_class_fpr);
701 jit_movi_f(regno, u);
702 jit_movr_f_w(_A0 - ((v->u.w - 4) >> 1), regno);
703 jit_unget_reg(regno);
706 jit_movi_f(_F12 - ((v->u.w - 4) >> 1), u);
709 regno = jit_get_reg(jit_class_fpr);
710 jit_movi_f(regno, u);
711 jit_stxi_f(v->u.w, _FP, regno);
712 jit_unget_reg(regno);
718 _jit_getarg_d(jit_state_t *_jit, jit_int32_t u, jit_node_t *v)
720 assert(v->code == jit_code_arg_d);
721 jit_inc_synth_wp(getarg_d, u, v);
723 if (jit_arg_reg_p(v->u.w))
724 jit_movr_d(u, _F12 - v->u.w);
725 else if (jit_arg_reg_p(v->u.w - 8))
726 jit_movr_d_w(_A0 - v->u.w - 8, u);
729 jit_movr_ww_d(u, _A0 - v->u.w, _A0 - (v->u.w + 1));
731 jit_movr_d(u, _F12 - ((v->u.w - 4) >> 1));
734 jit_ldxi_d(u, _FP, v->u.w);
739 _jit_putargr_d(jit_state_t *_jit, jit_int32_t u, jit_node_t *v)
741 assert(v->code == jit_code_arg_d);
742 jit_inc_synth_wp(putargr_d, u, v);
744 if (jit_arg_reg_p(v->u.w))
745 jit_movr_d(_F12 - v->u.w, u);
746 else if (jit_arg_reg_p(v->u.w - 8))
747 jit_movr_d_w(_A0 - v->u.w - 8, u);
750 jit_movr_d_ww(_A0 - v->u.w, _A0 - (v->u.w + 1), u);
752 jit_movr_d(_F12 - ((v->u.w - 4) >> 1), u);
755 jit_stxi_d(v->u.w, _FP, u);
760 _jit_putargi_d(jit_state_t *_jit, jit_float64_t u, jit_node_t *v)
763 assert(v->code == jit_code_arg_d);
764 jit_inc_synth_dp(putargi_d, u, v);
766 if (jit_arg_reg_p(v->u.w))
767 jit_movi_d(_F12 - v->u.w, u);
768 else if (jit_arg_reg_p(v->u.w - 8)) {
769 regno = jit_get_reg(jit_class_fpr);
770 jit_movi_d(regno, u);
771 jit_movr_d_w(_A0 - v->u.w - 8, u);
772 jit_unget_reg(regno);
776 regno = jit_get_reg(jit_class_fpr);
777 jit_movi_d(regno, u);
778 jit_movr_d_ww(_A0 - v->u.w, _A0 - (v->u.w + 1), regno);
779 jit_unget_reg(regno);
782 jit_movi_d(_F12 - ((v->u.w - 4) >> 1), u);
785 regno = jit_get_reg(jit_class_fpr);
786 jit_movi_d(regno, u);
787 jit_stxi_d(v->u.w, _FP, regno);
788 jit_unget_reg(regno);
794 _jit_pushargr(jit_state_t *_jit, jit_int32_t u)
796 jit_inc_synth_w(pushargr, u);
799 assert(_jitc->function);
800 if (jit_arg_reg_p(_jitc->function->call.argi)) {
801 jit_movr(_A0 - _jitc->function->call.argi, u);
802 ++_jitc->function->call.argi;
805 jit_stxi(_jitc->function->call.size + WORD_ADJUST, JIT_SP, u);
806 _jitc->function->call.size += STACK_SLOT;
810 assert(_jitc->function);
811 offset = _jitc->function->call.size >> STACK_SHIFT;
812 _jitc->function->call.argi = 1;
813 if (jit_arg_reg_p(offset))
814 jit_movr(_A0 - offset, u);
816 jit_stxi(_jitc->function->call.size, JIT_SP, u);
817 _jitc->function->call.size += STACK_SLOT;
823 _jit_pushargi(jit_state_t *_jit, jit_word_t u)
829 assert(_jitc->function);
830 jit_inc_synth_w(pushargi, u);
833 if (jit_arg_reg_p(_jitc->function->call.argi)) {
834 jit_movi(_A0 - _jitc->function->call.argi, u);
835 ++_jitc->function->call.argi;
838 regno = jit_get_reg(jit_class_gpr);
840 jit_stxi(_jitc->function->call.size + WORD_ADJUST, JIT_SP, regno);
841 _jitc->function->call.size += STACK_SLOT;
842 jit_unget_reg(regno);
845 offset = _jitc->function->call.size >> STACK_SHIFT;
846 ++_jitc->function->call.argi;
847 if (jit_arg_reg_p(offset))
848 jit_movi(_A0 - offset, u);
850 regno = jit_get_reg(jit_class_gpr);
852 jit_stxi(_jitc->function->call.size, JIT_SP, regno);
853 jit_unget_reg(regno);
855 _jitc->function->call.size += STACK_SLOT;
861 _jit_pushargr_f(jit_state_t *_jit, jit_int32_t u)
866 assert(_jitc->function);
867 jit_inc_synth_w(pushargr_f, u);
870 if (jit_arg_reg_p(_jitc->function->call.argi)) {
871 if (!(_jitc->function->call.call & jit_call_varargs))
872 jit_movr_f(_F12 - _jitc->function->call.argi, u);
874 jit_movr_f_w(_A0 - _jitc->function->call.argi, u);
875 ++_jitc->function->call.argi;
878 jit_stxi_f(_jitc->function->call.size, JIT_SP, u);
879 _jitc->function->call.size += STACK_SLOT;
882 offset = _jitc->function->call.size >> STACK_SHIFT;
883 if (offset < 2 && !_jitc->function->call.argi &&
884 !(_jitc->function->call.call & jit_call_varargs)) {
885 ++_jitc->function->call.argf;
886 jit_movr_f(_F12 - offset, u);
888 else if (offset < 4) {
889 ++_jitc->function->call.argi;
890 jit_movr_f_w(_A0 - offset, u);
893 jit_stxi_f(_jitc->function->call.size, JIT_SP, u);
894 _jitc->function->call.size += STACK_SLOT;
900 _jit_pushargi_f(jit_state_t *_jit, jit_float32_t u)
906 assert(_jitc->function);
907 jit_inc_synth_f(pushargi_f, u);
910 if (jit_arg_reg_p(_jitc->function->call.argi)) {
911 if (!(_jitc->function->call.call & jit_call_varargs))
912 jit_movi_f(_F12 - _jitc->function->call.argi, u);
914 jit_movi_f_w(_A0 - _jitc->function->call.argi, u);
915 ++_jitc->function->call.argi;
918 regno = jit_get_reg(jit_class_fpr);
919 jit_movi_f(regno, u);
920 jit_stxi_f(_jitc->function->call.size, JIT_SP, regno);
921 _jitc->function->call.size += STACK_SLOT;
922 jit_unget_reg(regno);
925 offset = _jitc->function->call.size >> STACK_SHIFT;
926 if (offset < 2 && !_jitc->function->call.argi &&
927 !(_jitc->function->call.call & jit_call_varargs)) {
928 ++_jitc->function->call.argf;
929 jit_movi_f(_F12 - offset, u);
931 else if (offset < 4) {
932 ++_jitc->function->call.argi;
933 jit_movi_f_w(_A0 - offset, u);
936 regno = jit_get_reg(jit_class_fpr);
937 jit_movi_f(regno, u);
938 jit_stxi_f(_jitc->function->call.size, JIT_SP, regno);
939 jit_unget_reg(regno);
941 _jitc->function->call.size += STACK_SLOT;
947 _jit_pushargr_d(jit_state_t *_jit, jit_int32_t u)
953 assert(_jitc->function);
954 jit_inc_synth_w(pushargr_d, u);
957 if (jit_arg_reg_p(_jitc->function->call.argi)) {
958 if (!(_jitc->function->call.call & jit_call_varargs))
959 jit_movr_d(_F12 - _jitc->function->call.argi, u);
961 jit_movr_d_w(_A0 - _jitc->function->call.argi, u);
962 ++_jitc->function->call.argi;
965 jit_stxi_d(_jitc->function->call.size, JIT_SP, u);
966 _jitc->function->call.size += STACK_SLOT;
969 adjust = !!_jitc->function->call.argi;
970 if (_jitc->function->call.size & 7) {
971 _jitc->function->call.size += 4;
974 offset = _jitc->function->call.size >> STACK_SHIFT;
976 if (adjust || (_jitc->function->call.call & jit_call_varargs)) {
977 jit_movr_d_ww(_A0 - offset, _A0 - (offset + 1), u);
978 _jitc->function->call.argi += 2;
981 jit_movr_d(_F12 - (offset >> 1), u);
982 ++_jitc->function->call.argf;
986 jit_stxi_d(_jitc->function->call.size, JIT_SP, u);
987 _jitc->function->call.size += sizeof(jit_float64_t);
993 _jit_pushargi_d(jit_state_t *_jit, jit_float64_t u)
1000 assert(_jitc->function);
1001 jit_inc_synth_d(pushargi_d, u);
1004 if (jit_arg_reg_p(_jitc->function->call.argi)) {
1005 if (!(_jitc->function->call.call & jit_call_varargs))
1006 jit_movi_d(_F12 - _jitc->function->call.argi, u);
1008 jit_movi_d_w(_A0 - _jitc->function->call.argi, u);
1009 ++_jitc->function->call.argi;
1012 regno = jit_get_reg(jit_class_fpr);
1013 jit_movi_d(regno, u);
1014 jit_stxi_d(_jitc->function->call.size, JIT_SP, regno);
1015 _jitc->function->call.size += STACK_SLOT;
1016 jit_unget_reg(regno);
1019 adjust = !!_jitc->function->call.argi;
1020 if (_jitc->function->call.size & 7) {
1021 _jitc->function->call.size += 4;
1024 offset = _jitc->function->call.size >> STACK_SHIFT;
1026 if (adjust || (_jitc->function->call.call & jit_call_varargs)) {
1027 jit_movi_d_ww(_A0 - offset, _A0 - (offset + 1), u);
1028 _jitc->function->call.argi += 2;
1031 jit_movi_d(_F12 - (offset >> 1), u);
1032 ++_jitc->function->call.argf;
1036 regno = jit_get_reg(jit_class_fpr);
1037 jit_movi_d(regno, u);
1038 jit_stxi_d(_jitc->function->call.size, JIT_SP, regno);
1039 jit_unget_reg(regno);
1041 _jitc->function->call.size += sizeof(jit_float64_t);
1047 _jit_regarg_p(jit_state_t *_jit, jit_node_t *node, jit_int32_t regno)
1051 spec = jit_class(_rvs[regno].spec);
1052 if (spec & jit_class_arg) {
1053 if (spec & jit_class_gpr) {
1054 regno = _A0 - regno;
1055 if (regno >= 0 && regno < node->v.w)
1058 else if (spec & jit_class_fpr) {
1059 regno = _F12 - regno;
1060 if (regno >= 0 && regno < node->w.w)
1069 _jit_finishr(jit_state_t *_jit, jit_int32_t r0)
1072 assert(_jitc->function);
1073 jit_inc_synth_w(finishr, r0);
1074 if (_jitc->function->self.alen < _jitc->function->call.size)
1075 _jitc->function->self.alen = _jitc->function->call.size;
1076 call = jit_callr(r0);
1077 call->v.w = _jitc->function->self.argi;
1079 call->w.w = call->v.w;
1081 call->w.w = _jitc->function->self.argf;
1083 _jitc->function->call.argi = _jitc->function->call.argf =
1084 _jitc->function->call.size = 0;
1090 _jit_finishi(jit_state_t *_jit, jit_pointer_t i0)
1094 assert(_jitc->function);
1095 jit_inc_synth_w(finishi, (jit_word_t)i0);
1096 if (_jitc->function->self.alen < _jitc->function->call.size)
1097 _jitc->function->self.alen = _jitc->function->call.size;
1098 node = jit_movi(_T9, (jit_word_t)i0);
1099 call = jit_callr(_T9);
1100 call->v.w = _jitc->function->call.argi;
1102 call->w.w = call->v.w;
1104 call->w.w = _jitc->function->call.argf;
1106 _jitc->function->call.argi = _jitc->function->call.argf =
1107 _jitc->function->call.size = 0;
1114 _jit_retval_c(jit_state_t *_jit, jit_int32_t r0)
1116 jit_extr_c(r0, JIT_RET);
1120 _jit_retval_uc(jit_state_t *_jit, jit_int32_t r0)
1122 jit_extr_uc(r0, JIT_RET);
1126 _jit_retval_s(jit_state_t *_jit, jit_int32_t r0)
1128 jit_extr_s(r0, JIT_RET);
1132 _jit_retval_us(jit_state_t *_jit, jit_int32_t r0)
1134 jit_extr_us(r0, JIT_RET);
1138 _jit_retval_i(jit_state_t *_jit, jit_int32_t r0)
1140 #if __WORDSIZE == 32
1142 jit_movr(r0, JIT_RET);
1144 jit_extr_i(r0, JIT_RET);
1148 #if __WORDSIZE == 64
1150 _jit_retval_ui(jit_state_t *_jit, jit_int32_t r0)
1152 jit_extr_ui(r0, JIT_RET);
1156 _jit_retval_l(jit_state_t *_jit, jit_int32_t r0)
1159 jit_movr(r0, JIT_RET);
1164 _jit_retval_f(jit_state_t *_jit, jit_int32_t r0)
1167 jit_movr_f(r0, JIT_FRET);
1171 _jit_retval_d(jit_state_t *_jit, jit_int32_t r0)
1174 jit_movr_d(r0, JIT_FRET);
1178 _emit_code(jit_state_t *_jit)
1188 #if DEVEL_DISASSEMBLER
1191 jit_int32_t patch_offset;
1193 #if DEVEL_DISASSEMBLER
1197 _jitc->function = NULL;
1199 jit_reglive_setup();
1203 undo.patch_offset = 0;
1204 #define case_rr(name, type) \
1205 case jit_code_##name##r##type: \
1206 name##r##type(rn(node->u.w), rn(node->v.w)); \
1208 #define case_rw(name, type) \
1209 case jit_code_##name##i##type: \
1210 name##i##type(rn(node->u.w), node->v.w); \
1212 #define case_wr(name, type) \
1213 case jit_code_##name##i##type: \
1214 name##i##type(node->u.w, rn(node->v.w)); \
1216 #define case_rrr(name, type) \
1217 case jit_code_##name##r##type: \
1218 name##r##type(rn(node->u.w), \
1219 rn(node->v.w), rn(node->w.w)); \
1221 #define case_rrw(name, type) \
1222 case jit_code_##name##i##type: \
1223 name##i##type(rn(node->u.w), rn(node->v.w), node->w.w); \
1225 #define case_rrrr(name, type) \
1226 case jit_code_##name##r##type: \
1227 name##r##type(rn(node->u.q.l), rn(node->u.q.h), \
1228 rn(node->v.w), rn(node->w.w)); \
1230 #define case_rrrw(name, type) \
1231 case jit_code_##name##i##type: \
1232 name##i##type(rn(node->u.q.l), rn(node->u.q.h), \
1233 rn(node->v.w), node->w.w); \
1235 #define case_rrf(name, type, size) \
1236 case jit_code_##name##i##type: \
1237 assert(node->flag & jit_flag_data); \
1238 name##i##type(rn(node->u.w), rn(node->v.w), \
1239 (jit_float##size##_t *)node->w.n->u.w); \
1241 #define case_wrr(name, type) \
1242 case jit_code_##name##i##type: \
1243 name##i##type(node->u.w, rn(node->v.w), rn(node->w.w)); \
1245 #define case_brr(name, type) \
1246 case jit_code_##name##r##type: \
1248 assert(temp->code == jit_code_label || \
1249 temp->code == jit_code_epilog); \
1250 if (temp->flag & jit_flag_patch) \
1251 name##r##type(temp->u.w, rn(node->v.w), \
1254 word = name##r##type(_jit->pc.w, \
1255 rn(node->v.w), rn(node->w.w)); \
1256 patch(word, node); \
1259 #define case_brw(name, type) \
1260 case jit_code_##name##i##type: \
1262 assert(temp->code == jit_code_label || \
1263 temp->code == jit_code_epilog); \
1264 if (temp->flag & jit_flag_patch) \
1265 name##i##type(temp->u.w, \
1266 rn(node->v.w), node->w.w); \
1268 word = name##i##type(_jit->pc.w, \
1269 rn(node->v.w), node->w.w); \
1270 patch(word, node); \
1273 #define case_brf(name, type, size) \
1274 case jit_code_##name##i##type: \
1276 assert(temp->code == jit_code_label || \
1277 temp->code == jit_code_epilog); \
1278 if (temp->flag & jit_flag_patch) \
1279 name##i##type(temp->u.w, rn(node->v.w), \
1280 (jit_float##size##_t *)node->w.n->u.w); \
1282 word = name##i##type(_jit->pc.w, rn(node->v.w), \
1283 (jit_float##size##_t *)node->w.n->u.w); \
1284 patch(word, node); \
1287 #if DEVEL_DISASSEMBLER
1290 for (node = _jitc->head; node; node = node->next) {
1291 if (_jit->pc.uc >= _jitc->code.end)
1294 #if DEVEL_DISASSEMBLER
1295 node->offset = (jit_uword_t)_jit->pc.w - (jit_uword_t)prevw;
1298 value = jit_classify(node->code);
1299 jit_regarg_set(node, value);
1300 switch (node->code) {
1301 case jit_code_align:
1302 /* Must align to a power of two */
1303 assert(!(node->u.w & (node->u.w - 1)));
1304 if ((word = _jit->pc.w & (node->u.w - 1)))
1305 nop(node->u.w - word);
1307 case jit_code_note: case jit_code_name:
1308 node->u.w = _jit->pc.w;
1310 case jit_code_label:
1311 /* remember label is defined */
1312 node->flag |= jit_flag_patch;
1313 node->u.w = _jit->pc.w;
1332 case_rrrr(qmul, _u);
1333 case_rrrw(qmul, _u);
1340 case_rrrr(qdiv, _u);
1341 case_rrrw(qdiv, _u);
1358 case_rr(trunc, _f_i);
1359 case_rr(trunc, _d_i);
1360 #if __WORDSIZE == 64
1361 case_rr(trunc, _f_l);
1362 case_rr(trunc, _d_l);
1374 #if __WORDSIZE == 64
1390 #if __WORDSIZE == 64
1402 #if __WORDSIZE == 64
1412 #if __WORDSIZE == 64
1418 #if __WORDSIZE == 64
1421 case_rr(bswap, _us);
1422 case_rr(bswap, _ui);
1423 #if __WORDSIZE == 64
1424 case_rr(bswap, _ul);
1430 #if __WORDSIZE == 64
1435 casr(rn(node->u.w), rn(node->v.w),
1436 rn(node->w.q.l), rn(node->w.q.h));
1439 casi(rn(node->u.w), node->v.w,
1440 rn(node->w.q.l), rn(node->w.q.h));
1446 if (node->flag & jit_flag_node) {
1448 if (temp->code == jit_code_data ||
1449 (temp->code == jit_code_label &&
1450 (temp->flag & jit_flag_patch)))
1451 movi(rn(node->u.w), temp->u.w);
1453 assert(temp->code == jit_code_label ||
1454 temp->code == jit_code_epilog);
1455 word = movi_p(rn(node->u.w), node->v.w);
1460 movi(rn(node->u.w), node->v.w);
1506 case_brr(boadd, _u);
1507 case_brw(boadd, _u);
1510 case_brr(bxadd, _u);
1511 case_brw(bxadd, _u);
1514 case_brr(bosub, _u);
1515 case_brw(bosub, _u);
1518 case_brr(bxsub, _u);
1519 case_brw(bxsub, _u);
1525 case_rrf(add, _f, 32);
1527 case_rrf(sub, _f, 32);
1528 case_rrf(rsb, _f, 32);
1530 case_rrf(mul, _f, 32);
1532 case_rrf(div, _f, 32);
1546 case jit_code_movi_f:
1547 assert(node->flag & jit_flag_data);
1548 movi_f(rn(node->u.w), (jit_float32_t *)node->v.n->u.w);
1552 case_rrf(lt, _f, 32);
1554 case_rrf(le, _f, 32);
1556 case_rrf(eq, _f, 32);
1558 case_rrf(ge, _f, 32);
1560 case_rrf(gt, _f, 32);
1562 case_rrf(ne, _f, 32);
1564 case_rrf(unlt, _f, 32);
1566 case_rrf(unle, _f, 32);
1568 case_rrf(uneq, _f, 32);
1570 case_rrf(unge, _f, 32);
1572 case_rrf(ungt, _f, 32);
1574 case_rrf(ltgt, _f, 32);
1576 case_rrf(ord, _f, 32);
1577 case_rrr(unord, _f);
1578 case_rrf(unord, _f, 32);
1580 case_brf(blt, _f, 32);
1582 case_brf(ble, _f, 32);
1584 case_brf(beq, _f, 32);
1586 case_brf(bge, _f, 32);
1588 case_brf(bgt, _f, 32);
1590 case_brf(bne, _f, 32);
1591 case_brr(bunlt, _f);
1592 case_brf(bunlt, _f, 32);
1593 case_brr(bunle, _f);
1594 case_brf(bunle, _f, 32);
1595 case_brr(buneq, _f);
1596 case_brf(buneq, _f, 32);
1597 case_brr(bunge, _f);
1598 case_brf(bunge, _f, 32);
1599 case_brr(bungt, _f);
1600 case_brf(bungt, _f, 32);
1601 case_brr(bltgt, _f);
1602 case_brf(bltgt, _f, 32);
1604 case_brf(bord, _f, 32);
1605 case_brr(bunord, _f);
1606 case_brf(bunord, _f, 32);
1608 case_rrf(add, _d, 64);
1610 case_rrf(sub, _d, 64);
1611 case_rrf(rsb, _d, 64);
1613 case_rrf(mul, _d, 64);
1615 case_rrf(div, _d, 64);
1629 case jit_code_movi_d:
1630 assert(node->flag & jit_flag_data);
1631 movi_d(rn(node->u.w), (jit_float64_t *)node->v.n->u.w);
1635 case_rrf(lt, _d, 64);
1637 case_rrf(le, _d, 64);
1639 case_rrf(eq, _d, 64);
1641 case_rrf(ge, _d, 64);
1643 case_rrf(gt, _d, 64);
1645 case_rrf(ne, _d, 64);
1647 case_rrf(unlt, _d, 64);
1649 case_rrf(unle, _d, 64);
1651 case_rrf(uneq, _d, 64);
1653 case_rrf(unge, _d, 64);
1655 case_rrf(ungt, _d, 64);
1657 case_rrf(ltgt, _d, 64);
1659 case_rrf(ord, _d, 64);
1660 case_rrr(unord, _d);
1661 case_rrf(unord, _d, 64);
1663 case_brf(blt, _d, 64);
1665 case_brf(ble, _d, 64);
1667 case_brf(beq, _d, 64);
1669 case_brf(bge, _d, 64);
1671 case_brf(bgt, _d, 64);
1673 case_brf(bne, _d, 64);
1674 case_brr(bunlt, _d);
1675 case_brf(bunlt, _d, 64);
1676 case_brr(bunle, _d);
1677 case_brf(bunle, _d, 64);
1678 case_brr(buneq, _d);
1679 case_brf(buneq, _d, 64);
1680 case_brr(bunge, _d);
1681 case_brf(bunge, _d, 64);
1682 case_brr(bungt, _d);
1683 case_brf(bungt, _d, 64);
1684 case_brr(bltgt, _d);
1685 case_brf(bltgt, _d, 64);
1687 case_brf(bord, _d, 64);
1688 case_brr(bunord, _d);
1689 case_brf(bunord, _d, 64);
1691 jmpr(rn(node->u.w));
1694 if (node->flag & jit_flag_node) {
1696 assert(temp->code == jit_code_label ||
1697 temp->code == jit_code_epilog);
1698 if (temp->flag & jit_flag_patch)
1701 word = jmpi(_jit->pc.w);
1708 case jit_code_callr:
1709 callr(rn(node->u.w));
1711 case jit_code_calli:
1712 if (node->flag & jit_flag_node) {
1714 assert(temp->code == jit_code_label ||
1715 temp->code == jit_code_epilog);
1716 word = calli_p(temp->u.w);
1717 if (!(temp->flag & jit_flag_patch))
1723 case jit_code_prolog:
1724 _jitc->function = _jitc->functions.ptr + node->w.w;
1726 undo.word = _jit->pc.w;
1727 #if DEVEL_DISASSEMBLER
1730 undo.patch_offset = _jitc->patches.offset;
1735 case jit_code_epilog:
1736 assert(_jitc->function == _jitc->functions.ptr + node->w.w);
1738 for (temp = undo.node->next;
1739 temp != node; temp = temp->next) {
1740 if (temp->code == jit_code_label ||
1741 temp->code == jit_code_epilog)
1742 temp->flag &= ~jit_flag_patch;
1744 temp->flag &= ~jit_flag_patch;
1746 _jit->pc.w = undo.word;
1747 #if DEVEL_DISASSEMBLER
1750 _jitc->patches.offset = undo.patch_offset;
1751 goto restart_function;
1753 /* remember label is defined */
1754 node->flag |= jit_flag_patch;
1755 node->u.w = _jit->pc.w;
1757 _jitc->function = NULL;
1760 case jit_code_movr_w_f:
1761 movr_w_f(rn(node->u.w), rn(node->v.w));
1764 case jit_code_movr_f_w:
1765 movr_f_w(rn(node->u.w), rn(node->v.w));
1767 case jit_code_movi_f_w:
1768 assert(node->flag & jit_flag_data);
1769 movi_f_w(rn(node->u.w), (jit_float32_t *)node->v.n->u.w);
1772 case jit_code_movr_d_w:
1773 movr_d_w(rn(node->u.w), rn(node->v.w));
1775 case jit_code_movi_d_w:
1776 assert(node->flag & jit_flag_data);
1777 movi_d_w(rn(node->u.w), (jit_float64_t *)node->v.n->u.w);
1780 case jit_code_movr_ww_d:
1781 movr_ww_d(rn(node->u.w), rn(node->v.w), rn(node->w.w));
1783 case jit_code_movr_d_ww:
1784 movr_d_ww(rn(node->u.w), rn(node->v.w), rn(node->w.w));
1786 case jit_code_movi_d_ww:
1787 assert(node->flag & jit_flag_data);
1788 movi_d_ww(rn(node->u.w), rn(node->v.w),
1789 (jit_float64_t *)node->w.n->u.w);
1792 case jit_code_va_start:
1793 vastart(rn(node->u.w));
1795 case jit_code_va_arg:
1796 vaarg(rn(node->u.w), rn(node->v.w));
1798 case jit_code_va_arg_d:
1799 vaarg_d(rn(node->u.w), rn(node->v.w));
1802 case jit_code_arg: case jit_code_ellipsis:
1803 case jit_code_va_push:
1804 case jit_code_allocai: case jit_code_allocar:
1805 case jit_code_arg_f: case jit_code_arg_d:
1806 case jit_code_va_end:
1808 case jit_code_retr: case jit_code_reti:
1809 case jit_code_retr_f: case jit_code_reti_f:
1810 case jit_code_retr_d: case jit_code_reti_d:
1811 case jit_code_getarg_c: case jit_code_getarg_uc:
1812 case jit_code_getarg_s: case jit_code_getarg_us:
1813 case jit_code_getarg_i:
1814 #if __WORDSIZE == 64
1815 case jit_code_getarg_ui: case jit_code_getarg_l:
1817 case jit_code_getarg_f: case jit_code_getarg_d:
1818 case jit_code_putargr: case jit_code_putargi:
1819 case jit_code_putargr_f: case jit_code_putargi_f:
1820 case jit_code_putargr_d: case jit_code_putargi_d:
1821 case jit_code_pushargr: case jit_code_pushargi:
1822 case jit_code_pushargr_f: case jit_code_pushargi_f:
1823 case jit_code_pushargr_d: case jit_code_pushargi_d:
1824 case jit_code_retval_c: case jit_code_retval_uc:
1825 case jit_code_retval_s: case jit_code_retval_us:
1826 case jit_code_retval_i:
1827 #if __WORDSIZE == 64
1828 case jit_code_retval_ui: case jit_code_retval_l:
1830 case jit_code_retval_f: case jit_code_retval_d:
1831 case jit_code_prepare:
1832 case jit_code_finishr: case jit_code_finishi:
1837 if (jit_carry != _NOREG) {
1838 switch (node->code) {
1840 case jit_code_addcr: case jit_code_addci:
1841 case jit_code_addxr: case jit_code_addxi:
1842 case jit_code_subcr: case jit_code_subci:
1843 case jit_code_subxr: case jit_code_subxi:
1846 jit_unget_reg(jit_carry);
1851 jit_regarg_clr(node, value);
1852 assert(_jitc->regarg == 0 ||
1853 (jit_carry != _NOREG && _jitc->regarg == (1 << jit_carry)));
1854 assert(_jitc->synth == 0);
1855 /* update register live state */
1869 for (offset = 0; offset < _jitc->patches.offset; offset++) {
1870 node = _jitc->patches.ptr[offset].node;
1871 word = node->code == jit_code_movi ? node->v.n->u.w : node->u.n->u.w;
1872 patch_at(_jitc->patches.ptr[offset].inst, word);
1875 jit_flush(_jit->code.ptr, _jit->pc.uc);
1877 return (_jit->code.ptr);
1881 # include "jit_rewind.c"
1882 # include "jit_mips-cpu.c"
1883 # include "jit_mips-fpu.c"
1887 jit_flush(void *fptr, void *tptr)
1889 #if defined(__linux__)
1892 s = sysconf(_SC_PAGE_SIZE);
1893 f = (jit_word_t)fptr & -s;
1894 t = (((jit_word_t)tptr) + s - 1) & -s;
1895 _flush_cache((void *)f, t - f, ICACHE);
1900 _emit_ldxi(jit_state_t *_jit, jit_int32_t r0, jit_int32_t r1, jit_word_t i0)
1902 ldxi(rn(r0), rn(r1), i0);
1906 _emit_stxi(jit_state_t *_jit, jit_word_t i0, jit_int32_t r0, jit_int32_t r1)
1908 stxi(i0, rn(r0), rn(r1));
1912 _emit_ldxi_d(jit_state_t *_jit, jit_int32_t r0, jit_int32_t r1, jit_word_t i0)
1914 ldxi_d(rn(r0), rn(r1), i0);
1918 _emit_stxi_d(jit_state_t *_jit, jit_word_t i0, jit_int32_t r0, jit_int32_t r1)
1920 stxi_d(i0, rn(r0), rn(r1));
1924 _patch(jit_state_t *_jit, jit_word_t instr, jit_node_t *node)
1928 assert(node->flag & jit_flag_node);
1929 if (node->code == jit_code_movi)
1930 flag = node->v.n->flag;
1932 flag = node->u.n->flag;
1933 assert(!(flag & jit_flag_patch));
1934 if (_jitc->patches.offset >= _jitc->patches.length) {
1935 jit_realloc((jit_pointer_t *)&_jitc->patches.ptr,
1936 _jitc->patches.length * sizeof(jit_patch_t),
1937 (_jitc->patches.length + 1024) * sizeof(jit_patch_t));
1938 _jitc->patches.length += 1024;
1940 _jitc->patches.ptr[_jitc->patches.offset].inst = instr;
1941 _jitc->patches.ptr[_jitc->patches.offset].node = node;
1942 ++_jitc->patches.offset;