2 * Copyright (C) 2013-2023 Free Software Foundation, Inc.
4 * This file is part of GNU lightning.
6 * GNU lightning is free software; you can redistribute it and/or modify it
7 * under the terms of the GNU Lesser General Public License as published
8 * by the Free Software Foundation; either version 3, or (at your option)
11 * GNU lightning is distributed in the hope that it will be useful, but
12 * WITHOUT ANY WARRANTY; without even the implied warranty of MERCHANTABILITY
13 * or FITNESS FOR A PARTICULAR PURPOSE. See the GNU Lesser General Public
14 * License for more details.
17 * Paulo Cesar Pereira de Andrade
21 * align16(lr+fp+x19+x2[0-8]+v8+v9+v1[0-15]) */
22 #define stack_framesize 160
24 #define jit_arg_reg_p(i) ((i) >= 0 && (i) < 8)
25 #define jit_arg_f_reg_p(i) ((i) >= 0 && (i) < 8)
28 typedef jit_pointer_t jit_va_list_t;
30 typedef struct jit_qreg {
35 #define va_gp_top_offset offsetof(jit_va_list_t, q0)
36 #define va_fp_top_offset sizeof(jit_va_list_t)
37 typedef struct jit_va_list {
67 #define compute_framesize() _compute_framesize(_jit)
68 static void _compute_framesize(jit_state_t*);
69 #define patch(instr, node) _patch(_jit, instr, node)
70 static void _patch(jit_state_t*,jit_word_t,jit_node_t*);
73 extern void __clear_cache(void *, void *);
76 # include "jit_aarch64-cpu.c"
77 # include "jit_aarch64-fpu.c"
83 jit_register_t _rvs[] = {
84 { rc(gpr) | 0x08, "x8" },
88 { rc(gpr) | 0x12, "x18" },
90 { rc(gpr) | 0x11, "x17" },
91 { rc(gpr) | 0x10, "x16" },
92 { rc(gpr) | 0x09, "x9" },
93 { rc(gpr) | 0x0a, "x10" },
94 { rc(gpr) | 0x0b, "x11" },
95 { rc(gpr) | 0x0c, "x12" },
96 { rc(gpr) | 0x0d, "x13" },
97 { rc(gpr) | 0x0e, "x14" },
98 { rc(gpr) | 0x0f, "x15" },
99 { rc(sav) | rc(gpr) | 0x13, "x19" },
100 { rc(sav) | rc(gpr) | 0x14, "x20" },
101 { rc(sav) | rc(gpr) | 0x15, "x21" },
102 { rc(sav) | rc(gpr) | 0x16, "x22" },
103 { rc(sav) | rc(gpr) | 0x17, "x23" },
104 { rc(sav) | rc(gpr) | 0x18, "x24" },
105 { rc(sav) | rc(gpr) | 0x19, "x25" },
106 { rc(sav) | rc(gpr) | 0x1a, "x26" },
107 { rc(sav) | rc(gpr) | 0x1b, "x27" },
108 { rc(sav) | rc(gpr) | 0x1c, "x28" },
112 { rc(arg) | rc(gpr) | 0x07, "x7" },
113 { rc(arg) | rc(gpr) | 0x06, "x6" },
114 { rc(arg) | rc(gpr) | 0x05, "x5" },
115 { rc(arg) | rc(gpr) | 0x04, "x4" },
116 { rc(arg) | rc(gpr) | 0x03, "x3" },
117 { rc(arg) | rc(gpr) | 0x02, "x2" },
118 { rc(arg) | rc(gpr) | 0x01, "x1" },
119 { rc(arg) | rc(gpr) | 0x00, "x0" },
120 { rc(fpr) | 0x1f, "v31" },
121 { rc(fpr) | 0x1e, "v30" },
122 { rc(fpr) | 0x1d, "v29" },
123 { rc(fpr) | 0x1c, "v28" },
124 { rc(fpr) | 0x1b, "v27" },
125 { rc(fpr) | 0x1a, "v26" },
126 { rc(fpr) | 0x19, "v25" },
127 { rc(fpr) | 0x18, "v24" },
128 { rc(fpr) | 0x17, "v23" },
129 { rc(fpr) | 0x16, "v22" },
130 { rc(fpr) | 0x15, "v21" },
131 { rc(fpr) | 0x14, "v20" },
132 { rc(fpr) | 0x13, "v19" },
133 { rc(fpr) | 0x12, "v18" },
134 { rc(fpr) | 0x11, "v17" },
135 { rc(fpr) | 0x10, "v16" },
136 { rc(sav) | rc(fpr) | 0x08, "v8" },
137 { rc(sav) | rc(fpr) | 0x09, "v9" },
138 { rc(sav) | rc(fpr) | 0x0a, "v10" },
139 { rc(sav) | rc(fpr) | 0x0b, "v11" },
140 { rc(sav) | rc(fpr) | 0x0c, "v12" },
141 { rc(sav) | rc(fpr) | 0x0d, "v13" },
142 { rc(sav) | rc(fpr) | 0x0e, "v14" },
143 { rc(sav) | rc(fpr) | 0x0f, "v15" },
144 { rc(arg) | rc(fpr) | 0x07, "v7" },
145 { rc(arg) | rc(fpr) | 0x06, "v6" },
146 { rc(arg) | rc(fpr) | 0x05, "v5" },
147 { rc(arg) | rc(fpr) | 0x04, "v4" },
148 { rc(arg) | rc(fpr) | 0x03, "v3" },
149 { rc(arg) | rc(fpr) | 0x02, "v2" },
150 { rc(arg) | rc(fpr) | 0x01, "v1" },
151 { rc(arg) | rc(fpr) | 0x00, "v0" },
152 { _NOREG, "<none>" },
155 static jit_int32_t iregs[] = {
156 _R19, _R20, _R21, _R22, _R23, _R24, _R25, _R26, _R27, _R28
159 static jit_int32_t fregs[] = {
160 _V8, _V9, _V10, _V11, _V12, _V13, _V14, _V15
172 _jit_init(jit_state_t *_jit)
174 _jitc->reglen = jit_size(_rvs) - 1;
178 _jit_prolog(jit_state_t *_jit)
184 assert(jit_regset_cmp_ui(&_jitc->regarg, 0) == 0);
185 jit_regset_set_ui(&_jitc->regsav, 0);
186 offset = _jitc->functions.offset;
187 if (offset >= _jitc->functions.length) {
188 jit_realloc((jit_pointer_t *)&_jitc->functions.ptr,
189 _jitc->functions.length * sizeof(jit_function_t),
190 (_jitc->functions.length + 16) * sizeof(jit_function_t));
191 _jitc->functions.length += 16;
193 _jitc->function = _jitc->functions.ptr + _jitc->functions.offset++;
194 _jitc->function->self.size = stack_framesize;
195 _jitc->function->self.argi = _jitc->function->self.argf =
196 _jitc->function->self.alen = 0;
197 _jitc->function->self.aoff = 0;
198 _jitc->function->self.call = jit_call_default;
199 jit_alloc((jit_pointer_t *)&_jitc->function->regoff,
200 _jitc->reglen * sizeof(jit_int32_t));
202 /* _no_link here does not mean the jit_link() call can be removed
204 * _jitc->function->prolog = jit_new_node(jit_code_prolog);
206 _jitc->function->prolog = jit_new_node_no_link(jit_code_prolog);
207 jit_link(_jitc->function->prolog);
208 _jitc->function->prolog->w.w = offset;
209 _jitc->function->epilog = jit_new_node_no_link(jit_code_epilog);
211 * v: offset in blocks vector
212 * w: offset in functions vector
214 _jitc->function->epilog->w.w = offset;
216 jit_regset_new(&_jitc->function->regset);
220 _jit_allocai(jit_state_t *_jit, jit_int32_t length)
222 assert(_jitc->function);
225 case 0: case 1: break;
226 case 2: _jitc->function->self.aoff &= -2; break;
227 case 3: case 4: _jitc->function->self.aoff &= -4; break;
228 default: _jitc->function->self.aoff &= -8; break;
230 _jitc->function->self.aoff -= length;
231 if (!_jitc->realize) {
232 jit_inc_synth_ww(allocai, _jitc->function->self.aoff, length);
235 return (_jitc->function->self.aoff);
239 _jit_allocar(jit_state_t *_jit, jit_int32_t u, jit_int32_t v)
242 assert(_jitc->function);
243 jit_inc_synth_ww(allocar, u, v);
244 if (!_jitc->function->allocar) {
245 _jitc->function->aoffoff = jit_allocai(sizeof(jit_int32_t));
246 _jitc->function->allocar = 1;
248 r0 = jit_get_reg(jit_class_gpr);
250 jit_andi(r0, r0, -16);
251 jit_ldxi_i(u, JIT_FP, _jitc->function->aoffoff);
253 /* Cannot "addr sp, sp, reg" because in this context "sp" is "[w|x]zr",
254 * the zero register */
256 jit_addr(JIT_SP, JIT_SP, r0);
258 r1 = jit_get_reg(jit_class_gpr);
259 /* note that "mov r1, sp" does not work, but the proper encoding
260 * can be triggered before actually emiting with "add r1, sp, 0" */
261 jit_addi(r1, JIT_SP, 0);
262 jit_addr(r1, r1, r0);
263 jit_addi(JIT_SP, r1, 0);
266 jit_stxi_i(_jitc->function->aoffoff, JIT_FP, u);
272 _jit_ret(jit_state_t *_jit)
275 assert(_jitc->function);
279 jit_patch_at(instr, _jitc->function->epilog);
284 _jit_retr(jit_state_t *_jit, jit_int32_t u, jit_code_t code)
286 jit_code_inc_synth_w(code, u);
287 jit_movr(JIT_RET, u);
293 _jit_reti(jit_state_t *_jit, jit_word_t u, jit_code_t code)
295 jit_code_inc_synth_w(code, u);
296 jit_movi(JIT_RET, u);
302 _jit_retr_f(jit_state_t *_jit, jit_int32_t u)
304 jit_inc_synth_w(retr_f, u);
306 jit_movr_f(JIT_FRET, u);
314 _jit_reti_f(jit_state_t *_jit, jit_float32_t u)
316 jit_inc_synth_f(reti_f, u);
317 jit_movi_f(JIT_FRET, u);
323 _jit_retr_d(jit_state_t *_jit, jit_int32_t u)
325 jit_inc_synth_w(retr_d, u);
327 jit_movr_d(JIT_FRET, u);
335 _jit_reti_d(jit_state_t *_jit, jit_float64_t u)
337 jit_inc_synth_d(reti_d, u);
338 jit_movi_d(JIT_FRET, u);
344 _jit_epilog(jit_state_t *_jit)
346 assert(_jitc->function);
347 assert(_jitc->function->epilog->next == NULL);
348 jit_link(_jitc->function->epilog);
349 _jitc->function = NULL;
353 _jit_arg_register_p(jit_state_t *_jit, jit_node_t *u)
355 if (u->code >= jit_code_arg_c && u->code <= jit_code_arg)
356 return (jit_arg_reg_p(u->u.w));
357 assert(u->code == jit_code_arg_f || u->code == jit_code_arg_d);
358 return (jit_arg_f_reg_p(u->u.w));
362 _jit_ellipsis(jit_state_t *_jit)
364 jit_inc_synth(ellipsis);
366 if (_jitc->prepare) {
368 assert(!(_jitc->function->call.call & jit_call_varargs));
369 _jitc->function->call.call |= jit_call_varargs;
373 assert(!(_jitc->function->self.call & jit_call_varargs));
374 _jitc->function->self.call |= jit_call_varargs;
377 /* Allocate va_list like object in the stack,
378 * with enough space to save all argument
379 * registers, and use fixed offsets for them. */
380 _jitc->function->vaoff = jit_allocai(sizeof(jit_va_list_t));
382 /* Initialize gp offset in save area. */
383 if (jit_arg_reg_p(_jitc->function->self.argi))
384 _jitc->function->vagp = (8 - _jitc->function->self.argi) * -8;
386 _jitc->function->vagp = 0;
388 /* Initialize fp offset in save area. */
389 if (jit_arg_f_reg_p(_jitc->function->self.argf))
390 _jitc->function->vafp = (8 - _jitc->function->self.argf) * -16;
392 _jitc->function->vafp = 0;
399 _jit_va_push(jit_state_t *_jit, jit_int32_t u)
401 jit_inc_synth_w(va_push, u);
407 _jit_arg(jit_state_t *_jit, jit_code_t code)
411 assert(_jitc->function);
412 assert(!(_jitc->function->self.call & jit_call_varargs));
413 if (jit_arg_reg_p(_jitc->function->self.argi))
414 offset = _jitc->function->self.argi++;
416 #if PACKED_STACK || STRONG_TYPE_CHECKING
417 assert(code >= jit_code_arg_c && code <= jit_code_arg);
420 _jitc->function->self.size +=
421 _jitc->function->self.size & ((1 << (code - jit_code_arg_c)) - 1);
423 offset = _jitc->function->self.size;
425 _jitc->function->self.size += 1 << (code - jit_code_arg_c);
427 _jitc->function->self.size += sizeof(jit_word_t);
431 node = jit_new_node_ww(code, offset,
432 ++_jitc->function->self.argn);
438 _jit_arg_f(jit_state_t *_jit)
442 assert(_jitc->function);
443 assert(!(_jitc->function->self.call & jit_call_varargs));
444 if (jit_arg_f_reg_p(_jitc->function->self.argf))
445 offset = _jitc->function->self.argf++;
448 _jitc->function->self.size +=
449 _jitc->function->self.size & (sizeof(jit_float32_t) - 1);
451 offset = _jitc->function->self.size;
453 _jitc->function->self.size += sizeof(jit_float32_t);
455 _jitc->function->self.size += sizeof(jit_word_t);
459 node = jit_new_node_ww(jit_code_arg_f, offset,
460 ++_jitc->function->self.argn);
466 _jit_arg_d(jit_state_t *_jit)
470 assert(_jitc->function);
471 assert(!(_jitc->function->self.call & jit_call_varargs));
472 if (jit_arg_f_reg_p(_jitc->function->self.argf))
473 offset = _jitc->function->self.argf++;
476 _jitc->function->self.size +=
477 _jitc->function->self.size & (sizeof(jit_float64_t) - 1);
479 offset = _jitc->function->self.size;
480 _jitc->function->self.size += sizeof(jit_float64_t);
483 node = jit_new_node_ww(jit_code_arg_d, offset,
484 ++_jitc->function->self.argn);
490 _jit_getarg_c(jit_state_t *_jit, jit_int32_t u, jit_node_t *v)
492 assert_arg_type(v->code, jit_code_arg_c);
493 jit_inc_synth_wp(getarg_c, u, v);
494 if (jit_arg_reg_p(v->u.w)) {
496 jit_movr(u, JIT_RA0 - v->u.w);
498 jit_extr_c(u, JIT_RA0 - v->u.w);
502 jit_node_t *node = jit_ldxi_c(u, JIT_FP, v->u.w);
503 jit_link_alist(node);
509 _jit_getarg_uc(jit_state_t *_jit, jit_int32_t u, jit_node_t *v)
511 assert_arg_type(v->code, jit_code_arg_c);
512 jit_inc_synth_wp(getarg_uc, u, v);
513 if (jit_arg_reg_p(v->u.w)) {
515 jit_movr(u, JIT_RA0 - v->u.w);
517 jit_extr_uc(u, JIT_RA0 - v->u.w);
521 jit_node_t *node = jit_ldxi_uc(u, JIT_FP, v->u.w);
522 jit_link_alist(node);
528 _jit_getarg_s(jit_state_t *_jit, jit_int32_t u, jit_node_t *v)
530 assert_arg_type(v->code, jit_code_arg_s);
531 jit_inc_synth_wp(getarg_s, u, v);
532 if (jit_arg_reg_p(v->u.w)) {
534 jit_movr(u, JIT_RA0 - v->u.w);
536 jit_extr_s(u, JIT_RA0 - v->u.w);
540 jit_node_t *node = jit_ldxi_s(u, JIT_FP, v->u.w);
541 jit_link_alist(node);
547 _jit_getarg_us(jit_state_t *_jit, jit_int32_t u, jit_node_t *v)
549 assert_arg_type(v->code, jit_code_arg_s);
550 jit_inc_synth_wp(getarg_us, u, v);
551 if (jit_arg_reg_p(v->u.w)) {
553 jit_movr(u, JIT_RA0 - v->u.w);
555 jit_extr_us(u, JIT_RA0 - v->u.w);
559 jit_node_t *node = jit_ldxi_us(u, JIT_FP, v->u.w);
560 jit_link_alist(node);
566 _jit_getarg_i(jit_state_t *_jit, jit_int32_t u, jit_node_t *v)
568 assert_arg_type(v->code, jit_code_arg_i);
569 jit_inc_synth_wp(getarg_i, u, v);
570 if (jit_arg_reg_p(v->u.w)) {
571 #if PACKED_STACK || __WORDSIZE == 32
572 jit_movr(u, JIT_RA0 - v->u.w);
574 jit_extr_i(u, JIT_RA0 - v->u.w);
578 jit_node_t *node = jit_ldxi_i(u, JIT_FP, v->u.w);
579 jit_link_alist(node);
586 _jit_getarg_ui(jit_state_t *_jit, jit_int32_t u, jit_node_t *v)
588 assert_arg_type(v->code, jit_code_arg_i);
589 jit_inc_synth_wp(getarg_ui, u, v);
590 if (jit_arg_reg_p(v->u.w)) {
592 jit_movr(u, JIT_RA0 - v->u.w);
594 jit_extr_ui(u, JIT_RA0 - v->u.w);
598 jit_node_t *node = jit_ldxi_ui(u, JIT_FP, v->u.w);
599 jit_link_alist(node);
605 _jit_getarg_l(jit_state_t *_jit, jit_int32_t u, jit_node_t *v)
607 assert_arg_type(v->code, jit_code_arg_l);
608 jit_inc_synth_wp(getarg_l, u, v);
609 if (jit_arg_reg_p(v->u.w))
610 jit_movr(u, JIT_RA0 - v->u.w);
612 jit_node_t *node = jit_ldxi_l(u, JIT_FP, v->u.w);
613 jit_link_alist(node);
620 _jit_putargr(jit_state_t *_jit, jit_int32_t u, jit_node_t *v, jit_code_t code)
622 assert_putarg_type(code, v->code);
623 jit_code_inc_synth_wp(code, u, v);
624 if (jit_arg_reg_p(v->u.w)) {
625 jit_int32_t regno = JIT_RA0 - v->u.w;
628 case jit_code_putargr_c: jit_extr_c(regno, u); break;
629 case jit_code_putargr_uc: jit_extr_uc(regno, u); break;
630 case jit_code_putargr_s: jit_extr_s(regno, u); break;
631 case jit_code_putargr_us: jit_extr_us(regno, u); break;
632 # if __WORDISZE == 32
633 case jit_code_putargr_i: jit_movr(regno, u); break;
635 case jit_code_putargr_i: jit_extr_i(regno, u); break;
636 case jit_code_putargr_ui: jit_extr_ui(regno, u); break;
637 case jit_code_putargr_l: jit_movr(regno, u); break;
639 default: abort(); break;
649 case jit_code_putargr_c: case jit_code_putargr_uc:
650 node = jit_stxi_c(v->u.w, JIT_FP, u); break;
651 case jit_code_putargr_s: case jit_code_putargr_us:
652 node = jit_stxi_s(v->u.w, JIT_FP, u); break;
653 # if __WORDSIZE == 32
654 case jit_code_putargr_i:
655 node = jit_stxi(v->u.w, JIT_FP, u); break;
657 case jit_code_putargr_i: case jit_code_putargr_ui:
658 node = jit_stxi_i(v->u.w, JIT_FP, u); break;
659 case jit_code_putargr_l:
660 node = jit_stxi(v->u.w, JIT_FP, u); break;
662 default: abort(); break;
665 node = jit_stxi(v->u.w, JIT_FP, u);
667 jit_link_alist(node);
673 _jit_putargi(jit_state_t *_jit, jit_word_t u, jit_node_t *v, jit_code_t code)
676 assert_putarg_type(code, v->code);
677 jit_code_inc_synth_wp(code, u, v);
680 case jit_code_putargi_c: u = (jit_int8_t)u; break;
681 case jit_code_putargi_uc: u = (jit_uint8_t)u; break;
682 case jit_code_putargi_s: u = (jit_int16_t)u; break;
683 case jit_code_putargi_us: u = (jit_uint16_t)u; break;
684 # if __WORDSIZE == 32
685 case jit_code_putargi_i: break;
687 case jit_code_putargi_i: u = (jit_int32_t)u; break;
688 case jit_code_putargi_ui: u = (jit_uint32_t)u; break;
689 case jit_code_putargi_l: break;
691 default: abort(); break;
694 if (jit_arg_reg_p(v->u.w))
695 jit_movi(JIT_RA0 - v->u.w, u);
698 regno = jit_get_reg(jit_class_gpr);
702 case jit_code_putargi_c: case jit_code_putargi_uc:
703 node = jit_stxi_c(v->u.w, JIT_FP, regno); break;
704 case jit_code_putargi_s: case jit_code_putargi_us:
705 node = jit_stxi_s(v->u.w, JIT_FP, regno); break;
706 # if __WORDSIZE == 32
707 case jit_code_putargi_i:
708 node = jit_stxi(v->u.w, JIT_FP, regno); break;
710 case jit_code_putargi_i: case jit_code_putargi_ui:
711 node = jit_stxi_i(v->u.w, JIT_FP, regno); break;
712 case jit_code_putargi_l:
713 node = jit_stxi(v->u.w, JIT_FP, regno); break;
715 default: abort(); break;
718 node = jit_stxi(v->u.w, JIT_FP, regno);
720 jit_link_alist(node);
721 jit_unget_reg(regno);
727 _jit_getarg_f(jit_state_t *_jit, jit_int32_t u, jit_node_t *v)
729 assert(v->code == jit_code_arg_f);
730 jit_inc_synth_wp(getarg_f, u, v);
731 if (jit_arg_reg_p(v->u.w))
732 jit_movr_f(u, JIT_FA0 - v->u.w);
734 jit_node_t *node = jit_ldxi_f(u, JIT_FP, v->u.w);
735 jit_link_alist(node);
741 _jit_putargr_f(jit_state_t *_jit, jit_int32_t u, jit_node_t *v)
743 assert(v->code == jit_code_arg_f);
744 jit_inc_synth_wp(putargr_f, u, v);
745 if (jit_arg_f_reg_p(v->u.w))
746 jit_movr_f(JIT_FA0 - v->u.w, u);
748 jit_node_t *node = jit_stxi_f(v->u.w, JIT_FP, u);
749 jit_link_alist(node);
755 _jit_putargi_f(jit_state_t *_jit, jit_float32_t u, jit_node_t *v)
758 assert(v->code == jit_code_arg_f);
759 jit_inc_synth_fp(putargi_f, u, v);
760 if (jit_arg_f_reg_p(v->u.w))
761 jit_movi_f(JIT_FA0 - v->u.w, u);
764 regno = jit_get_reg(jit_class_fpr);
765 jit_movi_f(regno, u);
766 node = jit_stxi_f(v->u.w, JIT_FP, regno);
767 jit_link_alist(node);
768 jit_unget_reg(regno);
774 _jit_getarg_d(jit_state_t *_jit, jit_int32_t u, jit_node_t *v)
776 assert(v->code == jit_code_arg_d);
777 jit_inc_synth_wp(getarg_d, u, v);
778 if (jit_arg_f_reg_p(v->u.w))
779 jit_movr_d(u, JIT_FA0 - v->u.w);
781 jit_node_t *node = jit_ldxi_d(u, JIT_FP, v->u.w);
782 jit_link_alist(node);
788 _jit_putargr_d(jit_state_t *_jit, jit_int32_t u, jit_node_t *v)
790 assert(v->code == jit_code_arg_d);
791 jit_inc_synth_wp(putargr_d, u, v);
792 if (jit_arg_reg_p(v->u.w))
793 jit_movr_d(JIT_FA0 - v->u.w, u);
795 jit_node_t *node = jit_stxi_d(v->u.w, JIT_FP, u);
796 jit_link_alist(node);
802 _jit_putargi_d(jit_state_t *_jit, jit_float64_t u, jit_node_t *v)
805 assert(v->code == jit_code_arg_d);
806 jit_inc_synth_dp(putargi_d, u, v);
807 if (jit_arg_reg_p(v->u.w))
808 jit_movi_d(JIT_FA0 - v->u.w, u);
811 regno = jit_get_reg(jit_class_fpr);
812 jit_movi_d(regno, u);
813 node = jit_stxi_d(v->u.w, JIT_FP, regno);
814 jit_link_alist(node);
815 jit_unget_reg(regno);
821 _jit_pushargr(jit_state_t *_jit, jit_int32_t u, jit_code_t code)
823 assert(_jitc->function);
824 jit_code_inc_synth_w(code, u);
826 if (jit_arg_reg_p(_jitc->function->call.argi)) {
827 jit_int32_t regno = JIT_RA0 - _jitc->function->call.argi;
830 case jit_code_pushargr_c: jit_extr_c(regno, u); break;
831 case jit_code_pushargr_uc: jit_extr_uc(regno, u); break;
832 case jit_code_pushargr_s: jit_extr_s(regno, u); break;
833 case jit_code_pushargr_us: jit_extr_us(regno, u); break;
834 # if __WORDISZE == 32
835 case jit_code_pushargr_i: jit_movr(regno, u); break;
837 case jit_code_pushargr_i: jit_extr_i(regno, u); break;
838 case jit_code_pushargr_ui: jit_extr_ui(regno, u); break;
839 case jit_code_pushargr_l: jit_movr(regno, u); break;
841 default: abort(); break;
847 if (_jitc->function->call.call & jit_call_varargs) {
848 assert(code == jit_code_pushargr);
849 jit_stxi(_jitc->function->call.size, JIT_SP, u);
850 _jitc->function->call.size += sizeof(jit_word_t);
853 ++_jitc->function->call.argi;
857 _jitc->function->call.size +=
858 _jitc->function->call.size &
859 ((1 << ((code - jit_code_pushargr_c) >> 2)) - 1);
861 case jit_code_pushargr_c: case jit_code_pushargr_uc:
862 jit_stxi_c(_jitc->function->call.size, JIT_SP, u);
864 case jit_code_pushargr_s: case jit_code_pushargr_us:
865 jit_stxi_s(_jitc->function->call.size, JIT_SP, u);
867 # if __WORDSIZE == 32
868 case jit_code_pushargr_i:
869 jit_stxi(_jitc->function->call.size, JIT_SP, u);
872 case jit_code_pushargr_i: case jit_code_pushargr_ui:
873 jit_stxi_i(_jitc->function->call.size, JIT_SP, u);
875 case jit_code_pushargr_l:
876 jit_stxi(_jitc->function->call.size, JIT_SP, u);
883 _jitc->function->call.size += 1 << ((code - jit_code_pushargr_c) >> 2);
885 jit_stxi(_jitc->function->call.size, JIT_SP, u);
886 _jitc->function->call.size += sizeof(jit_word_t);
894 _jit_pushargi(jit_state_t *_jit, jit_word_t u, jit_code_t code)
897 assert(_jitc->function);
898 jit_code_inc_synth_w(code, u);
902 case jit_code_pushargi_c: u = (jit_int8_t)u; break;
903 case jit_code_pushargi_uc: u = (jit_uint8_t)u; break;
904 case jit_code_pushargi_s: u = (jit_int16_t)u; break;
905 case jit_code_pushargi_us: u = (jit_uint16_t)u; break;
906 # if __WORDSIZE == 32
907 case jit_code_pushargi_i: break;
909 case jit_code_pushargi_i: u = (jit_int32_t)u; break;
910 case jit_code_pushargi_ui: u = (jit_uint32_t)u; break;
911 case jit_code_pushargi_l: break;
913 default: abort(); break;
916 if (jit_arg_reg_p(_jitc->function->call.argi)) {
917 regno = JIT_RA0 - _jitc->function->call.argi;
920 if (_jitc->function->call.call & jit_call_varargs) {
921 assert(code == jit_code_pushargi);
922 jit_stxi(_jitc->function->call.size, JIT_SP, regno);
923 _jitc->function->call.size += sizeof(jit_word_t);
926 ++_jitc->function->call.argi;
929 regno = jit_get_reg(jit_class_gpr);
932 _jitc->function->call.size +=
933 _jitc->function->call.size &
934 ((1 << ((code - jit_code_pushargr_c) >> 2)) - 1);
936 case jit_code_pushargi_c: case jit_code_pushargi_uc:
937 jit_stxi_c(_jitc->function->call.size, JIT_SP, regno);
939 case jit_code_pushargi_s: case jit_code_pushargi_us:
940 jit_stxi_s(_jitc->function->call.size, JIT_SP, regno);
942 # if __WORDSIZE == 32
943 case jit_code_pushargi_i:
944 jit_stxi(_jitc->function->call.size, JIT_SP, regno);
947 case jit_code_pushargi_i: case jit_code_pushargi_ui:
948 jit_stxi_i(_jitc->function->call.size, JIT_SP, regno);
950 case jit_code_pushargi_l:
951 jit_stxi(_jitc->function->call.size, JIT_SP, regno);
958 _jitc->function->call.size += 1 << ((code - jit_code_pushargr_c) >> 2);
960 jit_stxi(_jitc->function->call.size, JIT_SP, regno);
961 _jitc->function->call.size += sizeof(jit_word_t);
963 jit_unget_reg(regno);
970 _jit_pushargr_f(jit_state_t *_jit, jit_int32_t u)
972 assert(_jitc->function);
973 jit_inc_synth_w(pushargr_f, u);
975 if (jit_arg_f_reg_p(_jitc->function->call.argf)) {
976 jit_movr_f(JIT_FA0 - _jitc->function->call.argf, u);
978 if (_jitc->function->call.call & jit_call_varargs) {
979 assert(sizeof(jit_float32_t) == sizeof(jit_word_t));
980 jit_stxi_f(_jitc->function->call.size, JIT_SP,
981 JIT_FA0 - _jitc->function->call.argf);
982 _jitc->function->call.size += sizeof(jit_word_t);
985 ++_jitc->function->call.argf;
989 _jitc->function->call.size +=
990 _jitc->function->call.size & (sizeof(jit_float32_t) - 1);
991 jit_stxi_f(_jitc->function->call.size, JIT_SP, u);
992 _jitc->function->call.size += sizeof(jit_float32_t);
994 jit_stxi_f(_jitc->function->call.size, JIT_SP, u);
995 _jitc->function->call.size += sizeof(jit_word_t);
1003 _jit_pushargi_f(jit_state_t *_jit, jit_float32_t u)
1006 assert(_jitc->function);
1007 jit_inc_synth_f(pushargi_f, u);
1009 if (jit_arg_f_reg_p(_jitc->function->call.argf)) {
1010 jit_movi_f(JIT_FA0 - _jitc->function->call.argf, u);
1012 if (_jitc->function->call.call & jit_call_varargs) {
1013 assert(sizeof(jit_float32_t) == sizeof(jit_word_t));
1014 jit_stxi_f(_jitc->function->call.size, JIT_SP,
1015 JIT_FA0 - _jitc->function->call.argf);
1016 _jitc->function->call.size += sizeof(jit_word_t);
1019 ++_jitc->function->call.argf;
1022 regno = jit_get_reg(jit_class_fpr);
1023 jit_movi_f(regno, u);
1025 _jitc->function->call.size +=
1026 _jitc->function->call.size & (sizeof(jit_float32_t) - 1);
1027 jit_stxi_f(_jitc->function->call.size, JIT_SP, regno);
1028 _jitc->function->call.size += sizeof(jit_float32_t);
1030 jit_stxi_f(_jitc->function->call.size, JIT_SP, regno);
1031 _jitc->function->call.size += sizeof(jit_word_t);
1033 jit_unget_reg(regno);
1040 _jit_pushargr_d(jit_state_t *_jit, jit_int32_t u)
1042 assert(_jitc->function);
1043 jit_inc_synth_w(pushargr_d, u);
1045 if (jit_arg_f_reg_p(_jitc->function->call.argf)) {
1046 jit_movr_d(JIT_FA0 - _jitc->function->call.argf, u);
1048 if (_jitc->function->call.call & jit_call_varargs) {
1049 assert(sizeof(jit_float64_t) == sizeof(jit_word_t));
1050 jit_stxi_d(_jitc->function->call.size, JIT_SP,
1051 JIT_FA0 - _jitc->function->call.argf);
1052 _jitc->function->call.size += sizeof(jit_float64_t);
1055 ++_jitc->function->call.argf;
1059 _jitc->function->call.size +=
1060 _jitc->function->call.size & (sizeof(jit_float64_t) - 1);
1062 jit_stxi_d(_jitc->function->call.size, JIT_SP, u);
1063 _jitc->function->call.size += sizeof(jit_float64_t);
1070 _jit_pushargi_d(jit_state_t *_jit, jit_float64_t u)
1073 assert(_jitc->function);
1074 jit_inc_synth_d(pushargi_d, u);
1076 if (jit_arg_f_reg_p(_jitc->function->call.argf)) {
1077 jit_movi_d(JIT_FA0 - _jitc->function->call.argf, u);
1079 if (_jitc->function->call.call & jit_call_varargs) {
1080 assert(sizeof(jit_float64_t) == sizeof(jit_word_t));
1081 jit_stxi_d(_jitc->function->call.size, JIT_SP,
1082 JIT_FA0 - _jitc->function->call.argf);
1083 _jitc->function->call.size += sizeof(jit_float64_t);
1086 ++_jitc->function->call.argf;
1089 regno = jit_get_reg(jit_class_fpr);
1090 jit_movi_d(regno, u);
1092 _jitc->function->call.size +=
1093 _jitc->function->call.size & (sizeof(jit_float64_t) - 1);
1095 jit_stxi_d(_jitc->function->call.size, JIT_SP, regno);
1096 jit_unget_reg(regno);
1097 _jitc->function->call.size += sizeof(jit_float64_t);
1104 _jit_regarg_p(jit_state_t *_jit, jit_node_t *node, jit_int32_t regno)
1107 spec = jit_class(_rvs[regno].spec);
1108 if (spec & jit_class_arg) {
1109 regno = JIT_RA0 - regno;
1110 if (regno >= 0 && regno < node->v.w)
1112 if (spec & jit_class_fpr) {
1113 regno = JIT_FA0 - regno;
1114 if (regno >= 0 && regno < node->w.w)
1123 _jit_finishr(jit_state_t *_jit, jit_int32_t r0)
1126 assert(_jitc->function);
1128 jit_inc_synth_w(finishr, r0);
1130 _jitc->function->call.size +=
1131 _jitc->function->call.size & (sizeof(jit_word_t) - 1);
1133 if (_jitc->function->self.alen < _jitc->function->call.size)
1134 _jitc->function->self.alen = _jitc->function->call.size;
1135 node = jit_callr(r0);
1136 node->v.w = _jitc->function->self.argi;
1137 node->w.w = _jitc->function->call.argf;
1138 _jitc->function->call.argi = _jitc->function->call.argf =
1139 _jitc->function->call.size = 0;
1145 _jit_finishi(jit_state_t *_jit, jit_pointer_t i0)
1148 assert(_jitc->function);
1150 jit_inc_synth_w(finishi, (jit_word_t)i0);
1152 _jitc->function->call.size +=
1153 _jitc->function->call.size & (sizeof(jit_word_t) - 1);
1155 if (_jitc->function->self.alen < _jitc->function->call.size)
1156 _jitc->function->self.alen = _jitc->function->call.size;
1157 node = jit_calli(i0);
1158 node->v.w = _jitc->function->call.argi;
1159 node->w.w = _jitc->function->call.argf;
1160 _jitc->function->call.argi = _jitc->function->call.argf =
1161 _jitc->function->call.size = 0;
1168 _jit_retval_c(jit_state_t *_jit, jit_int32_t r0)
1170 jit_inc_synth_w(retval_c, r0);
1171 jit_extr_c(r0, JIT_RET);
1176 _jit_retval_uc(jit_state_t *_jit, jit_int32_t r0)
1178 jit_inc_synth_w(retval_uc, r0);
1179 jit_extr_uc(r0, JIT_RET);
1184 _jit_retval_s(jit_state_t *_jit, jit_int32_t r0)
1186 jit_inc_synth_w(retval_s, r0);
1187 jit_extr_s(r0, JIT_RET);
1192 _jit_retval_us(jit_state_t *_jit, jit_int32_t r0)
1194 jit_inc_synth_w(retval_us, r0);
1195 jit_extr_us(r0, JIT_RET);
1200 _jit_retval_i(jit_state_t *_jit, jit_int32_t r0)
1202 jit_inc_synth_w(retval_i, r0);
1203 #if __WORDSIZE == 32
1204 jit_movr(r0, JIT_RET);
1206 jit_extr_i(r0, JIT_RET);
1211 #if __WORDSIZE == 64
1213 _jit_retval_ui(jit_state_t *_jit, jit_int32_t r0)
1215 jit_inc_synth_w(retval_ui, r0);
1216 jit_extr_ui(r0, JIT_RET);
1221 _jit_retval_l(jit_state_t *_jit, jit_int32_t r0)
1223 jit_inc_synth_w(retval_l, r0);
1224 jit_movr(r0, JIT_RET);
1230 _jit_retval_f(jit_state_t *_jit, jit_int32_t r0)
1232 jit_inc_synth_w(retval_f, r0);
1234 jit_movr_f(r0, JIT_FRET);
1239 _jit_retval_d(jit_state_t *_jit, jit_int32_t r0)
1241 jit_inc_synth_w(retval_d, r0);
1243 jit_movr_d(r0, JIT_FRET);
1248 _emit_code(jit_state_t *_jit)
1259 jit_function_t func;
1260 #if DEVEL_DISASSEMBLER
1263 jit_int32_t const_offset;
1264 jit_int32_t patch_offset;
1266 #if DEVEL_DISASSEMBLER
1270 _jitc->function = NULL;
1272 jit_reglive_setup();
1276 undo.const_offset = undo.patch_offset = 0;
1277 # define assert_data(node) /**/
1278 #define case_rr(name, type) \
1279 case jit_code_##name##r##type: \
1280 name##r##type(rn(node->u.w), rn(node->v.w)); \
1282 #define case_rw(name, type) \
1283 case jit_code_##name##i##type: \
1284 name##i##type(rn(node->u.w), node->v.w); \
1286 #define case_wr(name, type) \
1287 case jit_code_##name##i##type: \
1288 name##i##type(node->u.w, rn(node->v.w)); \
1290 #define case_rrr(name, type) \
1291 case jit_code_##name##r##type: \
1292 name##r##type(rn(node->u.w), \
1293 rn(node->v.w), rn(node->w.w)); \
1295 #define case_rrrr(name, type) \
1296 case jit_code_##name##r##type: \
1297 name##r##type(rn(node->u.q.l), rn(node->u.q.h), \
1298 rn(node->v.w), rn(node->w.w)); \
1300 #define case_rrw(name, type) \
1301 case jit_code_##name##i##type: \
1302 name##i##type(rn(node->u.w), rn(node->v.w), node->w.w); \
1304 #define case_rrrw(name, type) \
1305 case jit_code_##name##i##type: \
1306 name##i##type(rn(node->u.q.l), rn(node->u.q.h), \
1307 rn(node->v.w), node->w.w); \
1309 #define case_rrf(name) \
1310 case jit_code_##name##i_f: \
1311 assert_data(node); \
1312 name##i_f(rn(node->u.w), rn(node->v.w), node->w.f); \
1314 #define case_rrd(name) \
1315 case jit_code_##name##i_d: \
1316 assert_data(node); \
1317 name##i_d(rn(node->u.w), rn(node->v.w), node->w.d); \
1319 #define case_wrr(name, type) \
1320 case jit_code_##name##i##type: \
1321 name##i##type(node->u.w, rn(node->v.w), rn(node->w.w)); \
1323 #define case_brr(name, type) \
1324 case jit_code_##name##r##type: \
1326 assert(temp->code == jit_code_label || \
1327 temp->code == jit_code_epilog); \
1328 if (temp->flag & jit_flag_patch) \
1329 name##r##type(temp->u.w, rn(node->v.w), \
1332 word = name##r##type(_jit->pc.w, \
1333 rn(node->v.w), rn(node->w.w)); \
1334 patch(word, node); \
1337 #define case_brw(name, type) \
1338 case jit_code_##name##i##type: \
1340 assert(temp->code == jit_code_label || \
1341 temp->code == jit_code_epilog); \
1342 if (temp->flag & jit_flag_patch) \
1343 name##i##type(temp->u.w, \
1344 rn(node->v.w), node->w.w); \
1346 word = name##i##type(_jit->pc.w, \
1347 rn(node->v.w), node->w.w); \
1348 patch(word, node); \
1351 #define case_brf(name) \
1352 case jit_code_##name##i_f: \
1354 assert(temp->code == jit_code_label || \
1355 temp->code == jit_code_epilog); \
1356 if (temp->flag & jit_flag_patch) \
1357 name##i_f(temp->u.w, rn(node->v.w), node->w.f); \
1359 word = name##i_f(_jit->pc.w, rn(node->v.w), \
1361 patch(word, node); \
1364 #define case_brd(name) \
1365 case jit_code_##name##i_d: \
1367 assert(temp->code == jit_code_label || \
1368 temp->code == jit_code_epilog); \
1369 if (temp->flag & jit_flag_patch) \
1370 name##i_d(temp->u.w, rn(node->v.w), node->w.d); \
1372 word = name##i_d(_jit->pc.w, rn(node->v.w), \
1374 patch(word, node); \
1377 #if DEVEL_DISASSEMBLER
1380 for (node = _jitc->head; node; node = node->next) {
1381 if (_jit->pc.uc >= _jitc->code.end)
1384 #if DEVEL_DISASSEMBLER
1385 node->offset = (jit_uword_t)_jit->pc.w - (jit_uword_t)prevw;
1388 value = jit_classify(node->code);
1389 jit_regarg_set(node, value);
1390 switch (node->code) {
1391 case jit_code_align:
1392 /* Must align to a power of two */
1393 assert(!(node->u.w & (node->u.w - 1)));
1394 if ((word = _jit->pc.w & (node->u.w - 1)))
1395 nop(node->u.w - word);
1398 nop((node->u.w + 3) & ~3);
1400 case jit_code_note: case jit_code_name:
1401 node->u.w = _jit->pc.w;
1403 case jit_code_label:
1404 /* remember label is defined */
1405 node->flag |= jit_flag_patch;
1406 node->u.w = _jit->pc.w;
1425 case_rrrr(qmul, _u);
1426 case_rrrw(qmul, _u);
1433 case_rrrr(qdiv, _u);
1434 case_rrrw(qdiv, _u);
1457 case_rr(trunc, _f_i);
1458 case_rr(trunc, _d_i);
1459 case_rr(trunc, _f_l);
1460 case_rr(trunc, _d_l);
1508 case_rr(bswap, _us);
1509 case_rr(bswap, _ui);
1510 case_rr(bswap, _ul);
1518 casr(rn(node->u.w), rn(node->v.w),
1519 rn(node->w.q.l), rn(node->w.q.h));
1522 casi(rn(node->u.w), node->v.w,
1523 rn(node->w.q.l), rn(node->w.q.h));
1529 if (node->flag & jit_flag_node) {
1531 if (temp->code == jit_code_data ||
1532 (temp->code == jit_code_label &&
1533 (temp->flag & jit_flag_patch)))
1534 movi(rn(node->u.w), temp->u.w);
1536 assert(temp->code == jit_code_label ||
1537 temp->code == jit_code_epilog);
1538 word = movi_p(rn(node->u.w), temp->u.w);
1543 movi(rn(node->u.w), node->v.w);
1587 case_brr(boadd, _u);
1588 case_brw(boadd, _u);
1591 case_brr(bxadd, _u);
1592 case_brw(bxadd, _u);
1595 case_brr(bosub, _u);
1596 case_brw(bosub, _u);
1599 case_brr(bxsub, _u);
1600 case_brw(bxsub, _u);
1627 case jit_code_movi_f:
1629 movi_f(rn(node->u.w), node->v.f);
1658 case_rrr(unord, _f);
1672 case_brr(bunlt, _f);
1674 case_brr(bunle, _f);
1676 case_brr(buneq, _f);
1678 case_brr(bunge, _f);
1680 case_brr(bungt, _f);
1682 case_brr(bltgt, _f);
1686 case_brr(bunord, _f);
1710 case jit_code_movi_d:
1712 movi_d(rn(node->u.w), node->v.d);
1741 case_rrr(unord, _d);
1755 case_brr(bunlt, _d);
1757 case_brr(bunle, _d);
1759 case_brr(buneq, _d);
1761 case_brr(bunge, _d);
1763 case_brr(bungt, _d);
1765 case_brr(bltgt, _d);
1769 case_brr(bunord, _d);
1773 jmpr(rn(node->u.w));
1776 if (node->flag & jit_flag_node) {
1778 assert(temp->code == jit_code_label ||
1779 temp->code == jit_code_epilog);
1780 if (temp->flag & jit_flag_patch)
1783 word = _jit->code.length -
1784 (_jit->pc.uc - _jit->code.ptr);
1786 word = jmpi(_jit->pc.w);
1788 word = jmpi_p(_jit->pc.w);
1797 case jit_code_callr:
1799 callr(rn(node->u.w));
1801 case jit_code_calli:
1803 if (node->flag & jit_flag_node) {
1805 assert(temp->code == jit_code_label ||
1806 temp->code == jit_code_epilog);
1807 if (temp->flag & jit_flag_patch)
1810 word = _jit->code.length -
1811 (_jit->pc.uc - _jit->code.ptr);
1813 word = calli(_jit->pc.w);
1815 word = calli_p(_jit->pc.w);
1822 case jit_code_prolog:
1823 _jitc->function = _jitc->functions.ptr + node->w.w;
1825 undo.word = _jit->pc.w;
1826 memcpy(&undo.func, _jitc->function, sizeof(undo.func));
1827 #if DEVEL_DISASSEMBLER
1830 undo.patch_offset = _jitc->patches.offset;
1832 compute_framesize();
1837 case jit_code_epilog:
1838 assert(_jitc->function == _jitc->functions.ptr + node->w.w);
1840 for (temp = undo.node->next;
1841 temp != node; temp = temp->next) {
1842 if (temp->code == jit_code_label ||
1843 temp->code == jit_code_epilog)
1844 temp->flag &= ~jit_flag_patch;
1846 temp->flag &= ~jit_flag_patch;
1848 _jit->pc.w = undo.word;
1849 /* undo.func.self.aoff and undo.func.regset should not
1850 * be undone, as they will be further updated, and are
1851 * the reason of the undo. */
1852 undo.func.self.aoff = _jitc->function->frame +
1853 _jitc->function->self.aoff;
1854 undo.func.need_frame = _jitc->function->need_frame;
1855 jit_regset_set(&undo.func.regset, &_jitc->function->regset);
1856 /* allocar information also does not need to be undone */
1857 undo.func.aoffoff = _jitc->function->aoffoff;
1858 undo.func.allocar = _jitc->function->allocar;
1859 memcpy(_jitc->function, &undo.func, sizeof(undo.func));
1860 #if DEVEL_DISASSEMBLER
1863 _jitc->patches.offset = undo.patch_offset;
1865 goto restart_function;
1867 /* remember label is defined */
1868 node->flag |= jit_flag_patch;
1869 node->u.w = _jit->pc.w;
1871 _jitc->function = NULL;
1873 case jit_code_va_start:
1874 vastart(rn(node->u.w));
1876 case jit_code_va_arg:
1877 vaarg(rn(node->u.w), rn(node->v.w));
1879 case jit_code_va_arg_d:
1880 vaarg_d(rn(node->u.w), rn(node->v.w));
1882 case jit_code_live: case jit_code_ellipsis:
1883 case jit_code_va_push:
1884 case jit_code_allocai: case jit_code_allocar:
1885 case jit_code_arg_c: case jit_code_arg_s:
1886 case jit_code_arg_i:
1887 # if __WORDSIZE == 64
1888 case jit_code_arg_l:
1890 case jit_code_arg_f: case jit_code_arg_d:
1891 case jit_code_va_end:
1893 case jit_code_retr_c: case jit_code_reti_c:
1894 case jit_code_retr_uc: case jit_code_reti_uc:
1895 case jit_code_retr_s: case jit_code_reti_s:
1896 case jit_code_retr_us: case jit_code_reti_us:
1897 case jit_code_retr_i: case jit_code_reti_i:
1898 #if __WORDSIZE == 64
1899 case jit_code_retr_ui: case jit_code_reti_ui:
1900 case jit_code_retr_l: case jit_code_reti_l:
1902 case jit_code_retr_f: case jit_code_reti_f:
1903 case jit_code_retr_d: case jit_code_reti_d:
1904 case jit_code_getarg_c: case jit_code_getarg_uc:
1905 case jit_code_getarg_s: case jit_code_getarg_us:
1906 case jit_code_getarg_i: case jit_code_getarg_ui:
1907 case jit_code_getarg_l:
1908 case jit_code_getarg_f: case jit_code_getarg_d:
1909 case jit_code_putargr_c: case jit_code_putargi_c:
1910 case jit_code_putargr_uc: case jit_code_putargi_uc:
1911 case jit_code_putargr_s: case jit_code_putargi_s:
1912 case jit_code_putargr_us: case jit_code_putargi_us:
1913 case jit_code_putargr_i: case jit_code_putargi_i:
1914 #if __WORDSIZE == 64
1915 case jit_code_putargr_ui: case jit_code_putargi_ui:
1916 case jit_code_putargr_l: case jit_code_putargi_l:
1918 case jit_code_putargr_f: case jit_code_putargi_f:
1919 case jit_code_putargr_d: case jit_code_putargi_d:
1920 case jit_code_pushargr_c: case jit_code_pushargi_c:
1921 case jit_code_pushargr_uc: case jit_code_pushargi_uc:
1922 case jit_code_pushargr_s: case jit_code_pushargi_s:
1923 case jit_code_pushargr_us: case jit_code_pushargi_us:
1924 case jit_code_pushargr_i: case jit_code_pushargi_i:
1925 #if __WORDSIZE == 64
1926 case jit_code_pushargr_ui: case jit_code_pushargi_ui:
1927 case jit_code_pushargr_l: case jit_code_pushargi_l:
1929 case jit_code_pushargr_f: case jit_code_pushargi_f:
1930 case jit_code_pushargr_d: case jit_code_pushargi_d:
1931 case jit_code_retval_c: case jit_code_retval_uc:
1932 case jit_code_retval_s: case jit_code_retval_us:
1933 case jit_code_retval_i:
1934 #if __WORDSIZE == 64
1935 case jit_code_retval_ui: case jit_code_retval_l:
1937 case jit_code_retval_f: case jit_code_retval_d:
1938 case jit_code_prepare:
1939 case jit_code_finishr: case jit_code_finishi:
1944 jit_regarg_clr(node, value);
1945 assert(_jitc->regarg == 0 && _jitc->synth == 0);
1946 /* update register live state */
1958 for (offset = 0; offset < _jitc->patches.offset; offset++) {
1959 node = _jitc->patches.ptr[offset].node;
1960 word = _jitc->patches.ptr[offset].inst;
1961 value = node->code == jit_code_movi ? node->v.n->u.w : node->u.n->u.w;
1962 patch_at(word, value);
1965 jit_flush(_jit->code.ptr, _jit->pc.uc);
1967 return (_jit->code.ptr);
1971 # include "jit_aarch64-cpu.c"
1972 # include "jit_aarch64-fpu.c"
1976 jit_flush(void *fptr, void *tptr)
1978 #if defined(__GNUC__)
1981 s = sysconf(_SC_PAGE_SIZE);
1982 f = (jit_word_t)fptr & -s;
1983 t = (((jit_word_t)tptr) + s - 1) & -s;
1984 __clear_cache((void *)f, (void *)t);
1989 _emit_ldxi(jit_state_t *_jit, jit_int32_t r0, jit_int32_t r1, jit_word_t i0)
1991 ldxi(rn(r0), rn(r1), i0);
1995 _emit_stxi(jit_state_t *_jit, jit_word_t i0, jit_int32_t r0, jit_int32_t r1)
1997 stxi(i0, rn(r0), rn(r1));
2001 _emit_ldxi_d(jit_state_t *_jit, jit_int32_t r0, jit_int32_t r1, jit_word_t i0)
2003 ldxi_d(rn(r0), rn(r1), i0);
2007 _emit_stxi_d(jit_state_t *_jit, jit_word_t i0, jit_int32_t r0, jit_int32_t r1)
2009 stxi_d(i0, rn(r0), rn(r1));
2013 _compute_framesize(jit_state_t *_jit)
2016 _jitc->framesize = 16; /* ra+fp */
2017 for (reg = 0; reg < jit_size(iregs); reg++)
2018 if (jit_regset_tstbit(&_jitc->function->regset, iregs[reg]))
2019 _jitc->framesize += sizeof(jit_word_t);
2021 for (reg = 0; reg < jit_size(fregs); reg++)
2022 if (jit_regset_tstbit(&_jitc->function->regset, fregs[reg]))
2023 _jitc->framesize += sizeof(jit_float64_t);
2025 /* Make sure functions called have a 16 byte aligned stack */
2026 _jitc->framesize = (_jitc->framesize + 15) & -16;
2030 _patch(jit_state_t *_jit, jit_word_t instr, jit_node_t *node)
2034 assert(node->flag & jit_flag_node);
2035 if (node->code == jit_code_movi)
2036 flag = node->v.n->flag;
2038 flag = node->u.n->flag;
2039 assert(!(flag & jit_flag_patch));
2040 if (_jitc->patches.offset >= _jitc->patches.length) {
2041 jit_realloc((jit_pointer_t *)&_jitc->patches.ptr,
2042 _jitc->patches.length * sizeof(jit_patch_t),
2043 (_jitc->patches.length + 1024) * sizeof(jit_patch_t));
2044 _jitc->patches.length += 1024;
2046 _jitc->patches.ptr[_jitc->patches.offset].inst = instr;
2047 _jitc->patches.ptr[_jitc->patches.offset].node = node;
2048 ++_jitc->patches.offset;