2 * Copyright (C) 2013-2019 Free Software Foundation, Inc.
4 * This file is part of GNU lightning.
6 * GNU lightning is free software; you can redistribute it and/or modify it
7 * under the terms of the GNU Lesser General Public License as published
8 * by the Free Software Foundation; either version 3, or (at your option)
11 * GNU lightning is distributed in the hope that it will be useful, but
12 * WITHOUT ANY WARRANTY; without even the implied warranty of MERCHANTABILITY
13 * or FITNESS FOR A PARTICULAR PURPOSE. See the GNU Lesser General Public
14 * License for more details.
17 * Paulo Cesar Pereira de Andrade
20 #define jit_arg_reg_p(i) ((i) >= 0 && (i) < 8)
21 #define jit_arg_f_reg_p(i) ((i) >= 0 && (i) < 8)
23 typedef struct jit_qreg {
28 #define va_gp_top_offset offsetof(jit_va_list_t, q0)
29 #define va_fp_top_offset sizeof(jit_va_list_t)
30 typedef struct jit_va_list {
59 #define patch(instr, node) _patch(_jit, instr, node)
60 static void _patch(jit_state_t*,jit_word_t,jit_node_t*);
63 extern void __clear_cache(void *, void *);
66 # include "jit_aarch64-cpu.c"
67 # include "jit_aarch64-fpu.c"
73 jit_register_t _rvs[] = {
74 { rc(gpr) | 0x08, "x8" },
75 { rc(gpr) | 0x12, "x18" },
76 { rc(gpr) | 0x11, "x17" },
77 { rc(gpr) | 0x10, "x16" },
78 { rc(gpr) | 0x09, "x9" },
79 { rc(gpr) | 0x0a, "x10" },
80 { rc(gpr) | 0x0b, "x11" },
81 { rc(gpr) | 0x0c, "x12" },
82 { rc(gpr) | 0x0d, "x13" },
83 { rc(gpr) | 0x0e, "x14" },
84 { rc(gpr) | 0x0f, "x15" },
85 { rc(sav) | rc(gpr) | 0x13, "x19" },
86 { rc(sav) | rc(gpr) | 0x14, "x20" },
87 { rc(sav) | rc(gpr) | 0x15, "x21" },
88 { rc(sav) | rc(gpr) | 0x16, "x22" },
89 { rc(sav) | rc(gpr) | 0x17, "x23" },
90 { rc(sav) | rc(gpr) | 0x18, "x24" },
91 { rc(sav) | rc(gpr) | 0x19, "x25" },
92 { rc(sav) | rc(gpr) | 0x1a, "x26" },
93 { rc(sav) | rc(gpr) | 0x1b, "x27" },
94 { rc(sav) | rc(gpr) | 0x1c, "x28" },
98 { rc(arg) | rc(gpr) | 0x07, "x7" },
99 { rc(arg) | rc(gpr) | 0x06, "x6" },
100 { rc(arg) | rc(gpr) | 0x05, "x5" },
101 { rc(arg) | rc(gpr) | 0x04, "x4" },
102 { rc(arg) | rc(gpr) | 0x03, "x3" },
103 { rc(arg) | rc(gpr) | 0x02, "x2" },
104 { rc(arg) | rc(gpr) | 0x01, "x1" },
105 { rc(arg) | rc(gpr) | 0x00, "x0" },
106 { rc(fpr) | 0x1f, "v31" },
107 { rc(fpr) | 0x1e, "v30" },
108 { rc(fpr) | 0x1d, "v29" },
109 { rc(fpr) | 0x1c, "v28" },
110 { rc(fpr) | 0x1b, "v27" },
111 { rc(fpr) | 0x1a, "v26" },
112 { rc(fpr) | 0x19, "v25" },
113 { rc(fpr) | 0x18, "v24" },
114 { rc(fpr) | 0x17, "v23" },
115 { rc(fpr) | 0x16, "v22" },
116 { rc(fpr) | 0x15, "v21" },
117 { rc(fpr) | 0x14, "v20" },
118 { rc(fpr) | 0x13, "v19" },
119 { rc(fpr) | 0x12, "v18" },
120 { rc(fpr) | 0x11, "v17" },
121 { rc(fpr) | 0x10, "v16" },
122 { rc(sav) | rc(fpr) | 0x08, "v8" },
123 { rc(sav) | rc(fpr) | 0x09, "v9" },
124 { rc(sav) | rc(fpr) | 0x0a, "v10" },
125 { rc(sav) | rc(fpr) | 0x0b, "v11" },
126 { rc(sav) | rc(fpr) | 0x0c, "v12" },
127 { rc(sav) | rc(fpr) | 0x0d, "v13" },
128 { rc(sav) | rc(fpr) | 0x0e, "v14" },
129 { rc(sav) | rc(fpr) | 0x0f, "v15" },
130 { rc(arg) | rc(fpr) | 0x07, "v7" },
131 { rc(arg) | rc(fpr) | 0x06, "v6" },
132 { rc(arg) | rc(fpr) | 0x05, "v5" },
133 { rc(arg) | rc(fpr) | 0x04, "v4" },
134 { rc(arg) | rc(fpr) | 0x03, "v3" },
135 { rc(arg) | rc(fpr) | 0x02, "v2" },
136 { rc(arg) | rc(fpr) | 0x01, "v1" },
137 { rc(arg) | rc(fpr) | 0x00, "v0" },
138 { _NOREG, "<none>" },
150 _jit_init(jit_state_t *_jit)
152 _jitc->reglen = jit_size(_rvs) - 1;
156 _jit_prolog(jit_state_t *_jit)
162 assert(jit_regset_cmp_ui(&_jitc->regarg, 0) == 0);
163 jit_regset_set_ui(&_jitc->regsav, 0);
164 offset = _jitc->functions.offset;
165 if (offset >= _jitc->functions.length) {
166 jit_realloc((jit_pointer_t *)&_jitc->functions.ptr,
167 _jitc->functions.length * sizeof(jit_function_t),
168 (_jitc->functions.length + 16) * sizeof(jit_function_t));
169 _jitc->functions.length += 16;
171 _jitc->function = _jitc->functions.ptr + _jitc->functions.offset++;
172 _jitc->function->self.size = stack_framesize;
173 _jitc->function->self.argi = _jitc->function->self.argf =
174 _jitc->function->self.alen = 0;
175 _jitc->function->self.aoff = 0;
176 _jitc->function->self.call = jit_call_default;
177 jit_alloc((jit_pointer_t *)&_jitc->function->regoff,
178 _jitc->reglen * sizeof(jit_int32_t));
180 /* _no_link here does not mean the jit_link() call can be removed
182 * _jitc->function->prolog = jit_new_node(jit_code_prolog);
184 _jitc->function->prolog = jit_new_node_no_link(jit_code_prolog);
185 jit_link(_jitc->function->prolog);
186 _jitc->function->prolog->w.w = offset;
187 _jitc->function->epilog = jit_new_node_no_link(jit_code_epilog);
189 * v: offset in blocks vector
190 * w: offset in functions vector
192 _jitc->function->epilog->w.w = offset;
194 jit_regset_new(&_jitc->function->regset);
198 _jit_allocai(jit_state_t *_jit, jit_int32_t length)
200 assert(_jitc->function);
202 case 0: case 1: break;
203 case 2: _jitc->function->self.aoff &= -2; break;
204 case 3: case 4: _jitc->function->self.aoff &= -4; break;
205 default: _jitc->function->self.aoff &= -8; break;
207 _jitc->function->self.aoff -= length;
208 if (!_jitc->realize) {
209 jit_inc_synth_ww(allocai, _jitc->function->self.aoff, length);
212 return (_jitc->function->self.aoff);
216 _jit_allocar(jit_state_t *_jit, jit_int32_t u, jit_int32_t v)
219 assert(_jitc->function);
220 jit_inc_synth_ww(allocar, u, v);
221 if (!_jitc->function->allocar) {
222 _jitc->function->aoffoff = jit_allocai(sizeof(jit_int32_t));
223 _jitc->function->allocar = 1;
225 r0 = jit_get_reg(jit_class_gpr);
227 jit_andi(r0, r0, -16);
228 jit_ldxi_i(u, JIT_FP, _jitc->function->aoffoff);
230 /* Cannot "addr sp, sp, reg" because in this context "sp" is "[w|x]zr",
231 * the zero register */
233 jit_addr(JIT_SP, JIT_SP, r0);
235 r1 = jit_get_reg(jit_class_gpr);
236 /* note that "mov r1, sp" does not work, but the proper encoding
237 * can be triggered before actually emiting with "add r1, sp, 0" */
238 jit_addi(r1, JIT_SP, 0);
239 jit_addr(r1, r1, r0);
240 jit_addi(JIT_SP, r1, 0);
243 jit_stxi_i(_jitc->function->aoffoff, JIT_FP, u);
249 _jit_ret(jit_state_t *_jit)
252 assert(_jitc->function);
256 jit_patch_at(instr, _jitc->function->epilog);
261 _jit_retr(jit_state_t *_jit, jit_int32_t u)
263 jit_inc_synth_w(retr, u);
265 jit_movr(JIT_RET, u);
272 _jit_reti(jit_state_t *_jit, jit_word_t u)
274 jit_inc_synth_w(reti, u);
275 jit_movi(JIT_RET, u);
281 _jit_retr_f(jit_state_t *_jit, jit_int32_t u)
283 jit_inc_synth_w(retr_f, u);
285 jit_movr_f(JIT_FRET, u);
293 _jit_reti_f(jit_state_t *_jit, jit_float32_t u)
295 jit_inc_synth_f(reti_f, u);
296 jit_movi_f(JIT_FRET, u);
302 _jit_retr_d(jit_state_t *_jit, jit_int32_t u)
304 jit_inc_synth_w(retr_d, u);
306 jit_movr_d(JIT_FRET, u);
314 _jit_reti_d(jit_state_t *_jit, jit_float64_t u)
316 jit_inc_synth_d(reti_d, u);
317 jit_movi_d(JIT_FRET, u);
323 _jit_epilog(jit_state_t *_jit)
325 assert(_jitc->function);
326 assert(_jitc->function->epilog->next == NULL);
327 jit_link(_jitc->function->epilog);
328 _jitc->function = NULL;
332 _jit_arg_register_p(jit_state_t *_jit, jit_node_t *u)
334 if (u->code == jit_code_arg)
335 return (jit_arg_reg_p(u->u.w));
336 assert(u->code == jit_code_arg_f || u->code == jit_code_arg_d);
337 return (jit_arg_f_reg_p(u->u.w));
341 _jit_ellipsis(jit_state_t *_jit)
343 jit_inc_synth(ellipsis);
344 if (_jitc->prepare) {
346 assert(!(_jitc->function->call.call & jit_call_varargs));
347 _jitc->function->call.call |= jit_call_varargs;
351 assert(!(_jitc->function->self.call & jit_call_varargs));
352 _jitc->function->self.call |= jit_call_varargs;
354 /* Allocate va_list like object in the stack,
355 * with enough space to save all argument
356 * registers, and use fixed offsets for them. */
357 _jitc->function->vaoff = jit_allocai(sizeof(jit_va_list_t));
359 /* Initialize gp offset in save area. */
360 if (jit_arg_reg_p(_jitc->function->self.argi))
361 _jitc->function->vagp = (8 - _jitc->function->self.argi) * -8;
363 _jitc->function->vagp = 0;
365 /* Initialize fp offset in save area. */
366 if (jit_arg_f_reg_p(_jitc->function->self.argf))
367 _jitc->function->vafp = (8 - _jitc->function->self.argf) * -16;
369 _jitc->function->vafp = 0;
375 _jit_va_push(jit_state_t *_jit, jit_int32_t u)
377 jit_inc_synth_w(va_push, u);
383 _jit_arg(jit_state_t *_jit)
387 assert(_jitc->function);
388 assert(!(_jitc->function->self.call & jit_call_varargs));
389 if (jit_arg_reg_p(_jitc->function->self.argi))
390 offset = _jitc->function->self.argi++;
392 offset = _jitc->function->self.size;
393 _jitc->function->self.size += sizeof(jit_word_t);
395 node = jit_new_node_ww(jit_code_arg, offset,
396 ++_jitc->function->self.argn);
402 _jit_arg_f(jit_state_t *_jit)
406 assert(_jitc->function);
407 assert(!(_jitc->function->self.call & jit_call_varargs));
408 if (jit_arg_f_reg_p(_jitc->function->self.argf))
409 offset = _jitc->function->self.argf++;
411 offset = _jitc->function->self.size;
412 _jitc->function->self.size += sizeof(jit_word_t);
414 node = jit_new_node_ww(jit_code_arg_f, offset,
415 ++_jitc->function->self.argn);
421 _jit_arg_d(jit_state_t *_jit)
425 assert(_jitc->function);
426 assert(!(_jitc->function->self.call & jit_call_varargs));
427 if (jit_arg_f_reg_p(_jitc->function->self.argf))
428 offset = _jitc->function->self.argf++;
430 offset = _jitc->function->self.size;
431 _jitc->function->self.size += sizeof(jit_word_t);
433 node = jit_new_node_ww(jit_code_arg_d, offset,
434 ++_jitc->function->self.argn);
440 _jit_getarg_c(jit_state_t *_jit, jit_int32_t u, jit_node_t *v)
442 assert(v->code == jit_code_arg);
443 jit_inc_synth_wp(getarg_c, u, v);
444 if (jit_arg_reg_p(v->u.w))
445 jit_extr_c(u, JIT_RA0 - v->u.w);
447 jit_ldxi_c(u, JIT_FP, v->u.w);
452 _jit_getarg_uc(jit_state_t *_jit, jit_int32_t u, jit_node_t *v)
454 assert(v->code == jit_code_arg);
455 jit_inc_synth_wp(getarg_uc, u, v);
456 if (jit_arg_reg_p(v->u.w))
457 jit_extr_uc(u, JIT_RA0 - v->u.w);
459 jit_ldxi_uc(u, JIT_FP, v->u.w);
464 _jit_getarg_s(jit_state_t *_jit, jit_int32_t u, jit_node_t *v)
466 assert(v->code == jit_code_arg);
467 jit_inc_synth_wp(getarg_s, u, v);
468 if (jit_arg_reg_p(v->u.w))
469 jit_extr_s(u, JIT_RA0 - v->u.w);
471 jit_ldxi_s(u, JIT_FP, v->u.w);
476 _jit_getarg_us(jit_state_t *_jit, jit_int32_t u, jit_node_t *v)
478 assert(v->code == jit_code_arg);
479 jit_inc_synth_wp(getarg_us, u, v);
480 if (jit_arg_reg_p(v->u.w))
481 jit_extr_us(u, JIT_RA0 - v->u.w);
483 jit_ldxi_us(u, JIT_FP, v->u.w);
488 _jit_getarg_i(jit_state_t *_jit, jit_int32_t u, jit_node_t *v)
490 assert(v->code == jit_code_arg);
491 jit_inc_synth_wp(getarg_i, u, v);
492 if (jit_arg_reg_p(v->u.w))
493 jit_extr_i(u, JIT_RA0 - v->u.w);
495 jit_ldxi_i(u, JIT_FP, v->u.w);
500 _jit_getarg_ui(jit_state_t *_jit, jit_int32_t u, jit_node_t *v)
502 assert(v->code == jit_code_arg);
503 jit_inc_synth_wp(getarg_ui, u, v);
504 if (jit_arg_reg_p(v->u.w))
505 jit_extr_ui(u, JIT_RA0 - v->u.w);
507 jit_ldxi_ui(u, JIT_FP, v->u.w);
512 _jit_getarg_l(jit_state_t *_jit, jit_int32_t u, jit_node_t *v)
514 assert(v->code == jit_code_arg);
515 jit_inc_synth_wp(getarg_l, u, v);
516 if (jit_arg_reg_p(v->u.w))
517 jit_movr(u, JIT_RA0 - v->u.w);
519 jit_ldxi_l(u, JIT_FP, v->u.w);
524 _jit_putargr(jit_state_t *_jit, jit_int32_t u, jit_node_t *v)
526 assert(v->code == jit_code_arg);
527 jit_inc_synth_wp(putargr, u, v);
528 if (jit_arg_reg_p(v->u.w))
529 jit_movr(JIT_RA0 - v->u.w, u);
531 jit_stxi(v->u.w, JIT_FP, u);
536 _jit_putargi(jit_state_t *_jit, jit_word_t u, jit_node_t *v)
539 assert(v->code == jit_code_arg);
540 jit_inc_synth_wp(putargi, u, v);
541 if (jit_arg_reg_p(v->u.w))
542 jit_movi(JIT_RA0 - v->u.w, u);
544 regno = jit_get_reg(jit_class_gpr);
546 jit_stxi(v->u.w, JIT_FP, regno);
547 jit_unget_reg(regno);
553 _jit_getarg_f(jit_state_t *_jit, jit_int32_t u, jit_node_t *v)
555 assert(v->code == jit_code_arg_f);
556 jit_inc_synth_wp(getarg_f, u, v);
557 if (jit_arg_reg_p(v->u.w))
558 jit_movr_f(u, JIT_FA0 - v->u.w);
560 jit_ldxi_f(u, JIT_FP, v->u.w);
565 _jit_putargr_f(jit_state_t *_jit, jit_int32_t u, jit_node_t *v)
567 assert(v->code == jit_code_arg_f);
568 jit_inc_synth_wp(putargr_f, u, v);
569 if (jit_arg_f_reg_p(v->u.w))
570 jit_movr_f(JIT_FA0 - v->u.w, u);
572 jit_stxi_f(v->u.w, JIT_FP, u);
577 _jit_putargi_f(jit_state_t *_jit, jit_float32_t u, jit_node_t *v)
580 assert(v->code == jit_code_arg_f);
581 jit_inc_synth_fp(putargi_f, u, v);
582 if (jit_arg_f_reg_p(v->u.w))
583 jit_movi_f(JIT_FA0 - v->u.w, u);
585 regno = jit_get_reg(jit_class_fpr);
586 jit_movi_f(regno, u);
587 jit_stxi_f(v->u.w, JIT_FP, regno);
588 jit_unget_reg(regno);
594 _jit_getarg_d(jit_state_t *_jit, jit_int32_t u, jit_node_t *v)
596 assert(v->code == jit_code_arg_d);
597 jit_inc_synth_wp(getarg_d, u, v);
598 if (jit_arg_f_reg_p(v->u.w))
599 jit_movr_d(u, JIT_FA0 - v->u.w);
601 jit_ldxi_d(u, JIT_FP, v->u.w);
606 _jit_putargr_d(jit_state_t *_jit, jit_int32_t u, jit_node_t *v)
608 assert(v->code == jit_code_arg_d);
609 jit_inc_synth_wp(putargr_d, u, v);
610 if (jit_arg_reg_p(v->u.w))
611 jit_movr_d(JIT_FA0 - v->u.w, u);
613 jit_stxi_d(v->u.w, JIT_FP, u);
618 _jit_putargi_d(jit_state_t *_jit, jit_float64_t u, jit_node_t *v)
621 assert(v->code == jit_code_arg_d);
622 jit_inc_synth_dp(putargi_d, u, v);
623 if (jit_arg_reg_p(v->u.w))
624 jit_movi_d(JIT_FA0 - v->u.w, u);
626 regno = jit_get_reg(jit_class_fpr);
627 jit_movi_d(regno, u);
628 jit_stxi_d(v->u.w, JIT_FP, regno);
629 jit_unget_reg(regno);
635 _jit_pushargr(jit_state_t *_jit, jit_int32_t u)
637 assert(_jitc->function);
638 jit_inc_synth_w(pushargr, u);
640 if (jit_arg_reg_p(_jitc->function->call.argi)) {
641 jit_movr(JIT_RA0 - _jitc->function->call.argi, u);
642 ++_jitc->function->call.argi;
645 jit_stxi(_jitc->function->call.size, JIT_SP, u);
646 _jitc->function->call.size += sizeof(jit_word_t);
652 _jit_pushargi(jit_state_t *_jit, jit_word_t u)
655 assert(_jitc->function);
656 jit_inc_synth_w(pushargi, u);
658 if (jit_arg_reg_p(_jitc->function->call.argi)) {
659 jit_movi(JIT_RA0 - _jitc->function->call.argi, u);
660 ++_jitc->function->call.argi;
663 regno = jit_get_reg(jit_class_gpr);
665 jit_stxi(_jitc->function->call.size, JIT_SP, regno);
666 jit_unget_reg(regno);
667 _jitc->function->call.size += sizeof(jit_word_t);
673 _jit_pushargr_f(jit_state_t *_jit, jit_int32_t u)
675 assert(_jitc->function);
676 jit_inc_synth_w(pushargr_f, u);
678 if (jit_arg_f_reg_p(_jitc->function->call.argf)) {
679 jit_movr_f(JIT_FA0 - _jitc->function->call.argf, u);
680 ++_jitc->function->call.argf;
683 jit_stxi_f(_jitc->function->call.size, JIT_SP, u);
684 _jitc->function->call.size += sizeof(jit_word_t);
690 _jit_pushargi_f(jit_state_t *_jit, jit_float32_t u)
693 assert(_jitc->function);
694 jit_inc_synth_f(pushargi_f, u);
696 if (jit_arg_f_reg_p(_jitc->function->call.argf)) {
697 jit_movi_f(JIT_FA0 - _jitc->function->call.argf, u);
698 ++_jitc->function->call.argf;
701 regno = jit_get_reg(jit_class_fpr);
702 jit_movi_f(regno, u);
703 jit_stxi_f(_jitc->function->call.size, JIT_SP, regno);
704 jit_unget_reg(regno);
705 _jitc->function->call.size += sizeof(jit_word_t);
711 _jit_pushargr_d(jit_state_t *_jit, jit_int32_t u)
713 assert(_jitc->function);
714 jit_inc_synth_w(pushargr_d, u);
716 if (jit_arg_f_reg_p(_jitc->function->call.argf)) {
717 jit_movr_d(JIT_FA0 - _jitc->function->call.argf, u);
718 ++_jitc->function->call.argf;
721 jit_stxi_d(_jitc->function->call.size, JIT_SP, u);
722 _jitc->function->call.size += sizeof(jit_word_t);
728 _jit_pushargi_d(jit_state_t *_jit, jit_float64_t u)
731 assert(_jitc->function);
732 jit_inc_synth_d(pushargi_d, u);
734 if (jit_arg_f_reg_p(_jitc->function->call.argf)) {
735 jit_movi_d(JIT_FA0 - _jitc->function->call.argf, u);
736 ++_jitc->function->call.argf;
739 regno = jit_get_reg(jit_class_fpr);
740 jit_movi_d(regno, u);
741 jit_stxi_d(_jitc->function->call.size, JIT_SP, regno);
742 jit_unget_reg(regno);
743 _jitc->function->call.size += sizeof(jit_word_t);
749 _jit_regarg_p(jit_state_t *_jit, jit_node_t *node, jit_int32_t regno)
752 spec = jit_class(_rvs[regno].spec);
753 if (spec & jit_class_arg) {
754 regno = JIT_RA0 - regno;
755 if (regno >= 0 && regno < node->v.w)
757 if (spec & jit_class_fpr) {
758 regno = JIT_FA0 - regno;
759 if (regno >= 0 && regno < node->w.w)
768 _jit_finishr(jit_state_t *_jit, jit_int32_t r0)
771 assert(_jitc->function);
772 jit_inc_synth_w(finishr, r0);
773 if (_jitc->function->self.alen < _jitc->function->call.size)
774 _jitc->function->self.alen = _jitc->function->call.size;
775 node = jit_callr(r0);
776 node->v.w = _jitc->function->self.argi;
777 node->w.w = _jitc->function->call.argf;
778 _jitc->function->call.argi = _jitc->function->call.argf =
779 _jitc->function->call.size = 0;
785 _jit_finishi(jit_state_t *_jit, jit_pointer_t i0)
788 assert(_jitc->function);
789 jit_inc_synth_w(finishi, (jit_word_t)i0);
790 if (_jitc->function->self.alen < _jitc->function->call.size)
791 _jitc->function->self.alen = _jitc->function->call.size;
792 node = jit_calli(i0);
793 node->v.w = _jitc->function->call.argi;
794 node->w.w = _jitc->function->call.argf;
795 _jitc->function->call.argi = _jitc->function->call.argf =
796 _jitc->function->call.size = 0;
803 _jit_retval_c(jit_state_t *_jit, jit_int32_t r0)
805 jit_inc_synth_w(retval_c, r0);
806 jit_extr_c(r0, JIT_RET);
811 _jit_retval_uc(jit_state_t *_jit, jit_int32_t r0)
813 jit_inc_synth_w(retval_uc, r0);
814 jit_extr_uc(r0, JIT_RET);
819 _jit_retval_s(jit_state_t *_jit, jit_int32_t r0)
821 jit_inc_synth_w(retval_s, r0);
822 jit_extr_s(r0, JIT_RET);
827 _jit_retval_us(jit_state_t *_jit, jit_int32_t r0)
829 jit_inc_synth_w(retval_us, r0);
830 jit_extr_us(r0, JIT_RET);
835 _jit_retval_i(jit_state_t *_jit, jit_int32_t r0)
837 jit_inc_synth_w(retval_i, r0);
838 jit_extr_i(r0, JIT_RET);
843 _jit_retval_ui(jit_state_t *_jit, jit_int32_t r0)
845 jit_inc_synth_w(retval_ui, r0);
846 jit_extr_ui(r0, JIT_RET);
851 _jit_retval_l(jit_state_t *_jit, jit_int32_t r0)
853 jit_inc_synth_w(retval_l, r0);
855 jit_movr(r0, JIT_RET);
860 _jit_retval_f(jit_state_t *_jit, jit_int32_t r0)
862 jit_inc_synth_w(retval_f, r0);
864 jit_movr_f(r0, JIT_FRET);
869 _jit_retval_d(jit_state_t *_jit, jit_int32_t r0)
871 jit_inc_synth_w(retval_d, r0);
873 jit_movr_d(r0, JIT_FRET);
878 _emit_code(jit_state_t *_jit)
889 #if DEVEL_DISASSEMBLER
892 jit_int32_t const_offset;
893 jit_int32_t patch_offset;
895 #if DEVEL_DISASSEMBLER
899 _jitc->function = NULL;
905 undo.const_offset = undo.patch_offset = 0;
906 # define assert_data(node) /**/
907 #define case_rr(name, type) \
908 case jit_code_##name##r##type: \
909 name##r##type(rn(node->u.w), rn(node->v.w)); \
911 #define case_rw(name, type) \
912 case jit_code_##name##i##type: \
913 name##i##type(rn(node->u.w), node->v.w); \
915 #define case_wr(name, type) \
916 case jit_code_##name##i##type: \
917 name##i##type(node->u.w, rn(node->v.w)); \
919 #define case_rrr(name, type) \
920 case jit_code_##name##r##type: \
921 name##r##type(rn(node->u.w), \
922 rn(node->v.w), rn(node->w.w)); \
924 #define case_rrrr(name, type) \
925 case jit_code_##name##r##type: \
926 name##r##type(rn(node->u.q.l), rn(node->u.q.h), \
927 rn(node->v.w), rn(node->w.w)); \
929 #define case_rrw(name, type) \
930 case jit_code_##name##i##type: \
931 name##i##type(rn(node->u.w), rn(node->v.w), node->w.w); \
933 #define case_rrrw(name, type) \
934 case jit_code_##name##i##type: \
935 name##i##type(rn(node->u.q.l), rn(node->u.q.h), \
936 rn(node->v.w), node->w.w); \
938 #define case_rrf(name) \
939 case jit_code_##name##i_f: \
941 name##i_f(rn(node->u.w), rn(node->v.w), node->w.f); \
943 #define case_rrd(name) \
944 case jit_code_##name##i_d: \
946 name##i_d(rn(node->u.w), rn(node->v.w), node->w.d); \
948 #define case_wrr(name, type) \
949 case jit_code_##name##i##type: \
950 name##i##type(node->u.w, rn(node->v.w), rn(node->w.w)); \
952 #define case_brr(name, type) \
953 case jit_code_##name##r##type: \
955 assert(temp->code == jit_code_label || \
956 temp->code == jit_code_epilog); \
957 if (temp->flag & jit_flag_patch) \
958 name##r##type(temp->u.w, rn(node->v.w), \
961 word = name##r##type(_jit->pc.w, \
962 rn(node->v.w), rn(node->w.w)); \
966 #define case_brw(name, type) \
967 case jit_code_##name##i##type: \
969 assert(temp->code == jit_code_label || \
970 temp->code == jit_code_epilog); \
971 if (temp->flag & jit_flag_patch) \
972 name##i##type(temp->u.w, \
973 rn(node->v.w), node->w.w); \
975 word = name##i##type(_jit->pc.w, \
976 rn(node->v.w), node->w.w); \
980 #define case_brf(name) \
981 case jit_code_##name##i_f: \
983 assert(temp->code == jit_code_label || \
984 temp->code == jit_code_epilog); \
985 if (temp->flag & jit_flag_patch) \
986 name##i_f(temp->u.w, rn(node->v.w), node->w.f); \
988 word = name##i_f(_jit->pc.w, rn(node->v.w), \
993 #define case_brd(name) \
994 case jit_code_##name##i_d: \
996 assert(temp->code == jit_code_label || \
997 temp->code == jit_code_epilog); \
998 if (temp->flag & jit_flag_patch) \
999 name##i_d(temp->u.w, rn(node->v.w), node->w.d); \
1001 word = name##i_d(_jit->pc.w, rn(node->v.w), \
1003 patch(word, node); \
1006 #if DEVEL_DISASSEMBLER
1009 for (node = _jitc->head; node; node = node->next) {
1010 if (_jit->pc.uc >= _jitc->code.end)
1013 #if DEVEL_DISASSEMBLER
1014 node->offset = (jit_uword_t)_jit->pc.w - (jit_uword_t)prevw;
1017 value = jit_classify(node->code);
1018 jit_regarg_set(node, value);
1019 switch (node->code) {
1020 case jit_code_align:
1021 assert(!(node->u.w & (node->u.w - 1)) &&
1022 node->u.w <= sizeof(jit_word_t));
1023 if (node->u.w == sizeof(jit_word_t) &&
1024 (word = _jit->pc.w & (sizeof(jit_word_t) - 1)))
1025 nop(sizeof(jit_word_t) - word);
1027 case jit_code_note: case jit_code_name:
1028 node->u.w = _jit->pc.w;
1030 case jit_code_label:
1031 /* remember label is defined */
1032 node->flag |= jit_flag_patch;
1033 node->u.w = _jit->pc.w;
1052 case_rrrr(qmul, _u);
1053 case_rrrw(qmul, _u);
1060 case_rrrr(qdiv, _u);
1061 case_rrrw(qdiv, _u);
1080 case_rr(trunc, _f_i);
1081 case_rr(trunc, _d_i);
1082 case_rr(trunc, _f_l);
1083 case_rr(trunc, _d_l);
1131 case_rr(bswap, _us);
1132 case_rr(bswap, _ui);
1133 case_rr(bswap, _ul);
1144 if (node->flag & jit_flag_node) {
1146 if (temp->code == jit_code_data ||
1147 (temp->code == jit_code_label &&
1148 (temp->flag & jit_flag_patch)))
1149 movi(rn(node->u.w), temp->u.w);
1151 assert(temp->code == jit_code_label ||
1152 temp->code == jit_code_epilog);
1153 word = movi_p(rn(node->u.w), temp->u.w);
1158 movi(rn(node->u.w), node->v.w);
1202 case_brr(boadd, _u);
1203 case_brw(boadd, _u);
1206 case_brr(bxadd, _u);
1207 case_brw(bxadd, _u);
1210 case_brr(bosub, _u);
1211 case_brw(bosub, _u);
1214 case_brr(bxsub, _u);
1215 case_brw(bxsub, _u);
1242 case jit_code_movi_f:
1244 movi_f(rn(node->u.w), node->v.f);
1273 case_rrr(unord, _f);
1287 case_brr(bunlt, _f);
1289 case_brr(bunle, _f);
1291 case_brr(buneq, _f);
1293 case_brr(bunge, _f);
1295 case_brr(bungt, _f);
1297 case_brr(bltgt, _f);
1301 case_brr(bunord, _f);
1325 case jit_code_movi_d:
1327 movi_d(rn(node->u.w), node->v.d);
1356 case_rrr(unord, _d);
1370 case_brr(bunlt, _d);
1372 case_brr(bunle, _d);
1374 case_brr(buneq, _d);
1376 case_brr(bunge, _d);
1378 case_brr(bungt, _d);
1380 case_brr(bltgt, _d);
1384 case_brr(bunord, _d);
1387 jmpr(rn(node->u.w));
1390 if (node->flag & jit_flag_node) {
1392 assert(temp->code == jit_code_label ||
1393 temp->code == jit_code_epilog);
1394 if (temp->flag & jit_flag_patch)
1397 word = jmpi_p(_jit->pc.w);
1404 case jit_code_callr:
1405 callr(rn(node->u.w));
1407 case jit_code_calli:
1408 if (node->flag & jit_flag_node) {
1410 assert(temp->code == jit_code_label ||
1411 temp->code == jit_code_epilog);
1412 if (temp->flag & jit_flag_patch)
1415 word = calli_p(_jit->pc.w);
1422 case jit_code_prolog:
1423 _jitc->function = _jitc->functions.ptr + node->w.w;
1425 undo.word = _jit->pc.w;
1426 #if DEVEL_DISASSEMBLER
1429 undo.patch_offset = _jitc->patches.offset;
1434 case jit_code_epilog:
1435 assert(_jitc->function == _jitc->functions.ptr + node->w.w);
1437 for (temp = undo.node->next;
1438 temp != node; temp = temp->next) {
1439 if (temp->code == jit_code_label ||
1440 temp->code == jit_code_epilog)
1441 temp->flag &= ~jit_flag_patch;
1443 temp->flag &= ~jit_flag_patch;
1445 _jit->pc.w = undo.word;
1446 #if DEVEL_DISASSEMBLER
1449 _jitc->patches.offset = undo.patch_offset;
1450 goto restart_function;
1452 /* remember label is defined */
1453 node->flag |= jit_flag_patch;
1454 node->u.w = _jit->pc.w;
1456 _jitc->function = NULL;
1458 case jit_code_va_start:
1459 vastart(rn(node->u.w));
1461 case jit_code_va_arg:
1462 vaarg(rn(node->u.w), rn(node->v.w));
1464 case jit_code_va_arg_d:
1465 vaarg_d(rn(node->u.w), rn(node->v.w));
1467 case jit_code_live: case jit_code_ellipsis:
1468 case jit_code_va_push:
1469 case jit_code_allocai: case jit_code_allocar:
1471 case jit_code_arg_f: case jit_code_arg_d:
1472 case jit_code_va_end:
1474 case jit_code_retr: case jit_code_reti:
1475 case jit_code_retr_f: case jit_code_reti_f:
1476 case jit_code_retr_d: case jit_code_reti_d:
1477 case jit_code_getarg_c: case jit_code_getarg_uc:
1478 case jit_code_getarg_s: case jit_code_getarg_us:
1479 case jit_code_getarg_i: case jit_code_getarg_ui:
1480 case jit_code_getarg_l:
1481 case jit_code_getarg_f: case jit_code_getarg_d:
1482 case jit_code_putargr: case jit_code_putargi:
1483 case jit_code_putargr_f: case jit_code_putargi_f:
1484 case jit_code_putargr_d: case jit_code_putargi_d:
1485 case jit_code_pushargr: case jit_code_pushargi:
1486 case jit_code_pushargr_f: case jit_code_pushargi_f:
1487 case jit_code_pushargr_d: case jit_code_pushargi_d:
1488 case jit_code_retval_c: case jit_code_retval_uc:
1489 case jit_code_retval_s: case jit_code_retval_us:
1490 case jit_code_retval_i:
1491 case jit_code_retval_ui: case jit_code_retval_l:
1492 case jit_code_retval_f: case jit_code_retval_d:
1493 case jit_code_prepare:
1494 case jit_code_finishr: case jit_code_finishi:
1499 jit_regarg_clr(node, value);
1500 assert(_jitc->regarg == 0 && _jitc->synth == 0);
1501 /* update register live state */
1513 for (offset = 0; offset < _jitc->patches.offset; offset++) {
1514 node = _jitc->patches.ptr[offset].node;
1515 word = _jitc->patches.ptr[offset].inst;
1516 value = node->code == jit_code_movi ? node->v.n->u.w : node->u.n->u.w;
1517 patch_at(word, value);
1520 jit_flush(_jit->code.ptr, _jit->pc.uc);
1522 return (_jit->code.ptr);
1526 # include "jit_aarch64-cpu.c"
1527 # include "jit_aarch64-fpu.c"
1531 jit_flush(void *fptr, void *tptr)
1533 #if defined(__GNUC__)
1536 s = sysconf(_SC_PAGE_SIZE);
1537 f = (jit_word_t)fptr & -s;
1538 t = (((jit_word_t)tptr) + s - 1) & -s;
1539 __clear_cache((void *)f, (void *)t);
1544 _emit_ldxi(jit_state_t *_jit, jit_int32_t r0, jit_int32_t r1, jit_word_t i0)
1546 ldxi(rn(r0), rn(r1), i0);
1550 _emit_stxi(jit_state_t *_jit, jit_word_t i0, jit_int32_t r0, jit_int32_t r1)
1552 stxi(i0, rn(r0), rn(r1));
1556 _emit_ldxi_d(jit_state_t *_jit, jit_int32_t r0, jit_int32_t r1, jit_word_t i0)
1558 ldxi_d(rn(r0), rn(r1), i0);
1562 _emit_stxi_d(jit_state_t *_jit, jit_word_t i0, jit_int32_t r0, jit_int32_t r1)
1564 stxi_d(i0, rn(r0), rn(r1));
1568 _patch(jit_state_t *_jit, jit_word_t instr, jit_node_t *node)
1572 assert(node->flag & jit_flag_node);
1573 if (node->code == jit_code_movi)
1574 flag = node->v.n->flag;
1576 flag = node->u.n->flag;
1577 assert(!(flag & jit_flag_patch));
1578 if (_jitc->patches.offset >= _jitc->patches.length) {
1579 jit_realloc((jit_pointer_t *)&_jitc->patches.ptr,
1580 _jitc->patches.length * sizeof(jit_patch_t),
1581 (_jitc->patches.length + 1024) * sizeof(jit_patch_t));
1582 _jitc->patches.length += 1024;
1584 _jitc->patches.ptr[_jitc->patches.offset].inst = instr;
1585 _jitc->patches.ptr[_jitc->patches.offset].node = node;
1586 ++_jitc->patches.offset;