2 * Copyright (C) 2019-2022 Free Software Foundation, Inc.
4 * This file is part of GNU lightning.
6 * GNU lightning is free software; you can redistribute it and/or modify it
7 * under the terms of the GNU Lesser General Public License as published
8 * by the Free Software Foundation; either version 3, or (at your option)
11 * GNU lightning is distributed in the hope that it will be useful, but
12 * WITHOUT ANY WARRANTY; without even the implied warranty of MERCHANTABILITY
13 * or FITNESS FOR A PARTICULAR PURPOSE. See the GNU Lesser General Public
14 * License for more details.
17 * Paulo Cesar Pereira de Andrade
20 #define jit_arg_reg_p(i) ((i) >= 0 && (i) < 8)
21 #define jit_arg_f_reg_p(i) ((i) >= 0 && (i) < 8)
26 typedef jit_pointer_t jit_va_list_t;
32 # define load_const(r0, i0) _load_const(_jit, r0, i0)
33 static void _load_const(jit_state_t*, jit_int32_t, jit_word_t);
34 static jit_word_t hash_const(jit_word_t);
35 # define put_const(i0) _put_const(_jit, i0)
36 static void _put_const(jit_state_t*, jit_word_t);
37 # define get_const(i0) _get_const(_jit, i0)
38 static jit_word_t _get_const(jit_state_t*, jit_word_t);
40 #define patch(instr, node) _patch(_jit, instr, node)
41 static void _patch(jit_state_t*,jit_word_t,jit_node_t*);
44 # include "jit_riscv-cpu.c"
45 # include "jit_riscv-fpu.c"
51 jit_register_t _rvs[] = {
56 #if 0 /* Pretend it does not exist, so _NOREG can be used in
60 { rc(gpr) | 0x05, "t0" },
61 { rc(gpr) | 0x06, "t1" },
62 { rc(gpr) | 0x07, "t2" },
63 { rc(gpr) | 0x1c, "t3" },
64 { rc(gpr) | 0x1d, "t4" },
65 { rc(gpr) | 0x1e, "t5" },
66 { rc(gpr) | 0x1f, "t6" },
68 { rc(sav) | rc(gpr) | 0x09, "s1" },
69 { rc(sav) | rc(gpr) | 0x12, "s2" },
70 { rc(sav) | rc(gpr) | 0x13, "s3" },
71 { rc(sav) | rc(gpr) | 0x14, "s4" },
72 { rc(sav) | rc(gpr) | 0x15, "s5" },
73 { rc(sav) | rc(gpr) | 0x16, "s6" },
74 { rc(sav) | rc(gpr) | 0x17, "s7" },
75 { rc(sav) | rc(gpr) | 0x18, "s8" },
76 { rc(sav) | rc(gpr) | 0x19, "s9" },
77 { rc(sav) | rc(gpr) | 0x1a, "s10" },
78 { rc(sav) | rc(gpr) | 0x1b, "s11" },
79 { rc(arg) | rc(gpr) | 0x11, "a7" },
80 { rc(arg) | rc(gpr) | 0x10, "a6" },
81 { rc(arg) | rc(gpr) | 0x0f, "a5" },
82 { rc(arg) | rc(gpr) | 0x0e, "a4" },
83 { rc(arg) | rc(gpr) | 0x0d, "a3" },
84 { rc(arg) | rc(gpr) | 0x0c, "a2" },
85 { rc(arg) | rc(gpr) | 0x0b, "a1" },
86 { rc(arg) | rc(gpr) | 0x0a, "a0" },
87 { rc(fpr) | 0x00, "ft0" },
88 { rc(fpr) | 0x01, "ft1" },
89 { rc(fpr) | 0x02, "ft2" },
90 { rc(fpr) | 0x03, "ft3" },
91 { rc(fpr) | 0x04, "ft4" },
92 { rc(fpr) | 0x05, "ft5" },
93 { rc(fpr) | 0x06, "ft6" },
94 { rc(fpr) | 0x07, "ft7" },
95 { rc(fpr) | 0x1c, "ft8" },
96 { rc(fpr) | 0x1d, "ft9" },
97 { rc(fpr) | 0x1e, "ft10" },
98 { rc(fpr) | 0x1f, "ft11" },
99 { rc(sav) | rc(fpr) | 0x08, "fs0" },
100 { rc(sav) | rc(fpr) | 0x09, "fs1" },
101 { rc(sav) | rc(fpr) | 0x12, "fs2" },
102 { rc(sav) | rc(fpr) | 0x13, "fs3" },
103 { rc(sav) | rc(fpr) | 0x14, "fs4" },
104 { rc(sav) | rc(fpr) | 0x15, "fs5" },
105 { rc(sav) | rc(fpr) | 0x16, "fs6" },
106 { rc(sav) | rc(fpr) | 0x17, "fs7" },
107 { rc(sav) | rc(fpr) | 0x18, "fs8" },
108 { rc(sav) | rc(fpr) | 0x19, "fs9" },
109 { rc(sav) | rc(fpr) | 0x1a, "fs10" },
110 { rc(sav) | rc(fpr) | 0x1b, "fs11" },
111 { rc(arg) | rc(fpr) | 0x11, "fa7" },
112 { rc(arg) | rc(fpr) | 0x10, "fa6" },
113 { rc(arg) | rc(fpr) | 0x0f, "fa5" },
114 { rc(arg) | rc(fpr) | 0x0e, "fa4" },
115 { rc(arg) | rc(fpr) | 0x0d, "fa3" },
116 { rc(arg) | rc(fpr) | 0x0c, "fa2" },
117 { rc(arg) | rc(fpr) | 0x0b, "fa1" },
118 { rc(arg) | rc(fpr) | 0x0a, "fa0" },
119 { _NOREG, "<none>" },
131 _jit_init(jit_state_t *_jit)
133 _jitc->reglen = jit_size(_rvs) - 1;
138 _jit_prolog(jit_state_t *_jit)
144 assert(jit_regset_cmp_ui(&_jitc->regarg, 0) == 0);
145 jit_regset_set_ui(&_jitc->regsav, 0);
146 offset = _jitc->functions.offset;
147 if (offset >= _jitc->functions.length) {
148 jit_realloc((jit_pointer_t *)&_jitc->functions.ptr,
149 _jitc->functions.length * sizeof(jit_function_t),
150 (_jitc->functions.length + 16) * sizeof(jit_function_t));
151 _jitc->functions.length += 16;
153 _jitc->function = _jitc->functions.ptr + _jitc->functions.offset++;
154 _jitc->function->self.size = stack_framesize;
155 _jitc->function->self.argi = _jitc->function->self.argf =
156 _jitc->function->self.alen = 0;
157 _jitc->function->self.aoff = 0;
158 _jitc->function->self.call = jit_call_default;
159 jit_alloc((jit_pointer_t *)&_jitc->function->regoff,
160 _jitc->reglen * sizeof(jit_int32_t));
162 /* _no_link here does not mean the jit_link() call can be removed
164 * _jitc->function->prolog = jit_new_node(jit_code_prolog);
166 _jitc->function->prolog = jit_new_node_no_link(jit_code_prolog);
167 jit_link(_jitc->function->prolog);
168 _jitc->function->prolog->w.w = offset;
169 _jitc->function->epilog = jit_new_node_no_link(jit_code_epilog);
171 * v: offset in blocks vector
172 * w: offset in functions vector
174 _jitc->function->epilog->w.w = offset;
176 jit_regset_new(&_jitc->function->regset);
180 _jit_allocai(jit_state_t *_jit, jit_int32_t length)
182 assert(_jitc->function);
184 case 0: case 1: break;
185 case 2: _jitc->function->self.aoff &= -2; break;
186 case 3: case 4: _jitc->function->self.aoff &= -4; break;
187 default: _jitc->function->self.aoff &= -8; break;
189 _jitc->function->self.aoff -= length;
190 if (!_jitc->realize) {
191 jit_inc_synth_ww(allocai, _jitc->function->self.aoff, length);
194 return (_jitc->function->self.aoff);
198 _jit_allocar(jit_state_t *_jit, jit_int32_t u, jit_int32_t v)
201 assert(_jitc->function);
202 jit_inc_synth_ww(allocar, u, v);
203 if (!_jitc->function->allocar) {
204 _jitc->function->aoffoff = jit_allocai(sizeof(jit_int32_t));
205 _jitc->function->allocar = 1;
207 r0 = jit_get_reg(jit_class_gpr);
209 jit_andi(r0, r0, -16);
210 jit_ldxi_i(u, JIT_FP, _jitc->function->aoffoff);
212 jit_addr(JIT_SP, JIT_SP, r0);
213 jit_stxi_i(_jitc->function->aoffoff, JIT_FP, u);
219 _jit_ret(jit_state_t *_jit)
222 assert(_jitc->function);
226 jit_patch_at(instr, _jitc->function->epilog);
231 _jit_retr(jit_state_t *_jit, jit_int32_t u)
233 jit_inc_synth_w(retr, u);
235 jit_movr(JIT_RET, u);
242 _jit_reti(jit_state_t *_jit, jit_word_t u)
244 jit_inc_synth_w(reti, u);
245 jit_movi(JIT_RET, u);
251 _jit_retr_f(jit_state_t *_jit, jit_int32_t u)
253 jit_inc_synth_w(retr_f, u);
255 jit_movr_f(JIT_FRET, u);
263 _jit_reti_f(jit_state_t *_jit, jit_float32_t u)
265 jit_inc_synth_f(reti_f, u);
266 jit_movi_f(JIT_FRET, u);
272 _jit_retr_d(jit_state_t *_jit, jit_int32_t u)
274 jit_inc_synth_w(retr_d, u);
276 jit_movr_d(JIT_FRET, u);
284 _jit_reti_d(jit_state_t *_jit, jit_float64_t u)
286 jit_inc_synth_d(reti_d, u);
287 jit_movi_d(JIT_FRET, u);
293 _jit_epilog(jit_state_t *_jit)
295 assert(_jitc->function);
296 assert(_jitc->function->epilog->next == NULL);
297 jit_link(_jitc->function->epilog);
298 _jitc->function = NULL;
302 _jit_arg_register_p(jit_state_t *_jit, jit_node_t *u)
304 if (u->code == jit_code_arg)
305 return (jit_arg_reg_p(u->u.w));
306 assert(u->code == jit_code_arg_f || u->code == jit_code_arg_d);
307 return (jit_arg_f_reg_p(u->u.w));
311 _jit_ellipsis(jit_state_t *_jit)
313 jit_inc_synth(ellipsis);
314 if (_jitc->prepare) {
316 assert(!(_jitc->function->call.call & jit_call_varargs));
317 _jitc->function->call.call |= jit_call_varargs;
321 assert(!(_jitc->function->self.call & jit_call_varargs));
322 _jitc->function->self.call |= jit_call_varargs;
323 _jitc->function->vagp = _jitc->function->self.argi;
329 _jit_va_push(jit_state_t *_jit, jit_int32_t u)
331 jit_inc_synth_w(va_push, u);
337 _jit_arg(jit_state_t *_jit)
341 assert(_jitc->function);
342 assert(!(_jitc->function->self.call & jit_call_varargs));
343 if (jit_arg_reg_p(_jitc->function->self.argi))
344 offset = _jitc->function->self.argi++;
346 offset = _jitc->function->self.size;
347 _jitc->function->self.size += sizeof(jit_word_t);
349 node = jit_new_node_ww(jit_code_arg, offset,
350 ++_jitc->function->self.argn);
356 _jit_arg_f(jit_state_t *_jit)
360 assert(_jitc->function);
361 assert(!(_jitc->function->self.call & jit_call_varargs));
362 if (jit_arg_f_reg_p(_jitc->function->self.argf))
363 offset = _jitc->function->self.argf++;
364 else if (jit_arg_reg_p(_jitc->function->self.argi)) {
365 offset = _jitc->function->self.argi++;
369 offset = _jitc->function->self.size;
370 _jitc->function->self.size += sizeof(jit_word_t);
372 node = jit_new_node_ww(jit_code_arg_f, offset,
373 ++_jitc->function->self.argn);
379 _jit_arg_d(jit_state_t *_jit)
383 assert(_jitc->function);
384 assert(!(_jitc->function->self.call & jit_call_varargs));
385 if (jit_arg_f_reg_p(_jitc->function->self.argf))
386 offset = _jitc->function->self.argf++;
387 else if (jit_arg_reg_p(_jitc->function->self.argi)) {
388 offset = _jitc->function->self.argi++;
392 offset = _jitc->function->self.size;
393 _jitc->function->self.size += sizeof(jit_word_t);
395 node = jit_new_node_ww(jit_code_arg_d, offset,
396 ++_jitc->function->self.argn);
402 _jit_getarg_c(jit_state_t *_jit, jit_int32_t u, jit_node_t *v)
404 assert(v->code == jit_code_arg);
405 jit_inc_synth_wp(getarg_c, u, v);
406 if (jit_arg_reg_p(v->u.w))
407 jit_extr_c(u, JIT_RA0 - v->u.w);
409 jit_ldxi_c(u, JIT_FP, v->u.w);
414 _jit_getarg_uc(jit_state_t *_jit, jit_int32_t u, jit_node_t *v)
416 assert(v->code == jit_code_arg);
417 jit_inc_synth_wp(getarg_uc, u, v);
418 if (jit_arg_reg_p(v->u.w))
419 jit_extr_uc(u, JIT_RA0 - v->u.w);
421 jit_ldxi_uc(u, JIT_FP, v->u.w);
426 _jit_getarg_s(jit_state_t *_jit, jit_int32_t u, jit_node_t *v)
428 assert(v->code == jit_code_arg);
429 jit_inc_synth_wp(getarg_s, u, v);
430 if (jit_arg_reg_p(v->u.w))
431 jit_extr_s(u, JIT_RA0 - v->u.w);
433 jit_ldxi_s(u, JIT_FP, v->u.w);
438 _jit_getarg_us(jit_state_t *_jit, jit_int32_t u, jit_node_t *v)
440 assert(v->code == jit_code_arg);
441 jit_inc_synth_wp(getarg_us, u, v);
442 if (jit_arg_reg_p(v->u.w))
443 jit_extr_us(u, JIT_RA0 - v->u.w);
445 jit_ldxi_us(u, JIT_FP, v->u.w);
450 _jit_getarg_i(jit_state_t *_jit, jit_int32_t u, jit_node_t *v)
452 assert(v->code == jit_code_arg);
453 jit_inc_synth_wp(getarg_i, u, v);
454 if (jit_arg_reg_p(v->u.w))
455 jit_extr_i(u, JIT_RA0 - v->u.w);
457 jit_ldxi_i(u, JIT_FP, v->u.w);
462 _jit_getarg_ui(jit_state_t *_jit, jit_int32_t u, jit_node_t *v)
464 assert(v->code == jit_code_arg);
465 jit_inc_synth_wp(getarg_ui, u, v);
466 if (jit_arg_reg_p(v->u.w))
467 jit_extr_ui(u, JIT_RA0 - v->u.w);
469 jit_ldxi_ui(u, JIT_FP, v->u.w);
474 _jit_getarg_l(jit_state_t *_jit, jit_int32_t u, jit_node_t *v)
476 assert(v->code == jit_code_arg);
477 jit_inc_synth_wp(getarg_l, u, v);
478 if (jit_arg_reg_p(v->u.w))
479 jit_movr(u, JIT_RA0 - v->u.w);
481 jit_ldxi_l(u, JIT_FP, v->u.w);
486 _jit_putargr(jit_state_t *_jit, jit_int32_t u, jit_node_t *v)
488 assert(v->code == jit_code_arg);
489 jit_inc_synth_wp(putargr, u, v);
490 if (jit_arg_reg_p(v->u.w))
491 jit_movr(JIT_RA0 - v->u.w, u);
493 jit_stxi(v->u.w, JIT_FP, u);
498 _jit_putargi(jit_state_t *_jit, jit_word_t u, jit_node_t *v)
501 assert(v->code == jit_code_arg);
502 jit_inc_synth_wp(putargi, u, v);
503 if (jit_arg_reg_p(v->u.w))
504 jit_movi(JIT_RA0 - v->u.w, u);
506 regno = jit_get_reg(jit_class_gpr);
508 jit_stxi(v->u.w, JIT_FP, regno);
509 jit_unget_reg(regno);
515 _jit_getarg_f(jit_state_t *_jit, jit_int32_t u, jit_node_t *v)
517 assert(v->code == jit_code_arg_f);
518 jit_inc_synth_wp(getarg_f, u, v);
519 if (jit_arg_f_reg_p(v->u.w))
520 jit_movr_f(u, JIT_FA0 - v->u.w);
521 else if (jit_arg_reg_p(v->u.w - 8))
522 jit_movr_w_f(u, JIT_RA0 - (v->u.w - 8));
524 jit_ldxi_f(u, JIT_FP, v->u.w);
529 _jit_putargr_f(jit_state_t *_jit, jit_int32_t u, jit_node_t *v)
531 assert(v->code == jit_code_arg_f);
532 jit_inc_synth_wp(putargr_f, u, v);
533 if (jit_arg_f_reg_p(v->u.w))
534 jit_movr_f(JIT_FA0 - v->u.w, u);
535 else if (jit_arg_reg_p(v->u.w - 8))
536 jit_movr_f_w(JIT_RA0 - (v->u.w - 8), u);
538 jit_stxi_f(v->u.w, JIT_FP, u);
543 _jit_putargi_f(jit_state_t *_jit, jit_float32_t u, jit_node_t *v)
546 assert(v->code == jit_code_arg_f);
547 jit_inc_synth_fp(putargi_f, u, v);
548 if (jit_arg_f_reg_p(v->u.w))
549 jit_movi_f(JIT_FA0 - v->u.w, u);
550 else if (jit_arg_reg_p(v->u.w - 8)) {
556 jit_movi(JIT_RA0 - (v->u.w - 8), uu.i);
559 regno = jit_get_reg(jit_class_fpr);
560 jit_movi_f(regno, u);
561 jit_stxi_f(v->u.w, JIT_FP, regno);
562 jit_unget_reg(regno);
568 _jit_getarg_d(jit_state_t *_jit, jit_int32_t u, jit_node_t *v)
570 assert(v->code == jit_code_arg_d);
571 jit_inc_synth_wp(getarg_d, u, v);
572 if (jit_arg_f_reg_p(v->u.w))
573 jit_movr_d(u, JIT_FA0 - v->u.w);
574 else if (jit_arg_reg_p(v->u.w - 8))
575 jit_movr_w_d(u, JIT_RA0 - (v->u.w - 8));
577 jit_ldxi_d(u, JIT_FP, v->u.w);
582 _jit_putargr_d(jit_state_t *_jit, jit_int32_t u, jit_node_t *v)
584 assert(v->code == jit_code_arg_d);
585 jit_inc_synth_wp(putargr_d, u, v);
586 if (jit_arg_reg_p(v->u.w))
587 jit_movr_d(JIT_FA0 - v->u.w, u);
588 else if (jit_arg_reg_p(v->u.w - 8))
589 jit_movr_d_w(JIT_RA0 - (v->u.w - 8), u);
591 jit_stxi_d(v->u.w, JIT_FP, u);
596 _jit_putargi_d(jit_state_t *_jit, jit_float64_t u, jit_node_t *v)
599 assert(v->code == jit_code_arg_d);
600 jit_inc_synth_dp(putargi_d, u, v);
601 if (jit_arg_reg_p(v->u.w))
602 jit_movi_d(JIT_FA0 - v->u.w, u);
603 else if (jit_arg_reg_p(v->u.w - 8)) {
609 jit_movi(JIT_RA0 - (v->u.w - 8), uu.w);
612 regno = jit_get_reg(jit_class_fpr);
613 jit_movi_d(regno, u);
614 jit_stxi_d(v->u.w, JIT_FP, regno);
615 jit_unget_reg(regno);
621 _jit_pushargr(jit_state_t *_jit, jit_int32_t u)
623 assert(_jitc->function);
624 jit_inc_synth_w(pushargr, u);
626 if (jit_arg_reg_p(_jitc->function->call.argi)) {
627 jit_movr(JIT_RA0 - _jitc->function->call.argi, u);
628 ++_jitc->function->call.argi;
631 jit_stxi(_jitc->function->call.size, JIT_SP, u);
632 _jitc->function->call.size += sizeof(jit_word_t);
638 _jit_pushargi(jit_state_t *_jit, jit_word_t u)
641 assert(_jitc->function);
642 jit_inc_synth_w(pushargi, u);
644 if (jit_arg_reg_p(_jitc->function->call.argi)) {
645 jit_movi(JIT_RA0 - _jitc->function->call.argi, u);
646 ++_jitc->function->call.argi;
649 regno = jit_get_reg(jit_class_gpr);
651 jit_stxi(_jitc->function->call.size, JIT_SP, regno);
652 jit_unget_reg(regno);
653 _jitc->function->call.size += sizeof(jit_word_t);
659 _jit_pushargr_f(jit_state_t *_jit, jit_int32_t u)
661 assert(_jitc->function);
662 jit_inc_synth_w(pushargr_f, u);
664 if (jit_arg_f_reg_p(_jitc->function->call.argf) &&
665 !(_jitc->function->call.call & jit_call_varargs)) {
666 jit_movr_f(JIT_FA0 - _jitc->function->call.argf, u);
667 ++_jitc->function->call.argf;
669 else if (jit_arg_reg_p(_jitc->function->call.argi)) {
670 jit_movr_f_w(JIT_RA0 - _jitc->function->call.argi, u);
671 ++_jitc->function->call.argi;
674 jit_stxi_f(_jitc->function->call.size, JIT_SP, u);
675 _jitc->function->call.size += sizeof(jit_word_t);
681 _jit_pushargi_f(jit_state_t *_jit, jit_float32_t u)
684 assert(_jitc->function);
685 jit_inc_synth_f(pushargi_f, u);
687 if (jit_arg_f_reg_p(_jitc->function->call.argf) &&
688 !(_jitc->function->call.call & jit_call_varargs)) {
689 jit_movi_f(JIT_FA0 - _jitc->function->call.argf, u);
690 ++_jitc->function->call.argf;
692 else if (jit_arg_reg_p(_jitc->function->call.argi)) {
693 jit_movi_f_w(JIT_RA0 - _jitc->function->call.argi, u);
694 ++_jitc->function->call.argi;
697 regno = jit_get_reg(jit_class_fpr);
698 jit_movi_f(regno, u);
699 jit_stxi_f(_jitc->function->call.size, JIT_SP, regno);
700 jit_unget_reg(regno);
701 _jitc->function->call.size += sizeof(jit_word_t);
707 _jit_pushargr_d(jit_state_t *_jit, jit_int32_t u)
709 assert(_jitc->function);
710 jit_inc_synth_w(pushargr_d, u);
712 if (jit_arg_f_reg_p(_jitc->function->call.argf) &&
713 !(_jitc->function->call.call & jit_call_varargs)) {
714 jit_movr_d(JIT_FA0 - _jitc->function->call.argf, u);
715 ++_jitc->function->call.argf;
717 else if (jit_arg_reg_p(_jitc->function->call.argi)) {
718 jit_movr_d_w(JIT_RA0 - _jitc->function->call.argi, u);
719 ++_jitc->function->call.argi;
722 jit_stxi_d(_jitc->function->call.size, JIT_SP, u);
723 _jitc->function->call.size += sizeof(jit_word_t);
729 _jit_pushargi_d(jit_state_t *_jit, jit_float64_t u)
732 assert(_jitc->function);
733 jit_inc_synth_d(pushargi_d, u);
735 if (jit_arg_f_reg_p(_jitc->function->call.argf) &&
736 !(_jitc->function->call.call & jit_call_varargs)) {
737 jit_movi_d(JIT_FA0 - _jitc->function->call.argf, u);
738 ++_jitc->function->call.argf;
740 else if (jit_arg_reg_p(_jitc->function->call.argi)) {
741 jit_movi_d_w(JIT_RA0 - _jitc->function->call.argi, u);
742 ++_jitc->function->call.argi;
745 regno = jit_get_reg(jit_class_fpr);
746 jit_movi_d(regno, u);
747 jit_stxi_d(_jitc->function->call.size, JIT_SP, regno);
748 jit_unget_reg(regno);
749 _jitc->function->call.size += sizeof(jit_word_t);
755 _jit_regarg_p(jit_state_t *_jit, jit_node_t *node, jit_int32_t regno)
758 spec = jit_class(_rvs[regno].spec);
759 if (spec & jit_class_arg) {
760 regno = JIT_RA0 - regno;
761 if (regno >= 0 && regno < node->v.w)
763 if (spec & jit_class_fpr) {
764 regno = JIT_FA0 - regno;
765 if (regno >= 0 && regno < node->w.w)
774 _jit_finishr(jit_state_t *_jit, jit_int32_t r0)
777 assert(_jitc->function);
778 jit_inc_synth_w(finishr, r0);
779 if (_jitc->function->self.alen < _jitc->function->call.size)
780 _jitc->function->self.alen = _jitc->function->call.size;
781 node = jit_callr(r0);
782 node->v.w = _jitc->function->self.argi;
783 node->w.w = _jitc->function->call.argf;
784 _jitc->function->call.argi = _jitc->function->call.argf =
785 _jitc->function->call.size = 0;
791 _jit_finishi(jit_state_t *_jit, jit_pointer_t i0)
794 assert(_jitc->function);
795 jit_inc_synth_w(finishi, (jit_word_t)i0);
796 if (_jitc->function->self.alen < _jitc->function->call.size)
797 _jitc->function->self.alen = _jitc->function->call.size;
798 node = jit_calli(i0);
799 node->v.w = _jitc->function->call.argi;
800 node->w.w = _jitc->function->call.argf;
801 _jitc->function->call.argi = _jitc->function->call.argf =
802 _jitc->function->call.size = 0;
809 _jit_retval_c(jit_state_t *_jit, jit_int32_t r0)
811 jit_inc_synth_w(retval_c, r0);
812 jit_extr_c(r0, JIT_RET);
817 _jit_retval_uc(jit_state_t *_jit, jit_int32_t r0)
819 jit_inc_synth_w(retval_uc, r0);
820 jit_extr_uc(r0, JIT_RET);
825 _jit_retval_s(jit_state_t *_jit, jit_int32_t r0)
827 jit_inc_synth_w(retval_s, r0);
828 jit_extr_s(r0, JIT_RET);
833 _jit_retval_us(jit_state_t *_jit, jit_int32_t r0)
835 jit_inc_synth_w(retval_us, r0);
836 jit_extr_us(r0, JIT_RET);
841 _jit_retval_i(jit_state_t *_jit, jit_int32_t r0)
843 jit_inc_synth_w(retval_i, r0);
844 jit_extr_i(r0, JIT_RET);
849 _jit_retval_ui(jit_state_t *_jit, jit_int32_t r0)
851 jit_inc_synth_w(retval_ui, r0);
852 jit_extr_ui(r0, JIT_RET);
857 _jit_retval_l(jit_state_t *_jit, jit_int32_t r0)
859 jit_inc_synth_w(retval_l, r0);
861 jit_movr(r0, JIT_RET);
866 _jit_retval_f(jit_state_t *_jit, jit_int32_t r0)
868 jit_inc_synth_w(retval_f, r0);
870 jit_movr_f(r0, JIT_FRET);
875 _jit_retval_d(jit_state_t *_jit, jit_int32_t r0)
877 jit_inc_synth_w(retval_d, r0);
879 jit_movr_d(r0, JIT_FRET);
884 _emit_code(jit_state_t *_jit)
895 #if DEVEL_DISASSEMBLER
898 jit_int32_t const_offset;
899 jit_int32_t patch_offset;
901 #if DEVEL_DISASSEMBLER
906 if (!_jitc->consts.hash.table) {
907 jit_alloc((jit_pointer_t *)&_jitc->consts.hash.table,
908 16 * sizeof(jit_const_t *));
909 _jitc->consts.hash.size = 16;
910 jit_alloc((jit_pointer_t *)&_jitc->consts.pool.ptr,
911 sizeof(jit_const_t *));
912 jit_alloc((jit_pointer_t *)_jitc->consts.pool.ptr,
913 1024 * sizeof(jit_const_t));
914 _jitc->consts.pool.length = 1;
916 /* Reset table if starting over jit generation */
918 memset(_jitc->consts.hash.table, 0,
919 _jitc->consts.hash.size * sizeof(jit_word_t));
920 for (offset = 0; offset < _jitc->consts.pool.length; offset++) {
922 jit_const_t *list = _jitc->consts.pool.ptr[offset];
923 for (i = 0; i < 1023; ++i, ++list)
924 list->next = list + 1;
925 if (offset + 1 < _jitc->consts.pool.length)
926 list->next = _jitc->consts.pool.ptr[offset + 1];
930 _jitc->consts.pool.list = _jitc->consts.pool.ptr[0];
931 _jitc->consts.hash.count = 0;
932 if (!_jitc->consts.vector.instrs) {
933 jit_alloc((jit_pointer_t *)&_jitc->consts.vector.instrs,
934 16 * sizeof(jit_word_t));
935 jit_alloc((jit_pointer_t *)&_jitc->consts.vector.values,
936 16 * sizeof(jit_word_t));
937 _jitc->consts.vector.length = 16;
939 _jitc->consts.vector.offset = 0;
942 _jitc->function = NULL;
948 undo.const_offset = undo.patch_offset = 0;
949 # define assert_data(node) /**/
950 #define case_rr(name, type) \
951 case jit_code_##name##r##type: \
952 name##r##type(rn(node->u.w), rn(node->v.w)); \
954 #define case_rw(name, type) \
955 case jit_code_##name##i##type: \
956 name##i##type(rn(node->u.w), node->v.w); \
958 #define case_wr(name, type) \
959 case jit_code_##name##i##type: \
960 name##i##type(node->u.w, rn(node->v.w)); \
962 #define case_rrr(name, type) \
963 case jit_code_##name##r##type: \
964 name##r##type(rn(node->u.w), \
965 rn(node->v.w), rn(node->w.w)); \
967 #define case_rrrr(name, type) \
968 case jit_code_##name##r##type: \
969 name##r##type(rn(node->u.q.l), rn(node->u.q.h), \
970 rn(node->v.w), rn(node->w.w)); \
972 #define case_rrw(name, type) \
973 case jit_code_##name##i##type: \
974 name##i##type(rn(node->u.w), rn(node->v.w), node->w.w); \
976 #define case_rrrw(name, type) \
977 case jit_code_##name##i##type: \
978 name##i##type(rn(node->u.q.l), rn(node->u.q.h), \
979 rn(node->v.w), node->w.w); \
981 #define case_rrf(name) \
982 case jit_code_##name##i_f: \
984 name##i_f(rn(node->u.w), rn(node->v.w), node->w.f); \
986 #define case_rrd(name) \
987 case jit_code_##name##i_d: \
989 name##i_d(rn(node->u.w), rn(node->v.w), node->w.d); \
991 #define case_wrr(name, type) \
992 case jit_code_##name##i##type: \
993 name##i##type(node->u.w, rn(node->v.w), rn(node->w.w)); \
995 #define case_brr(name, type) \
996 case jit_code_##name##r##type: \
998 assert(temp->code == jit_code_label || \
999 temp->code == jit_code_epilog); \
1000 if (temp->flag & jit_flag_patch) \
1001 name##r##type(temp->u.w, rn(node->v.w), \
1004 word = name##r##type(_jit->pc.w, \
1005 rn(node->v.w), rn(node->w.w)); \
1006 patch(word, node); \
1009 #define case_brw(name, type) \
1010 case jit_code_##name##i##type: \
1012 assert(temp->code == jit_code_label || \
1013 temp->code == jit_code_epilog); \
1014 if (temp->flag & jit_flag_patch) \
1015 name##i##type(temp->u.w, \
1016 rn(node->v.w), node->w.w); \
1018 word = name##i##type(_jit->pc.w, \
1019 rn(node->v.w), node->w.w); \
1020 patch(word, node); \
1023 #define case_brf(name) \
1024 case jit_code_##name##i_f: \
1026 assert(temp->code == jit_code_label || \
1027 temp->code == jit_code_epilog); \
1028 if (temp->flag & jit_flag_patch) \
1029 name##i_f(temp->u.w, rn(node->v.w), node->w.f); \
1031 word = name##i_f(_jit->pc.w, rn(node->v.w), \
1033 patch(word, node); \
1036 #define case_brd(name) \
1037 case jit_code_##name##i_d: \
1039 assert(temp->code == jit_code_label || \
1040 temp->code == jit_code_epilog); \
1041 if (temp->flag & jit_flag_patch) \
1042 name##i_d(temp->u.w, rn(node->v.w), node->w.d); \
1044 word = name##i_d(_jit->pc.w, rn(node->v.w), \
1046 patch(word, node); \
1049 #if DEVEL_DISASSEMBLER
1052 for (node = _jitc->head; node; node = node->next) {
1053 if (_jit->pc.uc >= _jitc->code.end)
1056 #if DEVEL_DISASSEMBLER
1057 node->offset = (jit_uword_t)_jit->pc.w - (jit_uword_t)prevw;
1060 value = jit_classify(node->code);
1061 jit_regarg_set(node, value);
1062 switch (node->code) {
1063 case jit_code_align:
1064 /* Must align to a power of two */
1065 assert(!(node->u.w & (node->u.w - 1)));
1066 if ((word = _jit->pc.w & (node->u.w - 1)))
1067 nop(node->u.w - word);
1069 case jit_code_note: case jit_code_name:
1070 node->u.w = _jit->pc.w;
1072 case jit_code_label:
1073 /* remember label is defined */
1074 node->flag |= jit_flag_patch;
1075 node->u.w = _jit->pc.w;
1094 case_rrrr(qmul, _u);
1095 case_rrrw(qmul, _u);
1102 case_rrrr(qdiv, _u);
1103 case_rrrw(qdiv, _u);
1122 case_rr(trunc, _f_i);
1123 case_rr(trunc, _d_i);
1124 case_rr(trunc, _f_l);
1125 case_rr(trunc, _d_l);
1173 case_rr(bswap, _us);
1174 case_rr(bswap, _ui);
1175 case_rr(bswap, _ul);
1183 casr(rn(node->u.w), rn(node->v.w),
1184 rn(node->w.q.l), rn(node->w.q.h));
1187 casi(rn(node->u.w), node->v.w,
1188 rn(node->w.q.l), rn(node->w.q.h));
1194 if (node->flag & jit_flag_node) {
1196 if (temp->code == jit_code_data ||
1197 (temp->code == jit_code_label &&
1198 (temp->flag & jit_flag_patch)))
1199 movi(rn(node->u.w), temp->u.w);
1201 assert(temp->code == jit_code_label ||
1202 temp->code == jit_code_epilog);
1203 word = movi_p(rn(node->u.w), temp->u.w);
1208 movi(rn(node->u.w), node->v.w);
1252 case_brr(boadd, _u);
1253 case_brw(boadd, _u);
1256 case_brr(bxadd, _u);
1257 case_brw(bxadd, _u);
1260 case_brr(bosub, _u);
1261 case_brw(bosub, _u);
1264 case_brr(bxsub, _u);
1265 case_brw(bxsub, _u);
1292 case jit_code_movi_f:
1294 movi_f(rn(node->u.w), node->v.f);
1323 case_rrr(unord, _f);
1337 case_brr(bunlt, _f);
1339 case_brr(bunle, _f);
1341 case_brr(buneq, _f);
1343 case_brr(bunge, _f);
1345 case_brr(bungt, _f);
1347 case_brr(bltgt, _f);
1351 case_brr(bunord, _f);
1375 case jit_code_movi_d:
1377 movi_d(rn(node->u.w), node->v.d);
1406 case_rrr(unord, _d);
1420 case_brr(bunlt, _d);
1422 case_brr(bunle, _d);
1424 case_brr(buneq, _d);
1426 case_brr(bunge, _d);
1428 case_brr(bungt, _d);
1430 case_brr(bltgt, _d);
1434 case_brr(bunord, _d);
1437 jmpr(rn(node->u.w));
1440 if (node->flag & jit_flag_node) {
1442 assert(temp->code == jit_code_label ||
1443 temp->code == jit_code_epilog);
1444 if (temp->flag & jit_flag_patch)
1447 word = jmpi_p(_jit->pc.w);
1454 case jit_code_callr:
1455 callr(rn(node->u.w));
1457 case jit_code_calli:
1458 if (node->flag & jit_flag_node) {
1460 assert(temp->code == jit_code_label ||
1461 temp->code == jit_code_epilog);
1462 if (temp->flag & jit_flag_patch)
1465 word = calli_p(_jit->pc.w);
1472 case jit_code_prolog:
1473 _jitc->function = _jitc->functions.ptr + node->w.w;
1475 undo.word = _jit->pc.w;
1476 #if DEVEL_DISASSEMBLER
1479 undo.patch_offset = _jitc->patches.offset;
1484 case jit_code_epilog:
1485 assert(_jitc->function == _jitc->functions.ptr + node->w.w);
1487 for (temp = undo.node->next;
1488 temp != node; temp = temp->next) {
1489 if (temp->code == jit_code_label ||
1490 temp->code == jit_code_epilog)
1491 temp->flag &= ~jit_flag_patch;
1493 temp->flag &= ~jit_flag_patch;
1495 _jit->pc.w = undo.word;
1496 #if DEVEL_DISASSEMBLER
1499 _jitc->patches.offset = undo.patch_offset;
1500 goto restart_function;
1502 /* remember label is defined */
1503 node->flag |= jit_flag_patch;
1504 node->u.w = _jit->pc.w;
1506 _jitc->function = NULL;
1508 case jit_code_movr_w_f:
1509 movr_w_f(rn(node->u.w), rn(node->v.w));
1511 case jit_code_movr_f_w:
1512 movr_f_w(rn(node->u.w), rn(node->v.w));
1514 case jit_code_movi_f_w:
1516 movi_f_w(rn(node->u.w), node->v.f);
1518 case jit_code_movr_w_d:
1519 movr_w_d(rn(node->u.w), rn(node->v.w));
1521 case jit_code_movr_d_w:
1522 movr_d_w(rn(node->u.w), rn(node->v.w));
1524 case jit_code_movi_d_w:
1526 movi_d_w(rn(node->u.w), node->v.d);
1528 case jit_code_va_start:
1529 vastart(rn(node->u.w));
1531 case jit_code_va_arg:
1532 vaarg(rn(node->u.w), rn(node->v.w));
1534 case jit_code_va_arg_d:
1535 vaarg_d(rn(node->u.w), rn(node->v.w));
1537 case jit_code_live: case jit_code_ellipsis:
1538 case jit_code_va_push:
1539 case jit_code_allocai: case jit_code_allocar:
1541 case jit_code_arg_f: case jit_code_arg_d:
1542 case jit_code_va_end:
1544 case jit_code_retr: case jit_code_reti:
1545 case jit_code_retr_f: case jit_code_reti_f:
1546 case jit_code_retr_d: case jit_code_reti_d:
1547 case jit_code_getarg_c: case jit_code_getarg_uc:
1548 case jit_code_getarg_s: case jit_code_getarg_us:
1549 case jit_code_getarg_i: case jit_code_getarg_ui:
1550 case jit_code_getarg_l:
1551 case jit_code_getarg_f: case jit_code_getarg_d:
1552 case jit_code_putargr: case jit_code_putargi:
1553 case jit_code_putargr_f: case jit_code_putargi_f:
1554 case jit_code_putargr_d: case jit_code_putargi_d:
1555 case jit_code_pushargr: case jit_code_pushargi:
1556 case jit_code_pushargr_f: case jit_code_pushargi_f:
1557 case jit_code_pushargr_d: case jit_code_pushargi_d:
1558 case jit_code_retval_c: case jit_code_retval_uc:
1559 case jit_code_retval_s: case jit_code_retval_us:
1560 case jit_code_retval_i:
1561 case jit_code_retval_ui: case jit_code_retval_l:
1562 case jit_code_retval_f: case jit_code_retval_d:
1563 case jit_code_prepare:
1564 case jit_code_finishr: case jit_code_finishi:
1569 if (jit_carry != _NOREG) {
1570 switch (node->code) {
1572 case jit_code_addcr: case jit_code_addci:
1573 case jit_code_addxr: case jit_code_addxi:
1574 case jit_code_subcr: case jit_code_subci:
1575 case jit_code_subxr: case jit_code_subxi:
1578 jit_unget_reg(jit_carry);
1583 jit_regarg_clr(node, value);
1584 assert(_jitc->regarg == 0 ||
1585 (jit_carry != _NOREG && _jitc->regarg == (1 << jit_carry)));
1586 assert(_jitc->synth == 0);
1587 /* update register live state */
1599 #if __WORDSIZE == 64
1600 /* Record all constants to be patched */
1601 for (offset = 0; offset < _jitc->patches.offset; offset++) {
1602 node = _jitc->patches.ptr[offset].node;
1603 value = node->code == jit_code_movi ? node->v.n->u.w : node->u.n->u.w;
1606 /* Record all direct constants */
1607 for (offset = 0; offset < _jitc->consts.vector.offset; offset++)
1608 put_const(_jitc->consts.vector.values[offset]);
1609 /* Now actually inject constants at the end of code buffer */
1610 if (_jitc->consts.hash.count) {
1612 /* Insert nop if aligned at 4 bytes */
1613 if (_jit->pc.w % sizeof(jit_word_t))
1614 nop(_jit->pc.w % sizeof(jit_word_t));
1615 for (offset = 0; offset < _jitc->consts.hash.size; offset++) {
1616 entry = _jitc->consts.hash.table[offset];
1617 for (; entry; entry = entry->next) {
1618 /* Make sure to not write out of bounds */
1619 if (_jit->pc.uc >= _jitc->code.end)
1621 entry->address = _jit->pc.w;
1622 *_jit->pc.ul++ = entry->value;
1628 for (offset = 0; offset < _jitc->patches.offset; offset++) {
1629 node = _jitc->patches.ptr[offset].node;
1630 word = _jitc->patches.ptr[offset].inst;
1631 value = node->code == jit_code_movi ? node->v.n->u.w : node->u.n->u.w;
1632 patch_at(word, value);
1635 #if __WORDSIZE == 64
1636 /* Patch direct complex constants */
1637 if (_jitc->consts.vector.instrs) {
1638 for (offset = 0; offset < _jitc->consts.vector.offset; offset++)
1639 patch_at(_jitc->consts.vector.instrs[offset],
1640 _jitc->consts.vector.values[offset]);
1641 jit_free((jit_pointer_t *)&_jitc->consts.vector.instrs);
1642 jit_free((jit_pointer_t *)&_jitc->consts.vector.values);
1645 /* Hash table no longer need */
1646 if (_jitc->consts.hash.table) {
1647 jit_free((jit_pointer_t *)&_jitc->consts.hash.table);
1648 for (offset = 0; offset < _jitc->consts.pool.length; offset++)
1649 jit_free((jit_pointer_t *)_jitc->consts.pool.ptr + offset);
1650 jit_free((jit_pointer_t *)&_jitc->consts.pool.ptr);
1654 jit_flush(_jit->code.ptr, _jit->pc.uc);
1656 return (_jit->code.ptr);
1660 # include "jit_riscv-cpu.c"
1661 # include "jit_riscv-fpu.c"
1665 _load_const(jit_state_t *_jit, jit_int32_t reg, jit_word_t value)
1667 if (_jitc->consts.vector.offset >= _jitc->consts.vector.length) {
1668 jit_word_t new_size = _jitc->consts.vector.length *
1669 2 * sizeof(jit_word_t);
1670 jit_realloc((jit_pointer_t *)&_jitc->consts.vector.instrs,
1671 _jitc->consts.vector.length * sizeof(jit_word_t), new_size);
1672 jit_realloc((jit_pointer_t *)&_jitc->consts.vector.values,
1673 _jitc->consts.vector.length * sizeof(jit_word_t), new_size);
1674 _jitc->consts.vector.length *= 2;
1676 _jitc->consts.vector.instrs[_jitc->consts.vector.offset] = _jit->pc.w;
1677 _jitc->consts.vector.values[_jitc->consts.vector.offset] = value;
1678 ++_jitc->consts.vector.offset;
1679 /* Resolve later the pc relative address */
1687 hash_const(jit_word_t value)
1689 const jit_uint8_t *ptr;
1691 for (i = key = 0, ptr = (jit_uint8_t *)&value; i < 4; ++i)
1692 key = (key << (key & 1)) ^ ptr[i];
1698 _put_const(jit_state_t *_jit, jit_word_t value)
1703 /* Check if already inserted in table */
1704 key = hash_const(value) % _jitc->consts.hash.size;
1705 for (entry = _jitc->consts.hash.table[key]; entry; entry = entry->next) {
1706 if (entry->value == value)
1710 /* Check if need to increase pool size */
1711 if (_jitc->consts.pool.list->next == NULL) {
1714 jit_word_t new_size = (_jitc->consts.pool.length + 1) *
1715 sizeof(jit_const_t*);
1716 jit_realloc((jit_pointer_t *)&_jitc->consts.pool.ptr,
1717 _jitc->consts.pool.length * sizeof(jit_const_t*), new_size);
1718 jit_alloc((jit_pointer_t *)
1719 _jitc->consts.pool.ptr + _jitc->consts.pool.length,
1720 1024 * sizeof(jit_const_t));
1721 list = _jitc->consts.pool.ptr[_jitc->consts.pool.length];
1722 _jitc->consts.pool.list->next = list;
1723 for (offset = 0; offset < 1023; ++offset, ++list)
1724 list->next = list + 1;
1726 ++_jitc->consts.pool.length;
1729 /* Rehash if more than 75% used table */
1730 if (_jitc->consts.hash.count > (_jitc->consts.hash.size / 4) * 3) {
1733 jit_const_t **table;
1734 jit_alloc((jit_pointer_t *)&table,
1735 _jitc->consts.hash.size * 2 * sizeof(jit_const_t *));
1736 for (i = 0; i < _jitc->consts.hash.size; ++i) {
1737 for (entry = _jitc->consts.hash.table[i]; entry; entry = next) {
1739 k = hash_const(entry->value) % (_jitc->consts.hash.size * 2);
1740 entry->next = table[k];
1744 jit_free((jit_pointer_t *)&_jitc->consts.hash.table);
1745 _jitc->consts.hash.size *= 2;
1746 _jitc->consts.hash.table = table;
1749 /* Insert in hash */
1750 entry = _jitc->consts.pool.list;
1751 _jitc->consts.pool.list = entry->next;
1752 ++_jitc->consts.hash.count;
1753 entry->value = value;
1754 entry->next = _jitc->consts.hash.table[key];
1755 _jitc->consts.hash.table[key] = entry;
1759 _get_const(jit_state_t *_jit, jit_word_t value)
1763 key = hash_const(value) % _jitc->consts.hash.size;
1764 for (entry = _jitc->consts.hash.table[key]; entry; entry = entry->next) {
1765 if (entry->value == value)
1766 return (entry->address);
1768 /* Only the final patch should call get_const() */
1773 jit_flush(void *fptr, void *tptr)
1775 #if defined(__GNUC__)
1778 s = sysconf(_SC_PAGE_SIZE);
1779 f = (jit_word_t)fptr & -s;
1780 t = (((jit_word_t)tptr) + s - 1) & -s;
1781 __clear_cache((void *)f, (void *)t);
1786 _emit_ldxi(jit_state_t *_jit, jit_int32_t r0, jit_int32_t r1, jit_word_t i0)
1788 ldxi(rn(r0), rn(r1), i0);
1792 _emit_stxi(jit_state_t *_jit, jit_word_t i0, jit_int32_t r0, jit_int32_t r1)
1794 stxi(i0, rn(r0), rn(r1));
1798 _emit_ldxi_d(jit_state_t *_jit, jit_int32_t r0, jit_int32_t r1, jit_word_t i0)
1800 ldxi_d(rn(r0), rn(r1), i0);
1804 _emit_stxi_d(jit_state_t *_jit, jit_word_t i0, jit_int32_t r0, jit_int32_t r1)
1806 stxi_d(i0, rn(r0), rn(r1));
1810 _patch(jit_state_t *_jit, jit_word_t instr, jit_node_t *node)
1814 assert(node->flag & jit_flag_node);
1815 if (node->code == jit_code_movi)
1816 flag = node->v.n->flag;
1818 flag = node->u.n->flag;
1819 assert(!(flag & jit_flag_patch));
1820 if (_jitc->patches.offset >= _jitc->patches.length) {
1821 jit_realloc((jit_pointer_t *)&_jitc->patches.ptr,
1822 _jitc->patches.length * sizeof(jit_patch_t),
1823 (_jitc->patches.length + 1024) * sizeof(jit_patch_t));
1824 _jitc->patches.length += 1024;
1826 _jitc->patches.ptr[_jitc->patches.offset].inst = instr;
1827 _jitc->patches.ptr[_jitc->patches.offset].node = node;
1828 ++_jitc->patches.offset;