2 * Copyright (C) 2019 Free Software Foundation, Inc.
4 * This file is part of GNU lightning.
6 * GNU lightning is free software; you can redistribute it and/or modify it
7 * under the terms of the GNU Lesser General Public License as published
8 * by the Free Software Foundation; either version 3, or (at your option)
11 * GNU lightning is distributed in the hope that it will be useful, but
12 * WITHOUT ANY WARRANTY; without even the implied warranty of MERCHANTABILITY
13 * or FITNESS FOR A PARTICULAR PURPOSE. See the GNU Lesser General Public
14 * License for more details.
17 * Paulo Cesar Pereira de Andrade
20 #define jit_arg_reg_p(i) ((i) >= 0 && (i) < 8)
21 #define jit_arg_f_reg_p(i) ((i) >= 0 && (i) < 8)
26 typedef jit_pointer_t jit_va_list_t;
31 #define patch(instr, node) _patch(_jit, instr, node)
32 static void _patch(jit_state_t*,jit_word_t,jit_node_t*);
35 # include "jit_riscv-cpu.c"
36 # include "jit_riscv-fpu.c"
42 jit_register_t _rvs[] = {
47 #if 0 /* Pretend it does not exist, so _NOREG can be used in
51 { rc(gpr) | 0x05, "t0" },
52 { rc(gpr) | 0x06, "t1" },
53 { rc(gpr) | 0x07, "t2" },
54 { rc(gpr) | 0x1c, "t3" },
55 { rc(gpr) | 0x1d, "t4" },
56 { rc(gpr) | 0x1e, "t5" },
57 { rc(gpr) | 0x1f, "t6" },
59 { rc(sav) | rc(gpr) | 0x09, "s1" },
60 { rc(sav) | rc(gpr) | 0x12, "s2" },
61 { rc(sav) | rc(gpr) | 0x13, "s3" },
62 { rc(sav) | rc(gpr) | 0x14, "s4" },
63 { rc(sav) | rc(gpr) | 0x15, "s5" },
64 { rc(sav) | rc(gpr) | 0x16, "s6" },
65 { rc(sav) | rc(gpr) | 0x17, "s7" },
66 { rc(sav) | rc(gpr) | 0x18, "s8" },
67 { rc(sav) | rc(gpr) | 0x19, "s9" },
68 { rc(sav) | rc(gpr) | 0x1a, "s10" },
69 { rc(sav) | rc(gpr) | 0x1b, "s11" },
70 { rc(arg) | rc(gpr) | 0x11, "a7" },
71 { rc(arg) | rc(gpr) | 0x10, "a6" },
72 { rc(arg) | rc(gpr) | 0x0f, "a5" },
73 { rc(arg) | rc(gpr) | 0x0e, "a4" },
74 { rc(arg) | rc(gpr) | 0x0d, "a3" },
75 { rc(arg) | rc(gpr) | 0x0c, "a2" },
76 { rc(arg) | rc(gpr) | 0x0b, "a1" },
77 { rc(arg) | rc(gpr) | 0x0a, "a0" },
78 { rc(fpr) | 0x00, "ft0" },
79 { rc(fpr) | 0x01, "ft1" },
80 { rc(fpr) | 0x02, "ft2" },
81 { rc(fpr) | 0x03, "ft3" },
82 { rc(fpr) | 0x04, "ft4" },
83 { rc(fpr) | 0x05, "ft5" },
84 { rc(fpr) | 0x06, "ft6" },
85 { rc(fpr) | 0x07, "ft7" },
86 { rc(fpr) | 0x1c, "ft8" },
87 { rc(fpr) | 0x1d, "ft9" },
88 { rc(fpr) | 0x1e, "ft10" },
89 { rc(fpr) | 0x1f, "ft11" },
90 { rc(sav) | rc(fpr) | 0x08, "fs0" },
91 { rc(sav) | rc(fpr) | 0x09, "fs1" },
92 { rc(sav) | rc(fpr) | 0x12, "fs2" },
93 { rc(sav) | rc(fpr) | 0x13, "fs3" },
94 { rc(sav) | rc(fpr) | 0x14, "fs4" },
95 { rc(sav) | rc(fpr) | 0x15, "fs5" },
96 { rc(sav) | rc(fpr) | 0x16, "fs6" },
97 { rc(sav) | rc(fpr) | 0x17, "fs7" },
98 { rc(sav) | rc(fpr) | 0x18, "fs8" },
99 { rc(sav) | rc(fpr) | 0x19, "fs9" },
100 { rc(sav) | rc(fpr) | 0x1a, "fs10" },
101 { rc(sav) | rc(fpr) | 0x1b, "fs11" },
102 { rc(arg) | rc(fpr) | 0x11, "fa7" },
103 { rc(arg) | rc(fpr) | 0x10, "fa6" },
104 { rc(arg) | rc(fpr) | 0x0f, "fa5" },
105 { rc(arg) | rc(fpr) | 0x0e, "fa4" },
106 { rc(arg) | rc(fpr) | 0x0d, "fa3" },
107 { rc(arg) | rc(fpr) | 0x0c, "fa2" },
108 { rc(arg) | rc(fpr) | 0x0b, "fa1" },
109 { rc(arg) | rc(fpr) | 0x0a, "fa0" },
110 { _NOREG, "<none>" },
122 _jit_init(jit_state_t *_jit)
124 _jitc->reglen = jit_size(_rvs) - 1;
129 _jit_prolog(jit_state_t *_jit)
135 assert(jit_regset_cmp_ui(&_jitc->regarg, 0) == 0);
136 jit_regset_set_ui(&_jitc->regsav, 0);
137 offset = _jitc->functions.offset;
138 if (offset >= _jitc->functions.length) {
139 jit_realloc((jit_pointer_t *)&_jitc->functions.ptr,
140 _jitc->functions.length * sizeof(jit_function_t),
141 (_jitc->functions.length + 16) * sizeof(jit_function_t));
142 _jitc->functions.length += 16;
144 _jitc->function = _jitc->functions.ptr + _jitc->functions.offset++;
145 _jitc->function->self.size = stack_framesize;
146 _jitc->function->self.argi = _jitc->function->self.argf =
147 _jitc->function->self.alen = 0;
148 _jitc->function->self.aoff = 0;
149 _jitc->function->self.call = jit_call_default;
150 jit_alloc((jit_pointer_t *)&_jitc->function->regoff,
151 _jitc->reglen * sizeof(jit_int32_t));
153 /* _no_link here does not mean the jit_link() call can be removed
155 * _jitc->function->prolog = jit_new_node(jit_code_prolog);
157 _jitc->function->prolog = jit_new_node_no_link(jit_code_prolog);
158 jit_link(_jitc->function->prolog);
159 _jitc->function->prolog->w.w = offset;
160 _jitc->function->epilog = jit_new_node_no_link(jit_code_epilog);
162 * v: offset in blocks vector
163 * w: offset in functions vector
165 _jitc->function->epilog->w.w = offset;
167 jit_regset_new(&_jitc->function->regset);
171 _jit_allocai(jit_state_t *_jit, jit_int32_t length)
173 assert(_jitc->function);
175 case 0: case 1: break;
176 case 2: _jitc->function->self.aoff &= -2; break;
177 case 3: case 4: _jitc->function->self.aoff &= -4; break;
178 default: _jitc->function->self.aoff &= -8; break;
180 _jitc->function->self.aoff -= length;
181 if (!_jitc->realize) {
182 jit_inc_synth_ww(allocai, _jitc->function->self.aoff, length);
185 return (_jitc->function->self.aoff);
189 _jit_allocar(jit_state_t *_jit, jit_int32_t u, jit_int32_t v)
192 assert(_jitc->function);
193 jit_inc_synth_ww(allocar, u, v);
194 if (!_jitc->function->allocar) {
195 _jitc->function->aoffoff = jit_allocai(sizeof(jit_int32_t));
196 _jitc->function->allocar = 1;
198 r0 = jit_get_reg(jit_class_gpr);
200 jit_andi(r0, r0, -16);
201 jit_ldxi_i(u, JIT_FP, _jitc->function->aoffoff);
203 jit_addr(JIT_SP, JIT_SP, r0);
204 jit_stxi_i(_jitc->function->aoffoff, JIT_FP, u);
210 _jit_ret(jit_state_t *_jit)
213 assert(_jitc->function);
217 jit_patch_at(instr, _jitc->function->epilog);
222 _jit_retr(jit_state_t *_jit, jit_int32_t u)
224 jit_inc_synth_w(retr, u);
226 jit_movr(JIT_RET, u);
233 _jit_reti(jit_state_t *_jit, jit_word_t u)
235 jit_inc_synth_w(reti, u);
236 jit_movi(JIT_RET, u);
242 _jit_retr_f(jit_state_t *_jit, jit_int32_t u)
244 jit_inc_synth_w(retr_f, u);
246 jit_movr_f(JIT_FRET, u);
254 _jit_reti_f(jit_state_t *_jit, jit_float32_t u)
256 jit_inc_synth_f(reti_f, u);
257 jit_movi_f(JIT_FRET, u);
263 _jit_retr_d(jit_state_t *_jit, jit_int32_t u)
265 jit_inc_synth_w(retr_d, u);
267 jit_movr_d(JIT_FRET, u);
275 _jit_reti_d(jit_state_t *_jit, jit_float64_t u)
277 jit_inc_synth_d(reti_d, u);
278 jit_movi_d(JIT_FRET, u);
284 _jit_epilog(jit_state_t *_jit)
286 assert(_jitc->function);
287 assert(_jitc->function->epilog->next == NULL);
288 jit_link(_jitc->function->epilog);
289 _jitc->function = NULL;
293 _jit_arg_register_p(jit_state_t *_jit, jit_node_t *u)
295 if (u->code == jit_code_arg)
296 return (jit_arg_reg_p(u->u.w));
297 assert(u->code == jit_code_arg_f || u->code == jit_code_arg_d);
298 return (jit_arg_f_reg_p(u->u.w));
302 _jit_ellipsis(jit_state_t *_jit)
304 jit_inc_synth(ellipsis);
305 if (_jitc->prepare) {
307 assert(!(_jitc->function->call.call & jit_call_varargs));
308 _jitc->function->call.call |= jit_call_varargs;
312 assert(!(_jitc->function->self.call & jit_call_varargs));
313 _jitc->function->self.call |= jit_call_varargs;
314 _jitc->function->vagp = _jitc->function->self.argi;
320 _jit_va_push(jit_state_t *_jit, jit_int32_t u)
322 jit_inc_synth_w(va_push, u);
328 _jit_arg(jit_state_t *_jit)
332 assert(_jitc->function);
333 assert(!(_jitc->function->self.call & jit_call_varargs));
334 if (jit_arg_reg_p(_jitc->function->self.argi))
335 offset = _jitc->function->self.argi++;
337 offset = _jitc->function->self.size;
338 _jitc->function->self.size += sizeof(jit_word_t);
340 node = jit_new_node_ww(jit_code_arg, offset,
341 ++_jitc->function->self.argn);
347 _jit_arg_f(jit_state_t *_jit)
351 assert(_jitc->function);
352 assert(!(_jitc->function->self.call & jit_call_varargs));
353 if (jit_arg_f_reg_p(_jitc->function->self.argf))
354 offset = _jitc->function->self.argf++;
355 else if (jit_arg_reg_p(_jitc->function->self.argi)) {
356 offset = _jitc->function->self.argi++;
360 offset = _jitc->function->self.size;
361 _jitc->function->self.size += sizeof(jit_word_t);
363 node = jit_new_node_ww(jit_code_arg_f, offset,
364 ++_jitc->function->self.argn);
370 _jit_arg_d(jit_state_t *_jit)
374 assert(_jitc->function);
375 assert(!(_jitc->function->self.call & jit_call_varargs));
376 if (jit_arg_f_reg_p(_jitc->function->self.argf))
377 offset = _jitc->function->self.argf++;
378 else if (jit_arg_reg_p(_jitc->function->self.argi)) {
379 offset = _jitc->function->self.argi++;
383 offset = _jitc->function->self.size;
384 _jitc->function->self.size += sizeof(jit_word_t);
386 node = jit_new_node_ww(jit_code_arg_d, offset,
387 ++_jitc->function->self.argn);
393 _jit_getarg_c(jit_state_t *_jit, jit_int32_t u, jit_node_t *v)
395 assert(v->code == jit_code_arg);
396 jit_inc_synth_wp(getarg_c, u, v);
397 if (jit_arg_reg_p(v->u.w))
398 jit_extr_c(u, JIT_RA0 - v->u.w);
400 jit_ldxi_c(u, JIT_FP, v->u.w);
405 _jit_getarg_uc(jit_state_t *_jit, jit_int32_t u, jit_node_t *v)
407 assert(v->code == jit_code_arg);
408 jit_inc_synth_wp(getarg_uc, u, v);
409 if (jit_arg_reg_p(v->u.w))
410 jit_extr_uc(u, JIT_RA0 - v->u.w);
412 jit_ldxi_uc(u, JIT_FP, v->u.w);
417 _jit_getarg_s(jit_state_t *_jit, jit_int32_t u, jit_node_t *v)
419 assert(v->code == jit_code_arg);
420 jit_inc_synth_wp(getarg_s, u, v);
421 if (jit_arg_reg_p(v->u.w))
422 jit_extr_s(u, JIT_RA0 - v->u.w);
424 jit_ldxi_s(u, JIT_FP, v->u.w);
429 _jit_getarg_us(jit_state_t *_jit, jit_int32_t u, jit_node_t *v)
431 assert(v->code == jit_code_arg);
432 jit_inc_synth_wp(getarg_us, u, v);
433 if (jit_arg_reg_p(v->u.w))
434 jit_extr_us(u, JIT_RA0 - v->u.w);
436 jit_ldxi_us(u, JIT_FP, v->u.w);
441 _jit_getarg_i(jit_state_t *_jit, jit_int32_t u, jit_node_t *v)
443 assert(v->code == jit_code_arg);
444 jit_inc_synth_wp(getarg_i, u, v);
445 if (jit_arg_reg_p(v->u.w))
446 jit_extr_i(u, JIT_RA0 - v->u.w);
448 jit_ldxi_i(u, JIT_FP, v->u.w);
453 _jit_getarg_ui(jit_state_t *_jit, jit_int32_t u, jit_node_t *v)
455 assert(v->code == jit_code_arg);
456 jit_inc_synth_wp(getarg_ui, u, v);
457 if (jit_arg_reg_p(v->u.w))
458 jit_extr_ui(u, JIT_RA0 - v->u.w);
460 jit_ldxi_ui(u, JIT_FP, v->u.w);
465 _jit_getarg_l(jit_state_t *_jit, jit_int32_t u, jit_node_t *v)
467 assert(v->code == jit_code_arg);
468 jit_inc_synth_wp(getarg_l, u, v);
469 if (jit_arg_reg_p(v->u.w))
470 jit_movr(u, JIT_RA0 - v->u.w);
472 jit_ldxi_l(u, JIT_FP, v->u.w);
477 _jit_putargr(jit_state_t *_jit, jit_int32_t u, jit_node_t *v)
479 assert(v->code == jit_code_arg);
480 jit_inc_synth_wp(putargr, u, v);
481 if (jit_arg_reg_p(v->u.w))
482 jit_movr(JIT_RA0 - v->u.w, u);
484 jit_stxi(v->u.w, JIT_FP, u);
489 _jit_putargi(jit_state_t *_jit, jit_word_t u, jit_node_t *v)
492 assert(v->code == jit_code_arg);
493 jit_inc_synth_wp(putargi, u, v);
494 if (jit_arg_reg_p(v->u.w))
495 jit_movi(JIT_RA0 - v->u.w, u);
497 regno = jit_get_reg(jit_class_gpr);
499 jit_stxi(v->u.w, JIT_FP, regno);
500 jit_unget_reg(regno);
506 _jit_getarg_f(jit_state_t *_jit, jit_int32_t u, jit_node_t *v)
508 assert(v->code == jit_code_arg_f);
509 jit_inc_synth_wp(getarg_f, u, v);
510 if (jit_arg_f_reg_p(v->u.w))
511 jit_movr_f(u, JIT_FA0 - v->u.w);
512 else if (jit_arg_reg_p(v->u.w - 8))
513 jit_movr_w_f(u, JIT_RA0 - (v->u.w - 8));
515 jit_ldxi_f(u, JIT_FP, v->u.w);
520 _jit_putargr_f(jit_state_t *_jit, jit_int32_t u, jit_node_t *v)
522 assert(v->code == jit_code_arg_f);
523 jit_inc_synth_wp(putargr_f, u, v);
524 if (jit_arg_f_reg_p(v->u.w))
525 jit_movr_f(JIT_FA0 - v->u.w, u);
526 else if (jit_arg_reg_p(v->u.w - 8))
527 jit_movr_f_w(JIT_RA0 - (v->u.w - 8), u);
529 jit_stxi_f(v->u.w, JIT_FP, u);
534 _jit_putargi_f(jit_state_t *_jit, jit_float32_t u, jit_node_t *v)
537 assert(v->code == jit_code_arg_f);
538 jit_inc_synth_fp(putargi_f, u, v);
539 if (jit_arg_f_reg_p(v->u.w))
540 jit_movi_f(JIT_FA0 - v->u.w, u);
541 else if (jit_arg_reg_p(v->u.w - 8)) {
547 jit_movi(JIT_RA0 - (v->u.w - 8), uu.i);
550 regno = jit_get_reg(jit_class_fpr);
551 jit_movi_f(regno, u);
552 jit_stxi_f(v->u.w, JIT_FP, regno);
553 jit_unget_reg(regno);
559 _jit_getarg_d(jit_state_t *_jit, jit_int32_t u, jit_node_t *v)
561 assert(v->code == jit_code_arg_d);
562 jit_inc_synth_wp(getarg_d, u, v);
563 if (jit_arg_f_reg_p(v->u.w))
564 jit_movr_d(u, JIT_FA0 - v->u.w);
565 else if (jit_arg_reg_p(v->u.w - 8))
566 jit_movr_w_d(u, JIT_RA0 - (v->u.w - 8));
568 jit_ldxi_d(u, JIT_FP, v->u.w);
573 _jit_putargr_d(jit_state_t *_jit, jit_int32_t u, jit_node_t *v)
575 assert(v->code == jit_code_arg_d);
576 jit_inc_synth_wp(putargr_d, u, v);
577 if (jit_arg_reg_p(v->u.w))
578 jit_movr_d(JIT_FA0 - v->u.w, u);
579 else if (jit_arg_reg_p(v->u.w - 8))
580 jit_movr_d_w(JIT_RA0 - (v->u.w - 8), u);
582 jit_stxi_d(v->u.w, JIT_FP, u);
587 _jit_putargi_d(jit_state_t *_jit, jit_float64_t u, jit_node_t *v)
590 assert(v->code == jit_code_arg_d);
591 jit_inc_synth_dp(putargi_d, u, v);
592 if (jit_arg_reg_p(v->u.w))
593 jit_movi_d(JIT_FA0 - v->u.w, u);
594 else if (jit_arg_reg_p(v->u.w - 8)) {
600 jit_movi(JIT_RA0 - (v->u.w - 8), uu.w);
603 regno = jit_get_reg(jit_class_fpr);
604 jit_movi_d(regno, u);
605 jit_stxi_d(v->u.w, JIT_FP, regno);
606 jit_unget_reg(regno);
612 _jit_pushargr(jit_state_t *_jit, jit_int32_t u)
614 assert(_jitc->function);
615 jit_inc_synth_w(pushargr, u);
617 if (jit_arg_reg_p(_jitc->function->call.argi)) {
618 jit_movr(JIT_RA0 - _jitc->function->call.argi, u);
619 ++_jitc->function->call.argi;
622 jit_stxi(_jitc->function->call.size, JIT_SP, u);
623 _jitc->function->call.size += sizeof(jit_word_t);
629 _jit_pushargi(jit_state_t *_jit, jit_word_t u)
632 assert(_jitc->function);
633 jit_inc_synth_w(pushargi, u);
635 if (jit_arg_reg_p(_jitc->function->call.argi)) {
636 jit_movi(JIT_RA0 - _jitc->function->call.argi, u);
637 ++_jitc->function->call.argi;
640 regno = jit_get_reg(jit_class_gpr);
642 jit_stxi(_jitc->function->call.size, JIT_SP, regno);
643 jit_unget_reg(regno);
644 _jitc->function->call.size += sizeof(jit_word_t);
650 _jit_pushargr_f(jit_state_t *_jit, jit_int32_t u)
652 assert(_jitc->function);
653 jit_inc_synth_w(pushargr_f, u);
655 if (jit_arg_f_reg_p(_jitc->function->call.argf) &&
656 !(_jitc->function->call.call & jit_call_varargs)) {
657 jit_movr_f(JIT_FA0 - _jitc->function->call.argf, u);
658 ++_jitc->function->call.argf;
660 else if (jit_arg_reg_p(_jitc->function->call.argi)) {
661 jit_movr_f_w(JIT_RA0 - _jitc->function->call.argi, u);
662 ++_jitc->function->call.argi;
665 jit_stxi_f(_jitc->function->call.size, JIT_SP, u);
666 _jitc->function->call.size += sizeof(jit_word_t);
672 _jit_pushargi_f(jit_state_t *_jit, jit_float32_t u)
675 assert(_jitc->function);
676 jit_inc_synth_f(pushargi_f, u);
678 if (jit_arg_f_reg_p(_jitc->function->call.argf) &&
679 !(_jitc->function->call.call & jit_call_varargs)) {
680 jit_movi_f(JIT_FA0 - _jitc->function->call.argf, u);
681 ++_jitc->function->call.argf;
683 else if (jit_arg_reg_p(_jitc->function->call.argi)) {
684 jit_movi_f_w(JIT_RA0 - _jitc->function->call.argi, u);
685 ++_jitc->function->call.argi;
688 regno = jit_get_reg(jit_class_fpr);
689 jit_movi_f(regno, u);
690 jit_stxi_f(_jitc->function->call.size, JIT_SP, regno);
691 jit_unget_reg(regno);
692 _jitc->function->call.size += sizeof(jit_word_t);
698 _jit_pushargr_d(jit_state_t *_jit, jit_int32_t u)
700 assert(_jitc->function);
701 jit_inc_synth_w(pushargr_d, u);
703 if (jit_arg_f_reg_p(_jitc->function->call.argf) &&
704 !(_jitc->function->call.call & jit_call_varargs)) {
705 jit_movr_d(JIT_FA0 - _jitc->function->call.argf, u);
706 ++_jitc->function->call.argf;
708 else if (jit_arg_reg_p(_jitc->function->call.argi)) {
709 jit_movr_d_w(JIT_RA0 - _jitc->function->call.argi, u);
710 ++_jitc->function->call.argi;
713 jit_stxi_d(_jitc->function->call.size, JIT_SP, u);
714 _jitc->function->call.size += sizeof(jit_word_t);
720 _jit_pushargi_d(jit_state_t *_jit, jit_float64_t u)
723 assert(_jitc->function);
724 jit_inc_synth_d(pushargi_d, u);
726 if (jit_arg_f_reg_p(_jitc->function->call.argf) &&
727 !(_jitc->function->call.call & jit_call_varargs)) {
728 jit_movi_d(JIT_FA0 - _jitc->function->call.argf, u);
729 ++_jitc->function->call.argf;
731 else if (jit_arg_reg_p(_jitc->function->call.argi)) {
732 jit_movi_d_w(JIT_RA0 - _jitc->function->call.argi, u);
733 ++_jitc->function->call.argi;
736 regno = jit_get_reg(jit_class_fpr);
737 jit_movi_d(regno, u);
738 jit_stxi_d(_jitc->function->call.size, JIT_SP, regno);
739 jit_unget_reg(regno);
740 _jitc->function->call.size += sizeof(jit_word_t);
746 _jit_regarg_p(jit_state_t *_jit, jit_node_t *node, jit_int32_t regno)
749 spec = jit_class(_rvs[regno].spec);
750 if (spec & jit_class_arg) {
751 regno = JIT_RA0 - regno;
752 if (regno >= 0 && regno < node->v.w)
754 if (spec & jit_class_fpr) {
755 regno = JIT_FA0 - regno;
756 if (regno >= 0 && regno < node->w.w)
765 _jit_finishr(jit_state_t *_jit, jit_int32_t r0)
768 assert(_jitc->function);
769 jit_inc_synth_w(finishr, r0);
770 if (_jitc->function->self.alen < _jitc->function->call.size)
771 _jitc->function->self.alen = _jitc->function->call.size;
772 node = jit_callr(r0);
773 node->v.w = _jitc->function->self.argi;
774 node->w.w = _jitc->function->call.argf;
775 _jitc->function->call.argi = _jitc->function->call.argf =
776 _jitc->function->call.size = 0;
782 _jit_finishi(jit_state_t *_jit, jit_pointer_t i0)
785 assert(_jitc->function);
786 jit_inc_synth_w(finishi, (jit_word_t)i0);
787 if (_jitc->function->self.alen < _jitc->function->call.size)
788 _jitc->function->self.alen = _jitc->function->call.size;
789 node = jit_calli(i0);
790 node->v.w = _jitc->function->call.argi;
791 node->w.w = _jitc->function->call.argf;
792 _jitc->function->call.argi = _jitc->function->call.argf =
793 _jitc->function->call.size = 0;
800 _jit_retval_c(jit_state_t *_jit, jit_int32_t r0)
802 jit_inc_synth_w(retval_c, r0);
803 jit_extr_c(r0, JIT_RET);
808 _jit_retval_uc(jit_state_t *_jit, jit_int32_t r0)
810 jit_inc_synth_w(retval_uc, r0);
811 jit_extr_uc(r0, JIT_RET);
816 _jit_retval_s(jit_state_t *_jit, jit_int32_t r0)
818 jit_inc_synth_w(retval_s, r0);
819 jit_extr_s(r0, JIT_RET);
824 _jit_retval_us(jit_state_t *_jit, jit_int32_t r0)
826 jit_inc_synth_w(retval_us, r0);
827 jit_extr_us(r0, JIT_RET);
832 _jit_retval_i(jit_state_t *_jit, jit_int32_t r0)
834 jit_inc_synth_w(retval_i, r0);
835 jit_extr_i(r0, JIT_RET);
840 _jit_retval_ui(jit_state_t *_jit, jit_int32_t r0)
842 jit_inc_synth_w(retval_ui, r0);
843 jit_extr_ui(r0, JIT_RET);
848 _jit_retval_l(jit_state_t *_jit, jit_int32_t r0)
850 jit_inc_synth_w(retval_l, r0);
852 jit_movr(r0, JIT_RET);
857 _jit_retval_f(jit_state_t *_jit, jit_int32_t r0)
859 jit_inc_synth_w(retval_f, r0);
861 jit_movr_f(r0, JIT_FRET);
866 _jit_retval_d(jit_state_t *_jit, jit_int32_t r0)
868 jit_inc_synth_w(retval_d, r0);
870 jit_movr_d(r0, JIT_FRET);
875 _emit_code(jit_state_t *_jit)
886 #if DEVEL_DISASSEMBLER
889 jit_int32_t const_offset;
890 jit_int32_t patch_offset;
892 #if DEVEL_DISASSEMBLER
896 _jitc->function = NULL;
902 undo.const_offset = undo.patch_offset = 0;
903 # define assert_data(node) /**/
904 #define case_rr(name, type) \
905 case jit_code_##name##r##type: \
906 name##r##type(rn(node->u.w), rn(node->v.w)); \
908 #define case_rw(name, type) \
909 case jit_code_##name##i##type: \
910 name##i##type(rn(node->u.w), node->v.w); \
912 #define case_wr(name, type) \
913 case jit_code_##name##i##type: \
914 name##i##type(node->u.w, rn(node->v.w)); \
916 #define case_rrr(name, type) \
917 case jit_code_##name##r##type: \
918 name##r##type(rn(node->u.w), \
919 rn(node->v.w), rn(node->w.w)); \
921 #define case_rrrr(name, type) \
922 case jit_code_##name##r##type: \
923 name##r##type(rn(node->u.q.l), rn(node->u.q.h), \
924 rn(node->v.w), rn(node->w.w)); \
926 #define case_rrw(name, type) \
927 case jit_code_##name##i##type: \
928 name##i##type(rn(node->u.w), rn(node->v.w), node->w.w); \
930 #define case_rrrw(name, type) \
931 case jit_code_##name##i##type: \
932 name##i##type(rn(node->u.q.l), rn(node->u.q.h), \
933 rn(node->v.w), node->w.w); \
935 #define case_rrf(name) \
936 case jit_code_##name##i_f: \
938 name##i_f(rn(node->u.w), rn(node->v.w), node->w.f); \
940 #define case_rrd(name) \
941 case jit_code_##name##i_d: \
943 name##i_d(rn(node->u.w), rn(node->v.w), node->w.d); \
945 #define case_wrr(name, type) \
946 case jit_code_##name##i##type: \
947 name##i##type(node->u.w, rn(node->v.w), rn(node->w.w)); \
949 #define case_brr(name, type) \
950 case jit_code_##name##r##type: \
952 assert(temp->code == jit_code_label || \
953 temp->code == jit_code_epilog); \
954 if (temp->flag & jit_flag_patch) \
955 name##r##type(temp->u.w, rn(node->v.w), \
958 word = name##r##type(_jit->pc.w, \
959 rn(node->v.w), rn(node->w.w)); \
963 #define case_brw(name, type) \
964 case jit_code_##name##i##type: \
966 assert(temp->code == jit_code_label || \
967 temp->code == jit_code_epilog); \
968 if (temp->flag & jit_flag_patch) \
969 name##i##type(temp->u.w, \
970 rn(node->v.w), node->w.w); \
972 word = name##i##type(_jit->pc.w, \
973 rn(node->v.w), node->w.w); \
977 #define case_brf(name) \
978 case jit_code_##name##i_f: \
980 assert(temp->code == jit_code_label || \
981 temp->code == jit_code_epilog); \
982 if (temp->flag & jit_flag_patch) \
983 name##i_f(temp->u.w, rn(node->v.w), node->w.f); \
985 word = name##i_f(_jit->pc.w, rn(node->v.w), \
990 #define case_brd(name) \
991 case jit_code_##name##i_d: \
993 assert(temp->code == jit_code_label || \
994 temp->code == jit_code_epilog); \
995 if (temp->flag & jit_flag_patch) \
996 name##i_d(temp->u.w, rn(node->v.w), node->w.d); \
998 word = name##i_d(_jit->pc.w, rn(node->v.w), \
1000 patch(word, node); \
1003 #if DEVEL_DISASSEMBLER
1006 for (node = _jitc->head; node; node = node->next) {
1007 if (_jit->pc.uc >= _jitc->code.end)
1010 #if DEVEL_DISASSEMBLER
1011 node->offset = (jit_uword_t)_jit->pc.w - (jit_uword_t)prevw;
1014 value = jit_classify(node->code);
1015 jit_regarg_set(node, value);
1016 switch (node->code) {
1017 case jit_code_align:
1018 assert(!(node->u.w & (node->u.w - 1)) &&
1019 node->u.w <= sizeof(jit_word_t));
1020 if (node->u.w == sizeof(jit_word_t) &&
1021 (word = _jit->pc.w & (sizeof(jit_word_t) - 1)))
1022 nop(sizeof(jit_word_t) - word);
1024 case jit_code_note: case jit_code_name:
1025 node->u.w = _jit->pc.w;
1027 case jit_code_label:
1028 /* remember label is defined */
1029 node->flag |= jit_flag_patch;
1030 node->u.w = _jit->pc.w;
1049 case_rrrr(qmul, _u);
1050 case_rrrw(qmul, _u);
1057 case_rrrr(qdiv, _u);
1058 case_rrrw(qdiv, _u);
1077 case_rr(trunc, _f_i);
1078 case_rr(trunc, _d_i);
1079 case_rr(trunc, _f_l);
1080 case_rr(trunc, _d_l);
1136 if (node->flag & jit_flag_node) {
1138 if (temp->code == jit_code_data ||
1139 (temp->code == jit_code_label &&
1140 (temp->flag & jit_flag_patch)))
1141 movi(rn(node->u.w), temp->u.w);
1143 assert(temp->code == jit_code_label ||
1144 temp->code == jit_code_epilog);
1145 word = movi_p(rn(node->u.w), temp->u.w);
1150 movi(rn(node->u.w), node->v.w);
1194 case_brr(boadd, _u);
1195 case_brw(boadd, _u);
1198 case_brr(bxadd, _u);
1199 case_brw(bxadd, _u);
1202 case_brr(bosub, _u);
1203 case_brw(bosub, _u);
1206 case_brr(bxsub, _u);
1207 case_brw(bxsub, _u);
1234 case jit_code_movi_f:
1236 movi_f(rn(node->u.w), node->v.f);
1265 case_rrr(unord, _f);
1279 case_brr(bunlt, _f);
1281 case_brr(bunle, _f);
1283 case_brr(buneq, _f);
1285 case_brr(bunge, _f);
1287 case_brr(bungt, _f);
1289 case_brr(bltgt, _f);
1293 case_brr(bunord, _f);
1317 case jit_code_movi_d:
1319 movi_d(rn(node->u.w), node->v.d);
1348 case_rrr(unord, _d);
1362 case_brr(bunlt, _d);
1364 case_brr(bunle, _d);
1366 case_brr(buneq, _d);
1368 case_brr(bunge, _d);
1370 case_brr(bungt, _d);
1372 case_brr(bltgt, _d);
1376 case_brr(bunord, _d);
1379 jmpr(rn(node->u.w));
1382 if (node->flag & jit_flag_node) {
1384 assert(temp->code == jit_code_label ||
1385 temp->code == jit_code_epilog);
1386 if (temp->flag & jit_flag_patch)
1389 word = jmpi_p(_jit->pc.w);
1396 case jit_code_callr:
1397 callr(rn(node->u.w));
1399 case jit_code_calli:
1400 if (node->flag & jit_flag_node) {
1402 assert(temp->code == jit_code_label ||
1403 temp->code == jit_code_epilog);
1404 if (temp->flag & jit_flag_patch)
1407 word = calli_p(_jit->pc.w);
1414 case jit_code_prolog:
1415 _jitc->function = _jitc->functions.ptr + node->w.w;
1417 undo.word = _jit->pc.w;
1418 #if DEVEL_DISASSEMBLER
1421 undo.patch_offset = _jitc->patches.offset;
1426 case jit_code_epilog:
1427 assert(_jitc->function == _jitc->functions.ptr + node->w.w);
1429 for (temp = undo.node->next;
1430 temp != node; temp = temp->next) {
1431 if (temp->code == jit_code_label ||
1432 temp->code == jit_code_epilog)
1433 temp->flag &= ~jit_flag_patch;
1435 temp->flag &= ~jit_flag_patch;
1437 _jit->pc.w = undo.word;
1438 #if DEVEL_DISASSEMBLER
1441 _jitc->patches.offset = undo.patch_offset;
1442 goto restart_function;
1444 /* remember label is defined */
1445 node->flag |= jit_flag_patch;
1446 node->u.w = _jit->pc.w;
1448 _jitc->function = NULL;
1450 case jit_code_movr_w_f:
1451 movr_w_f(rn(node->u.w), rn(node->v.w));
1453 case jit_code_movr_f_w:
1454 movr_f_w(rn(node->u.w), rn(node->v.w));
1456 case jit_code_movi_f_w:
1458 movi_f_w(rn(node->u.w), node->v.f);
1460 case jit_code_movr_w_d:
1461 movr_w_d(rn(node->u.w), rn(node->v.w));
1463 case jit_code_movr_d_w:
1464 movr_d_w(rn(node->u.w), rn(node->v.w));
1466 case jit_code_movi_d_w:
1468 movi_d_w(rn(node->u.w), node->v.d);
1470 case jit_code_va_start:
1471 vastart(rn(node->u.w));
1473 case jit_code_va_arg:
1474 vaarg(rn(node->u.w), rn(node->v.w));
1476 case jit_code_va_arg_d:
1477 vaarg_d(rn(node->u.w), rn(node->v.w));
1479 case jit_code_live: case jit_code_ellipsis:
1480 case jit_code_va_push:
1481 case jit_code_allocai: case jit_code_allocar:
1483 case jit_code_arg_f: case jit_code_arg_d:
1484 case jit_code_va_end:
1486 case jit_code_retr: case jit_code_reti:
1487 case jit_code_retr_f: case jit_code_reti_f:
1488 case jit_code_retr_d: case jit_code_reti_d:
1489 case jit_code_getarg_c: case jit_code_getarg_uc:
1490 case jit_code_getarg_s: case jit_code_getarg_us:
1491 case jit_code_getarg_i: case jit_code_getarg_ui:
1492 case jit_code_getarg_l:
1493 case jit_code_getarg_f: case jit_code_getarg_d:
1494 case jit_code_putargr: case jit_code_putargi:
1495 case jit_code_putargr_f: case jit_code_putargi_f:
1496 case jit_code_putargr_d: case jit_code_putargi_d:
1497 case jit_code_pushargr: case jit_code_pushargi:
1498 case jit_code_pushargr_f: case jit_code_pushargi_f:
1499 case jit_code_pushargr_d: case jit_code_pushargi_d:
1500 case jit_code_retval_c: case jit_code_retval_uc:
1501 case jit_code_retval_s: case jit_code_retval_us:
1502 case jit_code_retval_i:
1503 case jit_code_retval_ui: case jit_code_retval_l:
1504 case jit_code_retval_f: case jit_code_retval_d:
1505 case jit_code_prepare:
1506 case jit_code_finishr: case jit_code_finishi:
1511 if (jit_carry != _NOREG) {
1512 switch (node->code) {
1514 case jit_code_addcr: case jit_code_addci:
1515 case jit_code_addxr: case jit_code_addxi:
1516 case jit_code_subcr: case jit_code_subci:
1517 case jit_code_subxr: case jit_code_subxi:
1520 jit_unget_reg(jit_carry);
1525 jit_regarg_clr(node, value);
1526 assert(_jitc->regarg == 0 ||
1527 (jit_carry != _NOREG && _jitc->regarg == (1 << jit_carry)));
1528 assert(_jitc->synth == 0);
1529 /* update register live state */
1541 for (offset = 0; offset < _jitc->patches.offset; offset++) {
1542 node = _jitc->patches.ptr[offset].node;
1543 word = _jitc->patches.ptr[offset].inst;
1544 value = node->code == jit_code_movi ? node->v.n->u.w : node->u.n->u.w;
1545 patch_at(word, value);
1548 jit_flush(_jit->code.ptr, _jit->pc.uc);
1550 return (_jit->code.ptr);
1554 # include "jit_riscv-cpu.c"
1555 # include "jit_riscv-fpu.c"
1559 jit_flush(void *fptr, void *tptr)
1561 #if defined(__GNUC__)
1564 s = sysconf(_SC_PAGE_SIZE);
1565 f = (jit_word_t)fptr & -s;
1566 t = (((jit_word_t)tptr) + s - 1) & -s;
1567 __clear_cache((void *)f, (void *)t);
1572 _emit_ldxi(jit_state_t *_jit, jit_int32_t r0, jit_int32_t r1, jit_word_t i0)
1574 ldxi(rn(r0), rn(r1), i0);
1578 _emit_stxi(jit_state_t *_jit, jit_word_t i0, jit_int32_t r0, jit_int32_t r1)
1580 stxi(i0, rn(r0), rn(r1));
1584 _emit_ldxi_d(jit_state_t *_jit, jit_int32_t r0, jit_int32_t r1, jit_word_t i0)
1586 ldxi_d(rn(r0), rn(r1), i0);
1590 _emit_stxi_d(jit_state_t *_jit, jit_word_t i0, jit_int32_t r0, jit_int32_t r1)
1592 stxi_d(i0, rn(r0), rn(r1));
1596 _patch(jit_state_t *_jit, jit_word_t instr, jit_node_t *node)
1600 assert(node->flag & jit_flag_node);
1601 if (node->code == jit_code_movi)
1602 flag = node->v.n->flag;
1604 flag = node->u.n->flag;
1605 assert(!(flag & jit_flag_patch));
1606 if (_jitc->patches.offset >= _jitc->patches.length) {
1607 jit_realloc((jit_pointer_t *)&_jitc->patches.ptr,
1608 _jitc->patches.length * sizeof(jit_patch_t),
1609 (_jitc->patches.length + 1024) * sizeof(jit_patch_t));
1610 _jitc->patches.length += 1024;
1612 _jitc->patches.ptr[_jitc->patches.offset].inst = instr;
1613 _jitc->patches.ptr[_jitc->patches.offset].node = node;
1614 ++_jitc->patches.offset;