2 * Copyright (C) 2019 Free Software Foundation, Inc.
4 * This file is part of GNU lightning.
6 * GNU lightning is free software; you can redistribute it and/or modify it
7 * under the terms of the GNU Lesser General Public License as published
8 * by the Free Software Foundation; either version 3, or (at your option)
11 * GNU lightning is distributed in the hope that it will be useful, but
12 * WITHOUT ANY WARRANTY; without even the implied warranty of MERCHANTABILITY
13 * or FITNESS FOR A PARTICULAR PURPOSE. See the GNU Lesser General Public
14 * License for more details.
17 * Paulo Cesar Pereira de Andrade
20 #define jit_arg_reg_p(i) ((i) >= 0 && (i) < 8)
21 #define jit_arg_f_reg_p(i) ((i) >= 0 && (i) < 8)
26 typedef jit_pointer_t jit_va_list_t;
31 #define patch(instr, node) _patch(_jit, instr, node)
32 static void _patch(jit_state_t*,jit_word_t,jit_node_t*);
35 # include "jit_riscv-cpu.c"
36 # include "jit_riscv-fpu.c"
37 # include "jit_fallback.c"
43 jit_register_t _rvs[] = {
48 #if 0 /* Pretend it does not exist, so _NOREG can be used in
52 { rc(gpr) | 0x05, "t0" },
53 { rc(gpr) | 0x06, "t1" },
54 { rc(gpr) | 0x07, "t2" },
55 { rc(gpr) | 0x1c, "t3" },
56 { rc(gpr) | 0x1d, "t4" },
57 { rc(gpr) | 0x1e, "t5" },
58 { rc(gpr) | 0x1f, "t6" },
60 { rc(sav) | rc(gpr) | 0x09, "s1" },
61 { rc(sav) | rc(gpr) | 0x12, "s2" },
62 { rc(sav) | rc(gpr) | 0x13, "s3" },
63 { rc(sav) | rc(gpr) | 0x14, "s4" },
64 { rc(sav) | rc(gpr) | 0x15, "s5" },
65 { rc(sav) | rc(gpr) | 0x16, "s6" },
66 { rc(sav) | rc(gpr) | 0x17, "s7" },
67 { rc(sav) | rc(gpr) | 0x18, "s8" },
68 { rc(sav) | rc(gpr) | 0x19, "s9" },
69 { rc(sav) | rc(gpr) | 0x1a, "s10" },
70 { rc(sav) | rc(gpr) | 0x1b, "s11" },
71 { rc(arg) | rc(gpr) | 0x11, "a7" },
72 { rc(arg) | rc(gpr) | 0x10, "a6" },
73 { rc(arg) | rc(gpr) | 0x0f, "a5" },
74 { rc(arg) | rc(gpr) | 0x0e, "a4" },
75 { rc(arg) | rc(gpr) | 0x0d, "a3" },
76 { rc(arg) | rc(gpr) | 0x0c, "a2" },
77 { rc(arg) | rc(gpr) | 0x0b, "a1" },
78 { rc(arg) | rc(gpr) | 0x0a, "a0" },
79 { rc(fpr) | 0x00, "ft0" },
80 { rc(fpr) | 0x01, "ft1" },
81 { rc(fpr) | 0x02, "ft2" },
82 { rc(fpr) | 0x03, "ft3" },
83 { rc(fpr) | 0x04, "ft4" },
84 { rc(fpr) | 0x05, "ft5" },
85 { rc(fpr) | 0x06, "ft6" },
86 { rc(fpr) | 0x07, "ft7" },
87 { rc(fpr) | 0x1c, "ft8" },
88 { rc(fpr) | 0x1d, "ft9" },
89 { rc(fpr) | 0x1e, "ft10" },
90 { rc(fpr) | 0x1f, "ft11" },
91 { rc(sav) | rc(fpr) | 0x08, "fs0" },
92 { rc(sav) | rc(fpr) | 0x09, "fs1" },
93 { rc(sav) | rc(fpr) | 0x12, "fs2" },
94 { rc(sav) | rc(fpr) | 0x13, "fs3" },
95 { rc(sav) | rc(fpr) | 0x14, "fs4" },
96 { rc(sav) | rc(fpr) | 0x15, "fs5" },
97 { rc(sav) | rc(fpr) | 0x16, "fs6" },
98 { rc(sav) | rc(fpr) | 0x17, "fs7" },
99 { rc(sav) | rc(fpr) | 0x18, "fs8" },
100 { rc(sav) | rc(fpr) | 0x19, "fs9" },
101 { rc(sav) | rc(fpr) | 0x1a, "fs10" },
102 { rc(sav) | rc(fpr) | 0x1b, "fs11" },
103 { rc(arg) | rc(fpr) | 0x11, "fa7" },
104 { rc(arg) | rc(fpr) | 0x10, "fa6" },
105 { rc(arg) | rc(fpr) | 0x0f, "fa5" },
106 { rc(arg) | rc(fpr) | 0x0e, "fa4" },
107 { rc(arg) | rc(fpr) | 0x0d, "fa3" },
108 { rc(arg) | rc(fpr) | 0x0c, "fa2" },
109 { rc(arg) | rc(fpr) | 0x0b, "fa1" },
110 { rc(arg) | rc(fpr) | 0x0a, "fa0" },
111 { _NOREG, "<none>" },
123 _jit_init(jit_state_t *_jit)
125 _jitc->reglen = jit_size(_rvs) - 1;
130 _jit_prolog(jit_state_t *_jit)
136 assert(jit_regset_cmp_ui(&_jitc->regarg, 0) == 0);
137 jit_regset_set_ui(&_jitc->regsav, 0);
138 offset = _jitc->functions.offset;
139 if (offset >= _jitc->functions.length) {
140 jit_realloc((jit_pointer_t *)&_jitc->functions.ptr,
141 _jitc->functions.length * sizeof(jit_function_t),
142 (_jitc->functions.length + 16) * sizeof(jit_function_t));
143 _jitc->functions.length += 16;
145 _jitc->function = _jitc->functions.ptr + _jitc->functions.offset++;
146 _jitc->function->self.size = stack_framesize;
147 _jitc->function->self.argi = _jitc->function->self.argf =
148 _jitc->function->self.alen = 0;
149 _jitc->function->self.aoff = 0;
150 _jitc->function->self.call = jit_call_default;
151 jit_alloc((jit_pointer_t *)&_jitc->function->regoff,
152 _jitc->reglen * sizeof(jit_int32_t));
154 /* _no_link here does not mean the jit_link() call can be removed
156 * _jitc->function->prolog = jit_new_node(jit_code_prolog);
158 _jitc->function->prolog = jit_new_node_no_link(jit_code_prolog);
159 jit_link(_jitc->function->prolog);
160 _jitc->function->prolog->w.w = offset;
161 _jitc->function->epilog = jit_new_node_no_link(jit_code_epilog);
163 * v: offset in blocks vector
164 * w: offset in functions vector
166 _jitc->function->epilog->w.w = offset;
168 jit_regset_new(&_jitc->function->regset);
172 _jit_allocai(jit_state_t *_jit, jit_int32_t length)
174 assert(_jitc->function);
176 case 0: case 1: break;
177 case 2: _jitc->function->self.aoff &= -2; break;
178 case 3: case 4: _jitc->function->self.aoff &= -4; break;
179 default: _jitc->function->self.aoff &= -8; break;
181 _jitc->function->self.aoff -= length;
182 if (!_jitc->realize) {
183 jit_inc_synth_ww(allocai, _jitc->function->self.aoff, length);
186 return (_jitc->function->self.aoff);
190 _jit_allocar(jit_state_t *_jit, jit_int32_t u, jit_int32_t v)
193 assert(_jitc->function);
194 jit_inc_synth_ww(allocar, u, v);
195 if (!_jitc->function->allocar) {
196 _jitc->function->aoffoff = jit_allocai(sizeof(jit_int32_t));
197 _jitc->function->allocar = 1;
199 r0 = jit_get_reg(jit_class_gpr);
201 jit_andi(r0, r0, -16);
202 jit_ldxi_i(u, JIT_FP, _jitc->function->aoffoff);
204 jit_addr(JIT_SP, JIT_SP, r0);
205 jit_stxi_i(_jitc->function->aoffoff, JIT_FP, u);
211 _jit_ret(jit_state_t *_jit)
214 assert(_jitc->function);
218 jit_patch_at(instr, _jitc->function->epilog);
223 _jit_retr(jit_state_t *_jit, jit_int32_t u)
225 jit_inc_synth_w(retr, u);
227 jit_movr(JIT_RET, u);
234 _jit_reti(jit_state_t *_jit, jit_word_t u)
236 jit_inc_synth_w(reti, u);
237 jit_movi(JIT_RET, u);
243 _jit_retr_f(jit_state_t *_jit, jit_int32_t u)
245 jit_inc_synth_w(retr_f, u);
247 jit_movr_f(JIT_FRET, u);
255 _jit_reti_f(jit_state_t *_jit, jit_float32_t u)
257 jit_inc_synth_f(reti_f, u);
258 jit_movi_f(JIT_FRET, u);
264 _jit_retr_d(jit_state_t *_jit, jit_int32_t u)
266 jit_inc_synth_w(retr_d, u);
268 jit_movr_d(JIT_FRET, u);
276 _jit_reti_d(jit_state_t *_jit, jit_float64_t u)
278 jit_inc_synth_d(reti_d, u);
279 jit_movi_d(JIT_FRET, u);
285 _jit_epilog(jit_state_t *_jit)
287 assert(_jitc->function);
288 assert(_jitc->function->epilog->next == NULL);
289 jit_link(_jitc->function->epilog);
290 _jitc->function = NULL;
294 _jit_arg_register_p(jit_state_t *_jit, jit_node_t *u)
296 if (u->code == jit_code_arg)
297 return (jit_arg_reg_p(u->u.w));
298 assert(u->code == jit_code_arg_f || u->code == jit_code_arg_d);
299 return (jit_arg_f_reg_p(u->u.w));
303 _jit_ellipsis(jit_state_t *_jit)
305 jit_inc_synth(ellipsis);
306 if (_jitc->prepare) {
308 assert(!(_jitc->function->call.call & jit_call_varargs));
309 _jitc->function->call.call |= jit_call_varargs;
313 assert(!(_jitc->function->self.call & jit_call_varargs));
314 _jitc->function->self.call |= jit_call_varargs;
315 _jitc->function->vagp = _jitc->function->self.argi;
321 _jit_va_push(jit_state_t *_jit, jit_int32_t u)
323 jit_inc_synth_w(va_push, u);
329 _jit_arg(jit_state_t *_jit)
333 assert(_jitc->function);
334 assert(!(_jitc->function->self.call & jit_call_varargs));
335 if (jit_arg_reg_p(_jitc->function->self.argi))
336 offset = _jitc->function->self.argi++;
338 offset = _jitc->function->self.size;
339 _jitc->function->self.size += sizeof(jit_word_t);
341 node = jit_new_node_ww(jit_code_arg, offset,
342 ++_jitc->function->self.argn);
348 _jit_arg_f(jit_state_t *_jit)
352 assert(_jitc->function);
353 assert(!(_jitc->function->self.call & jit_call_varargs));
354 if (jit_arg_f_reg_p(_jitc->function->self.argf))
355 offset = _jitc->function->self.argf++;
356 else if (jit_arg_reg_p(_jitc->function->self.argi)) {
357 offset = _jitc->function->self.argi++;
361 offset = _jitc->function->self.size;
362 _jitc->function->self.size += sizeof(jit_word_t);
364 node = jit_new_node_ww(jit_code_arg_f, offset,
365 ++_jitc->function->self.argn);
371 _jit_arg_d(jit_state_t *_jit)
375 assert(_jitc->function);
376 assert(!(_jitc->function->self.call & jit_call_varargs));
377 if (jit_arg_f_reg_p(_jitc->function->self.argf))
378 offset = _jitc->function->self.argf++;
379 else if (jit_arg_reg_p(_jitc->function->self.argi)) {
380 offset = _jitc->function->self.argi++;
384 offset = _jitc->function->self.size;
385 _jitc->function->self.size += sizeof(jit_word_t);
387 node = jit_new_node_ww(jit_code_arg_d, offset,
388 ++_jitc->function->self.argn);
394 _jit_getarg_c(jit_state_t *_jit, jit_int32_t u, jit_node_t *v)
396 assert(v->code == jit_code_arg);
397 jit_inc_synth_wp(getarg_c, u, v);
398 if (jit_arg_reg_p(v->u.w))
399 jit_extr_c(u, JIT_RA0 - v->u.w);
401 jit_ldxi_c(u, JIT_FP, v->u.w);
406 _jit_getarg_uc(jit_state_t *_jit, jit_int32_t u, jit_node_t *v)
408 assert(v->code == jit_code_arg);
409 jit_inc_synth_wp(getarg_uc, u, v);
410 if (jit_arg_reg_p(v->u.w))
411 jit_extr_uc(u, JIT_RA0 - v->u.w);
413 jit_ldxi_uc(u, JIT_FP, v->u.w);
418 _jit_getarg_s(jit_state_t *_jit, jit_int32_t u, jit_node_t *v)
420 assert(v->code == jit_code_arg);
421 jit_inc_synth_wp(getarg_s, u, v);
422 if (jit_arg_reg_p(v->u.w))
423 jit_extr_s(u, JIT_RA0 - v->u.w);
425 jit_ldxi_s(u, JIT_FP, v->u.w);
430 _jit_getarg_us(jit_state_t *_jit, jit_int32_t u, jit_node_t *v)
432 assert(v->code == jit_code_arg);
433 jit_inc_synth_wp(getarg_us, u, v);
434 if (jit_arg_reg_p(v->u.w))
435 jit_extr_us(u, JIT_RA0 - v->u.w);
437 jit_ldxi_us(u, JIT_FP, v->u.w);
442 _jit_getarg_i(jit_state_t *_jit, jit_int32_t u, jit_node_t *v)
444 assert(v->code == jit_code_arg);
445 jit_inc_synth_wp(getarg_i, u, v);
446 if (jit_arg_reg_p(v->u.w))
447 jit_extr_i(u, JIT_RA0 - v->u.w);
449 jit_ldxi_i(u, JIT_FP, v->u.w);
454 _jit_getarg_ui(jit_state_t *_jit, jit_int32_t u, jit_node_t *v)
456 assert(v->code == jit_code_arg);
457 jit_inc_synth_wp(getarg_ui, u, v);
458 if (jit_arg_reg_p(v->u.w))
459 jit_extr_ui(u, JIT_RA0 - v->u.w);
461 jit_ldxi_ui(u, JIT_FP, v->u.w);
466 _jit_getarg_l(jit_state_t *_jit, jit_int32_t u, jit_node_t *v)
468 assert(v->code == jit_code_arg);
469 jit_inc_synth_wp(getarg_l, u, v);
470 if (jit_arg_reg_p(v->u.w))
471 jit_movr(u, JIT_RA0 - v->u.w);
473 jit_ldxi_l(u, JIT_FP, v->u.w);
478 _jit_putargr(jit_state_t *_jit, jit_int32_t u, jit_node_t *v)
480 assert(v->code == jit_code_arg);
481 jit_inc_synth_wp(putargr, u, v);
482 if (jit_arg_reg_p(v->u.w))
483 jit_movr(JIT_RA0 - v->u.w, u);
485 jit_stxi(v->u.w, JIT_FP, u);
490 _jit_putargi(jit_state_t *_jit, jit_word_t u, jit_node_t *v)
493 assert(v->code == jit_code_arg);
494 jit_inc_synth_wp(putargi, u, v);
495 if (jit_arg_reg_p(v->u.w))
496 jit_movi(JIT_RA0 - v->u.w, u);
498 regno = jit_get_reg(jit_class_gpr);
500 jit_stxi(v->u.w, JIT_FP, regno);
501 jit_unget_reg(regno);
507 _jit_getarg_f(jit_state_t *_jit, jit_int32_t u, jit_node_t *v)
509 assert(v->code == jit_code_arg_f);
510 jit_inc_synth_wp(getarg_f, u, v);
511 if (jit_arg_f_reg_p(v->u.w))
512 jit_movr_f(u, JIT_FA0 - v->u.w);
513 else if (jit_arg_reg_p(v->u.w - 8))
514 jit_movr_w_f(u, JIT_RA0 - (v->u.w - 8));
516 jit_ldxi_f(u, JIT_FP, v->u.w);
521 _jit_putargr_f(jit_state_t *_jit, jit_int32_t u, jit_node_t *v)
523 assert(v->code == jit_code_arg_f);
524 jit_inc_synth_wp(putargr_f, u, v);
525 if (jit_arg_f_reg_p(v->u.w))
526 jit_movr_f(JIT_FA0 - v->u.w, u);
527 else if (jit_arg_reg_p(v->u.w - 8))
528 jit_movr_f_w(JIT_RA0 - (v->u.w - 8), u);
530 jit_stxi_f(v->u.w, JIT_FP, u);
535 _jit_putargi_f(jit_state_t *_jit, jit_float32_t u, jit_node_t *v)
538 assert(v->code == jit_code_arg_f);
539 jit_inc_synth_fp(putargi_f, u, v);
540 if (jit_arg_f_reg_p(v->u.w))
541 jit_movi_f(JIT_FA0 - v->u.w, u);
542 else if (jit_arg_reg_p(v->u.w - 8)) {
548 jit_movi(JIT_RA0 - (v->u.w - 8), uu.i);
551 regno = jit_get_reg(jit_class_fpr);
552 jit_movi_f(regno, u);
553 jit_stxi_f(v->u.w, JIT_FP, regno);
554 jit_unget_reg(regno);
560 _jit_getarg_d(jit_state_t *_jit, jit_int32_t u, jit_node_t *v)
562 assert(v->code == jit_code_arg_d);
563 jit_inc_synth_wp(getarg_d, u, v);
564 if (jit_arg_f_reg_p(v->u.w))
565 jit_movr_d(u, JIT_FA0 - v->u.w);
566 else if (jit_arg_reg_p(v->u.w - 8))
567 jit_movr_w_d(u, JIT_RA0 - (v->u.w - 8));
569 jit_ldxi_d(u, JIT_FP, v->u.w);
574 _jit_putargr_d(jit_state_t *_jit, jit_int32_t u, jit_node_t *v)
576 assert(v->code == jit_code_arg_d);
577 jit_inc_synth_wp(putargr_d, u, v);
578 if (jit_arg_reg_p(v->u.w))
579 jit_movr_d(JIT_FA0 - v->u.w, u);
580 else if (jit_arg_reg_p(v->u.w - 8))
581 jit_movr_d_w(JIT_RA0 - (v->u.w - 8), u);
583 jit_stxi_d(v->u.w, JIT_FP, u);
588 _jit_putargi_d(jit_state_t *_jit, jit_float64_t u, jit_node_t *v)
591 assert(v->code == jit_code_arg_d);
592 jit_inc_synth_dp(putargi_d, u, v);
593 if (jit_arg_reg_p(v->u.w))
594 jit_movi_d(JIT_FA0 - v->u.w, u);
595 else if (jit_arg_reg_p(v->u.w - 8)) {
601 jit_movi(JIT_RA0 - (v->u.w - 8), uu.w);
604 regno = jit_get_reg(jit_class_fpr);
605 jit_movi_d(regno, u);
606 jit_stxi_d(v->u.w, JIT_FP, regno);
607 jit_unget_reg(regno);
613 _jit_pushargr(jit_state_t *_jit, jit_int32_t u)
615 assert(_jitc->function);
616 jit_inc_synth_w(pushargr, u);
618 if (jit_arg_reg_p(_jitc->function->call.argi)) {
619 jit_movr(JIT_RA0 - _jitc->function->call.argi, u);
620 ++_jitc->function->call.argi;
623 jit_stxi(_jitc->function->call.size, JIT_SP, u);
624 _jitc->function->call.size += sizeof(jit_word_t);
630 _jit_pushargi(jit_state_t *_jit, jit_word_t u)
633 assert(_jitc->function);
634 jit_inc_synth_w(pushargi, u);
636 if (jit_arg_reg_p(_jitc->function->call.argi)) {
637 jit_movi(JIT_RA0 - _jitc->function->call.argi, u);
638 ++_jitc->function->call.argi;
641 regno = jit_get_reg(jit_class_gpr);
643 jit_stxi(_jitc->function->call.size, JIT_SP, regno);
644 jit_unget_reg(regno);
645 _jitc->function->call.size += sizeof(jit_word_t);
651 _jit_pushargr_f(jit_state_t *_jit, jit_int32_t u)
653 assert(_jitc->function);
654 jit_inc_synth_w(pushargr_f, u);
656 if (jit_arg_f_reg_p(_jitc->function->call.argf) &&
657 !(_jitc->function->call.call & jit_call_varargs)) {
658 jit_movr_f(JIT_FA0 - _jitc->function->call.argf, u);
659 ++_jitc->function->call.argf;
661 else if (jit_arg_reg_p(_jitc->function->call.argi)) {
662 jit_movr_f_w(JIT_RA0 - _jitc->function->call.argi, u);
663 ++_jitc->function->call.argi;
666 jit_stxi_f(_jitc->function->call.size, JIT_SP, u);
667 _jitc->function->call.size += sizeof(jit_word_t);
673 _jit_pushargi_f(jit_state_t *_jit, jit_float32_t u)
676 assert(_jitc->function);
677 jit_inc_synth_f(pushargi_f, u);
679 if (jit_arg_f_reg_p(_jitc->function->call.argf) &&
680 !(_jitc->function->call.call & jit_call_varargs)) {
681 jit_movi_f(JIT_FA0 - _jitc->function->call.argf, u);
682 ++_jitc->function->call.argf;
684 else if (jit_arg_reg_p(_jitc->function->call.argi)) {
685 jit_movi_f_w(JIT_RA0 - _jitc->function->call.argi, u);
686 ++_jitc->function->call.argi;
689 regno = jit_get_reg(jit_class_fpr);
690 jit_movi_f(regno, u);
691 jit_stxi_f(_jitc->function->call.size, JIT_SP, regno);
692 jit_unget_reg(regno);
693 _jitc->function->call.size += sizeof(jit_word_t);
699 _jit_pushargr_d(jit_state_t *_jit, jit_int32_t u)
701 assert(_jitc->function);
702 jit_inc_synth_w(pushargr_d, u);
704 if (jit_arg_f_reg_p(_jitc->function->call.argf) &&
705 !(_jitc->function->call.call & jit_call_varargs)) {
706 jit_movr_d(JIT_FA0 - _jitc->function->call.argf, u);
707 ++_jitc->function->call.argf;
709 else if (jit_arg_reg_p(_jitc->function->call.argi)) {
710 jit_movr_d_w(JIT_RA0 - _jitc->function->call.argi, u);
711 ++_jitc->function->call.argi;
714 jit_stxi_d(_jitc->function->call.size, JIT_SP, u);
715 _jitc->function->call.size += sizeof(jit_word_t);
721 _jit_pushargi_d(jit_state_t *_jit, jit_float64_t u)
724 assert(_jitc->function);
725 jit_inc_synth_d(pushargi_d, u);
727 if (jit_arg_f_reg_p(_jitc->function->call.argf) &&
728 !(_jitc->function->call.call & jit_call_varargs)) {
729 jit_movi_d(JIT_FA0 - _jitc->function->call.argf, u);
730 ++_jitc->function->call.argf;
732 else if (jit_arg_reg_p(_jitc->function->call.argi)) {
733 jit_movi_d_w(JIT_RA0 - _jitc->function->call.argi, u);
734 ++_jitc->function->call.argi;
737 regno = jit_get_reg(jit_class_fpr);
738 jit_movi_d(regno, u);
739 jit_stxi_d(_jitc->function->call.size, JIT_SP, regno);
740 jit_unget_reg(regno);
741 _jitc->function->call.size += sizeof(jit_word_t);
747 _jit_regarg_p(jit_state_t *_jit, jit_node_t *node, jit_int32_t regno)
750 spec = jit_class(_rvs[regno].spec);
751 if (spec & jit_class_arg) {
752 regno = JIT_RA0 - regno;
753 if (regno >= 0 && regno < node->v.w)
755 if (spec & jit_class_fpr) {
756 regno = JIT_FA0 - regno;
757 if (regno >= 0 && regno < node->w.w)
766 _jit_finishr(jit_state_t *_jit, jit_int32_t r0)
769 assert(_jitc->function);
770 jit_inc_synth_w(finishr, r0);
771 if (_jitc->function->self.alen < _jitc->function->call.size)
772 _jitc->function->self.alen = _jitc->function->call.size;
773 node = jit_callr(r0);
774 node->v.w = _jitc->function->self.argi;
775 node->w.w = _jitc->function->call.argf;
776 _jitc->function->call.argi = _jitc->function->call.argf =
777 _jitc->function->call.size = 0;
783 _jit_finishi(jit_state_t *_jit, jit_pointer_t i0)
786 assert(_jitc->function);
787 jit_inc_synth_w(finishi, (jit_word_t)i0);
788 if (_jitc->function->self.alen < _jitc->function->call.size)
789 _jitc->function->self.alen = _jitc->function->call.size;
790 node = jit_calli(i0);
791 node->v.w = _jitc->function->call.argi;
792 node->w.w = _jitc->function->call.argf;
793 _jitc->function->call.argi = _jitc->function->call.argf =
794 _jitc->function->call.size = 0;
801 _jit_retval_c(jit_state_t *_jit, jit_int32_t r0)
803 jit_inc_synth_w(retval_c, r0);
804 jit_extr_c(r0, JIT_RET);
809 _jit_retval_uc(jit_state_t *_jit, jit_int32_t r0)
811 jit_inc_synth_w(retval_uc, r0);
812 jit_extr_uc(r0, JIT_RET);
817 _jit_retval_s(jit_state_t *_jit, jit_int32_t r0)
819 jit_inc_synth_w(retval_s, r0);
820 jit_extr_s(r0, JIT_RET);
825 _jit_retval_us(jit_state_t *_jit, jit_int32_t r0)
827 jit_inc_synth_w(retval_us, r0);
828 jit_extr_us(r0, JIT_RET);
833 _jit_retval_i(jit_state_t *_jit, jit_int32_t r0)
835 jit_inc_synth_w(retval_i, r0);
836 jit_extr_i(r0, JIT_RET);
841 _jit_retval_ui(jit_state_t *_jit, jit_int32_t r0)
843 jit_inc_synth_w(retval_ui, r0);
844 jit_extr_ui(r0, JIT_RET);
849 _jit_retval_l(jit_state_t *_jit, jit_int32_t r0)
851 jit_inc_synth_w(retval_l, r0);
853 jit_movr(r0, JIT_RET);
858 _jit_retval_f(jit_state_t *_jit, jit_int32_t r0)
860 jit_inc_synth_w(retval_f, r0);
862 jit_movr_f(r0, JIT_FRET);
867 _jit_retval_d(jit_state_t *_jit, jit_int32_t r0)
869 jit_inc_synth_w(retval_d, r0);
871 jit_movr_d(r0, JIT_FRET);
876 _emit_code(jit_state_t *_jit)
887 #if DEVEL_DISASSEMBLER
890 jit_int32_t const_offset;
891 jit_int32_t patch_offset;
893 #if DEVEL_DISASSEMBLER
897 _jitc->function = NULL;
903 undo.const_offset = undo.patch_offset = 0;
904 # define assert_data(node) /**/
905 #define case_rr(name, type) \
906 case jit_code_##name##r##type: \
907 name##r##type(rn(node->u.w), rn(node->v.w)); \
909 #define case_rw(name, type) \
910 case jit_code_##name##i##type: \
911 name##i##type(rn(node->u.w), node->v.w); \
913 #define case_wr(name, type) \
914 case jit_code_##name##i##type: \
915 name##i##type(node->u.w, rn(node->v.w)); \
917 #define case_rrr(name, type) \
918 case jit_code_##name##r##type: \
919 name##r##type(rn(node->u.w), \
920 rn(node->v.w), rn(node->w.w)); \
922 #define case_rrrr(name, type) \
923 case jit_code_##name##r##type: \
924 name##r##type(rn(node->u.q.l), rn(node->u.q.h), \
925 rn(node->v.w), rn(node->w.w)); \
927 #define case_rrw(name, type) \
928 case jit_code_##name##i##type: \
929 name##i##type(rn(node->u.w), rn(node->v.w), node->w.w); \
931 #define case_rrrw(name, type) \
932 case jit_code_##name##i##type: \
933 name##i##type(rn(node->u.q.l), rn(node->u.q.h), \
934 rn(node->v.w), node->w.w); \
936 #define case_rrf(name) \
937 case jit_code_##name##i_f: \
939 name##i_f(rn(node->u.w), rn(node->v.w), node->w.f); \
941 #define case_rrd(name) \
942 case jit_code_##name##i_d: \
944 name##i_d(rn(node->u.w), rn(node->v.w), node->w.d); \
946 #define case_wrr(name, type) \
947 case jit_code_##name##i##type: \
948 name##i##type(node->u.w, rn(node->v.w), rn(node->w.w)); \
950 #define case_brr(name, type) \
951 case jit_code_##name##r##type: \
953 assert(temp->code == jit_code_label || \
954 temp->code == jit_code_epilog); \
955 if (temp->flag & jit_flag_patch) \
956 name##r##type(temp->u.w, rn(node->v.w), \
959 word = name##r##type(_jit->pc.w, \
960 rn(node->v.w), rn(node->w.w)); \
964 #define case_brw(name, type) \
965 case jit_code_##name##i##type: \
967 assert(temp->code == jit_code_label || \
968 temp->code == jit_code_epilog); \
969 if (temp->flag & jit_flag_patch) \
970 name##i##type(temp->u.w, \
971 rn(node->v.w), node->w.w); \
973 word = name##i##type(_jit->pc.w, \
974 rn(node->v.w), node->w.w); \
978 #define case_brf(name) \
979 case jit_code_##name##i_f: \
981 assert(temp->code == jit_code_label || \
982 temp->code == jit_code_epilog); \
983 if (temp->flag & jit_flag_patch) \
984 name##i_f(temp->u.w, rn(node->v.w), node->w.f); \
986 word = name##i_f(_jit->pc.w, rn(node->v.w), \
991 #define case_brd(name) \
992 case jit_code_##name##i_d: \
994 assert(temp->code == jit_code_label || \
995 temp->code == jit_code_epilog); \
996 if (temp->flag & jit_flag_patch) \
997 name##i_d(temp->u.w, rn(node->v.w), node->w.d); \
999 word = name##i_d(_jit->pc.w, rn(node->v.w), \
1001 patch(word, node); \
1004 #if DEVEL_DISASSEMBLER
1007 for (node = _jitc->head; node; node = node->next) {
1008 if (_jit->pc.uc >= _jitc->code.end)
1011 #if DEVEL_DISASSEMBLER
1012 node->offset = (jit_uword_t)_jit->pc.w - (jit_uword_t)prevw;
1015 value = jit_classify(node->code);
1016 jit_regarg_set(node, value);
1017 switch (node->code) {
1018 case jit_code_align:
1019 assert(!(node->u.w & (node->u.w - 1)) &&
1020 node->u.w <= sizeof(jit_word_t));
1021 if (node->u.w == sizeof(jit_word_t) &&
1022 (word = _jit->pc.w & (sizeof(jit_word_t) - 1)))
1023 nop(sizeof(jit_word_t) - word);
1025 case jit_code_note: case jit_code_name:
1026 node->u.w = _jit->pc.w;
1028 case jit_code_label:
1029 /* remember label is defined */
1030 node->flag |= jit_flag_patch;
1031 node->u.w = _jit->pc.w;
1050 case_rrrr(qmul, _u);
1051 case_rrrw(qmul, _u);
1058 case_rrrr(qdiv, _u);
1059 case_rrrw(qdiv, _u);
1078 case_rr(trunc, _f_i);
1079 case_rr(trunc, _d_i);
1080 case_rr(trunc, _f_l);
1081 case_rr(trunc, _d_l);
1129 case_rr(bswap, _us);
1130 case_rr(bswap, _ui);
1131 case_rr(bswap, _ul);
1139 casr(rn(node->u.w), rn(node->v.w),
1140 rn(node->w.q.l), rn(node->w.q.h));
1143 casi(rn(node->u.w), node->v.w,
1144 rn(node->w.q.l), rn(node->w.q.h));
1150 if (node->flag & jit_flag_node) {
1152 if (temp->code == jit_code_data ||
1153 (temp->code == jit_code_label &&
1154 (temp->flag & jit_flag_patch)))
1155 movi(rn(node->u.w), temp->u.w);
1157 assert(temp->code == jit_code_label ||
1158 temp->code == jit_code_epilog);
1159 word = movi_p(rn(node->u.w), temp->u.w);
1164 movi(rn(node->u.w), node->v.w);
1208 case_brr(boadd, _u);
1209 case_brw(boadd, _u);
1212 case_brr(bxadd, _u);
1213 case_brw(bxadd, _u);
1216 case_brr(bosub, _u);
1217 case_brw(bosub, _u);
1220 case_brr(bxsub, _u);
1221 case_brw(bxsub, _u);
1248 case jit_code_movi_f:
1250 movi_f(rn(node->u.w), node->v.f);
1279 case_rrr(unord, _f);
1293 case_brr(bunlt, _f);
1295 case_brr(bunle, _f);
1297 case_brr(buneq, _f);
1299 case_brr(bunge, _f);
1301 case_brr(bungt, _f);
1303 case_brr(bltgt, _f);
1307 case_brr(bunord, _f);
1331 case jit_code_movi_d:
1333 movi_d(rn(node->u.w), node->v.d);
1362 case_rrr(unord, _d);
1376 case_brr(bunlt, _d);
1378 case_brr(bunle, _d);
1380 case_brr(buneq, _d);
1382 case_brr(bunge, _d);
1384 case_brr(bungt, _d);
1386 case_brr(bltgt, _d);
1390 case_brr(bunord, _d);
1393 jmpr(rn(node->u.w));
1396 if (node->flag & jit_flag_node) {
1398 assert(temp->code == jit_code_label ||
1399 temp->code == jit_code_epilog);
1400 if (temp->flag & jit_flag_patch)
1403 word = jmpi_p(_jit->pc.w);
1410 case jit_code_callr:
1411 callr(rn(node->u.w));
1413 case jit_code_calli:
1414 if (node->flag & jit_flag_node) {
1416 assert(temp->code == jit_code_label ||
1417 temp->code == jit_code_epilog);
1418 if (temp->flag & jit_flag_patch)
1421 word = calli_p(_jit->pc.w);
1428 case jit_code_prolog:
1429 _jitc->function = _jitc->functions.ptr + node->w.w;
1431 undo.word = _jit->pc.w;
1432 #if DEVEL_DISASSEMBLER
1435 undo.patch_offset = _jitc->patches.offset;
1440 case jit_code_epilog:
1441 assert(_jitc->function == _jitc->functions.ptr + node->w.w);
1443 for (temp = undo.node->next;
1444 temp != node; temp = temp->next) {
1445 if (temp->code == jit_code_label ||
1446 temp->code == jit_code_epilog)
1447 temp->flag &= ~jit_flag_patch;
1449 temp->flag &= ~jit_flag_patch;
1451 _jit->pc.w = undo.word;
1452 #if DEVEL_DISASSEMBLER
1455 _jitc->patches.offset = undo.patch_offset;
1456 goto restart_function;
1458 /* remember label is defined */
1459 node->flag |= jit_flag_patch;
1460 node->u.w = _jit->pc.w;
1462 _jitc->function = NULL;
1464 case jit_code_movr_w_f:
1465 movr_w_f(rn(node->u.w), rn(node->v.w));
1467 case jit_code_movr_f_w:
1468 movr_f_w(rn(node->u.w), rn(node->v.w));
1470 case jit_code_movi_f_w:
1472 movi_f_w(rn(node->u.w), node->v.f);
1474 case jit_code_movr_w_d:
1475 movr_w_d(rn(node->u.w), rn(node->v.w));
1477 case jit_code_movr_d_w:
1478 movr_d_w(rn(node->u.w), rn(node->v.w));
1480 case jit_code_movi_d_w:
1482 movi_d_w(rn(node->u.w), node->v.d);
1484 case jit_code_va_start:
1485 vastart(rn(node->u.w));
1487 case jit_code_va_arg:
1488 vaarg(rn(node->u.w), rn(node->v.w));
1490 case jit_code_va_arg_d:
1491 vaarg_d(rn(node->u.w), rn(node->v.w));
1493 case jit_code_live: case jit_code_ellipsis:
1494 case jit_code_va_push:
1495 case jit_code_allocai: case jit_code_allocar:
1497 case jit_code_arg_f: case jit_code_arg_d:
1498 case jit_code_va_end:
1500 case jit_code_retr: case jit_code_reti:
1501 case jit_code_retr_f: case jit_code_reti_f:
1502 case jit_code_retr_d: case jit_code_reti_d:
1503 case jit_code_getarg_c: case jit_code_getarg_uc:
1504 case jit_code_getarg_s: case jit_code_getarg_us:
1505 case jit_code_getarg_i: case jit_code_getarg_ui:
1506 case jit_code_getarg_l:
1507 case jit_code_getarg_f: case jit_code_getarg_d:
1508 case jit_code_putargr: case jit_code_putargi:
1509 case jit_code_putargr_f: case jit_code_putargi_f:
1510 case jit_code_putargr_d: case jit_code_putargi_d:
1511 case jit_code_pushargr: case jit_code_pushargi:
1512 case jit_code_pushargr_f: case jit_code_pushargi_f:
1513 case jit_code_pushargr_d: case jit_code_pushargi_d:
1514 case jit_code_retval_c: case jit_code_retval_uc:
1515 case jit_code_retval_s: case jit_code_retval_us:
1516 case jit_code_retval_i:
1517 case jit_code_retval_ui: case jit_code_retval_l:
1518 case jit_code_retval_f: case jit_code_retval_d:
1519 case jit_code_prepare:
1520 case jit_code_finishr: case jit_code_finishi:
1525 if (jit_carry != _NOREG) {
1526 switch (node->code) {
1528 case jit_code_addcr: case jit_code_addci:
1529 case jit_code_addxr: case jit_code_addxi:
1530 case jit_code_subcr: case jit_code_subci:
1531 case jit_code_subxr: case jit_code_subxi:
1534 jit_unget_reg(jit_carry);
1539 jit_regarg_clr(node, value);
1540 assert(_jitc->regarg == 0 ||
1541 (jit_carry != _NOREG && _jitc->regarg == (1 << jit_carry)));
1542 assert(_jitc->synth == 0);
1543 /* update register live state */
1555 for (offset = 0; offset < _jitc->patches.offset; offset++) {
1556 node = _jitc->patches.ptr[offset].node;
1557 word = _jitc->patches.ptr[offset].inst;
1558 value = node->code == jit_code_movi ? node->v.n->u.w : node->u.n->u.w;
1559 patch_at(word, value);
1562 jit_flush(_jit->code.ptr, _jit->pc.uc);
1564 return (_jit->code.ptr);
1568 # include "jit_riscv-cpu.c"
1569 # include "jit_riscv-fpu.c"
1570 # include "jit_fallback.c"
1574 jit_flush(void *fptr, void *tptr)
1576 #if defined(__GNUC__)
1579 s = sysconf(_SC_PAGE_SIZE);
1580 f = (jit_word_t)fptr & -s;
1581 t = (((jit_word_t)tptr) + s - 1) & -s;
1582 __clear_cache((void *)f, (void *)t);
1587 _emit_ldxi(jit_state_t *_jit, jit_int32_t r0, jit_int32_t r1, jit_word_t i0)
1589 ldxi(rn(r0), rn(r1), i0);
1593 _emit_stxi(jit_state_t *_jit, jit_word_t i0, jit_int32_t r0, jit_int32_t r1)
1595 stxi(i0, rn(r0), rn(r1));
1599 _emit_ldxi_d(jit_state_t *_jit, jit_int32_t r0, jit_int32_t r1, jit_word_t i0)
1601 ldxi_d(rn(r0), rn(r1), i0);
1605 _emit_stxi_d(jit_state_t *_jit, jit_word_t i0, jit_int32_t r0, jit_int32_t r1)
1607 stxi_d(i0, rn(r0), rn(r1));
1611 _patch(jit_state_t *_jit, jit_word_t instr, jit_node_t *node)
1615 assert(node->flag & jit_flag_node);
1616 if (node->code == jit_code_movi)
1617 flag = node->v.n->flag;
1619 flag = node->u.n->flag;
1620 assert(!(flag & jit_flag_patch));
1621 if (_jitc->patches.offset >= _jitc->patches.length) {
1622 jit_realloc((jit_pointer_t *)&_jitc->patches.ptr,
1623 _jitc->patches.length * sizeof(jit_patch_t),
1624 (_jitc->patches.length + 1024) * sizeof(jit_patch_t));
1625 _jitc->patches.length += 1024;
1627 _jitc->patches.ptr[_jitc->patches.offset].inst = instr;
1628 _jitc->patches.ptr[_jitc->patches.offset].node = node;
1629 ++_jitc->patches.offset;