2 * Copyright (C) 2012-2019 Free Software Foundation, Inc.
4 * This file is part of GNU lightning.
6 * GNU lightning is free software; you can redistribute it and/or modify it
7 * under the terms of the GNU Lesser General Public License as published
8 * by the Free Software Foundation; either version 3, or (at your option)
11 * GNU lightning is distributed in the hope that it will be useful, but
12 * WITHOUT ANY WARRANTY; without even the implied warranty of MERCHANTABILITY
13 * or FITNESS FOR A PARTICULAR PURPOSE. See the GNU Lesser General Public
14 * License for more details.
17 * Paulo Cesar Pereira de Andrade
20 #if defined(__linux__)
24 #define jit_arg_reg_p(i) ((i) >= 0 && (i) < 4)
25 #define jit_arg_f_reg_p(i) ((i) >= 0 && (i) < 16)
26 #define jit_arg_d_reg_p(i) ((i) >= 0 && (i) < 15)
28 #define arm_patch_node 0x80000000
29 #define arm_patch_word 0x40000000
30 #define arm_patch_jump 0x20000000
31 #define arm_patch_load 0x00000000
33 #define jit_fpr_p(rn) ((rn) > 15)
36 (stack_framesize - 16 + (jit_cpu.abi ? 64 : 0))
37 #define arg_offset(n) \
38 ((n) < 4 ? arg_base() + ((n) << 2) : (n))
40 /* Assume functions called never match jit instruction set, that is
41 * libc, gmp, mpfr, etc functions are in thumb mode and jit is in
42 * arm mode, what may cause a crash upon return of that function
43 * if generating jit for a relative jump.
45 #define jit_exchange_p() 1
47 /* FIXME is it really required to not touch _R10? */
52 typedef union _jit_thumb_t {
57 typedef jit_pointer_t jit_va_list;
62 #define jit_make_arg(node) _jit_make_arg(_jit,node)
63 static jit_node_t *_jit_make_arg(jit_state_t*,jit_node_t*);
64 #define jit_make_arg_f(node) _jit_make_arg_f(_jit,node)
65 static jit_node_t *_jit_make_arg_f(jit_state_t*,jit_node_t*);
66 #define jit_make_arg_d(node) _jit_make_arg_d(_jit,node)
67 static jit_node_t *_jit_make_arg_d(jit_state_t*,jit_node_t*);
68 #define jit_get_reg_pair() _jit_get_reg_pair(_jit)
69 static jit_int32_t _jit_get_reg_pair(jit_state_t*);
70 #define jit_unget_reg_pair(rn) _jit_unget_reg_pair(_jit,rn)
71 static void _jit_unget_reg_pair(jit_state_t*,jit_int32_t);
72 # define must_align_p(node) _must_align_p(_jit, node)
73 static jit_bool_t _must_align_p(jit_state_t*,jit_node_t*);
74 #define load_const(uniq,r0,i0) _load_const(_jit,uniq,r0,i0)
75 static void _load_const(jit_state_t*,jit_bool_t,jit_int32_t,jit_word_t);
76 #define flush_consts() _flush_consts(_jit)
77 static void _flush_consts(jit_state_t*);
78 #define invalidate_consts() _invalidate_consts(_jit)
79 static void _invalidate_consts(jit_state_t*);
80 #define patch(instr, node) _patch(_jit, instr, node)
81 static void _patch(jit_state_t*,jit_word_t,jit_node_t*);
85 extern void __clear_cache(void *, void *);
89 # include "jit_rewind.c"
90 # include "jit_arm-cpu.c"
91 # include "jit_arm-swf.c"
92 # include "jit_arm-vfp.c"
99 jit_register_t _rvs[] = {
100 { rc(gpr) | 0x0c, "ip" },
101 { rc(sav) | rc(gpr) | 0x04, "r4" },
102 { rc(sav) | rc(gpr) | 0x05, "r5" },
103 { rc(sav) | rc(gpr) | 0x06, "r6" },
104 { rc(sav) | rc(gpr) | 0x07, "r7" },
105 { rc(sav) | rc(gpr) | 0x08, "r8" },
106 { rc(sav) | rc(gpr) | 0x09, "r9" },
107 { rc(sav) | 0x0a, "sl" },
108 { rc(sav) | 0x0b, "fp" },
109 { rc(sav) | 0x0d, "sp" },
110 { rc(sav) | 0x0e, "lr" },
112 { rc(arg) | rc(gpr) | 0x03, "r3" },
113 { rc(arg) | rc(gpr) | 0x02, "r2" },
114 { rc(arg) | rc(gpr) | 0x01, "r1" },
115 { rc(arg) | rc(gpr) | 0x00, "r0" },
116 { rc(fpr) | 0x20, "d8" },
118 { rc(fpr) | 0x22, "d9" },
120 { rc(fpr) | 0x24, "d10" },
122 { rc(fpr) | 0x26, "d11" },
124 { rc(fpr) | 0x28, "d12" },
126 { rc(fpr) | 0x2a, "d13" },
128 { rc(fpr) | 0x2c, "d14" },
130 { rc(fpr) | 0x2e, "d15" },
132 { rc(arg) | 0x1f, "s15" },
133 { rc(arg)|rc(sft)|rc(fpr)|0x1e, "d7" },
134 { rc(arg) | 0x1d, "s13" },
135 { rc(arg)|rc(sft)|rc(fpr)|0x1c, "d6" },
136 { rc(arg) | 0x1b, "s11" },
137 { rc(arg)|rc(sft)|rc(fpr)|0x1a, "d5" },
138 { rc(arg) | 0x19, "s9" },
139 { rc(arg)|rc(sft)|rc(fpr)|0x18, "d4" },
140 { rc(arg) | 0x17, "s7" },
141 { rc(arg)|rc(sft)|rc(fpr)|0x16, "d3" },
142 { rc(arg) | 0x15, "s5" },
143 { rc(arg)|rc(sft)|rc(fpr)|0x14, "d2" },
144 { rc(arg) | 0x13, "s3" },
145 { rc(arg)|rc(sft)|rc(fpr)|0x12, "d1" },
146 { rc(arg) | 0x11, "s1" },
147 { rc(arg)|rc(sft)|rc(fpr)|0x10, "d0" },
148 { _NOREG, "<none>" },
157 #if defined(__linux__)
162 if ((fp = fopen("/proc/cpuinfo", "r")) != NULL) {
163 while (fgets(buf, sizeof(buf), fp)) {
164 if (strncmp(buf, "CPU architecture:", 17) == 0) {
165 jit_cpu.version = strtol(buf + 17, &ptr, 10);
167 if (*ptr == 'T' || *ptr == 't') {
171 else if (*ptr == 'E' || *ptr == 'e') {
179 else if (strncmp(buf, "Features\t:", 10) == 0) {
180 if ((ptr = strstr(buf + 10, "vfpv")))
181 jit_cpu.vfp = strtol(ptr + 4, NULL, 0);
182 if ((ptr = strstr(buf + 10, "neon")))
184 if ((ptr = strstr(buf + 10, "thumb")))
191 #if defined(__ARM_PCS_VFP)
194 if (!jit_cpu.version)
198 #if defined(__thumb2__)
201 /* armv6t2 todo (software float and thumb2) */
202 if (!jit_cpu.vfp && jit_cpu.thumb)
207 _jit_init(jit_state_t *_jit)
210 static jit_bool_t first = 1;
212 _jitc->reglen = jit_size(_rvs) - 1;
214 /* jit_get_cpu() should have been already called, and only once */
216 /* cause register to never be allocated, because simple
217 * software float only allocates stack space for 8 slots */
218 for (regno = _D8; regno < _D7; regno++)
219 _rvs[regno].spec = 0;
222 for (regno = _S15; regno <= _D0; regno++)
223 _rvs[regno].spec &= ~rc(arg);
230 _jit_prolog(jit_state_t *_jit)
236 assert(jit_regset_cmp_ui(&_jitc->regarg, 0) == 0);
237 jit_regset_set_ui(&_jitc->regsav, 0);
238 offset = _jitc->functions.offset;
239 if (offset >= _jitc->functions.length) {
240 jit_realloc((jit_pointer_t *)&_jitc->functions.ptr,
241 _jitc->functions.length * sizeof(jit_function_t),
242 (_jitc->functions.length + 16) * sizeof(jit_function_t));
243 _jitc->functions.length += 16;
245 _jitc->function = _jitc->functions.ptr + _jitc->functions.offset++;
246 _jitc->function->self.size = stack_framesize;
248 _jitc->function->self.size += 64;
249 _jitc->function->self.argi = _jitc->function->self.argf =
250 _jitc->function->self.alen = 0;
252 /* 8 soft float registers */
253 _jitc->function->self.aoff = -64;
255 _jitc->function->self.aoff = 0;
256 _jitc->function->self.call = jit_call_default;
257 jit_alloc((jit_pointer_t *)&_jitc->function->regoff,
258 _jitc->reglen * sizeof(jit_int32_t));
260 /* _no_link here does not mean the jit_link() call can be removed
262 * _jitc->function->prolog = jit_new_node(jit_code_prolog);
264 _jitc->function->prolog = jit_new_node_no_link(jit_code_prolog);
265 jit_link(_jitc->function->prolog);
266 _jitc->function->prolog->w.w = offset;
267 _jitc->function->epilog = jit_new_node_no_link(jit_code_epilog);
269 * v: offset in blocks vector
270 * w: offset in functions vector
272 _jitc->function->epilog->w.w = offset;
274 jit_regset_new(&_jitc->function->regset);
278 _jit_allocai(jit_state_t *_jit, jit_int32_t length)
280 assert(_jitc->function);
282 case 0: case 1: break;
283 case 2: _jitc->function->self.aoff &= -2; break;
284 case 3: case 4: _jitc->function->self.aoff &= -4; break;
285 default: _jitc->function->self.aoff &= -8; break;
287 _jitc->function->self.aoff -= length;
288 if (!_jitc->realize) {
289 jit_inc_synth_ww(allocai, _jitc->function->self.aoff, length);
292 return (_jitc->function->self.aoff);
296 _jit_allocar(jit_state_t *_jit, jit_int32_t u, jit_int32_t v)
299 assert(_jitc->function);
300 jit_inc_synth_ww(allocar, u, v);
301 if (!_jitc->function->allocar) {
302 _jitc->function->aoffoff = jit_allocai(sizeof(jit_int32_t));
303 _jitc->function->allocar = 1;
305 reg = jit_get_reg(jit_class_gpr);
307 jit_andi(reg, reg, -8);
308 jit_ldxi_i(u, JIT_FP, _jitc->function->aoffoff);
310 jit_addr(JIT_SP, JIT_SP, reg);
311 jit_stxi_i(_jitc->function->aoffoff, JIT_FP, u);
317 _jit_ret(jit_state_t *_jit)
320 assert(_jitc->function);
324 jit_patch_at(instr, _jitc->function->epilog);
329 _jit_retr(jit_state_t *_jit, jit_int32_t u)
331 jit_inc_synth_w(retr, u);
333 jit_movr(JIT_RET, u);
340 _jit_reti(jit_state_t *_jit, jit_word_t u)
342 jit_inc_synth_w(reti, u);
343 jit_movi(JIT_RET, u);
349 _jit_retr_f(jit_state_t *_jit, jit_int32_t u)
351 jit_inc_synth_w(retr_f, u);
354 jit_movr_f(JIT_FRET, u);
360 jit_movr_f_w(JIT_RET, u);
369 _jit_reti_f(jit_state_t *_jit, jit_float32_t u)
371 jit_inc_synth_f(reti_f, u);
373 jit_movi_f(JIT_FRET, u);
375 jit_movi_f_w(JIT_RET, u);
381 _jit_retr_d(jit_state_t *_jit, jit_int32_t u)
383 jit_inc_synth_w(retr_d, u);
386 jit_movr_d(JIT_FRET, u);
392 jit_movr_d_ww(JIT_RET, _R1, u);
401 _jit_reti_d(jit_state_t *_jit, jit_float64_t u)
403 jit_inc_synth_d(reti_d, u);
405 jit_movi_d(JIT_FRET, u);
407 jit_movi_d_ww(JIT_RET, _R1, u);
413 _jit_epilog(jit_state_t *_jit)
415 assert(_jitc->function);
416 assert(_jitc->function->epilog->next == NULL);
417 jit_link(_jitc->function->epilog);
418 _jitc->function = NULL;
422 _jit_arg_register_p(jit_state_t *_jit, jit_node_t *u)
424 if (u->code != jit_code_arg) {
425 if (u->code == jit_code_arg_f) {
427 return (jit_arg_f_reg_p(u->u.w));
430 assert(u->code == jit_code_arg_d);
432 return (jit_arg_d_reg_p(u->u.w));
435 return (jit_arg_reg_p(u->u.w));
439 _jit_make_arg(jit_state_t *_jit, jit_node_t *node)
442 if (jit_arg_reg_p(_jitc->function->self.argi))
443 offset = _jitc->function->self.argi++;
445 offset = _jitc->function->self.size;
446 _jitc->function->self.size += sizeof(jit_word_t);
448 if (node == (jit_node_t *)0)
449 node = jit_new_node(jit_code_arg);
453 node->v.w = ++_jitc->function->self.argn;
459 _jit_make_arg_f(jit_state_t *_jit, jit_node_t *node)
462 if (jit_cpu.abi && !(_jitc->function->self.call & jit_call_varargs)) {
463 if (jit_arg_f_reg_p(_jitc->function->self.argf)) {
464 offset = _jitc->function->self.argf++;
469 if (jit_arg_reg_p(_jitc->function->self.argi)) {
470 offset = _jitc->function->self.argi++;
474 offset = _jitc->function->self.size;
475 _jitc->function->self.size += sizeof(jit_float32_t);
477 if (node == (jit_node_t *)0)
478 node = jit_new_node(jit_code_arg_f);
482 node->v.w = ++_jitc->function->self.argn;
488 _jit_make_arg_d(jit_state_t *_jit, jit_node_t *node)
491 if (jit_cpu.abi && !(_jitc->function->self.call & jit_call_varargs)) {
492 if (jit_arg_d_reg_p(_jitc->function->self.argf)) {
493 if (_jitc->function->self.argf & 1)
494 ++_jitc->function->self.argf;
495 offset = _jitc->function->self.argf;
496 _jitc->function->self.argf += 2;
501 if (_jitc->function->self.argi & 1)
502 ++_jitc->function->self.argi;
503 if (jit_arg_reg_p(_jitc->function->self.argi)) {
504 offset = _jitc->function->self.argi;
505 _jitc->function->self.argi += 2;
509 if (_jitc->function->self.size & 7)
510 _jitc->function->self.size += 4;
511 offset = _jitc->function->self.size;
512 _jitc->function->self.size += sizeof(jit_float64_t);
514 if (node == (jit_node_t *)0)
515 node = jit_new_node(jit_code_arg_d);
519 node->v.w = ++_jitc->function->self.argn;
525 _jit_ellipsis(jit_state_t *_jit)
527 if (_jitc->prepare) {
528 assert(!(_jitc->function->call.call & jit_call_varargs));
529 _jitc->function->call.call |= jit_call_varargs;
530 if (jit_cpu.abi && _jitc->function->call.argf)
534 assert(!(_jitc->function->self.call & jit_call_varargs));
535 _jitc->function->self.call |= jit_call_varargs;
536 if (jit_cpu.abi && _jitc->function->self.argf)
538 /* First 4 stack addresses are always spilled r0-r3 */
539 if (jit_arg_reg_p(_jitc->function->self.argi))
540 _jitc->function->vagp = _jitc->function->self.argi * 4;
542 _jitc->function->vagp = 16;
544 jit_inc_synth(ellipsis);
553 _jit_va_push(jit_state_t *_jit, jit_int32_t u)
555 jit_inc_synth_w(va_push, u);
561 _jit_arg(jit_state_t *_jit)
563 assert(_jitc->function);
564 return (jit_make_arg((jit_node_t*)0));
568 _jit_arg_f(jit_state_t *_jit)
570 assert(_jitc->function);
571 return (jit_make_arg_f((jit_node_t*)0));
575 _jit_arg_d(jit_state_t *_jit)
577 assert(_jitc->function);
578 return (jit_make_arg_d((jit_node_t*)0));
582 _jit_getarg_c(jit_state_t *_jit, jit_int32_t u, jit_node_t *v)
584 assert(v->code == jit_code_arg);
585 jit_inc_synth_wp(getarg_c, u, v);
587 jit_ldxi_c(u, JIT_FP, arg_offset(v->u.w));
588 else if (jit_arg_reg_p(v->u.w))
589 jit_extr_c(u, JIT_RA0 - v->u.w);
591 jit_ldxi_c(u, JIT_FP, v->u.w);
596 _jit_getarg_uc(jit_state_t *_jit, jit_int32_t u, jit_node_t *v)
598 assert(v->code == jit_code_arg);
599 jit_inc_synth_wp(getarg_uc, u, v);
601 jit_ldxi_uc(u, JIT_FP, arg_offset(v->u.w));
602 else if (jit_arg_reg_p(v->u.w))
603 jit_extr_uc(u, JIT_RA0 - v->u.w);
605 jit_ldxi_uc(u, JIT_FP, v->u.w);
610 _jit_getarg_s(jit_state_t *_jit, jit_int32_t u, jit_node_t *v)
612 assert(v->code == jit_code_arg);
613 jit_inc_synth_wp(getarg_s, u, v);
615 jit_ldxi_s(u, JIT_FP, arg_offset(v->u.w));
616 else if (jit_arg_reg_p(v->u.w))
617 jit_extr_s(u, JIT_RA0 - v->u.w);
619 jit_ldxi_s(u, JIT_FP, v->u.w);
624 _jit_getarg_us(jit_state_t *_jit, jit_int32_t u, jit_node_t *v)
626 assert(v->code == jit_code_arg);
627 jit_inc_synth_wp(getarg_us, u, v);
629 jit_ldxi_us(u, JIT_FP, arg_offset(v->u.w));
630 else if (jit_arg_reg_p(v->u.w))
631 jit_extr_us(u, JIT_RA0 - v->u.w);
633 jit_ldxi_us(u, JIT_FP, v->u.w);
638 _jit_getarg_i(jit_state_t *_jit, jit_int32_t u, jit_node_t *v)
640 assert(v->code == jit_code_arg);
641 jit_inc_synth_wp(getarg_i, u, v);
643 jit_ldxi_i(u, JIT_FP, arg_offset(v->u.w));
644 else if (jit_arg_reg_p(v->u.w))
645 jit_movr(u, JIT_RA0 - v->u.w);
647 jit_ldxi_i(u, JIT_FP, v->u.w);
652 _jit_putargr(jit_state_t *_jit, jit_int32_t u, jit_node_t *v)
654 assert(v->code == jit_code_arg);
655 jit_inc_synth_wp(putargr, u, v);
657 jit_stxi(arg_offset(v->u.w), JIT_FP, u);
658 else if (jit_arg_reg_p(v->u.w))
659 jit_movr(JIT_RA0 - v->u.w, u);
661 jit_stxi(v->u.w, JIT_FP, u);
666 _jit_putargi(jit_state_t *_jit, jit_word_t u, jit_node_t *v)
669 assert(v->code == jit_code_arg);
670 jit_inc_synth_wp(putargi, u, v);
672 regno = jit_get_reg(jit_class_gpr);
674 jit_stxi(arg_offset(v->u.w), JIT_FP, regno);
675 jit_unget_reg(regno);
677 else if (jit_arg_reg_p(v->u.w))
678 jit_movi(JIT_RA0 - v->u.w, u);
680 regno = jit_get_reg(jit_class_gpr);
682 jit_stxi(v->u.w, JIT_FP, regno);
683 jit_unget_reg(regno);
689 _jit_getarg_f(jit_state_t *_jit, jit_int32_t u, jit_node_t *v)
691 assert(v->code == jit_code_arg_f);
692 jit_inc_synth_wp(getarg_f, u, v);
693 if (jit_cpu.abi && !(_jitc->function->self.call & jit_call_varargs)) {
694 if (jit_arg_f_reg_p(v->u.w))
695 jit_movr_f(u, JIT_FA0 - v->u.w);
697 jit_ldxi_f(u, JIT_FP, v->u.w);
699 else if (jit_swf_p())
700 jit_ldxi_f(u, JIT_FP, arg_offset(v->u.w));
702 if (jit_arg_reg_p(v->u.w))
703 jit_movr_w_f(u, JIT_RA0 - v->u.w);
705 jit_ldxi_f(u, JIT_FP, v->u.w);
711 _jit_putargr_f(jit_state_t *_jit, jit_int32_t u, jit_node_t *v)
713 assert(v->code == jit_code_arg_f);
714 jit_inc_synth_wp(putargr_f, u, v);
716 if (jit_arg_f_reg_p(v->u.w))
717 jit_movr_f(JIT_FA0 - v->u.w, u);
719 jit_stxi_f(v->u.w, JIT_FP, u);
721 else if (jit_swf_p())
722 jit_stxi_f(arg_offset(v->u.w), JIT_FP, u);
724 if (jit_arg_reg_p(v->u.w))
725 jit_movr_f_w(JIT_RA0 - v->u.w, u);
727 jit_stxi_f(v->u.w, JIT_FP, u);
733 _jit_putargi_f(jit_state_t *_jit, jit_float32_t u, jit_node_t *v)
736 assert(v->code == jit_code_arg_f);
737 jit_inc_synth_fp(putargi_f, u, v);
739 if (jit_arg_f_reg_p(v->u.w))
740 jit_movi_f(JIT_FA0 - v->u.w, u);
742 regno = jit_get_reg(jit_class_fpr);
743 jit_movi_f(regno, u);
744 jit_stxi_f(v->u.w, JIT_FP, regno);
745 jit_unget_reg(regno);
748 else if (jit_swf_p()) {
749 regno = jit_get_reg(jit_class_fpr);
750 jit_movi_f(regno, u);
751 jit_stxi_f(arg_offset(v->u.w), JIT_FP, regno);
752 jit_unget_reg(regno);
755 regno = jit_get_reg(jit_class_fpr);
756 jit_movi_f(regno, u);
757 if (jit_arg_reg_p(v->u.w))
758 jit_movr_f_w(JIT_RA0 - v->u.w, regno);
760 jit_stxi_f(v->u.w, JIT_FP, regno);
761 jit_unget_reg(regno);
767 _jit_getarg_d(jit_state_t *_jit, jit_int32_t u, jit_node_t *v)
769 assert(v->code == jit_code_arg_d);
770 jit_inc_synth_wp(getarg_d, u, v);
771 if (jit_cpu.abi && !(_jitc->function->self.call & jit_call_varargs)) {
772 if (jit_arg_f_reg_p(v->u.w))
773 jit_movr_d(u, JIT_FA0 - v->u.w);
775 jit_ldxi_d(u, JIT_FP, v->u.w);
777 else if (jit_swf_p())
778 jit_ldxi_d(u, JIT_FP, arg_offset(v->u.w));
780 if (jit_arg_reg_p(v->u.w))
781 jit_movr_ww_d(u, JIT_RA0 - v->u.w, JIT_RA0 - (v->u.w + 1));
783 jit_ldxi_d(u, JIT_FP, v->u.w);
789 _jit_putargr_d(jit_state_t *_jit, jit_int32_t u, jit_node_t *v)
791 assert(v->code == jit_code_arg_d);
792 jit_inc_synth_wp(putargr_d, u, v);
794 if (jit_arg_f_reg_p(v->u.w))
795 jit_movr_d(JIT_FA0 - v->u.w, u);
797 jit_stxi_d(v->u.w, JIT_FP, u);
799 else if (jit_swf_p())
800 jit_stxi_d(arg_offset(v->u.w), JIT_FP, u);
802 if (jit_arg_reg_p(v->u.w))
803 jit_movr_d_ww(JIT_RA0 - v->u.w, JIT_RA0 - (v->u.w + 1), u);
805 jit_stxi_d(v->u.w, JIT_FP, u);
811 _jit_putargi_d(jit_state_t *_jit, jit_float64_t u, jit_node_t *v)
814 assert(v->code == jit_code_arg_d);
815 jit_inc_synth_dp(putargi_d, u, v);
817 if (jit_arg_f_reg_p(v->u.w))
818 jit_movi_d(JIT_FA0 - v->u.w, u);
820 regno = jit_get_reg(jit_class_fpr);
821 jit_movi_d(regno, u);
822 jit_stxi_d(v->u.w, JIT_FP, regno);
823 jit_unget_reg(regno);
826 else if (jit_swf_p()) {
827 regno = jit_get_reg(jit_class_fpr);
828 jit_movi_d(regno, u);
829 jit_stxi_d(arg_offset(v->u.w), JIT_FP, regno);
830 jit_unget_reg(regno);
833 regno = jit_get_reg(jit_class_fpr);
834 jit_movi_d(regno, u);
835 if (jit_arg_reg_p(v->u.w))
836 jit_movr_d_ww(JIT_RA0 - v->u.w, JIT_RA0 - (v->u.w + 1), regno);
838 jit_stxi_d(v->u.w, JIT_FP, regno);
839 jit_unget_reg(regno);
845 _jit_pushargr(jit_state_t *_jit, jit_int32_t u)
847 assert(_jitc->function);
848 jit_inc_synth_w(pushargr, u);
850 if (jit_arg_reg_p(_jitc->function->call.argi)) {
851 jit_movr(JIT_RA0 - _jitc->function->call.argi, u);
852 ++_jitc->function->call.argi;
855 jit_stxi(_jitc->function->call.size, JIT_SP, u);
856 _jitc->function->call.size += sizeof(jit_word_t);
862 _jit_pushargi(jit_state_t *_jit, jit_word_t u)
865 assert(_jitc->function);
866 jit_inc_synth_w(pushargi, u);
868 if (jit_arg_reg_p(_jitc->function->call.argi)) {
869 jit_movi(JIT_RA0 - _jitc->function->call.argi, u);
870 ++_jitc->function->call.argi;
873 regno = jit_get_reg(jit_class_gpr);
875 jit_stxi(_jitc->function->call.size, JIT_SP, regno);
876 jit_unget_reg(regno);
877 _jitc->function->call.size += sizeof(jit_word_t);
883 _jit_pushargr_f(jit_state_t *_jit, jit_int32_t u)
885 assert(_jitc->function);
886 jit_inc_synth_w(pushargr_f, u);
888 if (jit_cpu.abi && !(_jitc->function->call.call & jit_call_varargs)) {
889 if (jit_arg_f_reg_p(_jitc->function->call.argf)) {
890 jit_movr_f(JIT_FA0 - _jitc->function->call.argf, u);
891 ++_jitc->function->call.argf;
896 if (jit_arg_reg_p(_jitc->function->call.argi)) {
897 jit_movr_f_w(JIT_RA0 - _jitc->function->call.argi, u);
898 ++_jitc->function->call.argi;
902 jit_stxi_f(_jitc->function->call.size, JIT_SP, u);
903 _jitc->function->call.size += sizeof(jit_word_t);
909 _jit_pushargi_f(jit_state_t *_jit, jit_float32_t u)
912 assert(_jitc->function);
913 jit_inc_synth_f(pushargi_f, u);
915 if (jit_cpu.abi && !(_jitc->function->call.call & jit_call_varargs)) {
916 if (jit_arg_f_reg_p(_jitc->function->call.argf)) {
917 /* cannot jit_movi_f in the argument register because
918 * float arguments are packed, and that would cause
919 * either an assertion in debug mode, or overwritting
921 regno = jit_get_reg(jit_class_fpr);
922 jit_movi_f(regno, u);
923 jit_movr_f(JIT_FA0 - _jitc->function->call.argf, regno);
924 jit_unget_reg(regno);
925 ++_jitc->function->call.argf;
930 if (jit_arg_reg_p(_jitc->function->call.argi)) {
931 jit_movi_f_w(JIT_RA0 - _jitc->function->call.argi, u);
932 ++_jitc->function->call.argi;
936 regno = jit_get_reg(jit_class_fpr);
937 jit_movi_f(regno, u);
938 jit_stxi_f(_jitc->function->call.size, JIT_SP, regno);
939 jit_unget_reg(regno);
940 _jitc->function->call.size += sizeof(jit_word_t);
946 _jit_pushargr_d(jit_state_t *_jit, jit_int32_t u)
948 assert(_jitc->function);
949 jit_inc_synth_w(pushargr_d, u);
951 if (jit_cpu.abi && !(_jitc->function->call.call & jit_call_varargs)) {
952 if (jit_arg_d_reg_p(_jitc->function->call.argf)) {
953 if (_jitc->function->call.argf & 1)
954 ++_jitc->function->call.argf;
955 jit_movr_d(JIT_FA0 - _jitc->function->call.argf, u);
956 _jitc->function->call.argf += 2;
961 if (_jitc->function->call.argi & 1)
962 ++_jitc->function->call.argi;
963 if (jit_arg_reg_p(_jitc->function->call.argi)) {
964 jit_movr_d_ww(JIT_RA0 - _jitc->function->call.argi,
965 JIT_RA0 - (_jitc->function->call.argi + 1),
967 _jitc->function->call.argi += 2;
971 if (_jitc->function->call.size & 7)
972 _jitc->function->call.size += 4;
973 jit_stxi_d(_jitc->function->call.size, JIT_SP, u);
974 _jitc->function->call.size += sizeof(jit_float64_t);
980 _jit_pushargi_d(jit_state_t *_jit, jit_float64_t u)
983 assert(_jitc->function);
984 jit_inc_synth_d(pushargi_d, u);
986 if (jit_cpu.abi && !(_jitc->function->call.call & jit_call_varargs)) {
987 if (jit_arg_d_reg_p(_jitc->function->call.argf)) {
988 if (_jitc->function->call.argf & 1)
989 ++_jitc->function->call.argf;
990 jit_movi_d(JIT_FA0 - _jitc->function->call.argf, u);
991 _jitc->function->call.argf += 2;
996 if (_jitc->function->call.argi & 1)
997 ++_jitc->function->call.argi;
998 if (jit_arg_reg_p(_jitc->function->call.argi)) {
999 jit_movi_d_ww(JIT_RA0 - _jitc->function->call.argi,
1000 JIT_RA0 - (_jitc->function->call.argi + 1),
1002 _jitc->function->call.argi += 2;
1006 if (_jitc->function->call.size & 7)
1007 _jitc->function->call.size += 4;
1008 regno = jit_get_reg(jit_class_fpr);
1009 jit_movi_d(regno, u);
1010 jit_stxi_d(_jitc->function->call.size, JIT_SP, regno);
1011 jit_unget_reg(regno);
1012 _jitc->function->call.size += sizeof(jit_float64_t);
1018 _jit_regarg_p(jit_state_t *_jit, jit_node_t *node, jit_int32_t regno)
1021 spec = jit_class(_rvs[regno].spec);
1022 if (spec & jit_class_arg) {
1023 regno = JIT_RA0 - regno;
1024 if (regno >= 0 && regno < node->v.w)
1026 if (jit_cpu.abi && spec & jit_class_fpr) {
1027 regno = JIT_FA0 - regno;
1028 if (regno >= 0 && regno < node->w.w)
1037 _jit_finishr(jit_state_t *_jit, jit_int32_t r0)
1040 assert(_jitc->function);
1041 jit_inc_synth_w(finishr, r0);
1042 if (_jitc->function->self.alen < _jitc->function->call.size)
1043 _jitc->function->self.alen = _jitc->function->call.size;
1044 node = jit_callr(r0);
1045 node->v.w = _jitc->function->self.argi;
1046 node->w.w = _jitc->function->call.argf;
1047 _jitc->function->call.argi = _jitc->function->call.argf =
1048 _jitc->function->call.size = 0;
1054 _jit_finishi(jit_state_t *_jit, jit_pointer_t i0)
1057 assert(_jitc->function);
1058 jit_inc_synth_w(finishi, (jit_word_t)i0);
1059 if (_jitc->function->self.alen < _jitc->function->call.size)
1060 _jitc->function->self.alen = _jitc->function->call.size;
1061 node = jit_calli(i0);
1062 node->v.w = _jitc->function->call.argi;
1063 node->w.w = _jitc->function->call.argf;
1064 _jitc->function->call.argi = _jitc->function->call.argf =
1065 _jitc->function->call.size = 0;
1072 _jit_retval_c(jit_state_t *_jit, jit_int32_t r0)
1074 jit_inc_synth_w(retval_c, r0);
1075 jit_extr_c(r0, JIT_RET);
1080 _jit_retval_uc(jit_state_t *_jit, jit_int32_t r0)
1082 jit_inc_synth_w(retval_uc, r0);
1083 jit_extr_uc(r0, JIT_RET);
1088 _jit_retval_s(jit_state_t *_jit, jit_int32_t r0)
1090 jit_inc_synth_w(retval_s, r0);
1091 jit_extr_s(r0, JIT_RET);
1096 _jit_retval_us(jit_state_t *_jit, jit_int32_t r0)
1098 jit_inc_synth_w(retval_us, r0);
1099 jit_extr_us(r0, JIT_RET);
1104 _jit_retval_i(jit_state_t *_jit, jit_int32_t r0)
1106 jit_inc_synth_w(retval_i, r0);
1108 jit_movr(r0, JIT_RET);
1113 _jit_retval_f(jit_state_t *_jit, jit_int32_t r0)
1115 jit_inc_synth_w(retval_f, r0);
1118 jit_movr_f(r0, JIT_FRET);
1120 else if (r0 != JIT_RET)
1121 jit_movr_w_f(r0, JIT_RET);
1126 _jit_retval_d(jit_state_t *_jit, jit_int32_t r0)
1128 jit_inc_synth_w(retval_d, r0);
1131 jit_movr_d(r0, JIT_FRET);
1133 else if (r0 != JIT_RET)
1134 jit_movr_ww_d(r0, JIT_RET, _R1);
1139 _emit_code(jit_state_t *_jit)
1150 #if DEVEL_DISASSEMBLER
1155 jit_int32_t info_offset;
1157 jit_int32_t const_offset;
1158 jit_int32_t patch_offset;
1160 #if DEVEL_DISASSEMBLER
1164 _jitc->function = NULL;
1167 jit_reglive_setup();
1169 _jitc->consts.data = NULL;
1170 _jitc->consts.offset = _jitc->consts.length = 0;
1179 undo.const_offset = undo.patch_offset = 0;
1180 # define assert_data(node) /**/
1181 #define case_rr(name, type) \
1182 case jit_code_##name##r##type: \
1183 name##r##type(rn(node->u.w), rn(node->v.w)); \
1185 #define case_rw(name, type) \
1186 case jit_code_##name##i##type: \
1187 name##i##type(rn(node->u.w), node->v.w); \
1189 #define case_vv(name, type) \
1190 case jit_code_##name##r##type: \
1192 swf_##name##r##type(rn(node->u.w), rn(node->v.w)); \
1194 vfp_##name##r##type(rn(node->u.w), rn(node->v.w)); \
1196 #define case_vw(name, type) \
1197 case jit_code_##name##i##type: \
1199 swf_##name##i##type(rn(node->u.w), node->v.w); \
1201 vfp_##name##i##type(rn(node->u.w), node->v.w); \
1203 #define case_wr(name, type) \
1204 case jit_code_##name##i##type: \
1205 name##i##type(node->u.w, rn(node->v.w)); \
1207 #define case_wv(name, type) \
1208 case jit_code_##name##i##type: \
1210 swf_##name##i##type(node->u.w, rn(node->v.w)); \
1212 vfp_##name##i##type(node->u.w, rn(node->v.w)); \
1214 #define case_rrr(name, type) \
1215 case jit_code_##name##r##type: \
1216 name##r##type(rn(node->u.w), \
1217 rn(node->v.w), rn(node->w.w)); \
1219 #define case_rrrr(name, type) \
1220 case jit_code_##name##r##type: \
1221 name##r##type(rn(node->u.q.l), rn(node->u.q.h), \
1222 rn(node->v.w), rn(node->w.w)); \
1224 #define case_vvv(name, type) \
1225 case jit_code_##name##r##type: \
1227 swf_##name##r##type(rn(node->u.w), \
1228 rn(node->v.w), rn(node->w.w)); \
1230 vfp_##name##r##type(rn(node->u.w), \
1231 rn(node->v.w), rn(node->w.w)); \
1233 #define case_rrw(name, type) \
1234 case jit_code_##name##i##type: \
1235 name##i##type(rn(node->u.w), rn(node->v.w), node->w.w); \
1237 #define case_rrrw(name, type) \
1238 case jit_code_##name##i##type: \
1239 name##i##type(rn(node->u.q.l), rn(node->u.q.h), \
1240 rn(node->v.w), node->w.w); \
1242 #define case_vvw(name, type) \
1243 case jit_code_##name##i##type: \
1245 swf_##name##i##type(rn(node->u.w), \
1246 rn(node->v.w), node->w.w); \
1248 vfp_##name##i##type(rn(node->u.w), \
1249 rn(node->v.w), node->w.w); \
1251 #define case_vvf(name) \
1252 case jit_code_##name##i_f: \
1253 assert_data(node); \
1255 swf_##name##i_f(rn(node->u.w), rn(node->v.w), \
1258 vfp_##name##i_f(rn(node->u.w), rn(node->v.w), \
1261 #define case_vvd(name) \
1262 case jit_code_##name##i_d: \
1263 assert_data(node); \
1265 swf_##name##i_d(rn(node->u.w), rn(node->v.w), \
1268 vfp_##name##i_d(rn(node->u.w), rn(node->v.w), \
1271 #define case_wrr(name, type) \
1272 case jit_code_##name##i##type: \
1273 name##i##type(node->u.w, rn(node->v.w), rn(node->w.w)); \
1275 #define case_wvv(name, type) \
1276 case jit_code_##name##i##type: \
1278 swf_##name##i##type(node->u.w, \
1279 rn(node->v.w), rn(node->w.w)); \
1281 vfp_##name##i##type(node->u.w, \
1282 rn(node->v.w), rn(node->w.w)); \
1284 #define case_brr(name, type) \
1285 case jit_code_##name##r##type: \
1287 assert(temp->code == jit_code_label || \
1288 temp->code == jit_code_epilog); \
1289 if (temp->flag & jit_flag_patch) \
1290 name##r##type(temp->u.w, rn(node->v.w), \
1293 word = name##r##type(_jit->pc.w, \
1294 rn(node->v.w), rn(node->w.w)); \
1295 patch(word, node); \
1298 #define case_bvv(name, type) \
1299 case jit_code_##name##r##type: \
1301 assert(temp->code == jit_code_label || \
1302 temp->code == jit_code_epilog); \
1303 if (temp->flag & jit_flag_patch) { \
1305 swf_##name##r##type(temp->u.w, rn(node->v.w), \
1308 vfp_##name##r##type(temp->u.w, rn(node->v.w), \
1313 word = swf_##name##r##type(_jit->pc.w, \
1317 word = vfp_##name##r##type(_jit->pc.w, \
1320 patch(word, node); \
1323 #define case_brw(name, type) \
1324 case jit_code_##name##i##type: \
1326 assert(temp->code == jit_code_label || \
1327 temp->code == jit_code_epilog); \
1328 if (temp->flag & jit_flag_patch) \
1329 name##i##type(temp->u.w, \
1330 rn(node->v.w), node->w.w); \
1332 word = name##i##type(_jit->pc.w, \
1333 rn(node->v.w), node->w.w); \
1334 patch(word, node); \
1337 #define case_bvf(name) \
1338 case jit_code_##name##i_f: \
1340 assert(temp->code == jit_code_label || \
1341 temp->code == jit_code_epilog); \
1342 if (temp->flag & jit_flag_patch) { \
1344 swf_##name##i_f(temp->u.w, rn(node->v.w), \
1347 vfp_##name##i_f(temp->u.w, rn(node->v.w), \
1352 word = swf_##name##i_f(_jit->pc.w, \
1356 word = vfp_##name##i_f(_jit->pc.w, \
1359 patch(word, node); \
1362 #define case_bvd(name) \
1363 case jit_code_##name##i_d: \
1365 assert(temp->code == jit_code_label || \
1366 temp->code == jit_code_epilog); \
1367 if (temp->flag & jit_flag_patch) { \
1369 swf_##name##i_d(temp->u.w, rn(node->v.w), \
1372 vfp_##name##i_d(temp->u.w, rn(node->v.w), \
1377 word = swf_##name##i_d(_jit->pc.w, \
1381 word = vfp_##name##i_d(_jit->pc.w, \
1384 patch(word, node); \
1387 #if DEVEL_DISASSEMBLER
1390 for (node = _jitc->head; node; node = node->next) {
1391 if (_jit->pc.uc >= _jitc->code.end)
1394 #if DEVEL_DISASSEMBLER
1395 node->offset = (jit_uword_t)_jit->pc.w - (jit_uword_t)prevw;
1398 value = jit_classify(node->code);
1399 jit_regarg_set(node, value);
1400 switch (node->code) {
1401 case jit_code_align:
1402 assert(!(node->u.w & (node->u.w - 1)) &&
1403 node->u.w <= sizeof(jit_word_t));
1404 if (node->u.w == sizeof(jit_word_t) &&
1405 (word = _jit->pc.w & (sizeof(jit_word_t) - 1)))
1406 nop(sizeof(jit_word_t) - word);
1408 case jit_code_note: case jit_code_name:
1409 if (must_align_p(node->next))
1411 node->u.w = _jit->pc.w;
1413 case jit_code_label:
1414 if (must_align_p(node->next))
1416 /* remember label is defined */
1417 node->flag |= jit_flag_patch;
1418 node->u.w = _jit->pc.w;
1437 case_rrrr(qmul, _u);
1438 case_rrrw(qmul, _u);
1445 case_rrrr(qdiv, _u);
1446 case_rrrw(qdiv, _u);
1465 case_vv(trunc, _f_i);
1466 case_vv(trunc, _d_i);
1507 if (node->flag & jit_flag_node) {
1509 if (temp->code == jit_code_data ||
1510 (temp->code == jit_code_label &&
1511 (temp->flag & jit_flag_patch)))
1512 movi(rn(node->u.w), temp->u.w);
1514 assert(temp->code == jit_code_label ||
1515 temp->code == jit_code_epilog);
1516 word = movi_p(rn(node->u.w), temp->u.w);
1521 movi(rn(node->u.w), node->v.w);
1565 case_brr(boadd, _u);
1566 case_brw(boadd, _u);
1569 case_brr(bxadd, _u);
1570 case_brw(bxadd, _u);
1573 case_brr(bosub, _u);
1574 case_brw(bosub, _u);
1577 case_brr(bxsub, _u);
1578 case_brw(bxsub, _u);
1605 case jit_code_movi_f:
1608 swf_movi_f(rn(node->u.w), node->v.f);
1610 vfp_movi_f(rn(node->u.w), node->v.f);
1639 case_vvv(unord, _f);
1653 case_bvv(bunlt, _f);
1655 case_bvv(bunle, _f);
1657 case_bvv(buneq, _f);
1659 case_bvv(bunge, _f);
1661 case_bvv(bungt, _f);
1663 case_bvv(bltgt, _f);
1667 case_bvv(bunord, _f);
1691 case jit_code_movi_d:
1694 swf_movi_d(rn(node->u.w), node->v.d);
1696 vfp_movi_d(rn(node->u.w), node->v.d);
1725 case_vvv(unord, _d);
1739 case_bvv(bunlt, _d);
1741 case_bvv(bunle, _d);
1743 case_bvv(buneq, _d);
1745 case_bvv(bunge, _d);
1747 case_bvv(bungt, _d);
1749 case_bvv(bltgt, _d);
1753 case_bvv(bunord, _d);
1756 jmpr(rn(node->u.w));
1760 if (node->flag & jit_flag_node) {
1762 assert(temp->code == jit_code_label ||
1763 temp->code == jit_code_epilog);
1764 if (temp->flag & jit_flag_patch)
1767 word = jmpi_p(_jit->pc.w, 1);
1775 case jit_code_callr:
1776 callr(rn(node->u.w));
1778 case jit_code_calli:
1779 if (node->flag & jit_flag_node) {
1781 assert(temp->code == jit_code_label ||
1782 temp->code == jit_code_epilog);
1783 if (temp->flag & jit_flag_patch)
1786 word = calli_p(_jit->pc.w);
1793 case jit_code_prolog:
1794 _jitc->function = _jitc->functions.ptr + node->w.w;
1796 undo.word = _jit->pc.w;
1797 #if DEVEL_DISASSEMBLER
1800 undo.data = _jitc->consts.data;
1801 undo.thumb = _jitc->thumb;
1802 undo.const_offset = _jitc->consts.offset;
1803 undo.patch_offset = _jitc->patches.offset;
1805 if (_jitc->data_info.ptr)
1806 undo.info_offset = _jitc->data_info.offset;
1812 case jit_code_epilog:
1813 assert(_jitc->function == _jitc->functions.ptr + node->w.w);
1815 for (temp = undo.node->next;
1816 temp != node; temp = temp->next) {
1817 if (temp->code == jit_code_label ||
1818 temp->code == jit_code_epilog)
1819 temp->flag &= ~jit_flag_patch;
1821 temp->flag &= ~jit_flag_patch;
1823 _jit->pc.w = undo.word;
1824 #if DEVEL_DISASSEMBLER
1827 invalidate_consts();
1828 _jitc->consts.data = undo.data;
1829 _jitc->thumb = undo.thumb;
1830 _jitc->consts.offset = undo.const_offset;
1831 _jitc->patches.offset = undo.patch_offset;
1833 if (_jitc->data_info.ptr)
1834 _jitc->data_info.offset = undo.info_offset;
1836 goto restart_function;
1838 /* remember label is defined */
1839 node->flag |= jit_flag_patch;
1840 node->u.w = _jit->pc.w;
1842 _jitc->function = NULL;
1845 case jit_code_movr_w_f:
1847 swf_movr_f(rn(node->u.w), rn(node->v.w));
1849 vfp_movr_f(rn(node->u.w), rn(node->v.w));
1851 case jit_code_movr_f_w:
1853 swf_movr_f(rn(node->u.w), rn(node->v.w));
1855 vfp_movr_f(rn(node->u.w), rn(node->v.w));
1857 case jit_code_movi_f_w:
1860 swf_movi_f(rn(node->u.w), node->v.f);
1862 vfp_movi_f(rn(node->u.w), node->v.f);
1864 case jit_code_movr_ww_d:
1866 swf_movr_d(rn(node->u.w), rn(node->v.w));
1868 vfp_movr_d(rn(node->u.w), rn(node->v.w));
1870 case jit_code_movr_d_ww:
1872 swf_movr_d(rn(node->u.w), rn(node->w.w));
1874 vfp_movr_d(rn(node->u.w), rn(node->w.w));
1876 case jit_code_movi_d_ww:
1879 swf_movi_d(rn(node->u.w), node->w.d);
1881 vfp_movi_d(rn(node->u.w), node->w.d);
1883 case jit_code_va_start:
1884 vastart(rn(node->u.w));
1886 case jit_code_va_arg:
1887 vaarg(rn(node->u.w), rn(node->v.w));
1889 case jit_code_va_arg_d:
1891 swf_vaarg_d(rn(node->u.w), rn(node->v.w));
1893 vfp_vaarg_d(rn(node->u.w), rn(node->v.w));
1895 case jit_code_live: case jit_code_ellipsis:
1896 case jit_code_va_push:
1897 case jit_code_allocai: case jit_code_allocar:
1899 case jit_code_arg_f: case jit_code_arg_d:
1900 case jit_code_va_end:
1902 case jit_code_retr: case jit_code_reti:
1903 case jit_code_retr_f: case jit_code_reti_f:
1904 case jit_code_retr_d: case jit_code_reti_d:
1905 case jit_code_getarg_c: case jit_code_getarg_uc:
1906 case jit_code_getarg_s: case jit_code_getarg_us:
1907 case jit_code_getarg_i:
1908 case jit_code_getarg_f: case jit_code_getarg_d:
1909 case jit_code_putargr: case jit_code_putargi:
1910 case jit_code_putargr_f: case jit_code_putargi_f:
1911 case jit_code_putargr_d: case jit_code_putargi_d:
1912 case jit_code_pushargr: case jit_code_pushargi:
1913 case jit_code_pushargr_f: case jit_code_pushargi_f:
1914 case jit_code_pushargr_d: case jit_code_pushargi_d:
1915 case jit_code_retval_c: case jit_code_retval_uc:
1916 case jit_code_retval_s: case jit_code_retval_us:
1917 case jit_code_retval_i:
1918 case jit_code_retval_f: case jit_code_retval_d:
1919 case jit_code_prepare:
1920 case jit_code_finishr: case jit_code_finishi:
1925 jit_regarg_clr(node, value);
1926 assert(_jitc->regarg == 0 && _jitc->synth == 0);
1927 /* update register live state */
1930 if (_jitc->consts.length &&
1931 (_jit->pc.uc - _jitc->consts.data >= 3968 ||
1932 (jit_uword_t)_jit->pc.uc -
1933 (jit_uword_t)_jitc->consts.patches[0] >= 3968)) {
1934 /* longest sequence should be 64 bytes, but preventively
1935 * do not let it go past 128 remaining bytes before a flush */
1937 node->next->code != jit_code_jmpi &&
1938 node->next->code != jit_code_jmpr &&
1939 node->next->code != jit_code_epilog) {
1940 /* insert a jump, flush constants and continue */
1942 assert(!jit_thumb_p());
1945 patch_at(arm_patch_jump, word, _jit->pc.w);
1970 for (offset = 0; offset < _jitc->patches.offset; offset++) {
1971 assert(_jitc->patches.ptr[offset].kind & arm_patch_node);
1972 node = _jitc->patches.ptr[offset].node;
1973 word = _jitc->patches.ptr[offset].inst;
1974 if (!jit_thumb_p() &&
1975 (node->code == jit_code_movi || node->code == jit_code_calli)) {
1976 /* calculate where to patch word */
1977 value = *(jit_int32_t *)word;
1978 assert((value & 0x0f700000) == ARM_LDRI);
1979 /* offset may become negative (-4) if last instruction
1980 * before unconditional branch and data following
1981 * FIXME can this cause issues in the preprocessor prefetch
1982 * or something else? should not, as the constants are after
1983 * an unconditional jump */
1984 if (value & ARM_P) value = value & 0x00000fff;
1985 else value = -(value & 0x00000fff);
1986 word = word + 8 + value;
1988 value = node->code == jit_code_movi ? node->v.n->u.w : node->u.n->u.w;
1989 patch_at(_jitc->patches.ptr[offset].kind & ~arm_patch_node, word, value);
1992 jit_flush(_jit->code.ptr, _jit->pc.uc);
1994 return (_jit->code.ptr);
1998 # include "jit_rewind.c"
1999 # include "jit_arm-cpu.c"
2000 # include "jit_arm-swf.c"
2001 # include "jit_arm-vfp.c"
2005 jit_flush(void *fptr, void *tptr)
2007 #if defined(__GNUC__)
2008 jit_uword_t i, f, t, s;
2010 s = sysconf(_SC_PAGE_SIZE);
2011 f = (jit_uword_t)fptr & -s;
2012 t = (((jit_uword_t)tptr) + s - 1) & -s;
2013 for (i = f; i < t; i += s)
2014 __clear_cache((void *)i, (void *)(i + s));
2019 _emit_ldxi(jit_state_t *_jit, jit_int32_t r0, jit_int32_t r1, jit_word_t i0)
2021 ldxi_i(rn(r0), rn(r1), i0);
2025 _emit_stxi(jit_state_t *_jit, jit_word_t i0, jit_int32_t r0, jit_int32_t r1)
2027 stxi_i(i0, rn(r0), rn(r1));
2031 _emit_ldxi_d(jit_state_t *_jit, jit_int32_t r0, jit_int32_t r1, jit_word_t i0)
2034 swf_ldxi_d(rn(r0), rn(r1), i0);
2036 vfp_ldxi_d(rn(r0), rn(r1), i0);
2040 _emit_stxi_d(jit_state_t *_jit, jit_word_t i0, jit_int32_t r0, jit_int32_t r1)
2043 swf_stxi_d(i0, rn(r0), rn(r1));
2045 vfp_stxi_d(i0, rn(r0), rn(r1));
2049 _jit_get_reg_pair(jit_state_t *_jit)
2051 /* bypass jit_get_reg() with argument or'ed with jit_class_chk
2052 * and try to find an consecutive, even free register pair, or
2053 * return JIT_NOREG if fail, as the cost of spills is greater
2054 * than splitting a double load/store in two operations. */
2055 if (jit_reg_free_p(_R0) && jit_reg_free_p(_R1)) {
2056 jit_regset_setbit(&_jitc->regarg, _R0);
2057 jit_regset_setbit(&_jitc->regarg, _R1);
2060 if (jit_reg_free_p(_R2) && jit_reg_free_p(_R3)) {
2061 jit_regset_setbit(&_jitc->regarg, _R2);
2062 jit_regset_setbit(&_jitc->regarg, _R3);
2065 if (jit_reg_free_p(_R4) && jit_reg_free_p(_R5)) {
2066 jit_regset_setbit(&_jitc->regarg, _R4);
2067 jit_regset_setbit(&_jitc->regarg, _R5);
2070 if (jit_reg_free_p(_R6) && jit_reg_free_p(_R7)) {
2071 jit_regset_setbit(&_jitc->regarg, _R6);
2072 jit_regset_setbit(&_jitc->regarg, _R7);
2075 if (jit_reg_free_p(_R8) && jit_reg_free_p(_R9)) {
2076 jit_regset_setbit(&_jitc->regarg, _R8);
2077 jit_regset_setbit(&_jitc->regarg, _R9);
2084 _jit_unget_reg_pair(jit_state_t *_jit, jit_int32_t reg)
2088 case _R0: jit_unget_reg(_R1); break;
2089 case _R2: jit_unget_reg(_R3); break;
2090 case _R4: jit_unget_reg(_R5); break;
2091 case _R6: jit_unget_reg(_R7); break;
2092 case _R8: jit_unget_reg(_R9); break;
2097 /* A prolog must be aligned at mod 4 bytes boundary.
2098 * This condition was not being required to be tested by
2099 * accident previously, but with the jit_frame and jit_tramp
2100 * code it is required */
2102 _must_align_p(jit_state_t *_jit, jit_node_t *node)
2104 if (jit_thumb_p() && (_jit->pc.w & 3)) {
2105 for (; node; node = node->next) {
2106 switch (node->code) {
2109 case jit_code_label:
2111 case jit_code_prolog:
2122 _load_const(jit_state_t *_jit, jit_bool_t uniq, jit_int32_t r0, jit_word_t i0)
2131 assert(!jit_thumb_p());
2133 /* use zero, a valid directly encoded immediate, to avoid the
2134 * need of a bitmask to know what offsets will be patched, so
2135 * that comparison will always fail for constants that cannot
2139 /* Actually, code is (currently at least) not self modifying,
2140 * so, any value reachable backwards is valid as a constant. */
2142 /* FIXME a quickly updateable/mutable hash table could be
2143 * better here, but most times only a few comparisons
2147 /* search in previous constant pool */
2148 if ((data = (jit_int32_t *)_jitc->consts.data)) {
2149 w = (jit_word_t)data;
2150 /* maximum backwards offset */
2151 base = (_jit->pc.w + 8) - 4092;
2153 /* can scan all possible available backward constants */
2156 base = (base - w) >> 2;
2157 size = _jitc->consts.size >> 2;
2158 for (offset = size - 1; offset >= base; offset--) {
2159 if (data[offset] == i0) {
2160 w = (jit_word_t)(data + offset);
2161 d = (_jit->pc.w + 8) - w;
2162 LDRIN(r0, _R15_REGNO, d);
2171 _jitc->consts.patches[_jitc->consts.offset++] = _jit->pc.w;
2172 /* (probably) positive forward offset */
2173 LDRI(r0, _R15_REGNO, 0);
2176 /* search already requested values */
2177 for (offset = 0; offset < _jitc->consts.length; offset++) {
2178 if (_jitc->consts.values[offset] == i0) {
2179 _jitc->consts.patches[_jitc->consts.offset++] = offset;
2186 /* cannot run out of space because of limited range
2187 * but assert anyway to catch logic errors */
2188 assert(_jitc->consts.length < 1024);
2189 assert(_jitc->consts.offset < 2048);
2191 _jitc->consts.patches[_jitc->consts.offset++] = _jitc->consts.length;
2192 _jitc->consts.values[_jitc->consts.length++] = i0;
2196 _flush_consts(jit_state_t *_jit)
2201 /* if no forward constants */
2202 if (!_jitc->consts.length)
2204 assert(!jit_thumb_p());
2206 _jitc->consts.data = _jit->pc.uc;
2207 _jitc->consts.size = _jitc->consts.length << 2;
2208 /* FIXME check will not overrun, otherwise, need to reallocate
2209 * code buffer and start over */
2210 jit_memcpy(_jitc->consts.data, _jitc->consts.values, _jitc->consts.size);
2211 _jit->pc.w += _jitc->consts.size;
2214 if (_jitc->data_info.ptr) {
2215 if (_jitc->data_info.offset >= _jitc->data_info.length) {
2216 jit_realloc((jit_pointer_t *)&_jitc->data_info.ptr,
2217 _jitc->data_info.length * sizeof(jit_data_info_t),
2218 (_jitc->data_info.length + 1024) *
2219 sizeof(jit_data_info_t));
2220 _jitc->data_info.length += 1024;
2222 _jitc->data_info.ptr[_jitc->data_info.offset].code = word;
2223 _jitc->data_info.ptr[_jitc->data_info.offset].length = _jitc->consts.size;
2224 ++_jitc->data_info.offset;
2228 for (offset = 0; offset < _jitc->consts.offset; offset += 2)
2229 patch_at(arm_patch_load, _jitc->consts.patches[offset],
2230 word + (_jitc->consts.patches[offset + 1] << 2));
2231 _jitc->consts.length = _jitc->consts.offset = 0;
2234 /* to be called if needing to start over a function */
2236 _invalidate_consts(jit_state_t *_jit)
2238 /* if no forward constants */
2239 if (_jitc->consts.length)
2240 _jitc->consts.length = _jitc->consts.offset = 0;
2244 _patch(jit_state_t *_jit, jit_word_t instr, jit_node_t *node)
2249 assert(node->flag & jit_flag_node);
2250 if (node->code == jit_code_movi) {
2251 flag = node->v.n->flag;
2252 kind = arm_patch_word;
2255 flag = node->u.n->flag;
2256 if (node->code == jit_code_calli ||
2257 (node->code == jit_code_jmpi && !(node->flag & jit_flag_node)))
2258 kind = arm_patch_word;
2260 kind = arm_patch_jump;
2262 assert(!(flag & jit_flag_patch));
2263 kind |= arm_patch_node;
2264 if (_jitc->patches.offset >= _jitc->patches.length) {
2265 jit_realloc((jit_pointer_t *)&_jitc->patches.ptr,
2266 _jitc->patches.length * sizeof(jit_patch_t),
2267 (_jitc->patches.length + 1024) * sizeof(jit_patch_t));
2268 _jitc->patches.length += 1024;
2270 _jitc->patches.ptr[_jitc->patches.offset].kind = kind;
2271 _jitc->patches.ptr[_jitc->patches.offset].inst = instr;
2272 _jitc->patches.ptr[_jitc->patches.offset].node = node;
2273 ++_jitc->patches.offset;