X-Git-Url: https://notaz.gp2x.de/cgi-bin/gitweb.cgi?a=blobdiff_plain;f=deps%2Flightning%2Flib%2Fjit_aarch64.c;h=bc7880098fcf9cb460901d33449a5a0a6086edbe;hb=016c6e93f6db684211f5c8b05433cb500715ba50;hp=b54d0070398e97bd03188705c4abbb06a4f8f813;hpb=c0c162422385a60ea7c8fa1dfe439e83e0a13d88;p=pcsx_rearmed.git diff --git a/deps/lightning/lib/jit_aarch64.c b/deps/lightning/lib/jit_aarch64.c index b54d0070..bc788009 100644 --- a/deps/lightning/lib/jit_aarch64.c +++ b/deps/lightning/lib/jit_aarch64.c @@ -1,5 +1,5 @@ /* - * Copyright (C) 2013-2022 Free Software Foundation, Inc. + * Copyright (C) 2013-2023 Free Software Foundation, Inc. * * This file is part of GNU lightning. * @@ -17,9 +17,16 @@ * Paulo Cesar Pereira de Andrade */ +/* callee save + * align16(lr+fp+x19+x2[0-8]+v8+v9+v1[0-15]) */ +#define stack_framesize 160 + #define jit_arg_reg_p(i) ((i) >= 0 && (i) < 8) #define jit_arg_f_reg_p(i) ((i) >= 0 && (i) < 8) +#if __APPLE__ +typedef jit_pointer_t jit_va_list_t; +#else typedef struct jit_qreg { jit_float64_t l; jit_float64_t h; @@ -52,10 +59,13 @@ typedef struct jit_va_list { jit_qreg_t q6; jit_qreg_t q7; } jit_va_list_t; +#endif /* * Prototypes */ +#define compute_framesize() _compute_framesize(_jit) +static void _compute_framesize(jit_state_t*); #define patch(instr, node) _patch(_jit, instr, node) static void _patch(jit_state_t*,jit_word_t,jit_node_t*); @@ -72,7 +82,11 @@ extern void __clear_cache(void *, void *); */ jit_register_t _rvs[] = { { rc(gpr) | 0x08, "x8" }, +#if __APPLE__ + { 0x12, "x18" }, +#else { rc(gpr) | 0x12, "x18" }, +#endif { rc(gpr) | 0x11, "x17" }, { rc(gpr) | 0x10, "x16" }, { rc(gpr) | 0x09, "x9" }, @@ -138,6 +152,14 @@ jit_register_t _rvs[] = { { _NOREG, "" }, }; +static jit_int32_t iregs[] = { + _R19, _R20, _R21, _R22, _R23, _R24, _R25, _R26, _R27, _R28 +}; + +static jit_int32_t fregs[] = { + _V8, _V9, _V10, _V11, _V12, _V13, _V14, _V15 +}; + /* * Implementation */ @@ -198,6 +220,7 @@ jit_int32_t _jit_allocai(jit_state_t *_jit, jit_int32_t length) { assert(_jitc->function); + jit_check_frame(); switch (length) { case 0: case 1: break; case 2: _jitc->function->self.aoff &= -2; break; @@ -258,20 +281,18 @@ _jit_ret(jit_state_t *_jit) } void -_jit_retr(jit_state_t *_jit, jit_int32_t u) +_jit_retr(jit_state_t *_jit, jit_int32_t u, jit_code_t code) { - jit_inc_synth_w(retr, u); - if (JIT_RET != u) - jit_movr(JIT_RET, u); - jit_live(JIT_RET); + jit_code_inc_synth_w(code, u); + jit_movr(JIT_RET, u); jit_ret(); jit_dec_synth(); } void -_jit_reti(jit_state_t *_jit, jit_word_t u) +_jit_reti(jit_state_t *_jit, jit_word_t u, jit_code_t code) { - jit_inc_synth_w(reti, u); + jit_code_inc_synth_w(code, u); jit_movi(JIT_RET, u); jit_ret(); jit_dec_synth(); @@ -331,7 +352,7 @@ _jit_epilog(jit_state_t *_jit) jit_bool_t _jit_arg_register_p(jit_state_t *_jit, jit_node_t *u) { - if (u->code == jit_code_arg) + if (u->code >= jit_code_arg_c && u->code <= jit_code_arg) return (jit_arg_reg_p(u->u.w)); assert(u->code == jit_code_arg_f || u->code == jit_code_arg_d); return (jit_arg_f_reg_p(u->u.w)); @@ -341,6 +362,7 @@ void _jit_ellipsis(jit_state_t *_jit) { jit_inc_synth(ellipsis); + jit_check_frame(); if (_jitc->prepare) { jit_link_prepare(); assert(!(_jitc->function->call.call & jit_call_varargs)); @@ -351,6 +373,7 @@ _jit_ellipsis(jit_state_t *_jit) assert(!(_jitc->function->self.call & jit_call_varargs)); _jitc->function->self.call |= jit_call_varargs; +#if !__APPLE_ /* Allocate va_list like object in the stack, * with enough space to save all argument * registers, and use fixed offsets for them. */ @@ -367,6 +390,7 @@ _jit_ellipsis(jit_state_t *_jit) _jitc->function->vafp = (8 - _jitc->function->self.argf) * -16; else _jitc->function->vafp = 0; +#endif } jit_dec_synth(); } @@ -380,7 +404,7 @@ _jit_va_push(jit_state_t *_jit, jit_int32_t u) } jit_node_t * -_jit_arg(jit_state_t *_jit) +_jit_arg(jit_state_t *_jit, jit_code_t code) { jit_node_t *node; jit_int32_t offset; @@ -389,10 +413,22 @@ _jit_arg(jit_state_t *_jit) if (jit_arg_reg_p(_jitc->function->self.argi)) offset = _jitc->function->self.argi++; else { +#if PACKED_STACK || STRONG_TYPE_CHECKING + assert(code >= jit_code_arg_c && code <= jit_code_arg); +#endif +#if PACKED_STACK + _jitc->function->self.size += + _jitc->function->self.size & ((1 << (code - jit_code_arg_c)) - 1); +#endif offset = _jitc->function->self.size; +#if PACKED_STACK + _jitc->function->self.size += 1 << (code - jit_code_arg_c); +#else _jitc->function->self.size += sizeof(jit_word_t); +#endif + jit_check_frame(); } - node = jit_new_node_ww(jit_code_arg, offset, + node = jit_new_node_ww(code, offset, ++_jitc->function->self.argn); jit_link_prolog(); return (node); @@ -408,8 +444,17 @@ _jit_arg_f(jit_state_t *_jit) if (jit_arg_f_reg_p(_jitc->function->self.argf)) offset = _jitc->function->self.argf++; else { +#if PACKED_STACK + _jitc->function->self.size += + _jitc->function->self.size & (sizeof(jit_float32_t) - 1); +#endif offset = _jitc->function->self.size; +#if PACKED_STACK + _jitc->function->self.size += sizeof(jit_float32_t); +#else _jitc->function->self.size += sizeof(jit_word_t); +#endif + jit_check_frame(); } node = jit_new_node_ww(jit_code_arg_f, offset, ++_jitc->function->self.argn); @@ -427,8 +472,13 @@ _jit_arg_d(jit_state_t *_jit) if (jit_arg_f_reg_p(_jitc->function->self.argf)) offset = _jitc->function->self.argf++; else { +#if PACKED_STACK + _jitc->function->self.size += + _jitc->function->self.size & (sizeof(jit_float64_t) - 1); +#endif offset = _jitc->function->self.size; - _jitc->function->self.size += sizeof(jit_word_t); + _jitc->function->self.size += sizeof(jit_float64_t); + jit_check_frame(); } node = jit_new_node_ww(jit_code_arg_d, offset, ++_jitc->function->self.argn); @@ -439,111 +489,235 @@ _jit_arg_d(jit_state_t *_jit) void _jit_getarg_c(jit_state_t *_jit, jit_int32_t u, jit_node_t *v) { - assert(v->code == jit_code_arg); + assert_arg_type(v->code, jit_code_arg_c); jit_inc_synth_wp(getarg_c, u, v); - if (jit_arg_reg_p(v->u.w)) + if (jit_arg_reg_p(v->u.w)) { +#if PACKED_STACK + jit_movr(u, JIT_RA0 - v->u.w); +#else jit_extr_c(u, JIT_RA0 - v->u.w); - else - jit_ldxi_c(u, JIT_FP, v->u.w); +#endif + } + else { + jit_node_t *node = jit_ldxi_c(u, JIT_FP, v->u.w); + jit_link_alist(node); + } jit_dec_synth(); } void _jit_getarg_uc(jit_state_t *_jit, jit_int32_t u, jit_node_t *v) { - assert(v->code == jit_code_arg); + assert_arg_type(v->code, jit_code_arg_c); jit_inc_synth_wp(getarg_uc, u, v); - if (jit_arg_reg_p(v->u.w)) + if (jit_arg_reg_p(v->u.w)) { +#if PACKED_STACK + jit_movr(u, JIT_RA0 - v->u.w); +#else jit_extr_uc(u, JIT_RA0 - v->u.w); - else - jit_ldxi_uc(u, JIT_FP, v->u.w); +#endif + } + else { + jit_node_t *node = jit_ldxi_uc(u, JIT_FP, v->u.w); + jit_link_alist(node); + } jit_dec_synth(); } void _jit_getarg_s(jit_state_t *_jit, jit_int32_t u, jit_node_t *v) { - assert(v->code == jit_code_arg); + assert_arg_type(v->code, jit_code_arg_s); jit_inc_synth_wp(getarg_s, u, v); - if (jit_arg_reg_p(v->u.w)) + if (jit_arg_reg_p(v->u.w)) { +#if PACKED_STACK + jit_movr(u, JIT_RA0 - v->u.w); +#else jit_extr_s(u, JIT_RA0 - v->u.w); - else - jit_ldxi_s(u, JIT_FP, v->u.w); +#endif + } + else { + jit_node_t *node = jit_ldxi_s(u, JIT_FP, v->u.w); + jit_link_alist(node); + } jit_dec_synth(); } void _jit_getarg_us(jit_state_t *_jit, jit_int32_t u, jit_node_t *v) { - assert(v->code == jit_code_arg); + assert_arg_type(v->code, jit_code_arg_s); jit_inc_synth_wp(getarg_us, u, v); - if (jit_arg_reg_p(v->u.w)) + if (jit_arg_reg_p(v->u.w)) { +#if PACKED_STACK + jit_movr(u, JIT_RA0 - v->u.w); +#else jit_extr_us(u, JIT_RA0 - v->u.w); - else - jit_ldxi_us(u, JIT_FP, v->u.w); +#endif + } + else { + jit_node_t *node = jit_ldxi_us(u, JIT_FP, v->u.w); + jit_link_alist(node); + } jit_dec_synth(); } void _jit_getarg_i(jit_state_t *_jit, jit_int32_t u, jit_node_t *v) { - assert(v->code == jit_code_arg); + assert_arg_type(v->code, jit_code_arg_i); jit_inc_synth_wp(getarg_i, u, v); - if (jit_arg_reg_p(v->u.w)) + if (jit_arg_reg_p(v->u.w)) { +#if PACKED_STACK || __WORDSIZE == 32 + jit_movr(u, JIT_RA0 - v->u.w); +#else jit_extr_i(u, JIT_RA0 - v->u.w); - else - jit_ldxi_i(u, JIT_FP, v->u.w); +#endif + } + else { + jit_node_t *node = jit_ldxi_i(u, JIT_FP, v->u.w); + jit_link_alist(node); + } jit_dec_synth(); } +#if __WORDSIZE == 64 void _jit_getarg_ui(jit_state_t *_jit, jit_int32_t u, jit_node_t *v) { - assert(v->code == jit_code_arg); + assert_arg_type(v->code, jit_code_arg_i); jit_inc_synth_wp(getarg_ui, u, v); - if (jit_arg_reg_p(v->u.w)) + if (jit_arg_reg_p(v->u.w)) { +#if PACKED_STACK + jit_movr(u, JIT_RA0 - v->u.w); +#else jit_extr_ui(u, JIT_RA0 - v->u.w); - else - jit_ldxi_ui(u, JIT_FP, v->u.w); +#endif + } + else { + jit_node_t *node = jit_ldxi_ui(u, JIT_FP, v->u.w); + jit_link_alist(node); + } jit_dec_synth(); } void _jit_getarg_l(jit_state_t *_jit, jit_int32_t u, jit_node_t *v) { - assert(v->code == jit_code_arg); + assert_arg_type(v->code, jit_code_arg_l); jit_inc_synth_wp(getarg_l, u, v); if (jit_arg_reg_p(v->u.w)) jit_movr(u, JIT_RA0 - v->u.w); - else - jit_ldxi_l(u, JIT_FP, v->u.w); + else { + jit_node_t *node = jit_ldxi_l(u, JIT_FP, v->u.w); + jit_link_alist(node); + } jit_dec_synth(); } +#endif void -_jit_putargr(jit_state_t *_jit, jit_int32_t u, jit_node_t *v) +_jit_putargr(jit_state_t *_jit, jit_int32_t u, jit_node_t *v, jit_code_t code) { - assert(v->code == jit_code_arg); - jit_inc_synth_wp(putargr, u, v); - if (jit_arg_reg_p(v->u.w)) - jit_movr(JIT_RA0 - v->u.w, u); - else - jit_stxi(v->u.w, JIT_FP, u); + assert_putarg_type(code, v->code); + jit_code_inc_synth_wp(code, u, v); + if (jit_arg_reg_p(v->u.w)) { + jit_int32_t regno = JIT_RA0 - v->u.w; +#if PACKED_STACK + switch (code) { + case jit_code_putargr_c: jit_extr_c(regno, u); break; + case jit_code_putargr_uc: jit_extr_uc(regno, u); break; + case jit_code_putargr_s: jit_extr_s(regno, u); break; + case jit_code_putargr_us: jit_extr_us(regno, u); break; +# if __WORDISZE == 32 + case jit_code_putargr_i: jit_movr(regno, u); break; +# else + case jit_code_putargr_i: jit_extr_i(regno, u); break; + case jit_code_putargr_ui: jit_extr_ui(regno, u); break; + case jit_code_putargr_l: jit_movr(regno, u); break; +# endif + default: abort(); break; + } +#else + jit_movr(regno, u); +#endif + } + else { + jit_node_t *node; +#if PACKED_STACK + switch (code) { + case jit_code_putargr_c: case jit_code_putargr_uc: + node = jit_stxi_c(v->u.w, JIT_FP, u); break; + case jit_code_putargr_s: case jit_code_putargr_us: + node = jit_stxi_s(v->u.w, JIT_FP, u); break; +# if __WORDSIZE == 32 + case jit_code_putargr_i: + node = jit_stxi(v->u.w, JIT_FP, u); break; +# else + case jit_code_putargr_i: case jit_code_putargr_ui: + node = jit_stxi_i(v->u.w, JIT_FP, u); break; + case jit_code_putargr_l: + node = jit_stxi(v->u.w, JIT_FP, u); break; +# endif + default: abort(); break; + } +#else + node = jit_stxi(v->u.w, JIT_FP, u); +#endif + jit_link_alist(node); + } jit_dec_synth(); } void -_jit_putargi(jit_state_t *_jit, jit_word_t u, jit_node_t *v) +_jit_putargi(jit_state_t *_jit, jit_word_t u, jit_node_t *v, jit_code_t code) { jit_int32_t regno; - assert(v->code == jit_code_arg); - jit_inc_synth_wp(putargi, u, v); + assert_putarg_type(code, v->code); + jit_code_inc_synth_wp(code, u, v); +#if PACKED_STACK + switch (code) { + case jit_code_putargi_c: u = (jit_int8_t)u; break; + case jit_code_putargi_uc: u = (jit_uint8_t)u; break; + case jit_code_putargi_s: u = (jit_int16_t)u; break; + case jit_code_putargi_us: u = (jit_uint16_t)u; break; +# if __WORDSIZE == 32 + case jit_code_putargi_i: break; +# else + case jit_code_putargi_i: u = (jit_int32_t)u; break; + case jit_code_putargi_ui: u = (jit_uint32_t)u; break; + case jit_code_putargi_l: break; +# endif + default: abort(); break; + } +#endif if (jit_arg_reg_p(v->u.w)) jit_movi(JIT_RA0 - v->u.w, u); else { + jit_node_t *node; regno = jit_get_reg(jit_class_gpr); jit_movi(regno, u); - jit_stxi(v->u.w, JIT_FP, regno); +#if PACKED_STACK + switch (code) { + case jit_code_putargi_c: case jit_code_putargi_uc: + node = jit_stxi_c(v->u.w, JIT_FP, regno); break; + case jit_code_putargi_s: case jit_code_putargi_us: + node = jit_stxi_s(v->u.w, JIT_FP, regno); break; +# if __WORDSIZE == 32 + case jit_code_putargi_i: + node = jit_stxi(v->u.w, JIT_FP, regno); break; +# else + case jit_code_putargi_i: case jit_code_putargi_ui: + node = jit_stxi_i(v->u.w, JIT_FP, regno); break; + case jit_code_putargi_l: + node = jit_stxi(v->u.w, JIT_FP, regno); break; +# endif + default: abort(); break; + } +#else + node = jit_stxi(v->u.w, JIT_FP, regno); +#endif + jit_link_alist(node); jit_unget_reg(regno); } jit_dec_synth(); @@ -556,8 +730,10 @@ _jit_getarg_f(jit_state_t *_jit, jit_int32_t u, jit_node_t *v) jit_inc_synth_wp(getarg_f, u, v); if (jit_arg_reg_p(v->u.w)) jit_movr_f(u, JIT_FA0 - v->u.w); - else - jit_ldxi_f(u, JIT_FP, v->u.w); + else { + jit_node_t *node = jit_ldxi_f(u, JIT_FP, v->u.w); + jit_link_alist(node); + } jit_dec_synth(); } @@ -568,8 +744,10 @@ _jit_putargr_f(jit_state_t *_jit, jit_int32_t u, jit_node_t *v) jit_inc_synth_wp(putargr_f, u, v); if (jit_arg_f_reg_p(v->u.w)) jit_movr_f(JIT_FA0 - v->u.w, u); - else - jit_stxi_f(v->u.w, JIT_FP, u); + else { + jit_node_t *node = jit_stxi_f(v->u.w, JIT_FP, u); + jit_link_alist(node); + } jit_dec_synth(); } @@ -582,9 +760,11 @@ _jit_putargi_f(jit_state_t *_jit, jit_float32_t u, jit_node_t *v) if (jit_arg_f_reg_p(v->u.w)) jit_movi_f(JIT_FA0 - v->u.w, u); else { + jit_node_t *node; regno = jit_get_reg(jit_class_fpr); jit_movi_f(regno, u); - jit_stxi_f(v->u.w, JIT_FP, regno); + node = jit_stxi_f(v->u.w, JIT_FP, regno); + jit_link_alist(node); jit_unget_reg(regno); } jit_dec_synth(); @@ -597,8 +777,10 @@ _jit_getarg_d(jit_state_t *_jit, jit_int32_t u, jit_node_t *v) jit_inc_synth_wp(getarg_d, u, v); if (jit_arg_f_reg_p(v->u.w)) jit_movr_d(u, JIT_FA0 - v->u.w); - else - jit_ldxi_d(u, JIT_FP, v->u.w); + else { + jit_node_t *node = jit_ldxi_d(u, JIT_FP, v->u.w); + jit_link_alist(node); + } jit_dec_synth(); } @@ -609,8 +791,10 @@ _jit_putargr_d(jit_state_t *_jit, jit_int32_t u, jit_node_t *v) jit_inc_synth_wp(putargr_d, u, v); if (jit_arg_reg_p(v->u.w)) jit_movr_d(JIT_FA0 - v->u.w, u); - else - jit_stxi_d(v->u.w, JIT_FP, u); + else { + jit_node_t *node = jit_stxi_d(v->u.w, JIT_FP, u); + jit_link_alist(node); + } jit_dec_synth(); } @@ -623,48 +807,161 @@ _jit_putargi_d(jit_state_t *_jit, jit_float64_t u, jit_node_t *v) if (jit_arg_reg_p(v->u.w)) jit_movi_d(JIT_FA0 - v->u.w, u); else { + jit_node_t *node; regno = jit_get_reg(jit_class_fpr); jit_movi_d(regno, u); - jit_stxi_d(v->u.w, JIT_FP, regno); + node = jit_stxi_d(v->u.w, JIT_FP, regno); + jit_link_alist(node); jit_unget_reg(regno); } jit_dec_synth(); } void -_jit_pushargr(jit_state_t *_jit, jit_int32_t u) +_jit_pushargr(jit_state_t *_jit, jit_int32_t u, jit_code_t code) { assert(_jitc->function); - jit_inc_synth_w(pushargr, u); + jit_code_inc_synth_w(code, u); jit_link_prepare(); if (jit_arg_reg_p(_jitc->function->call.argi)) { - jit_movr(JIT_RA0 - _jitc->function->call.argi, u); + jit_int32_t regno = JIT_RA0 - _jitc->function->call.argi; +#if PACKED_STACK + switch (code) { + case jit_code_pushargr_c: jit_extr_c(regno, u); break; + case jit_code_pushargr_uc: jit_extr_uc(regno, u); break; + case jit_code_pushargr_s: jit_extr_s(regno, u); break; + case jit_code_pushargr_us: jit_extr_us(regno, u); break; +# if __WORDISZE == 32 + case jit_code_pushargr_i: jit_movr(regno, u); break; +# else + case jit_code_pushargr_i: jit_extr_i(regno, u); break; + case jit_code_pushargr_ui: jit_extr_ui(regno, u); break; + case jit_code_pushargr_l: jit_movr(regno, u); break; +# endif + default: abort(); break; + } +#else + jit_movr(regno, u); +#endif +#if __APPLE__ + if (_jitc->function->call.call & jit_call_varargs) { + assert(code == jit_code_pushargr); + jit_stxi(_jitc->function->call.size, JIT_SP, u); + _jitc->function->call.size += sizeof(jit_word_t); + } +#endif ++_jitc->function->call.argi; } else { +#if PACKED_STACK + _jitc->function->call.size += + _jitc->function->call.size & + ((1 << ((code - jit_code_pushargr_c) >> 2)) - 1); + switch (code) { + case jit_code_pushargr_c: case jit_code_pushargr_uc: + jit_stxi_c(_jitc->function->call.size, JIT_SP, u); + break; + case jit_code_pushargr_s: case jit_code_pushargr_us: + jit_stxi_s(_jitc->function->call.size, JIT_SP, u); + break; +# if __WORDSIZE == 32 + case jit_code_pushargr_i: + jit_stxi(_jitc->function->call.size, JIT_SP, u); + break; +# else + case jit_code_pushargr_i: case jit_code_pushargr_ui: + jit_stxi_i(_jitc->function->call.size, JIT_SP, u); + break; + case jit_code_pushargr_l: + jit_stxi(_jitc->function->call.size, JIT_SP, u); + break; +# endif + default: + abort(); + break; + } + _jitc->function->call.size += 1 << ((code - jit_code_pushargr_c) >> 2); +#else jit_stxi(_jitc->function->call.size, JIT_SP, u); _jitc->function->call.size += sizeof(jit_word_t); +#endif + jit_check_frame(); } jit_dec_synth(); } void -_jit_pushargi(jit_state_t *_jit, jit_word_t u) +_jit_pushargi(jit_state_t *_jit, jit_word_t u, jit_code_t code) { jit_int32_t regno; assert(_jitc->function); - jit_inc_synth_w(pushargi, u); + jit_code_inc_synth_w(code, u); jit_link_prepare(); +#if PACKED_STACK + switch (code) { + case jit_code_pushargi_c: u = (jit_int8_t)u; break; + case jit_code_pushargi_uc: u = (jit_uint8_t)u; break; + case jit_code_pushargi_s: u = (jit_int16_t)u; break; + case jit_code_pushargi_us: u = (jit_uint16_t)u; break; +# if __WORDSIZE == 32 + case jit_code_pushargi_i: break; +# else + case jit_code_pushargi_i: u = (jit_int32_t)u; break; + case jit_code_pushargi_ui: u = (jit_uint32_t)u; break; + case jit_code_pushargi_l: break; +# endif + default: abort(); break; + } +#endif if (jit_arg_reg_p(_jitc->function->call.argi)) { - jit_movi(JIT_RA0 - _jitc->function->call.argi, u); + regno = JIT_RA0 - _jitc->function->call.argi; + jit_movi(regno, u); +#if __APPLE__ + if (_jitc->function->call.call & jit_call_varargs) { + assert(code == jit_code_pushargi); + jit_stxi(_jitc->function->call.size, JIT_SP, regno); + _jitc->function->call.size += sizeof(jit_word_t); + } +#endif ++_jitc->function->call.argi; } else { regno = jit_get_reg(jit_class_gpr); jit_movi(regno, u); +#if PACKED_STACK + _jitc->function->call.size += + _jitc->function->call.size & + ((1 << ((code - jit_code_pushargr_c) >> 2)) - 1); + switch (code) { + case jit_code_pushargi_c: case jit_code_pushargi_uc: + jit_stxi_c(_jitc->function->call.size, JIT_SP, regno); + break; + case jit_code_pushargi_s: case jit_code_pushargi_us: + jit_stxi_s(_jitc->function->call.size, JIT_SP, regno); + break; +# if __WORDSIZE == 32 + case jit_code_pushargi_i: + jit_stxi(_jitc->function->call.size, JIT_SP, regno); + break; +# else + case jit_code_pushargi_i: case jit_code_pushargi_ui: + jit_stxi_i(_jitc->function->call.size, JIT_SP, regno); + break; + case jit_code_pushargi_l: + jit_stxi(_jitc->function->call.size, JIT_SP, regno); + break; +# endif + default: + abort(); + break; + } + _jitc->function->call.size += 1 << ((code - jit_code_pushargr_c) >> 2); +#else jit_stxi(_jitc->function->call.size, JIT_SP, regno); - jit_unget_reg(regno); _jitc->function->call.size += sizeof(jit_word_t); +#endif + jit_unget_reg(regno); + jit_check_frame(); } jit_dec_synth(); } @@ -677,11 +974,27 @@ _jit_pushargr_f(jit_state_t *_jit, jit_int32_t u) jit_link_prepare(); if (jit_arg_f_reg_p(_jitc->function->call.argf)) { jit_movr_f(JIT_FA0 - _jitc->function->call.argf, u); +#if __APPLE__ + if (_jitc->function->call.call & jit_call_varargs) { + assert(sizeof(jit_float32_t) == sizeof(jit_word_t)); + jit_stxi_f(_jitc->function->call.size, JIT_SP, + JIT_FA0 - _jitc->function->call.argf); + _jitc->function->call.size += sizeof(jit_word_t); + } +#endif ++_jitc->function->call.argf; } else { +#if PACKED_STACK + _jitc->function->call.size += + _jitc->function->call.size & (sizeof(jit_float32_t) - 1); + jit_stxi_f(_jitc->function->call.size, JIT_SP, u); + _jitc->function->call.size += sizeof(jit_float32_t); +#else jit_stxi_f(_jitc->function->call.size, JIT_SP, u); _jitc->function->call.size += sizeof(jit_word_t); +#endif + jit_check_frame(); } jit_dec_synth(); } @@ -695,14 +1008,30 @@ _jit_pushargi_f(jit_state_t *_jit, jit_float32_t u) jit_link_prepare(); if (jit_arg_f_reg_p(_jitc->function->call.argf)) { jit_movi_f(JIT_FA0 - _jitc->function->call.argf, u); +#if __APPLE__ + if (_jitc->function->call.call & jit_call_varargs) { + assert(sizeof(jit_float32_t) == sizeof(jit_word_t)); + jit_stxi_f(_jitc->function->call.size, JIT_SP, + JIT_FA0 - _jitc->function->call.argf); + _jitc->function->call.size += sizeof(jit_word_t); + } +#endif ++_jitc->function->call.argf; } else { regno = jit_get_reg(jit_class_fpr); jit_movi_f(regno, u); +#if PACKED_STACK + _jitc->function->call.size += + _jitc->function->call.size & (sizeof(jit_float32_t) - 1); + jit_stxi_f(_jitc->function->call.size, JIT_SP, regno); + _jitc->function->call.size += sizeof(jit_float32_t); +#else jit_stxi_f(_jitc->function->call.size, JIT_SP, regno); - jit_unget_reg(regno); _jitc->function->call.size += sizeof(jit_word_t); +#endif + jit_unget_reg(regno); + jit_check_frame(); } jit_dec_synth(); } @@ -715,11 +1044,24 @@ _jit_pushargr_d(jit_state_t *_jit, jit_int32_t u) jit_link_prepare(); if (jit_arg_f_reg_p(_jitc->function->call.argf)) { jit_movr_d(JIT_FA0 - _jitc->function->call.argf, u); +#if __APPLE__ + if (_jitc->function->call.call & jit_call_varargs) { + assert(sizeof(jit_float64_t) == sizeof(jit_word_t)); + jit_stxi_d(_jitc->function->call.size, JIT_SP, + JIT_FA0 - _jitc->function->call.argf); + _jitc->function->call.size += sizeof(jit_float64_t); + } +#endif ++_jitc->function->call.argf; } else { +#if PACKED_STACK + _jitc->function->call.size += + _jitc->function->call.size & (sizeof(jit_float64_t) - 1); +#endif jit_stxi_d(_jitc->function->call.size, JIT_SP, u); - _jitc->function->call.size += sizeof(jit_word_t); + _jitc->function->call.size += sizeof(jit_float64_t); + jit_check_frame(); } jit_dec_synth(); } @@ -733,14 +1075,27 @@ _jit_pushargi_d(jit_state_t *_jit, jit_float64_t u) jit_link_prepare(); if (jit_arg_f_reg_p(_jitc->function->call.argf)) { jit_movi_d(JIT_FA0 - _jitc->function->call.argf, u); +#if __APPLE__ + if (_jitc->function->call.call & jit_call_varargs) { + assert(sizeof(jit_float64_t) == sizeof(jit_word_t)); + jit_stxi_d(_jitc->function->call.size, JIT_SP, + JIT_FA0 - _jitc->function->call.argf); + _jitc->function->call.size += sizeof(jit_float64_t); + } +#endif ++_jitc->function->call.argf; } else { regno = jit_get_reg(jit_class_fpr); jit_movi_d(regno, u); +#if PACKED_STACK + _jitc->function->call.size += + _jitc->function->call.size & (sizeof(jit_float64_t) - 1); +#endif jit_stxi_d(_jitc->function->call.size, JIT_SP, regno); jit_unget_reg(regno); - _jitc->function->call.size += sizeof(jit_word_t); + _jitc->function->call.size += sizeof(jit_float64_t); + jit_check_frame(); } jit_dec_synth(); } @@ -769,7 +1124,12 @@ _jit_finishr(jit_state_t *_jit, jit_int32_t r0) { jit_node_t *node; assert(_jitc->function); + jit_check_frame(); jit_inc_synth_w(finishr, r0); +#if PACKED_STACK + _jitc->function->call.size += + _jitc->function->call.size & (sizeof(jit_word_t) - 1); +#endif if (_jitc->function->self.alen < _jitc->function->call.size) _jitc->function->self.alen = _jitc->function->call.size; node = jit_callr(r0); @@ -786,7 +1146,12 @@ _jit_finishi(jit_state_t *_jit, jit_pointer_t i0) { jit_node_t *node; assert(_jitc->function); + jit_check_frame(); jit_inc_synth_w(finishi, (jit_word_t)i0); +#if PACKED_STACK + _jitc->function->call.size += + _jitc->function->call.size & (sizeof(jit_word_t) - 1); +#endif if (_jitc->function->self.alen < _jitc->function->call.size) _jitc->function->self.alen = _jitc->function->call.size; node = jit_calli(i0); @@ -835,10 +1200,15 @@ void _jit_retval_i(jit_state_t *_jit, jit_int32_t r0) { jit_inc_synth_w(retval_i, r0); +#if __WORDSIZE == 32 + jit_movr(r0, JIT_RET); +#else jit_extr_i(r0, JIT_RET); +#endif jit_dec_synth(); } +#if __WORDSIZE == 64 void _jit_retval_ui(jit_state_t *_jit, jit_int32_t r0) { @@ -851,10 +1221,10 @@ void _jit_retval_l(jit_state_t *_jit, jit_int32_t r0) { jit_inc_synth_w(retval_l, r0); - if (r0 != JIT_RET) - jit_movr(r0, JIT_RET); + jit_movr(r0, JIT_RET); jit_dec_synth(); } +#endif void _jit_retval_f(jit_state_t *_jit, jit_int32_t r0) @@ -886,6 +1256,7 @@ _emit_code(jit_state_t *_jit) jit_node_t *node; jit_uint8_t *data; jit_word_t word; + jit_function_t func; #if DEVEL_DISASSEMBLER jit_word_t prevw; #endif @@ -926,6 +1297,12 @@ _emit_code(jit_state_t *_jit) name##r##type(rn(node->u.q.l), rn(node->u.q.h), \ rn(node->v.w), rn(node->w.w)); \ break +#define case_rqr(name, type) \ + case jit_code_##name##r##type: \ + name##r##type(rn(node->u.w), rn(node->v.q.l), \ + rn(node->v.q.h), rn(node->w.w)); \ + case jit_code_##name##i##type: \ + break; #define case_rrw(name, type) \ case jit_code_##name##i##type: \ name##i##type(rn(node->u.w), rn(node->v.w), node->w.w); \ @@ -1023,6 +1400,9 @@ _emit_code(jit_state_t *_jit) if ((word = _jit->pc.w & (node->u.w - 1))) nop(node->u.w - word); break; + case jit_code_skip: + nop((node->u.w + 3) & ~3); + break; case jit_code_note: case jit_code_name: node->u.w = _jit->pc.w; break; @@ -1046,6 +1426,10 @@ _emit_code(jit_state_t *_jit) case_rrw(rsb,); case_rrr(mul,); case_rrw(mul,); + case_rrr(hmul,); + case_rrw(hmul,); + case_rrr(hmul, _u); + case_rrw(hmul, _u); case_rrrr(qmul,); case_rrrw(qmul,); case_rrrr(qmul, _u); @@ -1064,12 +1448,30 @@ _emit_code(jit_state_t *_jit) case_rrw(rem, _u); case_rrr(lsh,); case_rrw(lsh,); + case_rrrr(qlsh,); + case_rrrw(qlsh,); + case_rrrr(qlsh, _u); + case_rrrw(qlsh, _u); case_rrr(rsh,); case_rrw(rsh,); case_rrr(rsh, _u); case_rrw(rsh, _u); + case_rrrr(qrsh,); + case_rrrw(qrsh,); + case_rrrr(qrsh, _u); + case_rrrw(qrsh, _u); + case_rrr(lrot,); + case_rrw(lrot,); + case_rrr(rrot,); + case_rrw(rrot,); case_rr(neg,); case_rr(com,); + case_rr(clo,); + case_rr(clz,); + case_rr(cto,); + case_rr(ctz,); + case_rr(rbit,); + case_rr(popcnt,); case_rrr(and,); case_rrw(and,); case_rrr(or,); @@ -1108,6 +1510,18 @@ _emit_code(jit_state_t *_jit) case_rrw(ldx, _ui); case_rrr(ldx, _l); case_rrw(ldx, _l); + case jit_code_unldr: + unldr(rn(node->u.w), rn(node->v.w), node->w.w); + break; + case jit_code_unldi: + unldi(rn(node->u.w), node->v.w, node->w.w); + break; + case jit_code_unldr_u: + unldr_u(rn(node->u.w), rn(node->v.w), node->w.w); + break; + case jit_code_unldi_u: + unldi_u(rn(node->u.w), node->v.w, node->w.w); + break; case_rr(st, _c); case_wr(st, _c); case_rr(st, _s); @@ -1124,12 +1538,30 @@ _emit_code(jit_state_t *_jit) case_wrr(stx, _i); case_rrr(stx, _l); case_wrr(stx, _l); + case jit_code_unstr: + unstr(rn(node->u.w), rn(node->v.w), node->w.w); + break; + case jit_code_unsti: + unsti(node->u.w, rn(node->v.w), node->w.w); + break; case_rr(hton, _us); case_rr(hton, _ui); case_rr(hton, _ul); case_rr(bswap, _us); case_rr(bswap, _ui); case_rr(bswap, _ul); + case jit_code_extr: + extr(rn(node->u.w), rn(node->v.w), node->w.q.l, node->w.q.h); + break; + case jit_code_extr_u: + extr_u(rn(node->u.w), rn(node->v.w), node->w.q.l, node->w.q.h); + break; + case jit_code_depr: + depr(rn(node->u.w), rn(node->v.w), node->w.q.l, node->w.q.h); + break; + case jit_code_depi: + depi(rn(node->u.w), node->v.w, node->w.q.l, node->w.q.h); + break; case_rr(ext, _c); case_rr(ext, _uc); case_rr(ext, _s); @@ -1236,15 +1668,31 @@ _emit_code(jit_state_t *_jit) case_rr(abs, _f); case_rr(neg, _f); case_rr(sqrt, _f); + case_rqr(fma, _f); + case_rqr(fms, _f); + case_rqr(fnma, _f); + case_rqr(fnms, _f); case_rr(ext, _f); case_rr(ld, _f); case_rw(ld, _f); case_rrr(ldx, _f); case_rrw(ldx, _f); + case jit_code_unldr_x: + unldr_x(rn(node->u.w), rn(node->v.w), node->w.w); + break; + case jit_code_unldi_x: + unldi_x(rn(node->u.w), node->v.w, node->w.w); + break; case_rr(st, _f); case_wr(st, _f); case_rrr(stx, _f); case_wrr(stx, _f); + case jit_code_unstr_x: + unstr_x(rn(node->u.w), rn(node->v.w), node->w.w); + break; + case jit_code_unsti_x: + unsti_x(node->u.w, rn(node->v.w), node->w.w); + break; case_rr(mov, _f); case jit_code_movi_f: assert_data(node); @@ -1319,6 +1767,10 @@ _emit_code(jit_state_t *_jit) case_rr(abs, _d); case_rr(neg, _d); case_rr(sqrt, _d); + case_rqr(fma, _d); + case_rqr(fms, _d); + case_rqr(fnma, _d); + case_rqr(fnms, _d); case_rr(ext, _d); case_rr(ld, _d); case_rw(ld, _d); @@ -1391,6 +1843,7 @@ _emit_code(jit_state_t *_jit) case_brr(bunord, _d); case_brd(bunord); case jit_code_jmpr: + jit_check_frame(); jmpr(rn(node->u.w)); break; case jit_code_jmpi: @@ -1401,17 +1854,26 @@ _emit_code(jit_state_t *_jit) if (temp->flag & jit_flag_patch) jmpi(temp->u.w); else { - word = jmpi_p(_jit->pc.w); + word = _jit->code.length - + (_jit->pc.uc - _jit->code.ptr); + if (s26_p(word)) + word = jmpi(_jit->pc.w); + else + word = jmpi_p(_jit->pc.w); patch(word, node); } } - else + else { + jit_check_frame(); jmpi(node->u.w); + } break; case jit_code_callr: + jit_check_frame(); callr(rn(node->u.w)); break; case jit_code_calli: + jit_check_frame(); if (node->flag & jit_flag_node) { temp = node->u.n; assert(temp->code == jit_code_label || @@ -1419,7 +1881,12 @@ _emit_code(jit_state_t *_jit) if (temp->flag & jit_flag_patch) calli(temp->u.w); else { - word = calli_p(_jit->pc.w); + word = _jit->code.length - + (_jit->pc.uc - _jit->code.ptr); + if (s26_p(word)) + word = calli(_jit->pc.w); + else + word = calli_p(_jit->pc.w); patch(word, node); } } @@ -1430,11 +1897,14 @@ _emit_code(jit_state_t *_jit) _jitc->function = _jitc->functions.ptr + node->w.w; undo.node = node; undo.word = _jit->pc.w; + memcpy(&undo.func, _jitc->function, sizeof(undo.func)); #if DEVEL_DISASSEMBLER undo.prevw = prevw; #endif undo.patch_offset = _jitc->patches.offset; restart_function: + compute_framesize(); + patch_alist(0); _jitc->again = 0; prolog(node); break; @@ -1450,10 +1920,22 @@ _emit_code(jit_state_t *_jit) temp->flag &= ~jit_flag_patch; node = undo.node; _jit->pc.w = undo.word; + /* undo.func.self.aoff and undo.func.regset should not + * be undone, as they will be further updated, and are + * the reason of the undo. */ + undo.func.self.aoff = _jitc->function->frame + + _jitc->function->self.aoff; + undo.func.need_frame = _jitc->function->need_frame; + jit_regset_set(&undo.func.regset, &_jitc->function->regset); + /* allocar information also does not need to be undone */ + undo.func.aoffoff = _jitc->function->aoffoff; + undo.func.allocar = _jitc->function->allocar; + memcpy(_jitc->function, &undo.func, sizeof(undo.func)); #if DEVEL_DISASSEMBLER prevw = undo.prevw; #endif _jitc->patches.offset = undo.patch_offset; + patch_alist(1); goto restart_function; } /* remember label is defined */ @@ -1462,6 +1944,30 @@ _emit_code(jit_state_t *_jit) epilog(node); _jitc->function = NULL; break; + case jit_code_movr_w_f: + movr_w_f(rn(node->u.w), rn(node->v.w)); + break; + case jit_code_movr_f_w: + movr_f_w(rn(node->u.w), rn(node->v.w)); + break; + case jit_code_movi_f_w: + movi_f_w(rn(node->u.w), node->v.f); + break; + case jit_code_movi_w_f: + movi_w_f(rn(node->u.w), node->v.w); + break; + case jit_code_movr_w_d: + movr_w_d(rn(node->u.w), rn(node->v.w)); + break; + case jit_code_movr_d_w: + movr_d_w(rn(node->u.w), rn(node->v.w)); + break; + case jit_code_movi_d_w: + movi_d_w(rn(node->u.w), node->v.d); + break; + case jit_code_movi_w_d: + movi_w_d(rn(node->u.w), node->v.w); + break; case jit_code_va_start: vastart(rn(node->u.w)); break; @@ -1474,11 +1980,23 @@ _emit_code(jit_state_t *_jit) case jit_code_live: case jit_code_ellipsis: case jit_code_va_push: case jit_code_allocai: case jit_code_allocar: - case jit_code_arg: + case jit_code_arg_c: case jit_code_arg_s: + case jit_code_arg_i: +# if __WORDSIZE == 64 + case jit_code_arg_l: +# endif case jit_code_arg_f: case jit_code_arg_d: case jit_code_va_end: case jit_code_ret: - case jit_code_retr: case jit_code_reti: + case jit_code_retr_c: case jit_code_reti_c: + case jit_code_retr_uc: case jit_code_reti_uc: + case jit_code_retr_s: case jit_code_reti_s: + case jit_code_retr_us: case jit_code_reti_us: + case jit_code_retr_i: case jit_code_reti_i: +#if __WORDSIZE == 64 + case jit_code_retr_ui: case jit_code_reti_ui: + case jit_code_retr_l: case jit_code_reti_l: +#endif case jit_code_retr_f: case jit_code_reti_f: case jit_code_retr_d: case jit_code_reti_d: case jit_code_getarg_c: case jit_code_getarg_uc: @@ -1486,19 +2004,108 @@ _emit_code(jit_state_t *_jit) case jit_code_getarg_i: case jit_code_getarg_ui: case jit_code_getarg_l: case jit_code_getarg_f: case jit_code_getarg_d: - case jit_code_putargr: case jit_code_putargi: + case jit_code_putargr_c: case jit_code_putargi_c: + case jit_code_putargr_uc: case jit_code_putargi_uc: + case jit_code_putargr_s: case jit_code_putargi_s: + case jit_code_putargr_us: case jit_code_putargi_us: + case jit_code_putargr_i: case jit_code_putargi_i: +#if __WORDSIZE == 64 + case jit_code_putargr_ui: case jit_code_putargi_ui: + case jit_code_putargr_l: case jit_code_putargi_l: +#endif case jit_code_putargr_f: case jit_code_putargi_f: case jit_code_putargr_d: case jit_code_putargi_d: - case jit_code_pushargr: case jit_code_pushargi: + case jit_code_pushargr_c: case jit_code_pushargi_c: + case jit_code_pushargr_uc: case jit_code_pushargi_uc: + case jit_code_pushargr_s: case jit_code_pushargi_s: + case jit_code_pushargr_us: case jit_code_pushargi_us: + case jit_code_pushargr_i: case jit_code_pushargi_i: +#if __WORDSIZE == 64 + case jit_code_pushargr_ui: case jit_code_pushargi_ui: + case jit_code_pushargr_l: case jit_code_pushargi_l: +#endif case jit_code_pushargr_f: case jit_code_pushargi_f: case jit_code_pushargr_d: case jit_code_pushargi_d: case jit_code_retval_c: case jit_code_retval_uc: case jit_code_retval_s: case jit_code_retval_us: case jit_code_retval_i: +#if __WORDSIZE == 64 case jit_code_retval_ui: case jit_code_retval_l: +#endif case jit_code_retval_f: case jit_code_retval_d: case jit_code_prepare: case jit_code_finishr: case jit_code_finishi: + case jit_code_negi_f: case jit_code_absi_f: + case jit_code_sqrti_f: case jit_code_negi_d: + case jit_code_absi_d: case jit_code_sqrti_d: + break; + case jit_code_negi: + negi(rn(node->u.w), node->v.w); + break; + case jit_code_comi: + comi(rn(node->u.w), node->v.w); + break; + case jit_code_exti_c: + exti_c(rn(node->u.w), node->v.w); + break; + case jit_code_exti_uc: + exti_uc(rn(node->u.w), node->v.w); + break; + case jit_code_exti_s: + exti_s(rn(node->u.w), node->v.w); + break; + case jit_code_exti_us: + exti_us(rn(node->u.w), node->v.w); + break; + case jit_code_bswapi_us: + bswapi_us(rn(node->u.w), node->v.w); + break; + case jit_code_bswapi_ui: + bswapi_ui(rn(node->u.w), node->v.w); + break; + case jit_code_htoni_us: + htoni_us(rn(node->u.w), node->v.w); + break; + case jit_code_htoni_ui: + htoni_ui(rn(node->u.w), node->v.w); + break; +#if __WORDSIZE == 64 + case jit_code_exti_i: + exti_i(rn(node->u.w), node->v.w); + break; + case jit_code_exti_ui: + exti_ui(rn(node->u.w), node->v.w); + break; + case jit_code_bswapi_ul: + bswapi_ul(rn(node->u.w), node->v.w); + break; + case jit_code_htoni_ul: + htoni_ul(rn(node->u.w), node->v.w); + break; +#endif + case jit_code_cloi: + cloi(rn(node->u.w), node->v.w); + break; + case jit_code_clzi: + clzi(rn(node->u.w), node->v.w); + break; + case jit_code_ctoi: + ctoi(rn(node->u.w), node->v.w); + break; + case jit_code_ctzi: + ctzi(rn(node->u.w), node->v.w); + break; + case jit_code_rbiti: + rbiti(rn(node->u.w), node->v.w); + break; + case jit_code_popcnti: + popcnti(rn(node->u.w), node->v.w); + break; + case jit_code_exti: + exti(rn(node->u.w), node->v.w, node->w.q.l, node->w.q.h); + break; + case jit_code_exti_u: + exti_u(rn(node->u.w), node->v.w, node->w.q.l, node->w.q.h); break; default: abort(); @@ -1571,6 +2178,23 @@ _emit_stxi_d(jit_state_t *_jit, jit_word_t i0, jit_int32_t r0, jit_int32_t r1) stxi_d(i0, rn(r0), rn(r1)); } +static void +_compute_framesize(jit_state_t *_jit) +{ + jit_int32_t reg; + _jitc->framesize = 16; /* ra+fp */ + for (reg = 0; reg < jit_size(iregs); reg++) + if (jit_regset_tstbit(&_jitc->function->regset, iregs[reg])) + _jitc->framesize += sizeof(jit_word_t); + + for (reg = 0; reg < jit_size(fregs); reg++) + if (jit_regset_tstbit(&_jitc->function->regset, fregs[reg])) + _jitc->framesize += sizeof(jit_float64_t); + + /* Make sure functions called have a 16 byte aligned stack */ + _jitc->framesize = (_jitc->framesize + 15) & -16; +} + static void _patch(jit_state_t *_jit, jit_word_t instr, jit_node_t *node) {