X-Git-Url: https://notaz.gp2x.de/cgi-bin/gitweb.cgi?a=blobdiff_plain;f=deps%2Flightning%2Flib%2Fjit_arm.c;h=25aa7cbb628f9a21515c7d84b86e073a33c63772;hb=016c6e93f6db684211f5c8b05433cb500715ba50;hp=6b121bf323567cfe5fc5d62156a0cecd59731751;hpb=8622c9dcd0abc62601ff6c64f93f9294c520ed58;p=pcsx_rearmed.git diff --git a/deps/lightning/lib/jit_arm.c b/deps/lightning/lib/jit_arm.c index 6b121bf3..25aa7cbb 100644 --- a/deps/lightning/lib/jit_arm.c +++ b/deps/lightning/lib/jit_arm.c @@ -1,5 +1,5 @@ /* - * Copyright (C) 2012-2022 Free Software Foundation, Inc. + * Copyright (C) 2012-2023 Free Software Foundation, Inc. * * This file is part of GNU lightning. * @@ -21,6 +21,8 @@ # include #endif +#define stack_framesize 48 + #define jit_arg_reg_p(i) ((i) >= 0 && (i) < 4) #define jit_arg_f_reg_p(i) ((i) >= 0 && (i) < 16) #define jit_arg_d_reg_p(i) ((i) >= 0 && (i) < 15) @@ -28,12 +30,12 @@ #define arm_patch_node 0x80000000 #define arm_patch_word 0x40000000 #define arm_patch_jump 0x20000000 -#define arm_patch_load 0x00000000 +#define arm_patch_load 0x10000000 +#define arm_patch_call 0x08000000 #define jit_fpr_p(rn) ((rn) > 15) -#define arg_base() \ - (stack_framesize - 16 + (jit_cpu.abi ? 64 : 0)) +#define arg_base() (stack_framesize - 16) #define arg_offset(n) \ ((n) < 4 ? arg_base() + ((n) << 2) : (n)) @@ -42,10 +44,32 @@ * arm mode, what may cause a crash upon return of that function * if generating jit for a relative jump. */ -#define jit_exchange_p() 1 +#define jit_exchange_p() jit_cpu.exchange /* FIXME is it really required to not touch _R10? */ +#define CHECK_REG_ARGS() \ + do { \ + if (!_jitc->function->save_reg_args) \ + _jitc->again = _jitc->function->save_reg_args = 1; \ + } while (0) + +#define CHECK_SWF_OFFSET() \ + do { \ + if (!_jitc->function->swf_offset) { \ + _jitc->again = _jitc->function->save_reg_args = \ + _jitc->function->swf_offset = 1; \ + _jitc->function->self.aoff = -64; \ + } \ + } while (0) + +#define CHECK_RETURN() \ + do { \ + if (!_jitc->function->need_frame && \ + !_jitc->function->need_return) \ + _jitc->again = _jitc->function->need_return = 1; \ + } while (0) + /* * Types */ @@ -59,8 +83,8 @@ typedef jit_pointer_t jit_va_list; /* * Prototypes */ -#define jit_make_arg(node) _jit_make_arg(_jit,node) -static jit_node_t *_jit_make_arg(jit_state_t*,jit_node_t*); +#define jit_make_arg(node,code) _jit_make_arg(_jit,node,code) +static jit_node_t *_jit_make_arg(jit_state_t*,jit_node_t*,jit_code_t); #define jit_make_arg_f(node) _jit_make_arg_f(_jit,node) static jit_node_t *_jit_make_arg_f(jit_state_t*,jit_node_t*); #define jit_make_arg_d(node) _jit_make_arg_d(_jit,node) @@ -77,8 +101,10 @@ static void _load_const(jit_state_t*,jit_bool_t,jit_int32_t,jit_word_t); static void _flush_consts(jit_state_t*); #define invalidate_consts() _invalidate_consts(_jit) static void _invalidate_consts(jit_state_t*); -#define patch(instr, node) _patch(_jit, instr, node) -static void _patch(jit_state_t*,jit_word_t,jit_node_t*); +#define compute_framesize() _compute_framesize(_jit) +static void _compute_framesize(jit_state_t*); +#define patch(instr, node, kind) _patch(_jit, instr, node, kind) +static void _patch(jit_state_t*,jit_word_t,jit_node_t*,jit_int32_t); #if defined(__GNUC__) /* libgcc */ @@ -149,6 +175,10 @@ jit_register_t _rvs[] = { { _NOREG, "" }, }; +static jit_int32_t iregs[] = { + _R4, _R5, _R6, _R7, _R8, _R9, +}; + /* * Implementation */ @@ -202,6 +232,19 @@ jit_get_cpu(void) /* armv6t2 todo (software float and thumb2) */ if (!jit_cpu.vfp && jit_cpu.thumb) jit_cpu.thumb = 0; + /* FIXME need test environments for the below. For the moment just + * be very conservative */ + /* force generation of code assuming jit and function libraries called + * instruction set do not match */ + jit_cpu.exchange = 1; + /* do not generate hardware integer division by default */ + if (jit_cpu.version == 7) + jit_cpu.extend = 0; + + /* By default generate extra instructions for unaligned load/store. */ + jit_cpu.unaligned = 1; + /* Linux should only not handle unaligned vfp load/store */ + jit_cpu.vfp_unaligned = 1; } void @@ -245,15 +288,10 @@ _jit_prolog(jit_state_t *_jit) } _jitc->function = _jitc->functions.ptr + _jitc->functions.offset++; _jitc->function->self.size = stack_framesize; - if (jit_cpu.abi) - _jitc->function->self.size += 64; _jitc->function->self.argi = _jitc->function->self.argf = - _jitc->function->self.alen = 0; - if (jit_swf_p()) - /* 8 soft float registers */ - _jitc->function->self.aoff = -64; - else - _jitc->function->self.aoff = 0; + _jitc->function->self.alen = _jitc->function->self.aoff = 0; + _jitc->function->swf_offset = _jitc->function->save_reg_args = + _jitc->function->need_return = 0; _jitc->function->self.call = jit_call_default; jit_alloc((jit_pointer_t *)&_jitc->function->regoff, _jitc->reglen * sizeof(jit_int32_t)); @@ -279,6 +317,9 @@ jit_int32_t _jit_allocai(jit_state_t *_jit, jit_int32_t length) { assert(_jitc->function); + if (jit_swf_p()) + CHECK_SWF_OFFSET(); + jit_check_frame(); switch (length) { case 0: case 1: break; case 2: _jitc->function->self.aoff &= -2; break; @@ -327,20 +368,18 @@ _jit_ret(jit_state_t *_jit) } void -_jit_retr(jit_state_t *_jit, jit_int32_t u) +_jit_retr(jit_state_t *_jit, jit_int32_t u, jit_code_t code) { - jit_inc_synth_w(retr, u); - if (JIT_RET != u) - jit_movr(JIT_RET, u); - jit_live(JIT_RET); + jit_code_inc_synth_w(code, u); + jit_movr(JIT_RET, u); jit_ret(); jit_dec_synth(); } void -_jit_reti(jit_state_t *_jit, jit_word_t u) +_jit_reti(jit_state_t *_jit, jit_word_t u, jit_code_t code) { - jit_inc_synth_w(reti, u); + jit_code_inc_synth_w(code, u); jit_movi(JIT_RET, u); jit_ret(); jit_dec_synth(); @@ -422,7 +461,7 @@ _jit_epilog(jit_state_t *_jit) jit_bool_t _jit_arg_register_p(jit_state_t *_jit, jit_node_t *u) { - if (u->code != jit_code_arg) { + if (!(u->code >= jit_code_arg_c && u->code <= jit_code_arg)) { if (u->code == jit_code_arg_f) { if (jit_cpu.abi) return (jit_arg_f_reg_p(u->u.w)); @@ -437,7 +476,7 @@ _jit_arg_register_p(jit_state_t *_jit, jit_node_t *u) } static jit_node_t * -_jit_make_arg(jit_state_t *_jit, jit_node_t *node) +_jit_make_arg(jit_state_t *_jit, jit_node_t *node, jit_code_t code) { jit_int32_t offset; if (jit_arg_reg_p(_jitc->function->self.argi)) @@ -447,7 +486,7 @@ _jit_make_arg(jit_state_t *_jit, jit_node_t *node) _jitc->function->self.size += sizeof(jit_word_t); } if (node == (jit_node_t *)0) - node = jit_new_node(jit_code_arg); + node = jit_new_node(code); else link_node(node); node->u.w = offset; @@ -534,9 +573,10 @@ _jit_ellipsis(jit_state_t *_jit) else { assert(!(_jitc->function->self.call & jit_call_varargs)); _jitc->function->self.call |= jit_call_varargs; + CHECK_REG_ARGS(); if (jit_cpu.abi && _jitc->function->self.argf) rewind_prolog(); - /* First 4 stack addresses are always spilled r0-r3 */ + /* First 4 stack addresses need to be spilled r0-r3 */ if (jit_arg_reg_p(_jitc->function->self.argi)) _jitc->function->vagp = _jitc->function->self.argi * 4; else @@ -559,16 +599,21 @@ _jit_va_push(jit_state_t *_jit, jit_int32_t u) } jit_node_t * -_jit_arg(jit_state_t *_jit) +_jit_arg(jit_state_t *_jit, jit_code_t code) { assert(_jitc->function); - return (jit_make_arg((jit_node_t*)0)); + assert(!(_jitc->function->self.call & jit_call_varargs)); +#if STRONG_TYPE_CHECKING + assert(code >= jit_code_arg_c && code <= jit_code_arg); +#endif + return (jit_make_arg((jit_node_t*)0, code)); } jit_node_t * _jit_arg_f(jit_state_t *_jit) { assert(_jitc->function); + assert(!(_jitc->function->self.call & jit_call_varargs)); return (jit_make_arg_f((jit_node_t*)0)); } @@ -576,103 +621,141 @@ jit_node_t * _jit_arg_d(jit_state_t *_jit) { assert(_jitc->function); + assert(!(_jitc->function->self.call & jit_call_varargs)); return (jit_make_arg_d((jit_node_t*)0)); } void _jit_getarg_c(jit_state_t *_jit, jit_int32_t u, jit_node_t *v) { - assert(v->code == jit_code_arg); + jit_node_t *node = NULL; + assert_arg_type(v->code, jit_code_arg_c); jit_inc_synth_wp(getarg_c, u, v); if (jit_swf_p()) - jit_ldxi_c(u, JIT_FP, arg_offset(v->u.w)); + node = jit_ldxi_c(u, JIT_FP, arg_offset(v->u.w)); else if (jit_arg_reg_p(v->u.w)) jit_extr_c(u, JIT_RA0 - v->u.w); else - jit_ldxi_c(u, JIT_FP, v->u.w); + node = jit_ldxi_c(u, JIT_FP, v->u.w); + if (node) { + CHECK_REG_ARGS(); + jit_link_alist(node); + jit_check_frame(); + } jit_dec_synth(); } void _jit_getarg_uc(jit_state_t *_jit, jit_int32_t u, jit_node_t *v) { - assert(v->code == jit_code_arg); + jit_node_t *node = NULL; + assert_arg_type(v->code, jit_code_arg_c); jit_inc_synth_wp(getarg_uc, u, v); if (jit_swf_p()) - jit_ldxi_uc(u, JIT_FP, arg_offset(v->u.w)); + node = jit_ldxi_uc(u, JIT_FP, arg_offset(v->u.w)); else if (jit_arg_reg_p(v->u.w)) jit_extr_uc(u, JIT_RA0 - v->u.w); else - jit_ldxi_uc(u, JIT_FP, v->u.w); + node = jit_ldxi_uc(u, JIT_FP, v->u.w); + if (node) { + CHECK_REG_ARGS(); + jit_link_alist(node); + jit_check_frame(); + } jit_dec_synth(); } void _jit_getarg_s(jit_state_t *_jit, jit_int32_t u, jit_node_t *v) { - assert(v->code == jit_code_arg); + jit_node_t *node = NULL; + assert_arg_type(v->code, jit_code_arg_s); jit_inc_synth_wp(getarg_s, u, v); if (jit_swf_p()) - jit_ldxi_s(u, JIT_FP, arg_offset(v->u.w)); + node = jit_ldxi_s(u, JIT_FP, arg_offset(v->u.w)); else if (jit_arg_reg_p(v->u.w)) jit_extr_s(u, JIT_RA0 - v->u.w); else - jit_ldxi_s(u, JIT_FP, v->u.w); + node = jit_ldxi_s(u, JIT_FP, v->u.w); + if (node) { + CHECK_REG_ARGS(); + jit_link_alist(node); + jit_check_frame(); + } jit_dec_synth(); } void _jit_getarg_us(jit_state_t *_jit, jit_int32_t u, jit_node_t *v) { - assert(v->code == jit_code_arg); + jit_node_t *node = NULL; + assert_arg_type(v->code, jit_code_arg_s); jit_inc_synth_wp(getarg_us, u, v); if (jit_swf_p()) - jit_ldxi_us(u, JIT_FP, arg_offset(v->u.w)); + node = jit_ldxi_us(u, JIT_FP, arg_offset(v->u.w)); else if (jit_arg_reg_p(v->u.w)) jit_extr_us(u, JIT_RA0 - v->u.w); else - jit_ldxi_us(u, JIT_FP, v->u.w); + node = jit_ldxi_us(u, JIT_FP, v->u.w); + if (node) { + CHECK_REG_ARGS(); + jit_link_alist(node); + jit_check_frame(); + } jit_dec_synth(); } void _jit_getarg_i(jit_state_t *_jit, jit_int32_t u, jit_node_t *v) { - assert(v->code == jit_code_arg); + jit_node_t *node = NULL; + assert_arg_type(v->code, jit_code_arg_i); jit_inc_synth_wp(getarg_i, u, v); if (jit_swf_p()) - jit_ldxi_i(u, JIT_FP, arg_offset(v->u.w)); + node = jit_ldxi_i(u, JIT_FP, arg_offset(v->u.w)); else if (jit_arg_reg_p(v->u.w)) jit_movr(u, JIT_RA0 - v->u.w); else - jit_ldxi_i(u, JIT_FP, v->u.w); + node = jit_ldxi_i(u, JIT_FP, v->u.w); + if (node) { + CHECK_REG_ARGS(); + jit_link_alist(node); + jit_check_frame(); + } jit_dec_synth(); } void -_jit_putargr(jit_state_t *_jit, jit_int32_t u, jit_node_t *v) +_jit_putargr(jit_state_t *_jit, jit_int32_t u, jit_node_t *v, jit_code_t code) { - assert(v->code == jit_code_arg); - jit_inc_synth_wp(putargr, u, v); + jit_node_t *node = NULL; + assert_putarg_type(code, v->code); + jit_code_inc_synth_wp(code, u, v); if (jit_swf_p()) - jit_stxi(arg_offset(v->u.w), JIT_FP, u); + node = jit_stxi(arg_offset(v->u.w), JIT_FP, u); else if (jit_arg_reg_p(v->u.w)) jit_movr(JIT_RA0 - v->u.w, u); else - jit_stxi(v->u.w, JIT_FP, u); + node = jit_stxi(v->u.w, JIT_FP, u); + if (node) { + CHECK_REG_ARGS(); + jit_link_alist(node); + jit_check_frame(); + } jit_dec_synth(); } void -_jit_putargi(jit_state_t *_jit, jit_word_t u, jit_node_t *v) +_jit_putargi(jit_state_t *_jit, jit_word_t u, jit_node_t *v, jit_code_t code) { - jit_int32_t regno; - assert(v->code == jit_code_arg); - jit_inc_synth_wp(putargi, u, v); + jit_int32_t regno; + jit_node_t *node = NULL; + assert_putarg_type(code, v->code); + jit_code_inc_synth_wp(code, u, v); if (jit_swf_p()) { regno = jit_get_reg(jit_class_gpr); jit_movi(regno, u); - jit_stxi(arg_offset(v->u.w), JIT_FP, regno); + node = jit_stxi(arg_offset(v->u.w), JIT_FP, regno); jit_unget_reg(regno); } else if (jit_arg_reg_p(v->u.w)) @@ -680,30 +763,41 @@ _jit_putargi(jit_state_t *_jit, jit_word_t u, jit_node_t *v) else { regno = jit_get_reg(jit_class_gpr); jit_movi(regno, u); - jit_stxi(v->u.w, JIT_FP, regno); + node = jit_stxi(v->u.w, JIT_FP, regno); jit_unget_reg(regno); } + if (node) { + CHECK_REG_ARGS(); + jit_link_alist(node); + jit_check_frame(); + } jit_dec_synth(); } void _jit_getarg_f(jit_state_t *_jit, jit_int32_t u, jit_node_t *v) { + jit_node_t *node = NULL; assert(v->code == jit_code_arg_f); jit_inc_synth_wp(getarg_f, u, v); if (jit_cpu.abi && !(_jitc->function->self.call & jit_call_varargs)) { if (jit_arg_f_reg_p(v->u.w)) jit_movr_f(u, JIT_FA0 - v->u.w); else - jit_ldxi_f(u, JIT_FP, v->u.w); + node = jit_ldxi_f(u, JIT_FP, v->u.w); } else if (jit_swf_p()) - jit_ldxi_f(u, JIT_FP, arg_offset(v->u.w)); + node = jit_ldxi_f(u, JIT_FP, arg_offset(v->u.w)); else { if (jit_arg_reg_p(v->u.w)) jit_movr_w_f(u, JIT_RA0 - v->u.w); else - jit_ldxi_f(u, JIT_FP, v->u.w); + node = jit_ldxi_f(u, JIT_FP, v->u.w); + } + if (node) { + CHECK_REG_ARGS(); + jit_link_alist(node); + jit_check_frame(); } jit_dec_synth(); } @@ -711,21 +805,27 @@ _jit_getarg_f(jit_state_t *_jit, jit_int32_t u, jit_node_t *v) void _jit_putargr_f(jit_state_t *_jit, jit_int32_t u, jit_node_t *v) { + jit_node_t *node = NULL; assert(v->code == jit_code_arg_f); jit_inc_synth_wp(putargr_f, u, v); if (jit_cpu.abi) { if (jit_arg_f_reg_p(v->u.w)) jit_movr_f(JIT_FA0 - v->u.w, u); else - jit_stxi_f(v->u.w, JIT_FP, u); + node = jit_stxi_f(v->u.w, JIT_FP, u); } else if (jit_swf_p()) - jit_stxi_f(arg_offset(v->u.w), JIT_FP, u); + node = jit_stxi_f(arg_offset(v->u.w), JIT_FP, u); else { if (jit_arg_reg_p(v->u.w)) jit_movr_f_w(JIT_RA0 - v->u.w, u); else - jit_stxi_f(v->u.w, JIT_FP, u); + node = jit_stxi_f(v->u.w, JIT_FP, u); + } + if (node) { + CHECK_REG_ARGS(); + jit_link_alist(node); + jit_check_frame(); } jit_dec_synth(); } @@ -733,7 +833,8 @@ _jit_putargr_f(jit_state_t *_jit, jit_int32_t u, jit_node_t *v) void _jit_putargi_f(jit_state_t *_jit, jit_float32_t u, jit_node_t *v) { - jit_int32_t regno; + jit_int32_t regno; + jit_node_t *node = NULL; assert(v->code == jit_code_arg_f); jit_inc_synth_fp(putargi_f, u, v); if (jit_cpu.abi) { @@ -742,14 +843,14 @@ _jit_putargi_f(jit_state_t *_jit, jit_float32_t u, jit_node_t *v) else { regno = jit_get_reg(jit_class_fpr); jit_movi_f(regno, u); - jit_stxi_f(v->u.w, JIT_FP, regno); + node = jit_stxi_f(v->u.w, JIT_FP, regno); jit_unget_reg(regno); } } else if (jit_swf_p()) { regno = jit_get_reg(jit_class_fpr); jit_movi_f(regno, u); - jit_stxi_f(arg_offset(v->u.w), JIT_FP, regno); + node = jit_stxi_f(arg_offset(v->u.w), JIT_FP, regno); jit_unget_reg(regno); } else { @@ -758,30 +859,41 @@ _jit_putargi_f(jit_state_t *_jit, jit_float32_t u, jit_node_t *v) if (jit_arg_reg_p(v->u.w)) jit_movr_f_w(JIT_RA0 - v->u.w, regno); else - jit_stxi_f(v->u.w, JIT_FP, regno); + node = jit_stxi_f(v->u.w, JIT_FP, regno); jit_unget_reg(regno); } + if (node) { + CHECK_REG_ARGS(); + jit_link_alist(node); + jit_check_frame(); + } jit_dec_synth(); } void _jit_getarg_d(jit_state_t *_jit, jit_int32_t u, jit_node_t *v) { + jit_node_t *node = NULL; assert(v->code == jit_code_arg_d); jit_inc_synth_wp(getarg_d, u, v); if (jit_cpu.abi && !(_jitc->function->self.call & jit_call_varargs)) { if (jit_arg_f_reg_p(v->u.w)) jit_movr_d(u, JIT_FA0 - v->u.w); else - jit_ldxi_d(u, JIT_FP, v->u.w); + node = jit_ldxi_d(u, JIT_FP, v->u.w); } else if (jit_swf_p()) - jit_ldxi_d(u, JIT_FP, arg_offset(v->u.w)); + node = jit_ldxi_d(u, JIT_FP, arg_offset(v->u.w)); else { if (jit_arg_reg_p(v->u.w)) jit_movr_ww_d(u, JIT_RA0 - v->u.w, JIT_RA0 - (v->u.w + 1)); else - jit_ldxi_d(u, JIT_FP, v->u.w); + node = jit_ldxi_d(u, JIT_FP, v->u.w); + } + if (node) { + CHECK_REG_ARGS(); + jit_link_alist(node); + jit_check_frame(); } jit_dec_synth(); } @@ -789,21 +901,27 @@ _jit_getarg_d(jit_state_t *_jit, jit_int32_t u, jit_node_t *v) void _jit_putargr_d(jit_state_t *_jit, jit_int32_t u, jit_node_t *v) { + jit_node_t *node = NULL; assert(v->code == jit_code_arg_d); jit_inc_synth_wp(putargr_d, u, v); if (jit_cpu.abi) { if (jit_arg_f_reg_p(v->u.w)) jit_movr_d(JIT_FA0 - v->u.w, u); else - jit_stxi_d(v->u.w, JIT_FP, u); + node = jit_stxi_d(v->u.w, JIT_FP, u); } else if (jit_swf_p()) - jit_stxi_d(arg_offset(v->u.w), JIT_FP, u); + node = jit_stxi_d(arg_offset(v->u.w), JIT_FP, u); else { if (jit_arg_reg_p(v->u.w)) jit_movr_d_ww(JIT_RA0 - v->u.w, JIT_RA0 - (v->u.w + 1), u); else - jit_stxi_d(v->u.w, JIT_FP, u); + node = jit_stxi_d(v->u.w, JIT_FP, u); + } + if (node) { + CHECK_REG_ARGS(); + jit_link_alist(node); + jit_check_frame(); } jit_dec_synth(); } @@ -811,7 +929,8 @@ _jit_putargr_d(jit_state_t *_jit, jit_int32_t u, jit_node_t *v) void _jit_putargi_d(jit_state_t *_jit, jit_float64_t u, jit_node_t *v) { - jit_int32_t regno; + jit_int32_t regno; + jit_node_t *node = NULL; assert(v->code == jit_code_arg_d); jit_inc_synth_dp(putargi_d, u, v); if (jit_cpu.abi) { @@ -820,14 +939,14 @@ _jit_putargi_d(jit_state_t *_jit, jit_float64_t u, jit_node_t *v) else { regno = jit_get_reg(jit_class_fpr); jit_movi_d(regno, u); - jit_stxi_d(v->u.w, JIT_FP, regno); + node = jit_stxi_d(v->u.w, JIT_FP, regno); jit_unget_reg(regno); } } else if (jit_swf_p()) { regno = jit_get_reg(jit_class_fpr); jit_movi_d(regno, u); - jit_stxi_d(arg_offset(v->u.w), JIT_FP, regno); + node = jit_stxi_d(arg_offset(v->u.w), JIT_FP, regno); jit_unget_reg(regno); } else { @@ -836,17 +955,22 @@ _jit_putargi_d(jit_state_t *_jit, jit_float64_t u, jit_node_t *v) if (jit_arg_reg_p(v->u.w)) jit_movr_d_ww(JIT_RA0 - v->u.w, JIT_RA0 - (v->u.w + 1), regno); else - jit_stxi_d(v->u.w, JIT_FP, regno); + node = jit_stxi_d(v->u.w, JIT_FP, regno); jit_unget_reg(regno); } + if (node) { + CHECK_REG_ARGS(); + jit_link_alist(node); + jit_check_frame(); + } jit_dec_synth(); } void -_jit_pushargr(jit_state_t *_jit, jit_int32_t u) +_jit_pushargr(jit_state_t *_jit, jit_int32_t u, jit_code_t code) { assert(_jitc->function); - jit_inc_synth_w(pushargr, u); + jit_code_inc_synth_w(code, u); jit_link_prepare(); if (jit_arg_reg_p(_jitc->function->call.argi)) { jit_movr(JIT_RA0 - _jitc->function->call.argi, u); @@ -860,11 +984,11 @@ _jit_pushargr(jit_state_t *_jit, jit_int32_t u) } void -_jit_pushargi(jit_state_t *_jit, jit_word_t u) +_jit_pushargi(jit_state_t *_jit, jit_word_t u, jit_code_t code) { jit_int32_t regno; assert(_jitc->function); - jit_inc_synth_w(pushargi, u); + jit_code_inc_synth_w(code, u); jit_link_prepare(); if (jit_arg_reg_p(_jitc->function->call.argi)) { jit_movi(JIT_RA0 - _jitc->function->call.argi, u); @@ -1148,6 +1272,7 @@ _emit_code(jit_state_t *_jit) jit_node_t *node; jit_uint8_t *data; jit_word_t word; + jit_function_t func; #if DEVEL_DISASSEMBLER jit_word_t prevw; #endif @@ -1217,6 +1342,16 @@ _emit_code(jit_state_t *_jit) name##r##type(rn(node->u.w), \ rn(node->v.w), rn(node->w.w)); \ break +#define case_rqr(name, type) \ + case jit_code_##name##r##type: \ + if (jit_swf_p()) \ + swf_##name##r##type(rn(node->u.w), rn(node->v.q.l), \ + rn(node->v.q.h), rn(node->w.w));\ + else \ + vfp_##name##r##type(rn(node->u.w), rn(node->v.q.l), \ + rn(node->v.q.h), rn(node->w.w));\ + case jit_code_##name##i##type: \ + break #define case_rrrr(name, type) \ case jit_code_##name##r##type: \ name##r##type(rn(node->u.q.l), rn(node->u.q.h), \ @@ -1293,7 +1428,7 @@ _emit_code(jit_state_t *_jit) else { \ word = name##r##type(_jit->pc.w, \ rn(node->v.w), rn(node->w.w)); \ - patch(word, node); \ + patch(word, node, arm_patch_jump); \ } \ break #define case_bvv(name, type) \ @@ -1318,7 +1453,7 @@ _emit_code(jit_state_t *_jit) word = vfp_##name##r##type(_jit->pc.w, \ rn(node->v.w), \ rn(node->w.w)); \ - patch(word, node); \ + patch(word, node, arm_patch_jump); \ } \ break #define case_brw(name, type) \ @@ -1332,7 +1467,7 @@ _emit_code(jit_state_t *_jit) else { \ word = name##i##type(_jit->pc.w, \ rn(node->v.w), node->w.w); \ - patch(word, node); \ + patch(word, node, arm_patch_jump); \ } \ break; #define case_bvf(name) \ @@ -1357,7 +1492,7 @@ _emit_code(jit_state_t *_jit) word = vfp_##name##i_f(_jit->pc.w, \ rn(node->v.w), \ node->w.f); \ - patch(word, node); \ + patch(word, node, arm_patch_jump); \ } \ break #define case_bvd(name) \ @@ -1382,7 +1517,7 @@ _emit_code(jit_state_t *_jit) word = vfp_##name##i_d(_jit->pc.w, \ rn(node->v.w), \ node->w.d); \ - patch(word, node); \ + patch(word, node, arm_patch_jump); \ } \ break #if DEVEL_DISASSEMBLER @@ -1405,6 +1540,12 @@ _emit_code(jit_state_t *_jit) if ((word = _jit->pc.w & (node->u.w - 1))) nop(node->u.w - word); break; + case jit_code_skip: + if (jit_thumb_p()) + nop((node->u.w + 1) & ~1); + else + nop((node->u.w + 3) & ~3); + break; case jit_code_note: case jit_code_name: if (must_align_p(node->next)) nop(2); @@ -1432,6 +1573,10 @@ _emit_code(jit_state_t *_jit) case_rrw(rsb,); case_rrr(mul,); case_rrw(mul,); + case_rrr(hmul,); + case_rrw(hmul,); + case_rrr(hmul, _u); + case_rrw(hmul, _u); case_rrrr(qmul,); case_rrrw(qmul,); case_rrrr(qmul, _u); @@ -1450,12 +1595,38 @@ _emit_code(jit_state_t *_jit) case_rrw(rem, _u); case_rrr(lsh,); case_rrw(lsh,); +#define qlshr(r0, r1, r2, r3) fallback_qlshr(r0, r1, r2, r3) +#define qlshi(r0, r1, r2, i0) fallback_qlshi(r0, r1, r2, i0) +#define qlshr_u(r0, r1, r2, r3) fallback_qlshr_u(r0, r1, r2, r3) +#define qlshi_u(r0, r1, r2, i0) fallback_qlshi_u(r0, r1, r2, i0) + case_rrrr(qlsh,); + case_rrrw(qlsh,); + case_rrrr(qlsh, _u); + case_rrrw(qlsh, _u); case_rrr(rsh,); case_rrw(rsh,); case_rrr(rsh, _u); case_rrw(rsh, _u); +#define qrshr(r0, r1, r2, r3) fallback_qrshr(r0, r1, r2, r3) +#define qrshi(r0, r1, r2, i0) fallback_qrshi(r0, r1, r2, i0) +#define qrshr_u(r0, r1, r2, r3) fallback_qrshr_u(r0, r1, r2, r3) +#define qrshi_u(r0, r1, r2, i0) fallback_qrshi_u(r0, r1, r2, i0) + case_rrrr(qrsh,); + case_rrrw(qrsh,); + case_rrrr(qrsh, _u); + case_rrrw(qrsh, _u); + case_rrr(lrot,); + case_rrw(lrot,); + case_rrr(rrot,); + case_rrw(rrot,); case_rr(neg,); case_rr(com,); + case_rr(clo,); + case_rr(clz,); + case_rr(cto,); + case_rr(ctz,); + case_rr(rbit,); + case_rr(popcnt,); case_rrr(and,); case_rrw(and,); case_rrr(or,); @@ -1484,6 +1655,18 @@ _emit_code(jit_state_t *_jit) case_rrw(ldx, _us); case_rrr(ldx, _i); case_rrw(ldx, _i); + case jit_code_unldr: + unldr(rn(node->u.w), rn(node->v.w), node->w.w); + break; + case jit_code_unldi: + unldi(rn(node->u.w), node->v.w, node->w.w); + break; + case jit_code_unldr_u: + unldr_u(rn(node->u.w), rn(node->v.w), node->w.w); + break; + case jit_code_unldi_u: + unldi_u(rn(node->u.w), node->v.w, node->w.w); + break; case_rr(st, _c); case_wr(st, _c); case_rr(st, _s); @@ -1496,10 +1679,28 @@ _emit_code(jit_state_t *_jit) case_wrr(stx, _s); case_rrr(stx, _i); case_wrr(stx, _i); + case jit_code_unstr: + unstr(rn(node->u.w), rn(node->v.w), node->w.w); + break; + case jit_code_unsti: + unsti(node->u.w, rn(node->v.w), node->w.w); + break; case_rr(hton, _us); case_rr(hton, _ui); case_rr(bswap, _us); case_rr(bswap, _ui); + case jit_code_extr: + extr(rn(node->u.w), rn(node->v.w), node->w.q.l, node->w.q.h); + break; + case jit_code_extr_u: + extr_u(rn(node->u.w), rn(node->v.w), node->w.q.l, node->w.q.h); + break; + case jit_code_depr: + depr(rn(node->u.w), rn(node->v.w), node->w.q.l, node->w.q.h); + break; + case jit_code_depi: + depi(rn(node->u.w), node->v.w, node->w.q.l, node->w.q.h); + break; case_rr(ext, _c); case_rr(ext, _uc); case_rr(ext, _s); @@ -1526,7 +1727,7 @@ _emit_code(jit_state_t *_jit) assert(temp->code == jit_code_label || temp->code == jit_code_epilog); word = movi_p(rn(node->u.w), temp->u.w); - patch(word, node); + patch(word, node, arm_patch_word); } } else @@ -1604,15 +1805,43 @@ _emit_code(jit_state_t *_jit) case_vv(abs, _f); case_vv(neg, _f); case_vv(sqrt, _f); + case_rqr(fma, _f); + case_rqr(fms, _f); + case_rqr(fnma, _f); + case_rqr(fnms, _f); case_vv(ext, _f); case_vv(ld, _f); case_vw(ld, _f); case_vvv(ldx, _f); case_vvw(ldx, _f); + case jit_code_unldr_x: + if (jit_swf_p()) + swf_unldr_x(rn(node->u.w), rn(node->v.w), node->w.w); + else + vfp_unldr_x(rn(node->u.w), rn(node->v.w), node->w.w); + break; + case jit_code_unldi_x: + if (jit_swf_p()) + swf_unldi_x(rn(node->u.w), node->v.w, node->w.w); + else + vfp_unldi_x(rn(node->u.w), node->v.w, node->w.w); + break; case_vv(st, _f); case_wv(st, _f); case_vvv(stx, _f); case_wvv(stx, _f); + case jit_code_unstr_x: + if (jit_swf_p()) + swf_unstr_x(rn(node->u.w), rn(node->v.w), node->w.w); + else + vfp_unstr_x(rn(node->u.w), rn(node->v.w), node->w.w); + break; + case jit_code_unsti_x: + if (jit_swf_p()) + swf_unsti_x(node->u.w, rn(node->v.w), node->w.w); + else + vfp_unsti_x(node->u.w, rn(node->v.w), node->w.w); + break; case_vv(mov, _f); case jit_code_movi_f: assert_data(node); @@ -1690,6 +1919,10 @@ _emit_code(jit_state_t *_jit) case_vv(abs, _d); case_vv(neg, _d); case_vv(sqrt, _d); + case_rqr(fma, _d); + case_rqr(fms, _d); + case_rqr(fnma, _d); + case_rqr(fnms, _d); case_vv(ext, _d); case_vv(ld, _d); case_vw(ld, _d); @@ -1765,6 +1998,7 @@ _emit_code(jit_state_t *_jit) case_bvv(bunord, _d); case_bvd(bunord); case jit_code_jmpr: + jit_check_frame(); jmpr(rn(node->u.w)); flush_consts(); break; @@ -1776,36 +2010,59 @@ _emit_code(jit_state_t *_jit) if (temp->flag & jit_flag_patch) jmpi(temp->u.w); else { - word = jmpi_p(_jit->pc.w, 1); - patch(word, node); + word = _jit->code.length - + (_jit->pc.uc - _jit->code.ptr); + if (jit_thumb_p()) word >>= 1; + else word >>= 2; + word -= 2; + value = _s24P(word); + word = jmpi_p(_jit->pc.w, value); + patch(word, node, value ? + arm_patch_jump : arm_patch_word); } } - else + else { + jit_check_frame(); jmpi(node->u.w); + } flush_consts(); break; case jit_code_callr: + jit_check_frame(); callr(rn(node->u.w)); break; case jit_code_calli: if (node->flag & jit_flag_node) { + CHECK_RETURN(); temp = node->u.n; assert(temp->code == jit_code_label || temp->code == jit_code_epilog); if (temp->flag & jit_flag_patch) - calli(temp->u.w); + calli(temp->u.w, 0); else { - word = calli_p(_jit->pc.w); - patch(word, node); + word = _jit->code.length - + (_jit->pc.uc - _jit->code.ptr); + if (jit_exchange_p()) + word -= 8; + if (jit_thumb_p()) word >>= 1; + else word >>= 2; + word -= 2; + value = _s24P(word); + word = calli_p(_jit->pc.w, value); + patch(word, node, value ? + arm_patch_call : arm_patch_word); } } - else - calli(node->u.w); + else { + jit_check_frame(); + calli(node->u.w, jit_exchange_p()); + } break; case jit_code_prolog: _jitc->function = _jitc->functions.ptr + node->w.w; undo.node = node; undo.word = _jit->pc.w; + memcpy(&undo.func, _jitc->function, sizeof(undo.func)); #if DEVEL_DISASSEMBLER undo.prevw = prevw; #endif @@ -1819,6 +2076,8 @@ _emit_code(jit_state_t *_jit) #endif restart_function: _jitc->again = 0; + compute_framesize(); + patch_alist(0); prolog(node); break; case jit_code_epilog: @@ -1833,6 +2092,21 @@ _emit_code(jit_state_t *_jit) temp->flag &= ~jit_flag_patch; node = undo.node; _jit->pc.w = undo.word; + /* undo.func.self.aoff and undo.func.regset should not + * be undone, as they will be further updated, and are + * the reason of the undo. */ + undo.func.self.aoff = _jitc->function->frame + + _jitc->function->self.aoff; + undo.func.need_frame = _jitc->function->need_frame; + undo.func.need_return = _jitc->function->need_return; + jit_regset_set(&undo.func.regset, &_jitc->function->regset); + /* allocar information also does not need to be undone */ + undo.func.aoffoff = _jitc->function->aoffoff; + undo.func.allocar = _jitc->function->allocar; + /* swf_offset and check_reg_args must also not be undone */ + undo.func.swf_offset = _jitc->function->swf_offset; + undo.func.save_reg_args = _jitc->function->save_reg_args; + memcpy(_jitc->function, &undo.func, sizeof(undo.func)); #if DEVEL_DISASSEMBLER prevw = undo.prevw; #endif @@ -1845,6 +2119,7 @@ _emit_code(jit_state_t *_jit) if (_jitc->data_info.ptr) _jitc->data_info.offset = undo.info_offset; #endif + patch_alist(1); goto restart_function; } /* remember label is defined */ @@ -1856,41 +2131,46 @@ _emit_code(jit_state_t *_jit) break; case jit_code_movr_w_f: if (jit_swf_p()) - swf_movr_f(rn(node->u.w), rn(node->v.w)); + swf_movr_w_f(rn(node->u.w), rn(node->v.w)); else - vfp_movr_f(rn(node->u.w), rn(node->v.w)); + vfp_movr_w_f(rn(node->u.w), rn(node->v.w)); break; case jit_code_movr_f_w: if (jit_swf_p()) - swf_movr_f(rn(node->u.w), rn(node->v.w)); + swf_movr_f_w(rn(node->u.w), rn(node->v.w)); else - vfp_movr_f(rn(node->u.w), rn(node->v.w)); + vfp_movr_f_w(rn(node->u.w), rn(node->v.w)); break; case jit_code_movi_f_w: assert_data(node); + movi_f_w(rn(node->u.w), node->v.f); + break; + case jit_code_movi_w_f: if (jit_swf_p()) - swf_movi_f(rn(node->u.w), node->v.f); + swf_movi_w_f(rn(node->u.w), node->v.w); else - vfp_movi_f(rn(node->u.w), node->v.f); + vfp_movi_w_f(rn(node->u.w), node->v.w); break; case jit_code_movr_ww_d: if (jit_swf_p()) - swf_movr_d(rn(node->u.w), rn(node->v.w)); + swf_movr_ww_d(rn(node->u.w), rn(node->v.w), rn(node->w.w)); else - vfp_movr_d(rn(node->u.w), rn(node->v.w)); + vfp_movr_ww_d(rn(node->u.w), rn(node->v.w), rn(node->w.w)); break; case jit_code_movr_d_ww: if (jit_swf_p()) - swf_movr_d(rn(node->u.w), rn(node->w.w)); + swf_movr_d_ww(rn(node->u.w), rn(node->v.w), rn(node->w.w)); else - vfp_movr_d(rn(node->u.w), rn(node->w.w)); + vfp_movr_d_ww(rn(node->u.w), rn(node->v.w), rn(node->w.w)); break; case jit_code_movi_d_ww: - assert_data(node); + movi_d_ww(rn(node->u.w), rn(node->v.w), node->w.d); + break; + case jit_code_movi_ww_d: if (jit_swf_p()) - swf_movi_d(rn(node->u.w), node->w.d); + swf_movi_ww_d(rn(node->u.w), node->v.w, node->w.w); else - vfp_movi_d(rn(node->u.w), node->w.d); + vfp_movi_ww_d(rn(node->u.w), node->v.w, node->w.w); break; case jit_code_va_start: vastart(rn(node->u.w)); @@ -1907,21 +2187,34 @@ _emit_code(jit_state_t *_jit) case jit_code_live: case jit_code_ellipsis: case jit_code_va_push: case jit_code_allocai: case jit_code_allocar: - case jit_code_arg: + case jit_code_arg_c: case jit_code_arg_s: + case jit_code_arg_i: case jit_code_arg_f: case jit_code_arg_d: case jit_code_va_end: case jit_code_ret: - case jit_code_retr: case jit_code_reti: + case jit_code_retr_c: case jit_code_reti_c: + case jit_code_retr_uc: case jit_code_reti_uc: + case jit_code_retr_s: case jit_code_reti_s: + case jit_code_retr_us: case jit_code_reti_us: + case jit_code_retr_i: case jit_code_reti_i: case jit_code_retr_f: case jit_code_reti_f: case jit_code_retr_d: case jit_code_reti_d: case jit_code_getarg_c: case jit_code_getarg_uc: case jit_code_getarg_s: case jit_code_getarg_us: case jit_code_getarg_i: case jit_code_getarg_f: case jit_code_getarg_d: - case jit_code_putargr: case jit_code_putargi: + case jit_code_putargr_c: case jit_code_putargi_c: + case jit_code_putargr_uc: case jit_code_putargi_uc: + case jit_code_putargr_s: case jit_code_putargi_s: + case jit_code_putargr_us: case jit_code_putargi_us: + case jit_code_putargr_i: case jit_code_putargi_i: case jit_code_putargr_f: case jit_code_putargi_f: case jit_code_putargr_d: case jit_code_putargi_d: - case jit_code_pushargr: case jit_code_pushargi: + case jit_code_pushargr_c: case jit_code_pushargi_c: + case jit_code_pushargr_uc: case jit_code_pushargi_uc: + case jit_code_pushargr_s: case jit_code_pushargi_s: + case jit_code_pushargr_us: case jit_code_pushargi_us: + case jit_code_pushargr_i: case jit_code_pushargi_i: case jit_code_pushargr_f: case jit_code_pushargi_f: case jit_code_pushargr_d: case jit_code_pushargi_d: case jit_code_retval_c: case jit_code_retval_uc: @@ -1930,6 +2223,63 @@ _emit_code(jit_state_t *_jit) case jit_code_retval_f: case jit_code_retval_d: case jit_code_prepare: case jit_code_finishr: case jit_code_finishi: + case jit_code_negi_f: case jit_code_absi_f: + case jit_code_sqrti_f: case jit_code_negi_d: + case jit_code_absi_d: case jit_code_sqrti_d: + break; + case jit_code_negi: + negi(rn(node->u.w), node->v.w); + break; + case jit_code_comi: + comi(rn(node->u.w), node->v.w); + break; + case jit_code_exti_c: + exti_c(rn(node->u.w), node->v.w); + break; + case jit_code_exti_uc: + exti_uc(rn(node->u.w), node->v.w); + break; + case jit_code_exti_s: + exti_s(rn(node->u.w), node->v.w); + break; + case jit_code_exti_us: + exti_us(rn(node->u.w), node->v.w); + break; + case jit_code_bswapi_us: + bswapi_us(rn(node->u.w), node->v.w); + break; + case jit_code_bswapi_ui: + bswapi_ui(rn(node->u.w), node->v.w); + break; + case jit_code_htoni_us: + htoni_us(rn(node->u.w), node->v.w); + break; + case jit_code_htoni_ui: + htoni_ui(rn(node->u.w), node->v.w); + break; + case jit_code_cloi: + cloi(rn(node->u.w), node->v.w); + break; + case jit_code_clzi: + clzi(rn(node->u.w), node->v.w); + break; + case jit_code_ctoi: + ctoi(rn(node->u.w), node->v.w); + break; + case jit_code_ctzi: + ctzi(rn(node->u.w), node->v.w); + break; + case jit_code_rbiti: + rbiti(rn(node->u.w), node->v.w); + break; + case jit_code_popcnti: + popcnti(rn(node->u.w), node->v.w); + break; + case jit_code_exti: + exti(rn(node->u.w), node->v.w, node->w.q.l, node->w.q.h); + break; + case jit_code_exti_u: + exti_u(rn(node->u.w), node->v.w, node->w.q.l, node->w.q.h); break; default: abort(); @@ -1939,12 +2289,19 @@ _emit_code(jit_state_t *_jit) /* update register live state */ jit_reglive(node); +#if defined JIT_INSTR_MAX + word = 4096 - JIT_INSTR_MAX; +#else + word = 3968; +#endif + /* longest sequence should be 64 bytes, but preventively + * do not let it go past 256 remaining bytes before a flush */ + if (word > 3968) + word = 3968; if (_jitc->consts.length && - (_jit->pc.uc - _jitc->consts.data >= 3968 || + (_jit->pc.uc - _jitc->consts.data >= word || (jit_uword_t)_jit->pc.uc - - (jit_uword_t)_jitc->consts.patches[0] >= 3968)) { - /* longest sequence should be 64 bytes, but preventively - * do not let it go past 128 remaining bytes before a flush */ + (jit_uword_t)_jitc->consts.patches[0] >= word)) { if (node->next && node->next->code != jit_code_jmpi && node->next->code != jit_code_jmpr && @@ -1984,7 +2341,10 @@ _emit_code(jit_state_t *_jit) node = _jitc->patches.ptr[offset].node; word = _jitc->patches.ptr[offset].inst; if (!jit_thumb_p() && - (node->code == jit_code_movi || node->code == jit_code_calli)) { + (node->code == jit_code_movi || + (node->code == jit_code_calli && + (_jitc->patches.ptr[offset].kind & ~arm_patch_node) == + arm_patch_word))) { /* calculate where to patch word */ value = *(jit_int32_t *)word; assert((value & 0x0f700000) == ARM_LDRI); @@ -2254,24 +2614,31 @@ _invalidate_consts(jit_state_t *_jit) } static void -_patch(jit_state_t *_jit, jit_word_t instr, jit_node_t *node) +_compute_framesize(jit_state_t *_jit) +{ + jit_int32_t reg; + _jitc->framesize = sizeof(jit_word_t) * 2; /* lr+fp */ + for (reg = 0; reg < jit_size(iregs); reg++) + if (jit_regset_tstbit(&_jitc->function->regset, iregs[reg])) + _jitc->framesize += sizeof(jit_word_t); + + if (_jitc->function->save_reg_args) + _jitc->framesize += 16; + + /* Make sure functions called have a 8 byte aligned stack */ + _jitc->framesize = (_jitc->framesize + 7) & -8; +} + +static void +_patch(jit_state_t *_jit, jit_word_t instr, jit_node_t *node, jit_int32_t kind) { jit_int32_t flag; - jit_int32_t kind; assert(node->flag & jit_flag_node); - if (node->code == jit_code_movi) { + if (node->code == jit_code_movi) flag = node->v.n->flag; - kind = arm_patch_word; - } - else { + else flag = node->u.n->flag; - if (node->code == jit_code_calli || - (node->code == jit_code_jmpi && !(node->flag & jit_flag_node))) - kind = arm_patch_word; - else - kind = arm_patch_jump; - } assert(!(flag & jit_flag_patch)); kind |= arm_patch_node; if (_jitc->patches.offset >= _jitc->patches.length) {