[pcsx_rearmed.git] / deps / lightning / lib / jit_fallback.c

#if PROTO
#define fallback_save(r0)		_fallback_save(_jit, r0)
static void _fallback_save(jit_state_t*, jit_int32_t);
#define fallback_load(r0)		_fallback_load(_jit, r0)
static void _fallback_load(jit_state_t*, jit_int32_t);
#define fallback_save_regs(r0)		_fallback_save_regs(_jit, r0)
static void _fallback_save_regs(jit_state_t*, jit_int32_t);
#define fallback_load_regs(r0)		_fallback_load_regs(_jit, r0)
static void _fallback_load_regs(jit_state_t*, jit_int32_t);
#define fallback_calli(i0, i1)		_fallback_calli(_jit, i0, i1)
static void _fallback_calli(jit_state_t*, jit_word_t, jit_word_t);
#define fallback_casx(r0,r1,r2,r3,im)	_fallback_casx(_jit,r0,r1,r2,r3,im)
static void _fallback_casx(jit_state_t *, jit_int32_t, jit_int32_t,
			   jit_int32_t, jit_int32_t, jit_word_t);
#define fallback_clo(r0,r1)		_fallback_clo(_jit,r0,r1)
static void _fallback_clo(jit_state_t*, jit_int32_t, jit_int32_t);
#define fallback_clz(r0,r1)		_fallback_clz(_jit,r0,r1)
static void _fallback_clz(jit_state_t*, jit_int32_t, jit_int32_t);
#define fallback_cto(r0,r1)		_fallback_cto(_jit,r0,r1)
static void _fallback_cto(jit_state_t*, jit_int32_t, jit_int32_t);
#define fallback_ctz(r0,r1)		_fallback_ctz(_jit,r0,r1)
static void _fallback_ctz(jit_state_t*, jit_int32_t, jit_int32_t);
#  if defined(__ia64__)
#    define fallback_patch_jmpi(inst,lbl)				\
    do {								\
	sync();								\
	patch_at(jit_code_jmpi, inst, lbl);				\
    } while (0)
#  else
#    define fallback_patch_jmpi(inst,lbl) fallback_patch_at(inst,lbl)
#  endif
#  if defined(__arm__)
#    define fallback_patch_at(inst,lbl)	patch_at(arm_patch_jump,inst,lbl)
#  elif defined(__ia64__)
#    define fallback_patch_at(inst,lbl)					\
    do {								\
	sync();								\
	patch_at(jit_code_bnei, inst, lbl);				\
    } while (0);
#  else
#    define fallback_patch_at(inst,lbl)	patch_at(inst,lbl)
#  endif
#  if defined(__mips__)
#    define fallback_jmpi(i0)		jmpi(i0,1)
#  elif defined(__arm__)
#    define fallback_jmpi(i0)		jmpi_p(i0,1)
#  elif defined(__s390__) || defined(__s390x__)
#    define fallback_jmpi(i0)		jmpi(i0,1)
#  else
#    define fallback_jmpi(i0)		jmpi(i0)
#  endif
#  if defined(__mips__)
#    define fallback_bnei(i0,r0,i1)	bnei(i0,r0,i1)
#  elif defined(__s390__) || defined(__s390x__)
#    define fallback_bnei(i0,r0,i1)	bnei_p(i0,r0,i1)
#  else
#    define fallback_bnei(i0,r0,i1)	bnei(i0,r0,i1)
#  endif
#  if defined(__s390__) || defined(__s390x__)
#    define fallback_bmsr(i0,r0,r1)	bmsr_p(i0,r0,r1)
#  else
#    define fallback_bmsr(i0,r0,r1)	bmsr(i0,r0,r1)
#  endif
#endif

#if CODE
static void
_fallback_save(jit_state_t *_jit, jit_int32_t r0)
{
    jit_int32_t		offset, regno, spec;
    for (offset = 0; offset < JIT_R_NUM; offset++) {
	spec =  _rvs[offset].spec;
	regno = jit_regno(spec);
	if (regno == r0) {
	    if (!(spec & jit_class_sav))
		stxi(_jitc->function->regoff[JIT_R(offset)], rn(JIT_FP), regno);
	    break;
	}
    }
}

static void
_fallback_load(jit_state_t *_jit, jit_int32_t r0)
{
    jit_int32_t		offset, regno, spec;
    for (offset = 0; offset < JIT_R_NUM; offset++) {
	spec =  _rvs[offset].spec;
	regno = jit_regno(spec);
	if (regno == r0) {
	    if (!(spec & jit_class_sav))
		ldxi(regno, rn(JIT_FP), _jitc->function->regoff[JIT_R(offset)]);
	    break;
	}
    }
}

static void
_fallback_save_regs(jit_state_t *_jit, jit_int32_t r0)
{
    jit_int32_t		regno, spec;
    for (regno = 0; regno < _jitc->reglen; regno++) {
	spec =  _rvs[regno].spec;
	if ((jit_regset_tstbit(&_jitc->regarg, regno) ||
	     jit_regset_tstbit(&_jitc->reglive, regno)) &&
	    !(spec & jit_class_sav)) {
	    if (!_jitc->function->regoff[regno]) {
		_jitc->function->regoff[regno] =
		    jit_allocai(spec & jit_class_gpr ?
				sizeof(jit_word_t) : sizeof(jit_float64_t));
		_jitc->again = 1;
	    }
	    if ((spec & jit_class_gpr) && rn(regno) == r0)
		continue;
	    jit_regset_setbit(&_jitc->regsav, regno);
	    if (spec & jit_class_gpr)
		emit_stxi(_jitc->function->regoff[regno], JIT_FP, regno);
	    else
		emit_stxi_d(_jitc->function->regoff[regno], JIT_FP, regno);
	}
    }
}

static void
_fallback_load_regs(jit_state_t *_jit, jit_int32_t r0)
{
    jit_int32_t		regno, spec;
    for (regno = 0; regno < _jitc->reglen; regno++) {
	spec =  _rvs[regno].spec;
	if ((jit_regset_tstbit(&_jitc->regarg, regno) ||
	     jit_regset_tstbit(&_jitc->reglive, regno)) &&
	    !(spec & jit_class_sav)) {
	    if ((spec & jit_class_gpr) && rn(regno) == r0)
		continue;
	    jit_regset_setbit(&_jitc->regsav, regno);
	    if (spec & jit_class_gpr)
		emit_ldxi(regno, JIT_FP, _jitc->function->regoff[regno]);
	    else
		emit_ldxi_d(regno, JIT_FP, _jitc->function->regoff[regno]);
	}
    }
}

static void
_fallback_calli(jit_state_t *_jit, jit_word_t i0, jit_word_t i1)
{
#  if defined(__arm__)
    movi(rn(_R0), i1);
#  elif defined(__hppa__)
    movi(_R26_REGNO, i1);
#  endif
#  if defined(__arm__)
    calli(i0, jit_exchange_p());
#  elif defined(__mips__)
    calli(i0, 0);
#  elif defined(__powerpc__) && _CALL_SYSV
    calli(i0, 0);
#  elif defined(__s390__) || defined(__s390x__)
    calli(i0, 0);
#  else
    calli(i0);
#  endif
}

static void
_fallback_casx(jit_state_t *_jit, jit_int32_t r0, jit_int32_t r1,
	       jit_int32_t r2, jit_int32_t r3, jit_word_t i0)
{
    jit_int32_t		r1_reg, iscasi;
    jit_word_t		jump, done;
    /* XXX only attempts to fallback cas for lightning jit code */
    static pthread_mutex_t mutex = PTHREAD_MUTEX_INITIALIZER;
    if ((iscasi = r1 == _NOREG)) {
	r1_reg = jit_get_reg(jit_class_gpr|jit_class_sav);
	r1 = rn(r1_reg);
	movi(r1, i0);
    }
    fallback_save_regs(r0);
    fallback_calli((jit_word_t)pthread_mutex_lock, (jit_word_t)&mutex);
    fallback_load(r1);
    ldr(r0, r1);
    fallback_load(r2);
    eqr(r0, r0, r2);
    fallback_save(r0);
    jump = fallback_bnei(_jit->pc.w, r0, 1);
    fallback_load(r3);
#  if __WORDSIZE == 32
    str_i(r1, r3);
#  else
    str_l(r1, r3);
#  endif
    /* done: */
    done = _jit->pc.w;
    fallback_calli((jit_word_t)pthread_mutex_unlock, (jit_word_t)&mutex);
    fallback_load(r0);
    fallback_patch_at(jump, done);
    fallback_load_regs(r0);
    if (iscasi)
	jit_unget_reg(r1_reg);
}

static void
_fallback_clo(jit_state_t *_jit, jit_int32_t r0, jit_int32_t r1)
{
    jit_word_t		clz, done;
    comr(r0, r1);
    clz = fallback_bnei(_jit->pc.w, r0, 0);
    movi(r0, __WORDSIZE);
    done = fallback_jmpi(_jit->pc.w);
    fallback_patch_at(clz, _jit->pc.w);
    fallback_clz(r0, r0);
    fallback_patch_jmpi(done, _jit->pc.w);
}

static void
_fallback_clz(jit_state_t *_jit, jit_int32_t r0, jit_int32_t r1)
{
    jit_int32_t		r1_reg, r2, r2_reg;
    jit_word_t		clz, l32, l16, l8, l4, l2, l1;
    l32 = fallback_bnei(_jit->pc.w, r1, 0);
    movi(r0, __WORDSIZE);
    clz = fallback_jmpi(_jit->pc.w);
    fallback_patch_at(l32, _jit->pc.w);
    r2_reg = jit_get_reg(jit_class_gpr);
    r2 = rn(r2_reg);
    r1_reg = jit_get_reg(jit_class_gpr);
    movr(rn(r1_reg), r1);
    r1 = rn(r1_reg);
    movi(r0, 0);
#  if __WORDSIZE == 64
    movi(r2, 0xffffffff00000000UL);
    l32 = fallback_bmsr(_jit->pc.w, r1, r2);
    lshi(r1, r1, 32);
    addi(r0, r0, 32);
    fallback_patch_at(l32, _jit->pc.w);
    lshi(r2, r2, 16);
#  else
    movi(r2, 0xffff0000UL);
#  endif
    l16 = fallback_bmsr(_jit->pc.w, r1, r2);
    lshi(r1, r1, 16);
    addi(r0, r0, 16);
    fallback_patch_at(l16, _jit->pc.w);
    lshi(r2, r2, 8);
    l8 = fallback_bmsr(_jit->pc.w, r1, r2);
    lshi(r1, r1, 8);
    addi(r0, r0, 8);
    fallback_patch_at(l8, _jit->pc.w);
    lshi(r2, r2, 4);
    l4 = fallback_bmsr(_jit->pc.w, r1, r2);
    lshi(r1, r1, 4);
    addi(r0, r0, 4);
    fallback_patch_at(l4, _jit->pc.w);
    lshi(r2, r2, 2);
    l2 = fallback_bmsr(_jit->pc.w, r1, r2);
    lshi(r1, r1, 2);
    addi(r0, r0, 2);
    fallback_patch_at(l2, _jit->pc.w);
    lshi(r2, r2, 1);
    l1 = fallback_bmsr(_jit->pc.w, r1, r2);
    addi(r0, r0, 1);
    fallback_patch_at(l1, _jit->pc.w);
    fallback_patch_jmpi(clz, _jit->pc.w);
    jit_unget_reg(r2_reg);
    jit_unget_reg(r1_reg);
}

static void
_fallback_cto(jit_state_t *_jit, jit_int32_t r0, jit_int32_t r1)
{
    jit_word_t		ctz, done;
    comr(r0, r1);
    ctz = fallback_bnei(_jit->pc.w, r0, 0);
    movi(r0, __WORDSIZE);
    done = fallback_jmpi(_jit->pc.w);
    fallback_patch_at(ctz, _jit->pc.w);
    fallback_ctz(r0, r0);
    fallback_patch_jmpi(done, _jit->pc.w);
}

static void
_fallback_ctz(jit_state_t *_jit, jit_int32_t r0, jit_int32_t r1)
{
    jit_int32_t		r1_reg, r2, r2_reg;
    jit_word_t		ctz, l32, l16, l8, l4, l2, l1;
    l32 = fallback_bnei(_jit->pc.w, r1, 0);
    movi(r0, __WORDSIZE);
    ctz = fallback_jmpi(_jit->pc.w);
    fallback_patch_at(l32, _jit->pc.w);
    r2_reg = jit_get_reg(jit_class_gpr);
    r2 = rn(r2_reg);
    r1_reg = jit_get_reg(jit_class_gpr);
    movr(rn(r1_reg), r1);
    r1 = rn(r1_reg);
    movi(r0, 0);
#  if __WORDSIZE == 64
    movi(r2, 0xffffffffUL);
    l32 = fallback_bmsr(_jit->pc.w, r1, r2);
    rshi_u(r1, r1, 32);
    addi(r0, r0, 32);
    fallback_patch_at(l32, _jit->pc.w);
    rshi(r2, r2, 16);
#  else
    movi(r2, 0xffffUL);
#  endif
    l16 = fallback_bmsr(_jit->pc.w, r1, r2);
    rshi_u(r1, r1, 16);
    addi(r0, r0, 16);
    fallback_patch_at(l16, _jit->pc.w);
    rshi(r2, r2, 8);
    l8 = fallback_bmsr(_jit->pc.w, r1, r2);
    rshi_u(r1, r1, 8);
    addi(r0, r0, 8);
    fallback_patch_at(l8, _jit->pc.w);
    rshi(r2, r2, 4);
    l4 = fallback_bmsr(_jit->pc.w, r1, r2);
    rshi_u(r1, r1, 4);
    addi(r0, r0, 4);
    fallback_patch_at(l4, _jit->pc.w);
    rshi(r2, r2, 2);
    l2 = fallback_bmsr(_jit->pc.w, r1, r2);
    rshi_u(r1, r1, 2);
    addi(r0, r0, 2);
    fallback_patch_at(l2, _jit->pc.w);
    rshi(r2, r2, 1);
    l1 = fallback_bmsr(_jit->pc.w, r1, r2);
    addi(r0, r0, 1);
    fallback_patch_at(l1, _jit->pc.w);
    fallback_patch_jmpi(ctz, _jit->pc.w);
    jit_unget_reg(r2_reg);
    jit_unget_reg(r1_reg);
}
#endif
Commit	Line	Data
ba3814c1 PC	1	#if PROTO
	2	#define fallback_save(r0) _fallback_save(_jit, r0)
	3	static void _fallback_save(jit_state_t*, jit_int32_t);
	4	#define fallback_load(r0) _fallback_load(_jit, r0)
	5	static void _fallback_load(jit_state_t*, jit_int32_t);
	6	#define fallback_save_regs(r0) _fallback_save_regs(_jit, r0)
	7	static void _fallback_save_regs(jit_state_t*, jit_int32_t);
	8	#define fallback_load_regs(r0) _fallback_load_regs(_jit, r0)
	9	static void _fallback_load_regs(jit_state_t*, jit_int32_t);
	10	#define fallback_calli(i0, i1) _fallback_calli(_jit, i0, i1)
	11	static void _fallback_calli(jit_state_t*, jit_word_t, jit_word_t);
	12	#define fallback_casx(r0,r1,r2,r3,im) _fallback_casx(_jit,r0,r1,r2,r3,im)
	13	static void _fallback_casx(jit_state_t *, jit_int32_t, jit_int32_t,
	14	jit_int32_t, jit_int32_t, jit_word_t);
79bfeef6 PC	15	#define fallback_clo(r0,r1) _fallback_clo(_jit,r0,r1)
	16	static void _fallback_clo(jit_state_t*, jit_int32_t, jit_int32_t);
	17	#define fallback_clz(r0,r1) _fallback_clz(_jit,r0,r1)
	18	static void _fallback_clz(jit_state_t*, jit_int32_t, jit_int32_t);
	19	#define fallback_cto(r0,r1) _fallback_cto(_jit,r0,r1)
	20	static void _fallback_cto(jit_state_t*, jit_int32_t, jit_int32_t);
	21	#define fallback_ctz(r0,r1) _fallback_ctz(_jit,r0,r1)
	22	static void _fallback_ctz(jit_state_t*, jit_int32_t, jit_int32_t);
	23	# if defined(__ia64__)
	24	# define fallback_patch_jmpi(inst,lbl) \
	25	do { \
	26	sync(); \
	27	patch_at(jit_code_jmpi, inst, lbl); \
	28	} while (0)
	29	# else
	30	# define fallback_patch_jmpi(inst,lbl) fallback_patch_at(inst,lbl)
	31	# endif
	32	# if defined(__arm__)
	33	# define fallback_patch_at(inst,lbl) patch_at(arm_patch_jump,inst,lbl)
	34	# elif defined(__ia64__)
	35	# define fallback_patch_at(inst,lbl) \
	36	do { \
	37	sync(); \
	38	patch_at(jit_code_bnei, inst, lbl); \
	39	} while (0);
	40	# else
	41	# define fallback_patch_at(inst,lbl) patch_at(inst,lbl)
	42	# endif
	43	# if defined(__mips__)
	44	# define fallback_jmpi(i0) jmpi(i0,1)
	45	# elif defined(__arm__)
	46	# define fallback_jmpi(i0) jmpi_p(i0,1)
	47	# elif defined(__s390__) \|\| defined(__s390x__)
	48	# define fallback_jmpi(i0) jmpi(i0,1)
	49	# else
	50	# define fallback_jmpi(i0) jmpi(i0)
	51	# endif
	52	# if defined(__mips__)
	53	# define fallback_bnei(i0,r0,i1) bnei(i0,r0,i1)
	54	# elif defined(__s390__) \|\| defined(__s390x__)
	55	# define fallback_bnei(i0,r0,i1) bnei_p(i0,r0,i1)
	56	# else
	57	# define fallback_bnei(i0,r0,i1) bnei(i0,r0,i1)
	58	# endif
	59	# if defined(__s390__) \|\| defined(__s390x__)
	60	# define fallback_bmsr(i0,r0,r1) bmsr_p(i0,r0,r1)
	61	# else
	62	# define fallback_bmsr(i0,r0,r1) bmsr(i0,r0,r1)
	63	# endif
ba3814c1 PC	64	#endif
	65
	66	#if CODE
	67	static void
	68	_fallback_save(jit_state_t *_jit, jit_int32_t r0)
	69	{
	70	jit_int32_t offset, regno, spec;
	71	for (offset = 0; offset < JIT_R_NUM; offset++) {
	72	spec = _rvs[offset].spec;
	73	regno = jit_regno(spec);
	74	if (regno == r0) {
	75	if (!(spec & jit_class_sav))
c0c16242	76	stxi(_jitc->function->regoff[JIT_R(offset)], rn(JIT_FP), regno);
ba3814c1 PC	77	break;
	78	}
	79	}
	80	}
	81
	82	static void
	83	_fallback_load(jit_state_t *_jit, jit_int32_t r0)
	84	{
	85	jit_int32_t offset, regno, spec;
	86	for (offset = 0; offset < JIT_R_NUM; offset++) {
	87	spec = _rvs[offset].spec;
	88	regno = jit_regno(spec);
	89	if (regno == r0) {
	90	if (!(spec & jit_class_sav))
c0c16242	91	ldxi(regno, rn(JIT_FP), _jitc->function->regoff[JIT_R(offset)]);
ba3814c1 PC	92	break;
	93	}
	94	}
	95	}
	96
	97	static void
	98	_fallback_save_regs(jit_state_t *_jit, jit_int32_t r0)
	99	{
c0c16242 PC	100	jit_int32_t regno, spec;
c0c16242 PC	101	for (regno = 0; regno < _jitc->reglen; regno++) {
ba3814c1	102	spec = _rvs[regno].spec;
c0c16242 PC	103	if ((jit_regset_tstbit(&_jitc->regarg, regno) \|\|
	104	jit_regset_tstbit(&_jitc->reglive, regno)) &&
	105	!(spec & jit_class_sav)) {
ba3814c1 PC	106	if (!_jitc->function->regoff[regno]) {
ba3814c1 PC	107	_jitc->function->regoff[regno] =
c0c16242 PC	108	jit_allocai(spec & jit_class_gpr ?
c0c16242 PC	109	sizeof(jit_word_t) : sizeof(jit_float64_t));
ba3814c1 PC	110	_jitc->again = 1;
ba3814c1 PC	111	}
c0c16242 PC	112	if ((spec & jit_class_gpr) && rn(regno) == r0)
c0c16242 PC	113	continue;
ba3814c1	114	jit_regset_setbit(&_jitc->regsav, regno);
c0c16242 PC	115	if (spec & jit_class_gpr)
	116	emit_stxi(_jitc->function->regoff[regno], JIT_FP, regno);
	117	else
	118	emit_stxi_d(_jitc->function->regoff[regno], JIT_FP, regno);
ba3814c1 PC	119	}
	120	}
	121	}
	122
	123	static void
	124	_fallback_load_regs(jit_state_t *_jit, jit_int32_t r0)
	125	{
c0c16242 PC	126	jit_int32_t regno, spec;
c0c16242 PC	127	for (regno = 0; regno < _jitc->reglen; regno++) {
ba3814c1	128	spec = _rvs[regno].spec;
c0c16242 PC	129	if ((jit_regset_tstbit(&_jitc->regarg, regno) \|\|
	130	jit_regset_tstbit(&_jitc->reglive, regno)) &&
	131	!(spec & jit_class_sav)) {
	132	if ((spec & jit_class_gpr) && rn(regno) == r0)
	133	continue;
	134	jit_regset_setbit(&_jitc->regsav, regno);
	135	if (spec & jit_class_gpr)
	136	emit_ldxi(regno, JIT_FP, _jitc->function->regoff[regno]);
	137	else
	138	emit_ldxi_d(regno, JIT_FP, _jitc->function->regoff[regno]);
ba3814c1 PC	139	}
	140	}
	141	}
	142
	143	static void
	144	_fallback_calli(jit_state_t *_jit, jit_word_t i0, jit_word_t i1)
	145	{
c0c16242	146	# if defined(__arm__)
ba3814c1	147	movi(rn(_R0), i1);
ba3814c1 PC	148	# elif defined(__hppa__)
ba3814c1 PC	149	movi(_R26_REGNO, i1);
79bfeef6 PC	150	# endif
	151	# if defined(__arm__)
	152	calli(i0, jit_exchange_p());
	153	# elif defined(__mips__)
	154	calli(i0, 0);
	155	# elif defined(__powerpc__) && _CALL_SYSV
	156	calli(i0, 0);
	157	# elif defined(__s390__) \|\| defined(__s390x__)
	158	calli(i0, 0);
	159	# else
ba3814c1	160	calli(i0);
79bfeef6	161	# endif
ba3814c1 PC	162	}
	163
	164	static void
	165	_fallback_casx(jit_state_t *_jit, jit_int32_t r0, jit_int32_t r1,
	166	jit_int32_t r2, jit_int32_t r3, jit_word_t i0)
	167	{
	168	jit_int32_t r1_reg, iscasi;
	169	jit_word_t jump, done;
	170	/* XXX only attempts to fallback cas for lightning jit code */
	171	static pthread_mutex_t mutex = PTHREAD_MUTEX_INITIALIZER;
	172	if ((iscasi = r1 == _NOREG)) {
c0c16242	173	r1_reg = jit_get_reg(jit_class_gpr\|jit_class_sav);
ba3814c1 PC	174	r1 = rn(r1_reg);
	175	movi(r1, i0);
	176	}
	177	fallback_save_regs(r0);
	178	fallback_calli((jit_word_t)pthread_mutex_lock, (jit_word_t)&mutex);
	179	fallback_load(r1);
	180	ldr(r0, r1);
	181	fallback_load(r2);
	182	eqr(r0, r0, r2);
	183	fallback_save(r0);
79bfeef6	184	jump = fallback_bnei(_jit->pc.w, r0, 1);
ba3814c1 PC	185	fallback_load(r3);
	186	# if __WORDSIZE == 32
	187	str_i(r1, r3);
	188	# else
	189	str_l(r1, r3);
	190	# endif
	191	/* done: */
	192	done = _jit->pc.w;
	193	fallback_calli((jit_word_t)pthread_mutex_unlock, (jit_word_t)&mutex);
	194	fallback_load(r0);
79bfeef6	195	fallback_patch_at(jump, done);
ba3814c1 PC	196	fallback_load_regs(r0);
	197	if (iscasi)
	198	jit_unget_reg(r1_reg);
	199	}
79bfeef6 PC	200
	201	static void
	202	_fallback_clo(jit_state_t *_jit, jit_int32_t r0, jit_int32_t r1)
	203	{
	204	jit_word_t clz, done;
	205	comr(r0, r1);
	206	clz = fallback_bnei(_jit->pc.w, r0, 0);
	207	movi(r0, __WORDSIZE);
	208	done = fallback_jmpi(_jit->pc.w);
	209	fallback_patch_at(clz, _jit->pc.w);
	210	fallback_clz(r0, r0);
	211	fallback_patch_jmpi(done, _jit->pc.w);
	212	}
	213
	214	static void
	215	_fallback_clz(jit_state_t *_jit, jit_int32_t r0, jit_int32_t r1)
	216	{
	217	jit_int32_t r1_reg, r2, r2_reg;
	218	jit_word_t clz, l32, l16, l8, l4, l2, l1;
	219	l32 = fallback_bnei(_jit->pc.w, r1, 0);
	220	movi(r0, __WORDSIZE);
	221	clz = fallback_jmpi(_jit->pc.w);
	222	fallback_patch_at(l32, _jit->pc.w);
	223	r2_reg = jit_get_reg(jit_class_gpr);
	224	r2 = rn(r2_reg);
	225	r1_reg = jit_get_reg(jit_class_gpr);
	226	movr(rn(r1_reg), r1);
	227	r1 = rn(r1_reg);
	228	movi(r0, 0);
	229	# if __WORDSIZE == 64
	230	movi(r2, 0xffffffff00000000UL);
	231	l32 = fallback_bmsr(_jit->pc.w, r1, r2);
	232	lshi(r1, r1, 32);
	233	addi(r0, r0, 32);
	234	fallback_patch_at(l32, _jit->pc.w);
	235	lshi(r2, r2, 16);
	236	# else
	237	movi(r2, 0xffff0000UL);
	238	# endif
	239	l16 = fallback_bmsr(_jit->pc.w, r1, r2);
	240	lshi(r1, r1, 16);
	241	addi(r0, r0, 16);
	242	fallback_patch_at(l16, _jit->pc.w);
	243	lshi(r2, r2, 8);
	244	l8 = fallback_bmsr(_jit->pc.w, r1, r2);
	245	lshi(r1, r1, 8);
	246	addi(r0, r0, 8);
	247	fallback_patch_at(l8, _jit->pc.w);
	248	lshi(r2, r2, 4);
	249	l4 = fallback_bmsr(_jit->pc.w, r1, r2);
	250	lshi(r1, r1, 4);
	251	addi(r0, r0, 4);
	252	fallback_patch_at(l4, _jit->pc.w);
	253	lshi(r2, r2, 2);
	254	l2 = fallback_bmsr(_jit->pc.w, r1, r2);
	255	lshi(r1, r1, 2);
	256	addi(r0, r0, 2);
	257	fallback_patch_at(l2, _jit->pc.w);
	258	lshi(r2, r2, 1);
	259	l1 = fallback_bmsr(_jit->pc.w, r1, r2);
	260	addi(r0, r0, 1);
	261	fallback_patch_at(l1, _jit->pc.w);
	262	fallback_patch_jmpi(clz, _jit->pc.w);
	263	jit_unget_reg(r2_reg);
264	jit_unget_reg(r1_reg);
265	}
266
267	static void
268	_fallback_cto(jit_state_t *_jit, jit_int32_t r0, jit_int32_t r1)
269	{
270	jit_word_t ctz, done;
271	comr(r0, r1);
272	ctz = fallback_bnei(_jit->pc.w, r0, 0);
273	movi(r0, __WORDSIZE);
274	done = fallback_jmpi(_jit->pc.w);
275	fallback_patch_at(ctz, _jit->pc.w);
276	fallback_ctz(r0, r0);
277	fallback_patch_jmpi(done, _jit->pc.w);
278	}
279
280	static void
281	_fallback_ctz(jit_state_t *_jit, jit_int32_t r0, jit_int32_t r1)
282	{
283	jit_int32_t r1_reg, r2, r2_reg;
284	jit_word_t ctz, l32, l16, l8, l4, l2, l1;
285	l32 = fallback_bnei(_jit->pc.w, r1, 0);
286	movi(r0, __WORDSIZE);
287	ctz = fallback_jmpi(_jit->pc.w);
288	fallback_patch_at(l32, _jit->pc.w);
289	r2_reg = jit_get_reg(jit_class_gpr);
290	r2 = rn(r2_reg);
291	r1_reg = jit_get_reg(jit_class_gpr);
292	movr(rn(r1_reg), r1);
293	r1 = rn(r1_reg);
294	movi(r0, 0);
295	# if __WORDSIZE == 64
296	movi(r2, 0xffffffffUL);
297	l32 = fallback_bmsr(_jit->pc.w, r1, r2);
298	rshi_u(r1, r1, 32);
299	addi(r0, r0, 32);
300	fallback_patch_at(l32, _jit->pc.w);
301	rshi(r2, r2, 16);
302	# else
303	movi(r2, 0xffffUL);
304	# endif
305	l16 = fallback_bmsr(_jit->pc.w, r1, r2);
306	rshi_u(r1, r1, 16);
307	addi(r0, r0, 16);
308	fallback_patch_at(l16, _jit->pc.w);
309	rshi(r2, r2, 8);
310	l8 = fallback_bmsr(_jit->pc.w, r1, r2);
311	rshi_u(r1, r1, 8);
312	addi(r0, r0, 8);
313	fallback_patch_at(l8, _jit->pc.w);
314	rshi(r2, r2, 4);
315	l4 = fallback_bmsr(_jit->pc.w, r1, r2);
316	rshi_u(r1, r1, 4);
317	addi(r0, r0, 4);
318	fallback_patch_at(l4, _jit->pc.w);
319	rshi(r2, r2, 2);
320	l2 = fallback_bmsr(_jit->pc.w, r1, r2);
321	rshi_u(r1, r1, 2);
322	addi(r0, r0, 2);
323	fallback_patch_at(l2, _jit->pc.w);
324	rshi(r2, r2, 1);
325	l1 = fallback_bmsr(_jit->pc.w, r1, r2);
326	addi(r0, r0, 1);
327	fallback_patch_at(l1, _jit->pc.w);
328	fallback_patch_jmpi(ctz, _jit->pc.w);
329	jit_unget_reg(r2_reg);
330	jit_unget_reg(r1_reg);
331	}
ba3814c1	332	#endif