notaz.gp2x.de / pcsx_rearmed.git / blame
? search:
summary | shortlog | log | commit | commitdiff | tree
blob | blame (incremental) | history | HEAD
git subrepo pull (merge) --force deps/lightning
[pcsx_rearmed.git] / deps / lightning / lib / jit_ppc-cpu.c
CommitLineData
4a71579b 1/*
79bfeef6 2 * Copyright (C) 2012-2023 Free Software Foundation, Inc.
4a71579b
PC		 3 *
4 * This file is part of GNU lightning.
5 *
6 * GNU lightning is free software; you can redistribute it and/or modify it
7 * under the terms of the GNU Lesser General Public License as published
8 * by the Free Software Foundation; either version 3, or (at your option)
9 * any later version.
10 *
11 * GNU lightning is distributed in the hope that it will be useful, but
12 * WITHOUT ANY WARRANTY; without even the implied warranty of MERCHANTABILITY
13 * or FITNESS FOR A PARTICULAR PURPOSE. See the GNU Lesser General Public
14 * License for more details.
15 *
16 * Authors:
17 * Paulo Cesar Pereira de Andrade
18 */
19
20#if PROTO
21# if __WORDSIZE == 32
22# define gpr_save_area 72 /* r14~r31 = 18 * 4 */
23# if _CALL_SYSV
24# define params_offset (sizeof(jit_word_t) << 1)
25# else
26# define params_offset 24
27# endif
28# define can_sign_extend_int_p(im) 1
29# define can_zero_extend_int_p(im) 1
30# define fits_uint32_p(im) 1
31# else
32# define gpr_save_area 144 /* r14~r31 = 18 * 8 */
33# if _CALL_ELF == 2
34# define params_offset 32
35# else
36# define params_offset 48
37# endif
38# define can_sign_extend_int_p(im) \
39 (((im) >= 0 && (long)(im) <= 0x7fffffffL) || \
40 ((im) < 0 && (long)(im) >= -0x80000000L))
41# define can_zero_extend_int_p(im) \
42 ((im) >= 0 && (im) < 0x80000000L)
43# define fits_uint32_p(im) ((im & 0xffffffff00000000L) == 0)
44# endif
45# define fpr_save_area 64
46# define alloca_offset -(gpr_save_area + fpr_save_area)
47# define ii(i) *_jit->pc.ui++ = i
48# if __WORDSIZE == 32
49# define iw(i) *_jit->pc.ui++ = i
50# else
51# define iw(i) *_jit->pc.ul++ = i
52# endif
53# define can_sign_extend_short_p(im) ((im) >= -32768 && (im) <= 32767)
54# define can_zero_extend_short_p(im) ((im) >= 0 && (im) <= 65535)
55# define can_sign_extend_jump_p(im) ((im) >= -33554432 && (im) <= 33554431)
56# define _R0_REGNO 0
57# define _SP_REGNO 1
58# define _R2_REGNO 2
59# define _R11_REGNO 11
60# define _R12_REGNO 12
61# define _FP_REGNO 31
62# if __WORDSIZE == 32
63# define ldr(r0,r1) ldr_i(r0,r1)
ba86ff93 64# define ldi(r0,i0) ldi_i(r0,i0)
4a71579b
PC		 65# define ldxi(r0,r1,i0) ldxi_i(r0,r1,i0)
66# define ldxr(r0,r1,r2) ldxr_i(r0,r1,r2)
ba86ff93
PC		 67# define str(r0,r1) str_i(r0,r1)
68# define sti(i0,r0) sti_i(i0,r0)
4a71579b
PC		 69# define stxi(i0,r0,r1) stxi_i(i0,r0,r1)
70# define stxr(r0,r1,r2) stxr_i(r0,r1,r2)
71# else
72# define ldr(r0,r1) ldr_l(r0,r1)
ba86ff93 73# define ldi(r0,i0) ldi_l(r0,i0)
4a71579b
PC		 74# define ldxi(r0,r1,i0) ldxi_l(r0,r1,i0)
75# define ldxr(r0,r1,r2) ldxr_l(r0,r1,r2)
ba86ff93
PC		 76# define str(r0,r1) str_l(r0,r1)
77# define sti(i0,r0) sti_l(i0,r0)
4a71579b
PC		 78# define stxi(i0,r0,r1) stxi_l(i0,r0,r1)
79# define stxr(r0,r1,r2) stxr_l(r0,r1,r2)
80# endif
81# define FXO(o,d,a,b,e,x) _FXO(_jit,o,d,a,b,e,x,0)
82# define FXO_(o,d,a,b,e,x) _FXO(_jit,o,d,a,b,e,x,1)
83static void _FXO(jit_state_t*,int,int,int,int,int,int,int);
84# define FDs(o,d,a,s) _FDs(_jit,o,d,a,s)
85static void _FDs(jit_state_t*,int,int,int,int);
86# define FDu(o,d,a,s) _FDu(_jit,o,d,a,s)
87static void _FDu(jit_state_t*,int,int,int,int);
88# define FX(o,d,a,b,x) _FX(_jit,o,d,a,b,x,0)
89# define FX_(o,d,a,b,x) _FX(_jit,o,d,a,b,x,1)
90static void _FX(jit_state_t*,int,int,int,int,int,int);
91# define FI(o,t,a,k) _FI(_jit,o,t,a,k)
92static void _FI(jit_state_t*,int,int,int,int);
93# define FB(o,bo,bi,t,a,k) _FB(_jit,o,bo,bi,t,a,k)
94static void _FB(jit_state_t*,int,int,int,int,int,int);
95# define FXL(o,bo,bi,x) _FXL(_jit,o,bo,bi,x,0)
96# define FXL_(o,bo,bi,x) _FXL(_jit,o,bo,bi,x,1)
97static void _FXL(jit_state_t*,int,int,int,int,int);
98# define FC(o,d,l,a,b,x) _FC(_jit,o,d,l,a,b,x)
99static void _FC(jit_state_t*,int,int,int,int,int,int);
100# define FCI(o,d,l,a,s) _FCI(_jit,o,d,l,a,s)
101static void _FCI(jit_state_t*,int,int,int,int,int);
102# define FXFX(o,s,x,f) _FXFX(_jit,o,s,x,f)
103static void _FXFX(jit_state_t*,int,int,int,int);
104# define FM(o,s,a,h,b,e,r) _FM(_jit,o,s,a,h,b,e,r)
105static void _FM(jit_state_t*,int,int,int,int,int,int,int);
106# if __WORDSIZE == 64
107# define FMDS(o,s,a,b,e,x) _FMDS(_jit,o,s,a,b,e,x,0)
108# define FMDS_(o,s,a,b,e,x) _FMDS(_jit,o,s,a,b,e,x,1)
109static void _FMDS(jit_state_t*,int,int,int,int,int,int,int);
110# define FMD(o,s,a,h,b,x,i) _FMD(_jit,o,s,a,h,b,x,i,0)
111# define FMD_(o,s,a,h,b,x,i) _FMD(_jit,o,s,a,h,b,x,i,1)
112static void _FMD(jit_state_t*,int,int,int,int,int,int,int,int);
113# define FXS(o,d,a,h,x,i) _FXS(_jit,o,d,a,h,x,i,0)
114# define FXS_(o,d,a,h,x,i) _FXS(_jit,o,d,a,h,x,i,1)
115static void _FXS(jit_state_t*,int,int,int,int,int,int,int);
116# endif
117# define CR_0 0
118# define CR_1 1
119# define CR_2 2
120# define CR_3 3
121# define CR_4 4
122# define CR_5 5
123# define CR_6 6
124# define CR_7 7
125# define CR_LT 0
126# define CR_GT 1
127# define CR_EQ 2
128# define CR_SO 3
129# define CR_UN 3
130# define BCC_F 4
131# define BCC_T 12
132# define ADD(d,a,b) FXO(31,d,a,b,0,266)
133# define ADD_(d,a,b) FXO_(31,d,a,b,0,266)
134# define ADDO(d,a,b) FXO(31,d,a,b,1,266)
135# define ADDO_(d,a,b) FXO_(31,d,a,b,1,266)
136# define ADDC(d,a,b) FXO_(31,d,a,b,0,10)
137# define ADDC_(d,a,b) FXO_(31,d,a,b,0,10)
138# define ADDCO(d,a,b) FXO(31,d,a,b,1,10)
139# define ADDCO_(d,a,b) FXO_(31,d,a,b,1,10)
140# define ADDE(d,a,b) FXO(31,d,a,b,0,138)
141# define ADDE_(d,a,b) FXO_(31,d,a,b,0,138)
142# define ADDEO(d,a,b) FXO(31,d,a,b,1,138)
143# define ADDEO_(d,a,b) FXO_(31,d,a,b,1,138)
144# define ADDI(d,a,s) FDs(14,d,a,s)
145# define ADDIC(d,a,s) FDs(12,d,a,s)
146# define ADDIC_(d,a,s) FDs(13,d,a,s)
147# define ADDIS(d,a,s) FDs(15,d,a,s)
148# define LIS(d,s) ADDIS(d,0,s)
149# define ADDME(d,a) FXO(31,d,a,0,0,234)
150# define ADDME_(d,a) FXO_(31,d,a,0,0,234)
151# define ADDMEO(d,a) FXO(31,d,a,0,1,234)
152# define ADDMEO_(d,a) FXO_(31,d,a,0,1,234)
153# define ADDZE(d,a) FXO(31,d,a,0,0,202)
154# define ADDZE_(d,a) FXO_(31,d,a,0,0,202)
155# define ADDZEO(d,a) FXO(31,d,a,0,1,202)
156# define ADDZEO_(d,a) FXO_(31,d,a,0,1,202)
157# define AND(d,a,b) FX(31,a,d,b,28)
158# define ANDC(d,a,b) FXO(31,a,d,b,0,60)
159# define ANDC_(d,a,b) FXO_(31,a,d,b,0,60)
160# define AND_(d,a,b) FX_(31,a,b,d,28)
161# define ANDI_(d,a,u) FDu(28,a,d,u)
162# define ANDIS_(d,a,u) FDu(29,a,d,u)
163# define B(t) FI(18,t,0,0)
164# define BA(t) FI(18,t,1,0)
165# define BL(t) FI(18,t,0,1)
166# define BLA(t) FI(18,t,1,1)
167# define BC(o,i,t) FB(16,o,i,t,0,0)
168# define BCA(o,i,t) FB(16,o,i,t,1,0)
169# define BCL(o,i,t) FB(16,o,i,t,0,1)
170# define BCLA(o,i,t) FB(16,o,i,t,1,1)
171# define BLT(t) BC(BCC_T,CR_LT,t)
172# define BLE(t) BC(BCC_F,CR_GT,t)
173# define BEQ(t) BC(BCC_T,CR_EQ,t)
174# define BGE(t) BC(BCC_F,CR_LT,t)
175# define BGT(t) BC(BCC_T,CR_GT,t)
176# define BNE(t) BC(BCC_F,CR_EQ,t)
177# define BUN(t) BC(BCC_T,CR_UN,t)
178# define BNU(t) BC(BCC_F,CR_UN,t)
179# define BCCTR(o,i) FXL(19,o,i,528)
180# define BCCTRL(o,i) FXL_(19,o,i,528)
181# define BLTCTR() BCCTR(BCC_T,CR_LT)
182# define BLECTR() BCCTR(BCC_F,CR_GT)
183# define BEQCTR() BCCTR(BCC_T,CR_EQ)
184# define BGECTR() BCCTR(BCC_F,CR_LT)
185# define BGTCTR() BCCTR(BCC_T,CR_GT)
186# define BNECTR() BCCTR(BCC_F,CR_EQ)
187# define BCTR() BCCTR(20,0)
188# define BCTRL() BCCTRL(20,0)
189# define BCLR(o,i) FXL(19,o,i,16)
190# define BCLRL(o,i) FXL_(19,o,i,16)
191# define BLTLR() BCLR(BCC_T,CR_LT)
192# define BLELR() BCLR(BCC_F,CR_GT)
193# define BEQLR() BCLR(BCC_T,CR_EQ)
194# define BGELR() BCLR(BCC_F,CR_LT)
195# define BGTLR() BCLR(BCC_T,CR_GT)
196# define BNELR() BCLR(BCC_F,CR_EQ)
197# define BLR() BCLR(20,0)
198# define BLRL() BCLRL(20,0)
199# define XCMP(cr,l,a,b) FC(31,cr,l,a,b,0)
200# define CMPD(a,b) XCMP(0,1,a,b)
201# define CMPW(a,b) XCMP(0,0,a,b)
202# define XCMPI(cr,l,a,s) FCI(11,cr,l,a,s)
203# define CMPDI(a,s) XCMPI(0,1,a,s)
204# define CMPWI(a,s) XCMPI(0,0,a,s)
205# define XCMPL(cr,l,a,b) FC(31,cr,l,a,b,32)
206# define CMPLD(a,b) XCMPL(0,1,a,b)
207# define CMPLW(a,b) XCMPL(0,0,a,b)
208# define XCMPLI(cr,l,a,u) FCI(10,cr,l,a,u)
209# define CMPLDI(a,s) XCMPLI(0,1,a,s)
210# define CMPLWI(a,s) XCMPLI(0,0,a,s)
79bfeef6
PC		 211# if __WORDSIZE == 32
212# define CMPX(a,b) CMPW(a,b)
213# define CMPXI(a,s) CMPWI(a,s)
214# define CMPLX(a,b) CMPLW(a,b)
215# define CMPLXI(a,s) CMPLWI(a,s)
216# else
217# define CMPX(a,b) CMPD(a,b)
218# define CMPXI(a,s) CMPDI(a,s)
219# define CMPLX(a,b) CMPLD(a,b)
220# define CMPLXI(a,s) CMPLDI(a,s)
221# endif
4a71579b
PC		 222# define CNTLZW(a,s) FX(31,s,a,0,26)
223# define CNTLZW_(a,s) FX_(31,s,a,0,26)
79bfeef6
PC		 224# define CNTLZD(a,s) FX(31,s,a,0,58)
225# define CNTLZD_(a,s) FX_(31,s,a,0,58)
4a71579b
PC		 226# define CRAND(d,a,b) FX(19,d,a,b,257)
227# define CRANDC(d,a,b) FX(19,d,a,b,129)
228# define CREQV(d,a,b) FX(19,d,a,b,289)
229# define CRSET(d) CREQV(d,d,d)
230# define CRNAND(d,a,b) FX(19,d,a,b,225)
231# define CRNOR(d,a,b) FX(19,d,a,b,33)
232# define CRNOT(d,a) CRNOR(d,a,a)
233# define CROR(d,a,b) FX(19,d,a,b,449)
234# define CRMOVE(d,a) CROR(d,a,a)
235# define CRORC(d,a,b) FX(19,d,a,b,417)
236# define CRXOR(d,a,b) FX(19,d,a,b,193)
237# define CRCLR(d) CRXOR(d,d,d)
238# define DCBA(a,b) FX(31,0,a,b,758)
239# define DCBF(a,b) FX(31,0,a,b,86)
240# define DCBI(a,b) FX(31,0,a,b,470)
241# define DCBST(a,b) FX(31,0,a,b,54)
242# define DCBT(a,b) FX(31,0,a,b,278)
243# define DCBTST(a,b) FX(31,0,a,b,246)
244# define DCBZ(a,b) FX(31,0,a,b,1014)
245# define DIVW(d,a,b) FXO(31,d,a,b,0,491)
246# define DIVW_(d,a,b) FXO_(31,d,a,b,0,491)
247# define DIVWO(d,a,b) FXO(31,d,a,b,1,491)
248# define DIVWO_(d,a,b) FXO_(31,d,a,b,1,491)
249# define DIVWU(d,a,b) FXO(31,d,a,b,0,459)
250# define DIVWU_(d,a,b) FXO_(31,d,a,b,0,459)
251# define DIVWUO(d,a,b) FXO(31,d,a,b,1,459)
252# define DIVWUO_(d,a,b) FXO_(31,d,a,b,1,459)
253# define DIVD(d,a,b) FXO(31,d,a,b,0,489)
254# define DIVD_(d,a,b) FXO_(31,d,a,b,0,489)
255# define DIVDO(d,a,b) FXO(31,d,a,b,1,489)
256# define DIVDO_(d,a,b) FXO_(31,d,a,b,1,489)
257# define DIVDU(d,a,b) FXO(31,d,a,b,0,457)
258# define DIVDU_(d,a,b) FXO_(31,d,a,b,0,457)
259# define DIVDUO(d,a,b) FXO(31,d,a,b,1,457)
260# define DIVDUO_(d,a,b) FXO_(31,d,a,b,1,457)
261# define ECIWX(d,a,b) FX(31,d,a,b,310)
262# define ECOWX(s,a,b) FX(31,s,a,b,438)
263# define EIEIO() FX(31,0,0,0,854)
264# define EQV(d,a,b) FX(31,a,d,b,284)
265# define EQV_(d,a,b) FX_(31,a,d,b,284)
266# define EXTSB(d,a) FX(31,a,d,0,954)
267# define EXTSB_(d,a) FX_(31,a,d,0,954)
268# define EXTSH(d,a) FX(31,a,d,0,922)
269# define EXTSH_(d,a) FX_(31,a,d,0,922)
270# define EXTSW(d,a) FX(31,a,d,0,986)
271# define EXTSW_(d,a) FX_(31,a,d,0,986)
272# define ICIB(a,b) FX(31,0,a,b,982)
273# define ISYNC() FXL(19,0,0,150)
274# define LBZ(d,a,s) FDs(34,d,a,s)
275# define LBZU(d,a,s) FDs(35,d,a,s)
276# define LBZUX(d,a,b) FX(31,d,a,b,119)
277# define LBZX(d,a,b) FX(31,d,a,b,87)
278# define LHA(d,a,s) FDs(42,d,a,s)
279# define LHAU(d,a,s) FDs(43,d,a,s)
280# define LHAUX(d,a,b) FX(31,d,a,b,375)
281# define LHAX(d,a,b) FX(31,d,a,b,343)
ba3814c1 282# define LHBRX(d,a,b) FX(31,d,a,b,790)
4a71579b
PC		 283# define LHZ(d,a,s) FDs(40,d,a,s)
284# define LHZU(d,a,s) FDs(41,d,a,s)
285# define LHZUX(d,a,b) FX(31,d,a,b,311)
286# define LHZX(d,a,b) FX(31,d,a,b,279)
287# define LA(d,a,s) ADDI(d,a,s)
288# define LI(d,s) ADDI(d,0,s)
289# define LMW(d,a,s) FDs(46,d,a,s)
290# define LSWI(d,a,n) FX(31,d,a,n,597)
291# define LSWX(d,a,b) FX(31,d,a,b,533)
292# define LWARX(d,a,b) FX(31,d,a,b,20)
ba3814c1 293# define LDARX(d,a,b) FX(31,d,a,b,84)
4a71579b
PC		 294# define LWBRX(d,a,b) FX(31,d,a,b,534)
295# define LWA(d,a,s) FDs(58,d,a,s|2)
296# define LWAUX(d,a,b) FX(31,d,a,b,373)
297# define LWAX(d,a,b) FX(31,d,a,b,341)
298# define LWZ(d,a,s) FDs(32,d,a,s)
299# define LWZU(d,a,s) FDs(33,d,a,s)
300# define LWZUX(d,a,b) FX(31,d,a,b,55)
301# define LWZX(d,a,b) FX(31,d,a,b,23)
302# define LD(d,a,s) FDs(58,d,a,s)
303# define LDX(d,a,b) FX(31,d,a,b,21)
ba86ff93 304# define MCRF(d,s) FXL(19,((d)<<2),((s)<<2),0)
4a71579b
PC		 305# if DEBUG
306/* In case instruction is emulated, check the kernel can handle it.
307 Will only generate it if DEBUG is enabled.
308"""
309Chapter 6. Optional Facilities and Instructions that are being
310Phased Out of the Architecture
311...
3126.1 Move To Condition Register from XER
313The mcrxr instruction is being phased out of the archi-
314tecture. Its description is included here as an aid to
315constructing operating system code to emulate it.
316
317Move to Condition Register from XER
318X-form
319mcrxr BF
32031 BF // /// /// 512 /
3210 6 9 11 16 21 31
322CR(4xBF:4xBF+3) <- XER(32:35)
323XER(32:35) <- 0b0000
324The contents of XER(32:35) are copied to Condition Reg-
325ister field BF. XER(32:35) are set to zero.
326Special Registers Altered:
327CR field BF XER(32:35)
328
329Programming Note
330Warning: This instruction has been phased out of
331the architecture. Attempting to execute this
332instruction will cause the system illegal instruction
333error handler to be invoked
334"""
335 */
ba86ff93 336# define MCRXR(d) FX(31,((d)<<2),0,0,512)
4a71579b
PC		 337# else
338# define MCRXR(cr) _MCRXR(_jit,cr);
339static void _MCRXR(jit_state_t*, jit_int32_t);
340# endif
341# define MFCR(d) FX(31,d,0,0,19)
342# define MFMSR(d) FX(31,d,0,0,83)
ba86ff93 343# define MFSPR(d,s) FXFX(31,d,((s)<<5),339)
4a71579b
PC		 344# define MFXER(d) MFSPR(d,1)
345# define MFLR(d) MFSPR(d,8)
346# define MFCTR(d) MFSPR(d,9)
347# define MFSR(d,s) FX(31,d,s,0,595)
348# define MFSRIN(d,b) FX(31,d,0,b,659)
ba86ff93 349# define MFTB(d,x,y) FXFX(31,d,((x)|((y)<<5)),371)
4a71579b
PC		 350# define MFTBL(d) MFTB(d,8,12)
351# define MFTBU(d) MFTB(d,8,13)
ba86ff93 352# define MTCRF(c,s) FXFX(31,s,((c)<<1),144)
4a71579b
PC		 353# define MTCR(s) MTCRF(0xff,s)
354# define MTMSR(s) FX(31,s,0,0,146)
ba86ff93 355# define MTSPR(d,s) FXFX(31,d,((s)<<5),467)
4a71579b
PC		 356# define MTXER(d) MTSPR(d,1)
357# define MTLR(d) MTSPR(d,8)
358# define MTCTR(d) MTSPR(d,9)
ba86ff93
PC		 359# define MTSR(r,s) FX(31,((s)<<1),r,0,210)
360# define MTSRIN(r,b) FX(31,((r)<<1),0,b,242)
4a71579b
PC		 361# define MULLI(d,a,s) FDs(07,d,a,s)
362# define MULHW(d,a,b) FXO(31,d,a,b,0,75)
363# define MULHW_(d,a,b) FXO_(31,d,a,b,0,75)
364# define MULHWU(d,a,b) FXO(31,d,a,b,0,11)
365# define MULHWU_(d,a,b) FXO_(31,d,a,b,0,11)
366# define MULLW(d,a,b) FXO(31,d,a,b,0,235)
367# define MULLW_(d,a,b) FXO_(31,d,a,b,0,235)
368# define MULLWO(d,a,b) FXO(31,d,a,b,1,235)
369# define MULLWO_(d,a,b) FXO_(31,d,a,b,1,235)
370# define MULHD(d,a,b) FXO(31,d,a,b,0,73)
371# define MULHD_(d,a,b) FXO_(31,d,a,b,0,73)
372# define MULHDU(d,a,b) FXO(31,d,a,b,0,9)
373# define MULHDU_(d,a,b) FXO_(31,d,a,b,0,9)
374# define MULLD(d,a,b) FXO(31,d,a,b,0,233)
375# define MULLD_(d,a,b) FXO_(31,d,a,b,0,233)
376# define MULLDO(d,a,b) FXO(31,d,a,b,1,233)
377# define MULLDO_(d,a,b) FXO_(31,d,a,b,1,233)
378# define NAND(d,a,b) FX(31,a,d,b,476)
379# define NAND_(d,a,b) FX_(31,a,d,b,476)
380# define NEG(d,a) FXO(31,d,a,0,0,104)
381# define NEG_(d,a) FXO_(31,d,a,0,0,104)
382# define NEGO(d,a) FXO(31,d,a,0,1,104)
383# define NEGO_(d,a) FXO_(31,d,a,0,1,104)
384# define NOR(d,a,b) FX(31,a,d,b,124)
385# define NOR_(d,a,b) FX_(31,a,d,b,124)
386# define NOT(d,s) NOR(d,s,s)
387# define OR(d,a,b) FX(31,a,d,b,444)
388# define OR_(d,a,b) FX_(31,a,d,b,444)
389# define MR(d,a) OR(d,a,a)
390# define ORC(d,a,b) FX(31,a,d,b,412)
391# define ORC_(d,a,b) FX_(31,a,d,b,412)
392# define ORI(d,a,u) FDu(24,a,d,u)
393# define NOP() ORI(0,0,0)
394# define ORIS(d,a,u) FDu(25,a,d,u)
ba86ff93 395# define POPCNTB(a,s) FX(31,s,a,0,122)
4a71579b
PC		 396# define RFI() FXL(19,0,0,50)
397# define RLWIMI(d,s,h,b,e) FM(20,s,d,h,b,e,0)
398# define RLWIMI_(d,s,h,b,e) FM(20,s,d,h,b,e,1)
ba86ff93
PC		 399# define INSLWI(a,s,n,b) RLWIMI(a,s,(32-(b)),b,(((b)+(n))-1))
400# define INSRWI(a,s,n,b) RLWIMI(a,s,(32-((b)+(n))),b,(((b)+(n))-1))
4a71579b
PC		 401# define RLWINM(a,s,h,b,e) FM(21,s,a,h,b,e,0)
402# define RLWINM_(a,s,h,b,e) FM(21,s,a,h,b,e,1)
ba86ff93
PC		 403# define EXTLWI(a,s,n,b) RLWINM(a,s,b,0,((n)-1))
404# define EXTRWI(a,s,n,b) RLWINM(a,s,((b)+(n)),(32-(n)),31)
4a71579b 405# define ROTLWI(a,s,n) RLWINM(a,s,n,0,31)
ba86ff93
PC		 406# define ROTRWI(a,s,n) RLWINM(a,s,(32-(n)),0,31)
407# define SLWI(a,s,n) RLWINM(a,s,n,0,(31-(n)))
408# define SRWI(a,s,n) RLWINM(a,s,(32-(n)),n,31)
4a71579b 409# define CLRLWI(a,s,n) RLWINM(a,s,0,n,31)
ba86ff93
PC		 410# define CLRRWI(a,s,n) RLWINM(a,s,0,0,(31-(n)))
411# define CLRLSWI(a,s,b,n) RLWINM(a,s,n,((b)-(n)),(31-(n)))
4a71579b
PC		 412# define RLWNM(a,s,b,m,e) FM(23,s,a,b,m,e,0)
413# define RLWNM_(a,s,b,m,e) FM(23,s,a,b,m,e,1)
414# define ROTLW(a,s,b) RLWNM(a,s,b,0,31)
415# define SC() FDu(17,0,0,2)
416# define SLW(a,s,b) FX(31,s,a,b,24)
417# define SLW_(a,s,b) FX_(31,s,a,b,24)
418# define SRAW(a,s,b) FX(31,s,a,b,792)
419# define SRAW_(a,s,b) FX_(31,s,a,b,792)
420# define SRAWI(a,s,h) FX(31,s,a,h,824)
421# define SRAWI_(a,s,h) FX_(31,s,a,h,824)
422# define SRW(a,s,b) FX(31,s,a,b,536)
423# define SRW_(a,s,b) FX_(31,s,a,b,536)
424# if __WORDSIZE == 64
ba86ff93
PC		 425# define RLDICL(a,s,h,b) FMD(30,s,a,((h)&~32),b,0,((h)>>5))
426# define RLDICL_(a,s,h,b) FMD_(30,s,a,((h)&~32),b,0,((h)>>5))
427# define EXTRDI(x,y,n,b) RLDICL(x,y,((b)+(n)),(64-(n)))
428# define SRDI(x,y,n) RLDICL(x,y,(64-(n)),n)
4a71579b 429# define CLRLDI(x,y,n) RLDICL(x,y,0,n)
ba86ff93
PC		 430# define RLDICR(a,s,h,e) FMD(30,s,a,((h)&~32),e,1,((h)>>5))
431# define RLDICR_(a,s,h,e) FMD_(30,s,a,((h)&~32),e,1,((h)>>5))
432# define EXTLDI(x,y,n,b) RLDICR(x,y,b,((n)-1))
433# define SLDI(x,y,n) RLDICR(x,y,n,(63-(n)))
434# define CLRRDI(x,y,n) RLDICR(x,y,0,(63-(n)))
435# define RLDIC(a,s,h,b) FMD(30,s,a,((h)&~32),b,2,((h)>>5))
436# define RLDIC_(a,s,h,b) FMD_(30,s,a,((h)&~32),b,2,((h)>>5))
437# define CLRLSLDI(x,y,b,n) RLDIC(x,y,n,((b)-(n)))
4a71579b
PC		 438# define RLDCL(a,s,h,b) FMDS(30,s,a,h,b,8)
439# define RLDCL_(a,s,h,b) FMDS_(30,s,a,h,b,8)
440# define ROTLD(x,y,z) RLDCL(x,y,z,0)
441# define RLDCR(a,s,b,e) FMDS(30,s,a,b,e,0)
442# define RLDCR_(a,s,b,e) FMDS_(30,s,a,b,e,0)
ba86ff93
PC		 443# define RLDIMI(a,s,h,b) FMD(30,s,a,((h)&~32),b,3,((h)>>5))
444# define RLDIMI_(a,s,h,b) FMD_(30,s,a,((h)&~32),b,3,((h)>>5))
445# define INSLDI(x,y,n,b) RLDIMI(x,y,(64-(b)),(((b)+(n))-1))
446# define INSRDI(x,y,n,b) RLDIMI(x,y,(64-((b)+(n))),b)
4a71579b
PC		 447# define SLD(a,s,b) FX(31,s,a,b,27)
448# define SLD_(a,s,b) FX_(31,s,a,b,27)
449# define SRD(a,s,b) FX(31,s,a,b,539)
450# define SRD_(a,s,b) FX_(31,s,a,b,539)
ba86ff93
PC		 451# define SRADI(a,s,h) FXS(31,s,a,((h)&~32),413,((h)>>5))
452# define SRADI_(a,s,h) FXS_(31,s,a,((h)&~32),413,((h)>>5))
4a71579b
PC		 453# define SRAD(a,s,b) FX(31,s,a,b,794)
454# define SRAD_(a,s,b) FX_(31,s,a,b,794)
455# endif
456# define STB(s,a,d) FDs(38,s,a,d)
457# define STBU(s,a,d) FDs(39,s,a,d)
458# define STBUX(s,a,b) FX(31,s,a,b,247)
459# define STBX(s,a,b) FX(31,s,a,b,215)
460# define STH(s,a,d) FDs(44,s,a,d)
461# define STHBRX(s,a,b) FX(31,s,a,b,918)
462# define STHU(s,a,d) FDs(45,s,a,d)
463# define STHUX(s,a,b) FX(31,s,a,b,439)
464# define STHX(s,a,b) FX(31,s,a,b,407)
465# define STMW(s,a,d) FDs(47,s,a,d)
466# define STWSI(s,a,nb) FX(31,s,a,nb,725)
467# define STSWX(s,a,b) FX(31,s,a,b,661)
468# define STW(s,a,d) FDs(36,s,a,d)
469# define STWBRX(s,a,b) FX(31,s,a,b,662)
470# define STWCX_(s,a,b) FX_(31,s,a,b,150)
ba3814c1 471# define STDCX_(s,a,b) FX_(31,s,a,b,214)
4a71579b
PC		 472# define STWU(s,a,d) FDs(37,s,a,d)
473# define STWUX(s,a,b) FX(31,s,a,b,183)
474# define STWX(s,a,b) FX(31,s,a,b,151)
475# define STD(s,a,d) FDs(62,s,a,d)
476# define STDX(s,a,b) FX(31,s,a,b,149)
ba86ff93 477# define STDU(s,a,d) FDs(62,s,a,((d)|1))
4a71579b
PC		 478# define STDUX(s,a,b) FX(31,s,a,b,181)
479# define SUBF(d,a,b) FXO(31,d,a,b,0,40)
480# define SUBF_(d,a,b) FXO_(31,d,a,b,0,40)
481# define SUBFO(d,a,b) FXO(31,d,a,b,1,40)
482# define SUBFO_(d,a,b) FXO_(31,d,a,b,1,40)
483# define SUB(d,a,b) SUBF(d,b,a)
484# define SUB_(d,a,b) SUBF_(d,b,a)
485# define SUBO(d,a,b) SUBFO(d,b,a)
486# define SUBO_(d,a,b) SUBFO_(d,b,a)
ba86ff93
PC		 487# define SUBI(d,a,s) ADDI(d,a,-(s))
488# define SUBIS(d,a,s) ADDIS(d,a,-(s))
4a71579b
PC		 489# define SUBFC(d,a,b) FXO(31,d,a,b,0,8)
490# define SUBFC_(d,a,b) FXO_(31,d,a,b,0,8)
491# define SUBFCO(d,a,b) FXO(31,d,a,b,1,8)
492# define SUBFCO_(d,a,b) FXO_(31,d,a,b,1,8)
493# define SUBC(d,a,b) SUBFC(d,b,a)
ba86ff93
PC		 494# define SUBIC(d,a,s) ADDIC(d,a,-(s))
495# define SUBIC_(d,a,s) ADDIC_(d,a,-(s))
4a71579b
PC		 496# define SUBFE(d,a,b) FXO(31,d,a,b,0,136)
497# define SUBFE_(d,a,b) FXO_(31,d,a,b,0,136)
498# define SUBFEO(d,a,b) FXO(31,d,a,b,1,136)
499# define SUBFEO_(d,a,b) FXO_(31,d,a,b,1,136)
500# define SUBE(d,a,b) SUBFE(d,b,a)
501# define SUBFIC(d,a,s) FDs(8,d,a,s)
502# define SUBFME(d,a) FXO(31,d,a,0,0,232)
503# define SUBFME_(d,a) FXO_(31,d,a,0,0,232)
504# define SUBFMEO(d,a) FXO(31,d,a,0,1,232)
505# define SUBFMEO_(d,a) FXO_(31,d,a,0,1,232)
506# define SUBFZE(d,a) FXO(31,d,a,0,0,200)
507# define SUBFZE_(d,a) FXO_(31,d,a,0,0,200)
508# define SUBFZEO(d,a) FXO(31,d,a,0,1,200)
509# define SUBFZEO_(d,a) FXO_(31,d,a,0,1,200)
510# define SYNC() FX(31,0,0,0,598)
511# define TLBIA() FX(31,0,0,0,370)
512# define TLBIE(b) FX(31,0,0,b,306)
513# define TLBSYNC() FX(31,0,0,0,566)
514# define TW(t,a,b) FX(31,t,a,b,4)
515# define TWEQ(a,b) FX(31,4,a,b,4)
516# define TWLGE(a,b) FX(31,5,a,b,4)
517# define TRAP() FX(31,31,0,0,4)
518# define TWI(t,a,s) FDs(3,t,a,s)
519# define TWGTI(a,s) TWI(8,a,s)
520# define TWLLEI(a,s) TWI(6,a,s)
521# define XOR(d,a,b) FX(31,a,d,b,316)
522# define XOR_(d,a,b) FX_(31,a,d,b,316)
523# define XORI(s,a,u) FDu(26,a,s,u)
524# define XORIS(s,a,u) FDu(27,a,s,u)
525# define nop(c) _nop(_jit,c)
526static void _nop(jit_state_t*,jit_int32_t);
527# define movr(r0,r1) _movr(_jit,r0,r1)
528static void _movr(jit_state_t*,jit_int32_t,jit_int32_t);
529# define movi(r0,i0) _movi(_jit,r0,i0)
530static void _movi(jit_state_t*,jit_int32_t,jit_word_t);
e0659411
PC		 531# define movnr(r0,r1,r2) _movnr(_jit,r0,r1,r2)
532static void _movnr(jit_state_t*,jit_int32_t,jit_int32_t,jit_int32_t);
533# define movzr(r0,r1,r2) _movzr(_jit,r0,r1,r2)
534static void _movzr(jit_state_t*,jit_int32_t,jit_int32_t,jit_int32_t);
4a71579b
PC		 535# define movi_p(r0,i0) _movi_p(_jit,r0,i0)
536static jit_word_t _movi_p(jit_state_t*,jit_int32_t,jit_word_t);
ba3814c1
PC		 537# define casx(r0, r1, r2, r3, i0) _casx(_jit, r0, r1, r2, r3, i0)
538static void _casx(jit_state_t *_jit,jit_int32_t,jit_int32_t,
539 jit_int32_t,jit_int32_t,jit_word_t);
540#define casr(r0, r1, r2, r3) casx(r0, r1, r2, r3, 0)
541#define casi(r0, i0, r1, r2) casx(r0, _NOREG, r1, r2, i0)
4a71579b
PC		 542# define negr(r0,r1) NEG(r0,r1)
543# define comr(r0,r1) NOT(r0,r1)
79bfeef6
PC		 544# define clor(r0, r1) _clor(_jit, r0, r1)
545static void _clor(jit_state_t*, jit_int32_t, jit_int32_t);
546# if __WORDSIZE == 32
547# define clzr(r0, r1) CNTLZW(r0, r1)
548# else
549# define clzr(r0, r1) CNTLZD(r0, r1)
550# endif
551# define ctor(r0, r1) _ctor(_jit, r0, r1)
552static void _ctor(jit_state_t*, jit_int32_t, jit_int32_t);
553# define ctzr(r0, r1) _ctzr(_jit, r0, r1)
554static void _ctzr(jit_state_t*, jit_int32_t, jit_int32_t);
ba86ff93
PC		 555# define popcntr(r0, r1) _popcntr(_jit, r0, r1)
556static void _popcntr(jit_state_t*, jit_int32_t, jit_int32_t);
557# define extr(r0,r1,i0,i1) _extr(_jit,r0,r1,i0,i1)
558static void _extr(jit_state_t*,jit_int32_t,jit_int32_t,jit_word_t,jit_word_t);
559# define extr_u(r0,r1,i0,i1) _extr_u(_jit,r0,r1,i0,i1)
560static void _extr_u(jit_state_t*,jit_int32_t,jit_int32_t,jit_word_t,jit_word_t);
561# define depr(r0,r1,i0,i1) _depr(_jit,r0,r1,i0,i1)
562static void _depr(jit_state_t*,jit_int32_t,jit_int32_t,jit_word_t,jit_word_t);
4a71579b
PC		 563# define extr_c(r0,r1) EXTSB(r0,r1)
564# define extr_uc(r0,r1) ANDI_(r0,r1,0xff)
565# define extr_s(r0,r1) EXTSH(r0,r1)
566# define extr_us(r0,r1) ANDI_(r0,r1,0xffff)
567# if __WORDSIZE == 64
568# define extr_i(r0,r1) EXTSW(r0,r1)
569# define extr_ui(r0,r1) CLRLDI(r0,r1,32)
570# endif
ba3814c1
PC		 571# define bswapr_us_lh(r0,r1,no_flag) _bswapr_us(_jit,r0,r1,no_flag)
572# define bswapr_us(r0,r1) _bswapr_us(_jit,r0,r1,0)
573static void _bswapr_us(jit_state_t*,jit_int32_t,jit_int32_t,jit_bool_t);
574# define bswapr_ui_lw(r0,r1,no_flag) _bswapr_ui(_jit,r0,r1,no_flag)
575# define bswapr_ui(r0,r1) _bswapr_ui(_jit,r0,r1,0)
576static void _bswapr_ui(jit_state_t*,jit_int32_t,jit_int32_t,jit_bool_t);
40a44dcb
PC		 577# if __WORDSIZE == 64
578# define bswapr_ul(r0,r1) generic_bswapr_ul(_jit,r0,r1)
4a71579b
PC		 579# endif
580# define addr(r0,r1,r2) ADD(r0,r1,r2)
581# define addi(r0,r1,i0) _addi(_jit,r0,r1,i0)
582static void _addi(jit_state_t*,jit_int32_t,jit_int32_t,jit_word_t);
583# define addcr(r0,r1,r2) ADDC(r0,r1,r2)
584# define addci(r0,r1,i0) _addci(_jit,r0,r1,i0)
585static void _addci(jit_state_t*,jit_int32_t,jit_int32_t,jit_word_t);
586# define addxr(r0,r1,r2) ADDE(r0,r1,r2)
587# define addxi(r0,r1,i0) _addxi(_jit,r0,r1,i0)
588static void _addxi(jit_state_t*,jit_int32_t,jit_int32_t,jit_word_t);
589# define subr(r0,r1,r2) SUB(r0,r1,r2)
590# define subi(r0,r1,i0) _subi(_jit,r0,r1,i0)
591static void _subi(jit_state_t*,jit_int32_t,jit_int32_t,jit_word_t);
592# define subcr(r0,r1,r2) SUBC(r0,r1,r2)
593# define subci(r0,r1,i0) _subci(_jit,r0,r1,i0)
594static void _subci(jit_state_t*,jit_int32_t,jit_int32_t,jit_word_t);
595# define subxr(r0,r1,r2) SUBFE(r0,r2,r1)
596# define subxi(r0,r1,i0) _subxi(_jit,r0,r1,i0)
597static void _subxi(jit_state_t*,jit_int32_t,jit_int32_t,jit_word_t);
598# define rsbi(r0, r1, i0) _rsbi(_jit, r0, r1, i0)
599static void _rsbi(jit_state_t*,jit_int32_t,jit_int32_t,jit_word_t);
600# if __WORDSIZE == 32
601# define mulr(r0,r1,r2) MULLW(r0,r1,r2)
602# define mullr(r0,r1,r2) MULLW(r0,r1,r2)
ba86ff93
PC		 603# define hmulr(r0,r1,r2) MULHW(r0,r1,r2)
604# define hmulr_u(r0,r1,r2) MULHWU(r0,r1,r2)
4a71579b
PC		 605# else
606# define mulr(r0,r1,r2) MULLD(r0,r1,r2)
607# define mullr(r0,r1,r2) MULLD(r0,r1,r2)
ba86ff93
PC		 608# define hmulr(r0,r1,r2) MULHD(r0,r1,r2)
609# define hmulr_u(r0,r1,r2) MULHDU(r0,r1,r2)
4a71579b
PC		 610# endif
611# define muli(r0,r1,i0) _muli(_jit,r0,r1,i0)
612static void _muli(jit_state_t*,jit_int32_t,jit_int32_t,jit_word_t);
ba86ff93
PC		 613# define hmuli(r0,r1,i0) _hmuli(_jit,r0,r1,i0)
614static void _hmuli(jit_state_t*,jit_int32_t,jit_int32_t,jit_word_t);
615# define hmuli_u(r0,r1,i0) _hmuli_u(_jit,r0,r1,i0)
616static void _hmuli_u(jit_state_t*,jit_int32_t,jit_int32_t,jit_word_t);
4a71579b
PC		 617# define qmulr(r0,r1,r2,r3) iqmulr(r0,r1,r2,r3,1)
618# define qmulr_u(r0,r1,r2,r3) iqmulr(r0,r1,r2,r3,0)
619# define iqmulr(r0,r1,r2,r3,cc) _iqmulr(_jit,r0,r1,r2,r3,cc)
620static void _iqmulr(jit_state_t*,jit_int32_t,jit_int32_t,
621 jit_int32_t,jit_int32_t,jit_bool_t);
622# define qmuli(r0,r1,r2,i0) iqmuli(r0,r1,r2,i0,1)
623# define qmuli_u(r0,r1,r2,i0) iqmuli(r0,r1,r2,i0,0)
624# define iqmuli(r0,r1,r2,i0,cc) _iqmuli(_jit,r0,r1,r2,i0,cc)
625static void _iqmuli(jit_state_t*,jit_int32_t,jit_int32_t,
626 jit_int32_t,jit_word_t,jit_bool_t);
627# if __WORDSIZE == 32
628# define divr(r0,r1,r2) DIVW(r0,r1,r2)
629# else
630# define divr(r0,r1,r2) DIVD(r0,r1,r2)
631# endif
632# define divi(r0,r1,i0) _divi(_jit,r0,r1,i0)
633static void _divi(jit_state_t*,jit_int32_t,jit_int32_t,jit_word_t);
634# if __WORDSIZE == 32
635# define divr_u(r0,r1,r2) DIVWU(r0,r1,r2)
636# else
637# define divr_u(r0,r1,r2) DIVDU(r0,r1,r2)
638# endif
639# define divi_u(r0,r1,i0) _divi_u(_jit,r0,r1,i0)
640static void _divi_u(jit_state_t*,jit_int32_t,jit_int32_t,jit_word_t);
641# define qdivr(r0,r1,r2,r3) iqdivr(r0,r1,r2,r3,1)
642# define qdivr_u(r0,r1,r2,r3) iqdivr(r0,r1,r2,r3,0)
643# define iqdivr(r0,r1,r2,r3,cc) _iqdivr(_jit,r0,r1,r2,r3,cc)
644static void _iqdivr(jit_state_t*,jit_int32_t,jit_int32_t,
645 jit_int32_t,jit_int32_t,jit_bool_t);
646# define qdivi(r0,r1,r2,i0) iqdivi(r0,r1,r2,i0,1)
647# define qdivi_u(r0,r1,r2,i0) iqdivi(r0,r1,r2,i0,0)
648# define iqdivi(r0,r1,r2,i0,cc) _iqdivi(_jit,r0,r1,r2,i0,cc)
649static void _iqdivi(jit_state_t*,jit_int32_t,jit_int32_t,
650 jit_int32_t,jit_word_t,jit_bool_t);
651# define remr(r0,r1,r2) _remr(_jit,r0,r1,r2)
652static void _remr(jit_state_t*,jit_int32_t,jit_int32_t,jit_int32_t);
653# define remi(r0,r1,i0) _remi(_jit,r0,r1,i0)
654static void _remi(jit_state_t*,jit_int32_t,jit_int32_t,jit_word_t);
655# define remr_u(r0,r1,r2) _remr_u(_jit,r0,r1,r2)
656static void _remr_u(jit_state_t*,jit_int32_t,jit_int32_t,jit_int32_t);
657# define remi_u(r0,r1,i0) _remi_u(_jit,r0,r1,i0)
658static void _remi_u(jit_state_t*,jit_int32_t,jit_int32_t,jit_word_t);
659# define andr(r0,r1,r2) AND(r0,r1,r2)
660# define andi(r0,r1,i0) _andi(_jit,r0,r1,i0)
661static void _andi(jit_state_t*,jit_int32_t,jit_int32_t,jit_word_t);
662# define orr(r0,r1,r2) OR(r0,r1,r2)
663# define ori(r0,r1,i0) _ori(_jit,r0,r1,i0)
664static void _ori(jit_state_t*,jit_int32_t,jit_int32_t,jit_word_t);
665# define xorr(r0,r1,r2) XOR(r0,r1,r2)
666# define xori(r0,r1,i0) _xori(_jit,r0,r1,i0)
667static void _xori(jit_state_t*,jit_int32_t,jit_int32_t,jit_word_t);
668# if __WORDSIZE == 32
669# define lshr(r0,r1,r2) SLW(r0,r1,r2)
670# else
671# define lshr(r0,r1,r2) SLD(r0,r1,r2)
672# endif
ba86ff93
PC		 673#define qlshr(r0,r1,r2,r3) xlshr(1,r0,r1,r2,r3)
674#define xlshr(s,r0,r1,r2,r3) _xlshr(_jit,s,r0,r1,r2,r3)
675static void
676_xlshr(jit_state_t*,jit_bool_t,jit_int32_t,jit_int32_t,jit_int32_t,jit_int32_t);
4a71579b
PC		 677# define lshi(r0,r1,i0) _lshi(_jit,r0,r1,i0)
678static void _lshi(jit_state_t*,jit_int32_t,jit_int32_t,jit_word_t);
ba86ff93
PC		 679# define qlshi(r0, r1, r2, i0) xlshi(1, r0, r1, r2, i0)
680# define xlshi(s, r0, r1, r2, i0) _xlshi(_jit, s, r0, r1, r2, i0)
681static void
682_xlshi(jit_state_t*,jit_bool_t,jit_int32_t,jit_int32_t,jit_int32_t,jit_word_t);
683# define qlshr_u(r0, r1, r2, r3) xlshr(0, r0, r1, r2, r3)
684# define qlshi_u(r0, r1, r2, i0) xlshi(0, r0, r1, r2, i0)
4a71579b
PC		 685# if __WORDSIZE == 32
686# define rshr(r0,r1,r2) SRAW(r0,r1,r2)
687# else
688# define rshr(r0,r1,r2) SRAD(r0,r1,r2)
689# endif
690# define rshi(r0,r1,i0) _rshi(_jit,r0,r1,i0)
691static void _rshi(jit_state_t*,jit_int32_t,jit_int32_t,jit_word_t);
692# if __WORDSIZE == 32
693# define rshr_u(r0,r1,r2) SRW(r0,r1,r2)
694# else
695# define rshr_u(r0,r1,r2) SRD(r0,r1,r2)
696# endif
697# define rshi_u(r0,r1,i0) _rshi_u(_jit,r0,r1,i0)
698static void _rshi_u(jit_state_t*,jit_int32_t,jit_int32_t,jit_word_t);
ba86ff93
PC		 699# define qrshr(r0, r1, r2, r3) xrshr(1, r0, r1, r2, r3)
700# define qrshr_u(r0, r1, r2, r3) xrshr(0, r0, r1, r2, r3)
701# define xrshr(s, r0, r1, r2, r3) _xrshr(_jit, s, r0, r1, r2, r3)
702static void
703_xrshr(jit_state_t*,jit_bool_t,jit_int32_t,jit_int32_t,jit_int32_t,jit_int32_t);
704# define qrshi(r0, r1, r2, i0) xrshi(1, r0, r1, r2, i0)
705# define qrshi_u(r0, r1, r2, i0) xrshi(0, r0, r1, r2, i0)
706# define xrshi(s, r0, r1, r2, i0) _xrshi(_jit, s, r0, r1, r2, i0)
707static void
708_xrshi(jit_state_t*,jit_bool_t,jit_int32_t,jit_int32_t,jit_int32_t,jit_word_t);
709# if __WORDSIZE == 32
710# define lrotr(r0,r1,r2) ROTLW(r0,r1,r2)
711# else
712# define lrotr(r0,r1,r2) ROTLD(r0,r1,r2)
713# endif
714# define lroti(r0,r1,i0) _lroti(_jit,r0,r1,i0)
715static void _lroti(jit_state_t*,jit_int32_t,jit_int32_t,jit_word_t);
716# define rrotr(r0,r1,r2) _rrotr(_jit,r0,r1,r2)
717static void _rrotr(jit_state_t*,jit_int32_t,jit_int32_t,jit_int32_t);
718# define rroti(r0,r1,i0) _rroti(_jit,r0,r1,i0)
719static void _rroti(jit_state_t*,jit_int32_t,jit_int32_t,jit_word_t);
4a71579b
PC		 720# define ltr(r0,r1,r2) _ltr(_jit,r0,r1,r2)
721static void _ltr(jit_state_t*,jit_int32_t,jit_int32_t,jit_int32_t);
722# define lti(r0,r1,i0) _lti(_jit,r0,r1,i0)
723static void _lti(jit_state_t*,jit_int32_t,jit_int32_t,jit_word_t);
724# define ltr_u(r0,r1,r2) _ltr_u(_jit,r0,r1,r2)
725static void _ltr_u(jit_state_t*,jit_int32_t,jit_int32_t,jit_int32_t);
726# define lti_u(r0,r1,i0) _lti_u(_jit,r0,r1,i0)
727static void _lti_u(jit_state_t*,jit_int32_t,jit_int32_t,jit_word_t);
728# define ler(r0,r1,r2) _ler(_jit,r0,r1,r2)
729static void _ler(jit_state_t*,jit_int32_t,jit_int32_t,jit_int32_t);
730# define lei(r0,r1,i0) _lei(_jit,r0,r1,i0)
731static void _lei(jit_state_t*,jit_int32_t,jit_int32_t,jit_word_t);
732# define ler_u(r0,r1,r2) _ler_u(_jit,r0,r1,r2)
733static void _ler_u(jit_state_t*,jit_int32_t,jit_int32_t,jit_int32_t);
734# define lei_u(r0,r1,i0) _lei_u(_jit,r0,r1,i0)
735static void _lei_u(jit_state_t*,jit_int32_t,jit_int32_t,jit_word_t);
736# define eqr(r0,r1,r2) _eqr(_jit,r0,r1,r2)
737static void _eqr(jit_state_t*,jit_int32_t,jit_int32_t,jit_int32_t);
738# define eqi(r0,r1,i0) _eqi(_jit,r0,r1,i0)
739static void _eqi(jit_state_t*,jit_int32_t,jit_int32_t,jit_word_t);
740# define ger(r0,r1,r2) _ger(_jit,r0,r1,r2)
741static void _ger(jit_state_t*,jit_int32_t,jit_int32_t,jit_int32_t);
742# define gei(r0,r1,i0) _gei(_jit,r0,r1,i0)
743static void _gei(jit_state_t*,jit_int32_t,jit_int32_t,jit_word_t);
744# define ger_u(r0,r1,r2) _ger_u(_jit,r0,r1,r2)
745static void _ger_u(jit_state_t*,jit_int32_t,jit_int32_t,jit_int32_t);
746# define gei_u(r0,r1,i0) _gei_u(_jit,r0,r1,i0)
747static void _gei_u(jit_state_t*,jit_int32_t,jit_int32_t,jit_word_t);
748# define gtr(r0,r1,r2) _gtr(_jit,r0,r1,r2)
749static void _gtr(jit_state_t*,jit_int32_t,jit_int32_t,jit_int32_t);
750# define gti(r0,r1,i0) _gti(_jit,r0,r1,i0)
751static void _gti(jit_state_t*,jit_int32_t,jit_int32_t,jit_word_t);
752# define gtr_u(r0,r1,r2) _gtr_u(_jit,r0,r1,r2)
753static void _gtr_u(jit_state_t*,jit_int32_t,jit_int32_t,jit_int32_t);
754# define gti_u(r0,r1,i0) _gti_u(_jit,r0,r1,i0)
755static void _gti_u(jit_state_t*,jit_int32_t,jit_int32_t,jit_word_t);
756# define ner(r0,r1,r2) _ner(_jit,r0,r1,r2)
757static void _ner(jit_state_t*,jit_int32_t,jit_int32_t,jit_int32_t);
758# define nei(r0,r1,i0) _nei(_jit,r0,r1,i0)
759static void _nei(jit_state_t*,jit_int32_t,jit_int32_t,jit_word_t);
760#define bltr(i0,r0,r1) _bltr(_jit,i0,r0,r1)
761static jit_word_t _bltr(jit_state_t*,jit_word_t,jit_int32_t,jit_int32_t);
762#define blti(i0,r0,i1) _blti(_jit,i0,r0,i1)
763static jit_word_t _blti(jit_state_t*,jit_word_t,jit_int32_t,jit_word_t);
764#define bltr_u(i0,r0,r1) _bltr_u(_jit,i0,r0,r1)
765static jit_word_t _bltr_u(jit_state_t*,jit_word_t,jit_int32_t,jit_int32_t);
766#define blti_u(i0,r0,i1) _blti_u(_jit,i0,r0,i1)
767static jit_word_t _blti_u(jit_state_t*,jit_word_t,jit_int32_t,jit_word_t);
768#define bler(i0,r0,r1) _bler(_jit,i0,r0,r1)
769static jit_word_t _bler(jit_state_t*,jit_word_t,jit_int32_t,jit_int32_t);
770#define blei(i0,r0,i1) _blei(_jit,i0,r0,i1)
771static jit_word_t _blei(jit_state_t*,jit_word_t,jit_int32_t,jit_word_t);
772#define bler_u(i0,r0,r1) _bler_u(_jit,i0,r0,r1)
773static jit_word_t _bler_u(jit_state_t*,jit_word_t,jit_int32_t,jit_int32_t);
774#define blei_u(i0,r0,i1) _blei_u(_jit,i0,r0,i1)
775static jit_word_t _blei_u(jit_state_t*,jit_word_t,jit_int32_t,jit_word_t);
776#define beqr(i0,r0,r1) _beqr(_jit,i0,r0,r1)
777static jit_word_t _beqr(jit_state_t*,jit_word_t,jit_int32_t,jit_int32_t);
778#define beqi(i0,r0,i1) _beqi(_jit,i0,r0,i1)
779static jit_word_t _beqi(jit_state_t*,jit_word_t,jit_int32_t,jit_word_t);
780#define bger(i0,r0,r1) _bger(_jit,i0,r0,r1)
781static jit_word_t _bger(jit_state_t*,jit_word_t,jit_int32_t,jit_int32_t);
782#define bgei(i0,r0,i1) _bgei(_jit,i0,r0,i1)
783static jit_word_t _bgei(jit_state_t*,jit_word_t,jit_int32_t,jit_word_t);
784#define bger_u(i0,r0,r1) _bger_u(_jit,i0,r0,r1)
785static jit_word_t _bger_u(jit_state_t*,jit_word_t,jit_int32_t,jit_int32_t);
786#define bgei_u(i0,r0,i1) _bgei_u(_jit,i0,r0,i1)
787static jit_word_t _bgei_u(jit_state_t*,jit_word_t,jit_int32_t,jit_word_t);
788#define bgtr(i0,r0,r1) _bgtr(_jit,i0,r0,r1)
789static jit_word_t _bgtr(jit_state_t*,jit_word_t,jit_int32_t,jit_int32_t);
790#define bgti(i0,r0,i1) _bgti(_jit,i0,r0,i1)
791static jit_word_t _bgti(jit_state_t*,jit_word_t,jit_int32_t,jit_word_t);
792#define bgtr_u(i0,r0,r1) _bgtr_u(_jit,i0,r0,r1)
793static jit_word_t _bgtr_u(jit_state_t*,jit_word_t,jit_int32_t,jit_int32_t);
794#define bgti_u(i0,r0,i1) _bgti_u(_jit,i0,r0,i1)
795static jit_word_t _bgti_u(jit_state_t*,jit_word_t,jit_int32_t,jit_word_t);
796#define bner(i0,r0,r1) _bner(_jit,i0,r0,r1)
797static jit_word_t _bner(jit_state_t*,jit_word_t,jit_int32_t,jit_int32_t);
798#define bnei(i0,r0,i1) _bnei(_jit,i0,r0,i1)
799static jit_word_t _bnei(jit_state_t*,jit_word_t,jit_int32_t,jit_word_t);
800#define bmsr(i0,r0,r1) _bmsr(_jit,i0,r0,r1)
801static jit_word_t _bmsr(jit_state_t*,jit_word_t,jit_int32_t,jit_int32_t);
802#define bmsi(i0,r0,i1) _bmsi(_jit,i0,r0,i1)
803static jit_word_t _bmsi(jit_state_t*,jit_word_t,jit_int32_t,jit_word_t);
804#define bmcr(i0,r0,r1) _bmcr(_jit,i0,r0,r1)
805static jit_word_t _bmcr(jit_state_t*,jit_word_t,jit_int32_t,jit_int32_t);
806#define bmci(i0,r0,i1) _bmci(_jit,i0,r0,i1)
807static jit_word_t _bmci(jit_state_t*,jit_word_t,jit_int32_t,jit_word_t);
808#define boaddr(i0,r0,r1) _boaddr(_jit,i0,r0,r1)
809static jit_word_t _boaddr(jit_state_t*,jit_word_t,jit_int32_t,jit_int32_t);
810#define boaddi(i0,r0,i1) _boaddi(_jit,i0,r0,i1)
811static jit_word_t _boaddi(jit_state_t*,jit_word_t,jit_int32_t,jit_word_t);
812#define bxaddr(i0,r0,r1) _bxaddr(_jit,i0,r0,r1)
813static jit_word_t _bxaddr(jit_state_t*,jit_word_t,jit_int32_t,jit_int32_t);
814#define bxaddi(i0,r0,i1) _bxaddi(_jit,i0,r0,i1)
815static jit_word_t _bxaddi(jit_state_t*,jit_word_t,jit_int32_t,jit_word_t);
816#define bosubr(i0,r0,r1) _bosubr(_jit,i0,r0,r1)
817static jit_word_t _bosubr(jit_state_t*,jit_word_t,jit_int32_t,jit_int32_t);
818#define bosubi(i0,r0,i1) _bosubi(_jit,i0,r0,i1)
819static jit_word_t _bosubi(jit_state_t*,jit_word_t,jit_int32_t,jit_word_t);
820#define bxsubr(i0,r0,r1) _bxsubr(_jit,i0,r0,r1)
821static jit_word_t _bxsubr(jit_state_t*,jit_word_t,jit_int32_t,jit_int32_t);
822#define bxsubi(i0,r0,i1) _bxsubi(_jit,i0,r0,i1)
823static jit_word_t _bxsubi(jit_state_t*,jit_word_t,jit_int32_t,jit_word_t);
824#define boaddr_u(i0,r0,r1) _boaddr_u(_jit,i0,r0,r1)
825static jit_word_t _boaddr_u(jit_state_t*,jit_word_t,jit_int32_t,jit_int32_t);
826#define boaddi_u(i0,r0,i1) _boaddi_u(_jit,i0,r0,i1)
827static jit_word_t _boaddi_u(jit_state_t*,jit_word_t,jit_int32_t,jit_word_t);
828#define bxaddr_u(i0,r0,r1) _bxaddr_u(_jit,i0,r0,r1)
829static jit_word_t _bxaddr_u(jit_state_t*,jit_word_t,jit_int32_t,jit_int32_t);
830#define bxaddi_u(i0,r0,i1) _bxaddi_u(_jit,i0,r0,i1)
831static jit_word_t _bxaddi_u(jit_state_t*,jit_word_t,jit_int32_t,jit_word_t);
832#define bosubr_u(i0,r0,r1) _bosubr_u(_jit,i0,r0,r1)
833static jit_word_t _bosubr_u(jit_state_t*,jit_word_t,jit_int32_t,jit_int32_t);
834#define bosubi_u(i0,r0,i1) _bosubi_u(_jit,i0,r0,i1)
835static jit_word_t _bosubi_u(jit_state_t*,jit_word_t,jit_int32_t,jit_word_t);
836#define bxsubr_u(i0,r0,r1) _bxsubr_u(_jit,i0,r0,r1)
837static jit_word_t _bxsubr_u(jit_state_t*,jit_word_t,jit_int32_t,jit_int32_t);
838#define bxsubi_u(i0,r0,i1) _bxsubi_u(_jit,i0,r0,i1)
839static jit_word_t _bxsubi_u(jit_state_t*,jit_word_t,jit_int32_t,jit_word_t);
840# define ldr_c(r0,r1) _ldr_c(_jit,r0,r1)
841static void _ldr_c(jit_state_t*,jit_int32_t,jit_int32_t);
842# define ldi_c(r0,i0) _ldi_c(_jit,r0,i0)
843static void _ldi_c(jit_state_t*,jit_int32_t,jit_word_t);
844# define ldxr_c(r0,r1,i0) _ldxr_c(_jit,r0,r1,i0)
845static void _ldxr_c(jit_state_t*,jit_int32_t,jit_int32_t,jit_int32_t);
846# define ldxi_c(r0,r1,i0) _ldxi_c(_jit,r0,r1,i0)
847static void _ldxi_c(jit_state_t*,jit_int32_t,jit_int32_t,jit_word_t);
848# define ldr_uc(r0,r1) LBZX(r0, _R0_REGNO, r1)
849# define ldi_uc(r0,i0) _ldi_uc(_jit,r0,i0)
850static void _ldi_uc(jit_state_t*,jit_int32_t,jit_word_t);
851# define ldxr_uc(r0,r1,r2) _ldxr_uc(_jit,r0,r1,r2)
852static void _ldxr_uc(jit_state_t*,jit_int32_t,jit_int32_t,jit_int32_t);
853# define ldxi_uc(r0,r1,i0) _ldxi_uc(_jit,r0,r1,i0)
854static void _ldxi_uc(jit_state_t*,jit_int32_t,jit_int32_t,jit_word_t);
855# define ldr_s(r0,r1) LHAX(r0, _R0_REGNO, r1)
856# define ldi_s(r0,i0) _ldi_s(_jit,r0,i0)
857static void _ldi_s(jit_state_t*,jit_int32_t,jit_word_t);
858# define ldxr_s(r0,r1,i0) _ldxr_s(_jit,r0,r1,i0)
859static void _ldxr_s(jit_state_t*,jit_int32_t,jit_int32_t,jit_int32_t);
860# define ldxi_s(r0,r1,i0) _ldxi_s(_jit,r0,r1,i0)
861static void _ldxi_s(jit_state_t*,jit_int32_t,jit_int32_t,jit_word_t);
862# define ldr_us(r0,r1) LHZX(r0, _R0_REGNO, r1)
863# define ldi_us(r0,i0) _ldi_us(_jit,r0,i0)
864static void _ldi_us(jit_state_t*,jit_int32_t,jit_word_t);
865# define ldxr_us(r0,r1,i0) _ldxr_us(_jit,r0,r1,i0)
866static void _ldxr_us(jit_state_t*,jit_int32_t,jit_int32_t,jit_int32_t);
867# define ldxi_us(r0,r1,i0) _ldxi_us(_jit,r0,r1,i0)
868static void _ldxi_us(jit_state_t*,jit_int32_t,jit_int32_t,jit_word_t);
869# if __WORDSIZE == 32
870# define ldr_i(r0,r1) LWZX(r0, _R0_REGNO, r1)
871# else
872# define ldr_i(r0,r1) LWAX(r0, _R0_REGNO, r1)
873# endif
874# define ldi_i(r0,i0) _ldi_i(_jit,r0,i0)
875static void _ldi_i(jit_state_t*,jit_int32_t,jit_word_t);
876# define ldxr_i(r0,r1,i0) _ldxr_i(_jit,r0,r1,i0)
877static void _ldxr_i(jit_state_t*,jit_int32_t,jit_int32_t,jit_int32_t);
878# define ldxi_i(r0,r1,i0) _ldxi_i(_jit,r0,r1,i0)
879static void _ldxi_i(jit_state_t*,jit_int32_t,jit_int32_t,jit_word_t);
880# if __WORDSIZE == 64
881# define ldr_ui(r0,r1) LWZX(r0, _R0_REGNO, r1)
882# define ldi_ui(r0,i0) _ldi_ui(_jit,r0,i0)
883static void _ldi_ui(jit_state_t*,jit_int32_t,jit_word_t);
884# define ldxr_ui(r0,r1,i0) _ldxr_ui(_jit,r0,r1,i0)
885static void _ldxr_ui(jit_state_t*,jit_int32_t,jit_int32_t,jit_int32_t);
886# define ldxi_ui(r0,r1,i0) _ldxi_ui(_jit,r0,r1,i0)
887static void _ldxi_ui(jit_state_t*,jit_int32_t,jit_int32_t,jit_word_t);
888# define ldr_l(r0,r1) LDX(r0, _R0_REGNO, r1)
889# define ldi_l(r0,i0) _ldi_l(_jit,r0,i0)
890static void _ldi_l(jit_state_t*,jit_int32_t,jit_word_t);
891# define ldxr_l(r0,r1,i0) _ldxr_l(_jit,r0,r1,i0)
892static void _ldxr_l(jit_state_t*,jit_int32_t,jit_int32_t,jit_int32_t);
893# define ldxi_l(r0,r1,i0) _ldxi_l(_jit,r0,r1,i0)
894static void _ldxi_l(jit_state_t*,jit_int32_t,jit_int32_t,jit_word_t);
895# endif
896# define str_c(r0,r1) STBX(r1, _R0_REGNO, r0)
897# define sti_c(i0,r0) _sti_c(_jit,i0,r0)
898static void _sti_c(jit_state_t*,jit_word_t,jit_int32_t);
899# define stxr_c(r0,r1,r2) _stxr_c(_jit,r0,r1,r2)
900static void _stxr_c(jit_state_t*,jit_int32_t,jit_int32_t,jit_int32_t);
901# define stxi_c(i0,r0,r1) _stxi_c(_jit,i0,r0,r1)
902static void _stxi_c(jit_state_t*,jit_word_t,jit_int32_t,jit_int32_t);
903# define str_s(r0,r1) STHX(r1, _R0_REGNO, r0)
904# define sti_s(i0,r0) _sti_s(_jit,i0,r0)
905static void _sti_s(jit_state_t*,jit_word_t,jit_int32_t);
906# define stxr_s(r0,r1,r2) _stxr_s(_jit,r0,r1,r2)
907static void _stxr_s(jit_state_t*,jit_int32_t,jit_int32_t,jit_int32_t);
908# define stxi_s(i0,r0,r1) _stxi_s(_jit,i0,r0,r1)
909static void _stxi_s(jit_state_t*,jit_word_t,jit_int32_t,jit_int32_t);
910# define str_i(r0,r1) STWX(r1, _R0_REGNO, r0)
911# define sti_i(i0,r0) _sti_i(_jit,i0,r0)
912static void _sti_i(jit_state_t*,jit_word_t,jit_int32_t);
913# define stxr_i(r0,r1,r2) _stxr_i(_jit,r0,r1,r2)
914static void _stxr_i(jit_state_t*,jit_int32_t,jit_int32_t,jit_int32_t);
915# define stxi_i(i0,r0,r1) _stxi_i(_jit,i0,r0,r1)
916static void _stxi_i(jit_state_t*,jit_word_t,jit_int32_t,jit_int32_t);
917# if __WORDSIZE == 64
918# define str_l(r0,r1) STDX(r1, _R0_REGNO, r0)
919# define sti_l(i0,r0) _sti_l(_jit,i0,r0)
920static void _sti_l(jit_state_t*,jit_word_t,jit_int32_t);
921# define stxr_l(r0,r1,r2) _stxr_l(_jit,r0,r1,r2)
922static void _stxr_l(jit_state_t*,jit_int32_t,jit_int32_t,jit_int32_t);
923# define stxi_l(i0,r0,r1) _stxi_l(_jit,i0,r0,r1)
924static void _stxi_l(jit_state_t*,jit_word_t,jit_int32_t,jit_int32_t);
925# endif
926# define jmpr(r0) _jmpr(_jit,r0)
927static void _jmpr(jit_state_t*,jit_int32_t);
928# define jmpi(i0) _jmpi(_jit,i0)
929static jit_word_t _jmpi(jit_state_t*,jit_word_t);
930# define jmpi_p(i0) _jmpi_p(_jit,i0)
931static jit_word_t _jmpi_p(jit_state_t*,jit_word_t) maybe_unused;
932# if _CALL_SYSV
933# define callr(r0,i0) _callr(_jit,r0,i0)
934static void _callr(jit_state_t*,jit_int32_t,jit_int32_t);
935# define calli(i0,i1) _calli(_jit,i0,i1)
79bfeef6 936static jit_word_t _calli(jit_state_t*,jit_word_t,jit_int32_t);
4a71579b
PC		 937# define calli_p(i0,i1) _calli_p(_jit,i0,i1)
938static jit_word_t _calli_p(jit_state_t*,jit_word_t,jit_int32_t);
939# else
940# define callr(r0) _callr(_jit,r0)
941static void _callr(jit_state_t*,jit_int32_t);
942# define calli(i0) _calli(_jit,i0)
79bfeef6 943static jit_word_t _calli(jit_state_t*,jit_word_t);
4a71579b
PC		 944# define calli_p(i0) _calli_p(_jit,i0)
945static jit_word_t _calli_p(jit_state_t*,jit_word_t);
946#endif
947# define prolog(node) _prolog(_jit, node)
948static void _prolog(jit_state_t*, jit_node_t*);
949# define epilog(node) _epilog(_jit, node)
950static void _epilog(jit_state_t*, jit_node_t*);
951# define vastart(r0) _vastart(_jit, r0)
952static void _vastart(jit_state_t*, jit_int32_t);
953# define vaarg(r0, r1) _vaarg(_jit, r0, r1)
954static void _vaarg(jit_state_t*, jit_int32_t, jit_int32_t);
955# define vaarg_d(r0, r1) _vaarg_d(_jit, r0, r1)
956static void _vaarg_d(jit_state_t*, jit_int32_t, jit_int32_t);
957# define patch_at(i,l) _patch_at(_jit,i,l)
958static void _patch_at(jit_state_t*,jit_word_t,jit_word_t);
959#endif
960
961#if CODE
962# define _u16(v) ((v) & 0xffff)
963# define _u26(v) ((v) & 0x3ffffff)
964static void
965_FXO(jit_state_t *_jit, int o, int d, int a, int b, int e, int x, int r)
966{
967 assert(!(o & ~((1 << 6) - 1)));
968 assert(!(d & ~((1 << 5) - 1)));
969 assert(!(a & ~((1 << 5) - 1)));
970 assert(!(b & ~((1 << 5) - 1)));
971 assert(!(e & ~((1 << 1) - 1)));
972 assert(!(x & ~((1 << 9) - 1)));
973 assert(!(r & ~((1 << 1) - 1)));
974 ii((o<<26)|(d<<21)|(a<<16)|(b<<11)|(e<<10)|(x<<1)|r);
975}
976
977static void
978_FDs(jit_state_t *_jit, int o, int d, int a, int s)
979{
980 assert(!(o & ~((1 << 6) - 1)));
981 assert(!(d & ~((1 << 5) - 1)));
982 assert(!(a & ~((1 << 5) - 1)));
983 assert(can_sign_extend_short_p(s));
984 ii((o<<26)|(d<<21)|(a<<16)|_u16(s));
985}
986
987static void
988_FDu(jit_state_t *_jit, int o, int d, int a, int s)
989{
990 assert(!(o & ~((1 << 6) - 1)));
991 assert(!(d & ~((1 << 5) - 1)));
992 assert(!(a & ~((1 << 5) - 1)));
993 assert(can_zero_extend_short_p(s));
994 ii((o<<26)|(d<<21)|(a<<16)|_u16(s));
995}
996
997static void
998_FX(jit_state_t *_jit, int o, int s, int a, int b, int x, int r)
999{
1000 assert(!(o & ~((1 << 6) - 1)));
1001 assert(!(s & ~((1 << 5) - 1)));
1002 assert(!(a & ~((1 << 5) - 1)));
1003 assert(!(b & ~((1 << 5) - 1)));
1004 assert(!(x & ~((1 << 10) - 1)));
1005 assert(!(r & ~((1 << 1) - 1)));
1006 ii((o<<26)|(s<<21)|(a<<16)|(b<<11)|(x<<1)|r);
1007}
1008
1009static void
1010_FI(jit_state_t *_jit, int o, int t, int a, int k)
1011{
1012 assert(!(o & ~(( 1 << 6) - 1)));
1013 assert(!(t & 3) && can_sign_extend_jump_p(t));
1014 assert(!(a & ~(( 1 << 1) - 1)));
1015 assert(!(k & ~(( 1 << 1) - 1)));
1016 ii((o<<26)|_u26(t)|(a<<1)|k);
1017}
1018
1019static void
1020_FB(jit_state_t *_jit, int o, int bo, int bi, int t, int a, int k)
1021{
1022 assert(!( o & ~((1 << 6) - 1)));
1023 assert(!(bo & ~((1 << 5) - 1)));
1024 assert(!(bi & ~((1 << 5) - 1)));
1025 assert(!(t & 3) && can_sign_extend_short_p(t));
1026 assert(!(a & ~(( 1 << 1) - 1)));
1027 assert(!(k & ~(( 1 << 1) - 1)));
1028 ii((o<<26)|(bo<<21)|(bi<<16)|_u16(t)|(a<<1)|k);
1029}
1030
1031static void
1032_FXL(jit_state_t *_jit, int o, int bo, int bi, int x, int k)
1033{
1034 assert(!( o & ~((1 << 6) - 1)));
1035 assert(!(bo & ~((1 << 5) - 1)));
1036 assert(!(bi & ~((1 << 5) - 1)));
1037 assert(!(x & ~(( 1 << 10) - 1)));
1038 assert(!(k & ~(( 1 << 1) - 1)));
1039 ii((o<<26)|(bo<<21)|(bi<<16)|(x<<1)|k);
1040}
1041
1042static void
1043_FC(jit_state_t *_jit, int o, int d, int l, int a, int b, int x)
1044{
1045 assert(!(o & ~((1 << 6) - 1)));
1046 assert(!(d & ~((1 << 3) - 1)));
1047 assert(!(l & ~((1 << 1) - 1)));
1048 assert(!(a & ~((1 << 5) - 1)));
1049 assert(!(b & ~((1 << 5) - 1)));
1050 assert(!(x & ~((1 << 10) - 1)));
1051 ii((o<<26)|(d<<23)|(l<<21)|(a<<16)|(b<<11)|(x<<1));
1052}
1053
1054static void
1055_FCI(jit_state_t *_jit, int o, int d, int l, int a, int s)
1056{
1057 assert(!(o & ~((1 << 6) - 1)));
1058 assert(!(d & ~((1 << 3) - 1)));
1059 assert(!(l & ~((1 << 1) - 1)));
1060 assert(!(a & ~((1 << 5) - 1)));
1061 if (o == 11) assert(can_sign_extend_short_p(s));
1062 else if (o == 10) assert(can_zero_extend_short_p(s));
1063#if DEBUG
1064 else abort();
1065#endif
1066 ii((o<<26)|(d<<23)|(l<<21)|(a<<16)|_u16(s));
1067}
1068
1069static void
1070_FXFX(jit_state_t *_jit, int o, int d, int x, int f)
1071{
1072 assert(!(o & ~((1 << 6) - 1)));
1073 assert(!(d & ~((1 << 5) - 1)));
1074 assert(!(x & ~((1 << 10) - 1)));
1075 assert(!(f & ~((1 << 10) - 1)));
1076 ii((o<<26)|(d<<21)|(x<<11)|(f<<1));
1077}
1078
1079static void
1080_FM(jit_state_t *_jit, int o, int s, int a, int h, int b, int e, int r)
1081{
1082 assert(!(o & ~((1 << 6) - 1)));
1083 assert(!(s & ~((1 << 5) - 1)));
1084 assert(!(a & ~((1 << 5) - 1)));
1085 assert(!(h & ~((1 << 5) - 1)));
1086 assert(!(b & ~((1 << 5) - 1)));
1087 assert(!(e & ~((1 << 5) - 1)));
1088 assert(!(r & ~((1 << 1) - 1)));
1089 ii((o<<26)|(s<<21)|(a<<16)|(h<<11)|(b<<6)|(e<<1)|r);
1090}
1091
1092# if __WORDSIZE == 64
1093static void
1094_FMDS(jit_state_t *_jit, int o, int s, int a, int b, int e, int x, int r)
1095{
1096 assert(!(o & ~((1 << 6) - 1)));
1097 assert(!(s & ~((1 << 5) - 1)));
1098 assert(!(a & ~((1 << 5) - 1)));
1099 assert(!(b & ~((1 << 5) - 1)));
1100 assert(!(e & ~((1 << 6) - 1)));
1101 assert(!(x & ~((1 << 4) - 1)));
1102 assert(!(r & ~((1 << 1) - 1)));
1103 e = (e >> 5) | ((e << 1) & 63);
1104 ii((o<<26)|(s<<21)|(a<<16)|(b<<11)|(e<<5)|(x<<1)|r);
1105}
1106
1107static void
1108_FMD(jit_state_t *_jit, int o, int s, int a, int h, int e, int x, int i, int r)
1109{
1110 assert(!(o & ~((1 << 6) - 1)));
1111 assert(!(s & ~((1 << 5) - 1)));
1112 assert(!(a & ~((1 << 5) - 1)));
1113 assert(!(h & ~((1 << 5) - 1)));
1114 assert(!(e & ~((1 << 6) - 1)));
1115 assert(!(x & ~((1 << 3) - 1)));
1116 assert(!(i & ~((1 << 1) - 1)));
1117 assert(!(r & ~((1 << 1) - 1)));
1118 e = (e >> 5) | ((e << 1) & 63);
1119 ii((o<<26)|(s<<21)|(a<<16)|(h<<11)|(e<<5)|(x<<2)|(i<<1)|r);
1120}
1121
1122static void
1123_FXS(jit_state_t *_jit, int o, int s, int a, int h, int x, int i, int r)
1124{
1125 assert(!(o & ~((1 << 6) - 1)));
1126 assert(!(s & ~((1 << 5) - 1)));
1127 assert(!(a & ~((1 << 5) - 1)));
1128 assert(!(h & ~((1 << 5) - 1)));
1129 assert(!(x & ~((1 << 9) - 1)));
1130 assert(!(i & ~((1 << 1) - 1)));
1131 assert(!(r & ~((1 << 1) - 1)));
1132 ii((o<<26)|(s<<21)|(a<<16)|(h<<11)|(x<<2)|(i<<1)|r);
1133}
1134#endif
1135
1136#if !DEBUG
1137/*
1138 * Use the sequence commented at
1139 * http://tenfourfox.blogspot.com/2011/04/attention-g5-owners-your-javascript-no.html
1140 */
1141static void
1142_MCRXR(jit_state_t *_jit, jit_int32_t cr)
1143{
1144 jit_int32_t reg;
1145 reg = jit_get_reg(jit_class_gpr|jit_class_nospill);
1146 MFXER(rn(reg));
1147 MTCRF(128, rn(reg));
1148 RLWINM(rn(reg), rn(reg), 0, 0, 28);
1149 MTXER(rn(reg));
1150 jit_unget_reg(reg);
1151}
1152#endif
1153
1154static void
1155_nop(jit_state_t *_jit, jit_int32_t i0)
1156{
1157 for (; i0 > 0; i0 -= 4)
1158 NOP();
1159 assert(i0 == 0);
1160}
1161
1162static void
1163_movr(jit_state_t *_jit, jit_int32_t r0, jit_int32_t r1)
1164{
1165 if (r0 != r1)
1166 MR(r0, r1);
1167}
1168
1169static void
1170_movi(jit_state_t *_jit, jit_int32_t r0, jit_word_t i0)
1171{
1172 if (can_sign_extend_short_p(i0))
1173 LI(r0, i0);
1174 else {
1175 if (can_sign_extend_int_p(i0))
1176 LIS(r0, (jit_int16_t)(i0 >> 16));
1177 else if (can_zero_extend_int_p(i0)) {
1178 if (i0 & 0xffff0000) {
1179 ORI(r0, r0, (jit_uint16_t)(i0 >> 16));
1180 SLWI(r0, r0, 16);
1181 }
1182 }
1183# if __WORDSIZE == 64
1184 else {
1185 movi(r0, (jit_uint32_t)(i0 >> 32));
1186 if (i0 & 0xffff0000) {
1187 SLDI(r0, r0, 16);
1188 ORI(r0, r0, (jit_uint16_t)(i0 >> 16));
1189 SLDI(r0, r0, 16);
1190 }
1191 else
1192 SLDI(r0, r0, 32);
1193 }
1194# endif
1195 if (i0 & 0xffff)
1196 ORI(r0, r0, (jit_uint16_t)i0);
1197 }
1198}
1199
e0659411
PC		 1200static void
1201_movnr(jit_state_t *_jit, jit_int32_t r0, jit_int32_t r1, jit_int32_t r2)
1202{
79bfeef6 1203 CMPXI(r2, 0);
e0659411
PC		 1204 BEQ(8);
1205 MR(r0, r1);
1206}
1207
1208static void
1209_movzr(jit_state_t *_jit, jit_int32_t r0, jit_int32_t r1, jit_int32_t r2)
1210{
79bfeef6 1211 CMPXI(r2, 0);
e0659411
PC		 1212 BNE(8);
1213 MR(r0, r1);
1214}
1215
4a71579b
PC		 1216static jit_word_t
1217_movi_p(jit_state_t *_jit, jit_int32_t r0, jit_word_t i0)
1218{
1219 jit_word_t word = _jit->pc.w;
1220# if __WORDSIZE == 32
1221 LIS(r0, (jit_int16_t)(i0 >> 16));
1222 ORI(r0, r0, (jit_uint16_t)i0);
1223# else
1224 LIS(r0, (jit_int16_t)(i0 >> 48));
1225 ORI(r0, r0, (jit_uint16_t)(i0 >> 32));
1226 SLDI(r0, r0, 16);
1227 ORI(r0, r0, (jit_uint16_t)(i0 >> 16));
1228 SLDI(r0, r0, 16);
1229 ORI(r0, r0, (jit_uint16_t)i0);
1230# endif
1231 return (word);
1232}
1233
4a71579b 1234static void
ba3814c1
PC		 1235_casx(jit_state_t *_jit, jit_int32_t r0, jit_int32_t r1,
1236 jit_int32_t r2, jit_int32_t r3, jit_word_t i0)
4a71579b 1237{
ba3814c1
PC		 1238 jit_int32_t r1_reg, iscasi;
1239 jit_word_t retry, done, jump0, jump1;
1240 if ((iscasi = (r1 == _NOREG))) {
1241 r1_reg = jit_get_reg(jit_class_gpr);
1242 r1 = rn(r1_reg);
1243 movi(r1, i0);
1244 }
1245 SYNC();
1246 /* retry: */
1247 retry = _jit->pc.w;
1248# if __WORDSIZE == 32
1249 LWARX(r0, _R0_REGNO, r1);
1250# else
1251 LDARX(r0, _R0_REGNO, r1);
1252# endif
1253 jump0 = bner(_jit->pc.w, r0, r2); /* bne done r0 r2 */
1254# if __WORDSIZE == 32
1255 STWCX_(r3, _R0_REGNO, r1);
1256# else
1257 STDCX_(r3, _R0_REGNO, r1);
1258# endif
c0c16242
PC		 1259 jump1 = _jit->pc.w;
1260 BNE(0); /* BNE retry */
ba3814c1
PC		 1261 /* done: */
1262 done = _jit->pc.w;
1263 ISYNC();
1264 MFCR(r0);
c0c16242 1265 EXTRWI(r0, r0, 1, CR_EQ);
ba3814c1
PC		 1266 patch_at(jump0, done);
1267 patch_at(jump1, retry);
1268 if (iscasi)
1269 jit_unget_reg(r1_reg);
1270}
1271
79bfeef6 1272static void
ba86ff93
PC		 1273_clor(jit_state_t *_jit, jit_int32_t r0, jit_int32_t r1)
1274{
1275 comr(r0, r1);
1276 clzr(r0, r0);
1277}
1278
1279static void
1280_ctor(jit_state_t *_jit, jit_int32_t r0, jit_int32_t r1)
1281{
1282 comr(r0, r1);
1283 ctzr(r0, r0);
1284}
1285
1286static void
1287_ctzr(jit_state_t *_jit, jit_int32_t r0, jit_int32_t r1)
79bfeef6 1288{
ba86ff93 1289 jit_int32_t t0, t1;
79bfeef6
PC		 1290 t0 = jit_get_reg(jit_class_gpr);
1291 t1 = jit_get_reg(jit_class_gpr);
ba86ff93
PC		 1292 negr(rn(t0), r1);
1293 andr(rn(t0), rn(t0), r1);
1294 clzr(r0, rn(t0));
1295 xori(rn(t1), r0, __WORDSIZE - 1);
1296 movnr(r0, rn(t1), rn(t0));
79bfeef6 1297 jit_unget_reg(t0);
ba86ff93 1298 jit_unget_reg(t1);
79bfeef6
PC		 1299}
1300
1301static void
ba86ff93 1302_popcntr(jit_state_t *_jit, jit_int32_t r0, jit_int32_t r1)
79bfeef6 1303{
ba86ff93
PC		 1304 jit_int32_t reg;
1305 reg = jit_get_reg(jit_class_gpr);
1306 POPCNTB(r0, r1);
1307#if __WORDSIZE == 32
1308 movi(rn(reg), 0x01010101);
1309#else
1310 movi(rn(reg), 0x0101010101010101);
1311#endif
1312 mullr(r0, r0, rn(reg));
1313 rshi_u(r0, r0, __WORDSIZE - 8);
1314 jit_unget_reg(reg);
79bfeef6
PC		 1315}
1316
1317static void
ba86ff93
PC		 1318_extr(jit_state_t *_jit,
1319 jit_int32_t r0, jit_int32_t r1, jit_word_t i0 ,jit_word_t i1)
79bfeef6 1320{
ba86ff93
PC		 1321 assert(i0 >= 0 && i1 >= 1 && i0 + i1 <= __WORDSIZE);
1322 if ( i1 == __WORDSIZE)
1323 movr(r0, r1);
1324 else {
1325# if __BYTE_ORDER == __BIG_ENDIAN
1326 i0 = __WORDSIZE - (i0 + i1);
1327# endif
1328 if (__WORDSIZE - (i0 + i1)) {
1329 lshi(r0, r1, __WORDSIZE - (i0 + i1));
1330 rshi(r0, r0, __WORDSIZE - i1);
1331 }
1332 else
1333 rshi(r0, r1, __WORDSIZE - i1);
1334 }
79bfeef6
PC		 1335}
1336
1337static void
ba86ff93
PC		 1338_extr_u(jit_state_t *_jit,
1339 jit_int32_t r0, jit_int32_t r1, jit_word_t i0 ,jit_word_t i1)
79bfeef6 1340{
ba86ff93
PC		 1341 assert(i0 >= 0 && i1 >= 1 && i0 + i1 <= __WORDSIZE);
1342 if (i1 == __WORDSIZE)
1343 movr(r0, r1);
1344 else {
1345# if __BYTE_ORDER == __BIG_ENDIAN
1346 i0 = __WORDSIZE - (i0 + i1);
1347# endif
1348# if __WORDSIZE == 32
1349 RLWINM(r0, r1, (32 - i0) & 0x1f, 32 - i1, 31);
1350# else
1351 RLDICL(r0, r1, (64 - i0) & 0x3f, 64 - i1);
1352# endif
1353 }
1354}
1355
1356static void
1357_depr(jit_state_t *_jit,
1358 jit_int32_t r0, jit_int32_t r1, jit_word_t i0 ,jit_word_t i1)
1359{
1360 assert(i0 >= 0 && i1 >= 1 && i0 + i1 <= __WORDSIZE);
1361 if (i1 == __WORDSIZE)
1362 movr(r0, r1);
1363 else {
1364# if __BYTE_ORDER == __BIG_ENDIAN
1365 i0 = __WORDSIZE - (i0 + i1);
1366# endif
1367#if __WORDSIZE == 32
1368 RLWIMI(r0, r1, i0, 32 - (i0 + i1), 31 - i0);
1369#else
1370 RLDIMI(r0, r1, i0, 64 - (i0 + i1));
1371#endif
1372 }
79bfeef6
PC		 1373}
1374
ba3814c1
PC		 1375static void
1376_bswapr_us(jit_state_t *_jit, jit_int32_t r0, jit_int32_t r1, jit_bool_t no_flag)
1377{
1378 jit_int32_t reg, addr_reg;
1379
1380 /* Convert load followed by bswap to a single instruction */
1381 /* FIXME r0 and r1 do not need to be the same, only must check if
1382 * r1 was loaded in previous instruction */
1383 if (no_flag && r0 == r1) {
1384 if ((*(_jit->pc.ui - 1) & 0xffe007ff) == (0x7c00022e | r0 << 21)) {
1385 /* Convert LHZX to LHBRX */
1386 _jit->pc.ui--;
1387 LHBRX(r0, (*_jit->pc.ui >> 16) & 0x1f, (*_jit->pc.ui >> 11) & 0x1f);
1388 return;
1389 }
1390
1391 if ((*(_jit->pc.ui - 1) & 0xffe00000) == (0xa0000000 | r0 << 21)) {
1392 /* Convert LHZ to LHBRX */
1393 _jit->pc.ui--;
1394 addr_reg = (*_jit->pc.ui >> 16) & 0x1f;
1395
1396 reg = jit_get_reg(jit_class_gpr);
1397 LI(rn(reg), (short)*_jit->pc.ui);
1398 LHBRX(r0, rn(reg), addr_reg);
1399 jit_unget_reg(reg);
1400 return;
1401 }
1402 }
1403
40a44dcb
PC		 1404 if (r0 == r1) {
1405 RLWIMI(r0, r0, 16, 8, 15);
1406 RLWINM(r0, r0, 24, 16, 31);
1407 } else {
1408 RLWINM(r0, r1, 8, 16, 23);
1409 RLWIMI(r0, r1, 24, 24, 31);
1410 }
4a71579b
PC		 1411}
1412
1413static void
ba3814c1 1414_bswapr_ui(jit_state_t *_jit, jit_int32_t r0, jit_int32_t r1, jit_bool_t no_flag)
4a71579b 1415{
ba3814c1
PC		 1416 jit_int32_t reg, addr_reg;
1417
1418 /* Convert load followed by bswap to a single instruction */
1419 /* FIXME r0 and r1 do not need to be the same, only must check if
1420 * r1 was loaded in previous instruction */
1421 if (no_flag && r0 == r1) {
1422 if ((*(_jit->pc.ui - 1) & 0xffe007ff) == (0x7c00002e | r0 << 21)) {
1423 /* Convert LWZX to LWBRX */
1424 _jit->pc.ui--;
1425 LWBRX(r0, (*_jit->pc.ui >> 16) & 0x1f, (*_jit->pc.ui >> 11) & 0x1f);
1426 return;
1427 }
1428
1429 if ((*(_jit->pc.ui - 1) & 0xffe00000) == (0x80000000 | r0 << 21)) {
1430 /* Convert LWZ to LWBRX */
1431 _jit->pc.ui--;
1432 addr_reg = (*_jit->pc.ui >> 16) & 0x1f;
1433
1434 reg = jit_get_reg(jit_class_gpr);
1435 LI(rn(reg), (short)*_jit->pc.ui);
1436 LWBRX(r0, rn(reg), addr_reg);
1437 jit_unget_reg(reg);
1438 return;
1439 }
1440 }
1441
4a71579b
PC		 1442 reg = jit_get_reg(jit_class_gpr);
1443 ROTLWI(rn(reg), r1, 8);
1444 RLWIMI(rn(reg), r1, 24, 0, 7);
1445 RLWIMI(rn(reg), r1, 24, 16, 23);
519a9ea1 1446# if __WORDSIZE == 64
4a71579b 1447 CLRLDI(r0, rn(reg), 32);
519a9ea1
PC		 1448# else
1449 MR(r0,rn(reg));
1450# endif
4a71579b
PC		 1451 jit_unget_reg(reg);
1452}
1453
4a71579b
PC		 1454static void
1455_addi(jit_state_t *_jit, jit_int32_t r0, jit_int32_t r1, jit_word_t i0)
1456{
1457 jit_int32_t reg;
1458 if (can_sign_extend_short_p(i0))
1459 ADDI(r0, r1, i0);
1460 else if (can_zero_extend_int_p(i0) && !(i0 & 0x0000ffff))
1461 ADDIS(r0, r1, i0 >> 16);
1462 else {
1463 reg = jit_get_reg(jit_class_gpr);
1464 movi(rn(reg), i0);
1465 ADD(r0, r1, rn(reg));
1466 jit_unget_reg(reg);
1467 }
1468}
1469
1470static void
1471_addci(jit_state_t *_jit, jit_int32_t r0, jit_int32_t r1, jit_word_t i0)
1472{
1473 jit_int32_t reg;
1474 if (can_sign_extend_short_p(i0))
1475 ADDIC(r0, r1, i0);
1476 else {
1477 reg = jit_get_reg(jit_class_gpr);
1478 movi(rn(reg), i0);
1479 ADDC(r0, r1, rn(reg));
1480 jit_unget_reg(reg);
1481 }
1482}
1483
1484static void
1485_addxi(jit_state_t *_jit, jit_int32_t r0, jit_int32_t r1, jit_word_t i0)
1486{
1487 jit_int32_t reg;
1488 reg = jit_get_reg(jit_class_gpr);
1489 movi(rn(reg), i0);
1490 ADDE(r0, r1, rn(reg));
1491 jit_unget_reg(reg);
1492}
1493
1494static void
1495_subi(jit_state_t *_jit, jit_int32_t r0, jit_int32_t r1, jit_word_t i0)
1496{
1497 jit_int32_t reg;
1498 jit_word_t ni0 = -i0;
1499 if (can_sign_extend_short_p(ni0))
1500 ADDI(r0, r1, ni0);
1501 else if (can_zero_extend_int_p(ni0) && !(ni0 & 0x0000ffff))
1502 ADDIS(r0, r1, ni0 >> 16);
1503 else {
1504 reg = jit_get_reg(jit_class_gpr);
1505 movi(rn(reg), i0);
1506 SUB(r0, r1, rn(reg));
1507 jit_unget_reg(reg);
1508 }
1509}
1510
1511static void
1512_subci(jit_state_t *_jit, jit_int32_t r0, jit_int32_t r1, jit_word_t i0)
1513{
1514 jit_int32_t reg;
1515 reg = jit_get_reg(jit_class_gpr);
1516 movi(rn(reg), i0);
1517 SUBC(r0, r1, rn(reg));
1518 jit_unget_reg(reg);
1519}
1520
1521static void
1522_subxi(jit_state_t *_jit, jit_int32_t r0, jit_int32_t r1, jit_word_t i0)
1523{
1524 jit_int32_t reg;
1525 reg = jit_get_reg(jit_class_gpr);
1526 movi(rn(reg), i0);
1527 SUBE(r0, r1, rn(reg));
1528 jit_unget_reg(reg);
1529}
1530
1531static void
1532_rsbi(jit_state_t *_jit, jit_int32_t r0, jit_int32_t r1, jit_word_t i0)
1533{
1534 subi(r0, r1, i0);
1535 negr(r0, r0);
1536}
1537
1538static void
1539_muli(jit_state_t *_jit, jit_int32_t r0, jit_int32_t r1, jit_word_t i0)
1540{
1541 jit_int32_t reg;
ba86ff93
PC		 1542 /* NOTE verified and overflow is correctly computed.
1543 * No need to check for __WORDSIZE == 32.
1544 * Documented as a 32 bit instruction. */
4a71579b
PC		 1545 if (can_sign_extend_short_p(i0))
1546 MULLI(r0, r1, i0);
1547 else {
1548 reg = jit_get_reg(jit_class_gpr);
1549 movi(rn(reg), i0);
1550 mulr(r0, r1, rn(reg));
1551 jit_unget_reg(reg);
1552 }
1553}
1554
ba86ff93
PC		 1555static void
1556_hmuli(jit_state_t *_jit, jit_int32_t r0, jit_int32_t r1, jit_word_t i0)
1557{
1558 jit_int32_t reg;
1559 reg = jit_get_reg(jit_class_gpr);
1560 movi(rn(reg), i0);
1561 hmulr(r0, r1, rn(reg));
1562 jit_unget_reg(reg);
1563}
1564
1565static void
1566_hmuli_u(jit_state_t *_jit, jit_int32_t r0, jit_int32_t r1, jit_word_t i0)
1567{
1568 jit_int32_t reg;
1569 reg = jit_get_reg(jit_class_gpr);
1570 movi(rn(reg), i0);
1571 hmulr_u(r0, r1, rn(reg));
1572 jit_unget_reg(reg);
1573}
1574
4a71579b
PC		 1575static void
1576_iqmulr(jit_state_t *_jit, jit_int32_t r0, jit_int32_t r1,
1577 jit_int32_t r2, jit_int32_t r3, jit_bool_t sign)
1578{
1579 jit_int32_t reg;
1580 if (r0 == r2 || r0 == r3) {
1581 reg = jit_get_reg(jit_class_gpr);
1582 mullr(rn(reg), r2, r3);
1583 }
1584 else
1585 mullr(r0, r2, r3);
1586 if (sign)
ba86ff93 1587 hmulr(r1, r2, r3);
4a71579b 1588 else
ba86ff93 1589 hmulr_u(r1, r2, r3);
4a71579b
PC		 1590 if (r0 == r2 || r0 == r3) {
1591 movr(r0, rn(reg));
1592 jit_unget_reg(reg);
1593 }
1594}
1595
1596static void
1597_iqmuli(jit_state_t *_jit, jit_int32_t r0, jit_int32_t r1,
1598 jit_int32_t r2, jit_word_t i0, jit_bool_t sign)
1599{
1600 jit_int32_t reg;
1601 reg = jit_get_reg(jit_class_gpr);
1602 movi(rn(reg), i0);
1603 iqmulr(r0, r1, r2, rn(reg), sign);
1604 jit_unget_reg(reg);
1605}
1606
1607static void
1608_divi(jit_state_t *_jit, jit_int32_t r0, jit_int32_t r1, jit_word_t i0)
1609{
1610 jit_int32_t reg;
1611 reg = jit_get_reg(jit_class_gpr);
1612 movi(rn(reg), i0);
1613 divr(r0, r1, rn(reg));
1614 jit_unget_reg(reg);
1615}
1616
1617static void
1618_divi_u(jit_state_t *_jit, jit_int32_t r0, jit_int32_t r1, jit_word_t i0)
1619{
1620 jit_int32_t reg;
1621 reg = jit_get_reg(jit_class_gpr);
1622 movi(rn(reg), i0);
1623 divr_u(r0, r1, rn(reg));
1624 jit_unget_reg(reg);
1625}
1626
1627static void
1628_iqdivr(jit_state_t *_jit, jit_int32_t r0, jit_int32_t r1,
1629 jit_int32_t r2, jit_int32_t r3, jit_bool_t sign)
1630{
1631 jit_int32_t sv0, rg0;
1632 jit_int32_t sv1, rg1;
1633
1634 if (r0 == r2 || r0 == r3) {
1635 sv0 = jit_get_reg(jit_class_gpr);
1636 rg0 = rn(sv0);
1637 }
1638 else
1639 rg0 = r0;
1640 if (r1 == r2 || r1 == r3) {
1641 sv1 = jit_get_reg(jit_class_gpr);
1642 rg1 = rn(sv1);
1643 }
1644 else
1645 rg1 = r1;
1646
1647 if (sign)
1648 divr(rg0, r2, r3);
1649 else
1650 divr_u(rg0, r2, r3);
1651 mulr(rg1, r3, rg0);
1652 subr(rg1, r2, rg1);
1653 if (rg0 != r0) {
1654 movr(r0, rg0);
1655 jit_unget_reg(sv0);
1656 }
1657 if (rg1 != r1) {
1658 movr(r1, rg1);
1659 jit_unget_reg(sv1);
1660 }
1661}
1662
1663static void
1664_iqdivi(jit_state_t *_jit, jit_int32_t r0, jit_int32_t r1,
1665 jit_int32_t r2, jit_word_t i0, jit_bool_t sign)
1666{
1667 jit_int32_t reg;
1668 reg = jit_get_reg(jit_class_gpr);
1669 movi(rn(reg), i0);
1670 iqdivr(r0, r1, r2, rn(reg), sign);
1671 jit_unget_reg(reg);
1672}
1673
1674static void
1675_remr(jit_state_t *_jit, jit_int32_t r0, jit_int32_t r1, jit_int32_t r2)
1676{
1677 jit_int32_t reg;
1678 if (r0 == r1 || r0 == r2) {
1679 reg = jit_get_reg(jit_class_gpr);
1680 divr(rn(reg), r1, r2);
1681 mulr(rn(reg), r2, rn(reg));
1682 subr(r0, r1, rn(reg));
1683 jit_unget_reg(reg);
1684 }
1685 else {
1686 divr(r0, r1, r2);
1687 mulr(r0, r2, r0);
1688 subr(r0, r1, r0);
1689 }
1690}
1691
1692static void
1693_remi(jit_state_t *_jit, jit_int32_t r0, jit_int32_t r1, jit_word_t i0)
1694{
1695 jit_int32_t reg;
1696 reg = jit_get_reg(jit_class_gpr);
1697 movi(rn(reg), i0);
1698 remr(r0, r1, rn(reg));
1699 jit_unget_reg(reg);
1700}
1701
1702static void
1703_remr_u(jit_state_t *_jit, jit_int32_t r0, jit_int32_t r1, jit_int32_t r2)
1704{
1705 jit_int32_t reg;
1706 if (r0 == r1 || r0 == r2) {
1707 reg = jit_get_reg(jit_class_gpr);
1708 divr_u(rn(reg), r1, r2);
1709 mulr(rn(reg), r2, rn(reg));
1710 subr(r0, r1, rn(reg));
1711 jit_unget_reg(reg);
1712 }
1713 else {
1714 divr_u(r0, r1, r2);
1715 mulr(r0, r2, r0);
1716 subr(r0, r1, r0);
1717 }
1718}
1719
1720static void
1721_remi_u(jit_state_t *_jit, jit_int32_t r0, jit_int32_t r1, jit_word_t i0)
1722{
1723 jit_int32_t reg;
1724 reg = jit_get_reg(jit_class_gpr);
1725 movi(rn(reg), i0);
1726 remr_u(r0, r1, rn(reg));
1727 jit_unget_reg(reg);
1728}
1729
ba3814c1
PC		 1730# define is_mask(im) ((im) ? (__builtin_popcountl((im) + (1 << __builtin_ctzl(im))) <= 1) : 0)
1731
4a71579b
PC		 1732static void
1733_andi(jit_state_t *_jit, jit_int32_t r0, jit_int32_t r1, jit_word_t i0)
1734{
ba3814c1 1735 jit_int32_t reg, offt;
4a71579b
PC		 1736 if (can_zero_extend_short_p(i0))
1737 ANDI_(r0, r1, i0);
1738 else if (can_zero_extend_int_p(i0) && !(i0 & 0x0000ffff))
1739 ANDIS_(r0, r1, (jit_uword_t)i0 >> 16);
ba3814c1
PC		 1740 else if (__WORDSIZE == 32 && is_mask(i0)) {
1741 offt = __builtin_ctzl(i0);
1742 RLWINM(r0, r1, 0, 32 - offt - __builtin_popcountl(i0), 31 - offt);
1743 } else if (__WORDSIZE == 32 && is_mask(~i0)) {
1744 offt = __builtin_ctzl(~i0);
1745 RLWINM(r0, r1, 0, 32 - offt, 31 - offt - __builtin_popcountl(~i0));
1746 } else {
4a71579b
PC		 1747 reg = jit_get_reg(jit_class_gpr);
1748 movi(rn(reg), i0);
1749 AND(r0, r1, rn(reg));
1750 jit_unget_reg(reg);
1751 }
1752}
1753
1754static void
1755_ori(jit_state_t *_jit, jit_int32_t r0, jit_int32_t r1, jit_word_t i0)
1756{
1757 jit_int32_t reg;
1758 if (can_zero_extend_short_p(i0))
1759 ORI(r0, r1, i0);
1760 else if (can_zero_extend_int_p(i0) && !(i0 & 0x0000ffff))
1761 ORIS(r0, r1, (jit_uword_t)i0 >> 16);
1762 else {
1763 reg = jit_get_reg(jit_class_gpr);
1764 movi(rn(reg), i0);
1765 OR(r0, r1, rn(reg));
1766 jit_unget_reg(reg);
1767 }
1768}
1769
1770static void
1771_xori(jit_state_t *_jit, jit_int32_t r0, jit_int32_t r1, jit_word_t i0)
1772{
1773 jit_int32_t reg;
1774 if (can_zero_extend_short_p(i0))
1775 XORI(r0, r1, i0);
1776 else if (can_zero_extend_int_p(i0) && !(i0 & 0x0000ffff))
1777 XORIS(r0, r1, (jit_uword_t)i0 >> 16);
1778 else {
1779 reg = jit_get_reg(jit_class_gpr);
1780 movi(rn(reg), i0);
1781 XOR(r0, r1, rn(reg));
1782 jit_unget_reg(reg);
1783 }
1784}
1785
1786static void
1787_lshi(jit_state_t *_jit, jit_int32_t r0, jit_int32_t r1, jit_word_t i0)
1788{
1789 if (i0 == 0)
1790 movr(r0, r1);
1791 else {
1792# if __WORDSIZE == 32
1793 SLWI(r0, r1, i0);
1794# else
1795 SLDI(r0, r1, i0);
1796# endif
1797 }
1798}
1799
ba86ff93
PC		 1800static void
1801_xlshr(jit_state_t *_jit, jit_bool_t sign,
1802 jit_int32_t r0, jit_int32_t r1, jit_int32_t r2, jit_int32_t r3)
1803{
1804 jit_int32_t t0, s0, t2, s2, t3, s3;
1805 jit_word_t over, zero, done, done_over;
1806 s0 = jit_get_reg(jit_class_gpr);
1807 t0 = rn(s0);
1808 if (r0 == r2 || r1 == r2) {
1809 s2 = jit_get_reg(jit_class_gpr);
1810 t2 = rn(s2);
1811 movr(t2, r2);
1812 }
1813 else
1814 t2 = r2;
1815 if (r0 == r3 || r1 == r3) {
1816 s3 = jit_get_reg(jit_class_gpr);
1817 t3 = rn(s3);
1818 movr(t3, r3);
1819 }
1820 else
1821 t3 = r3;
1822 rsbi(t0, t3, __WORDSIZE);
1823 lshr(r0, t2, t3);
1824 if (sign)
1825 rshr(r1, t2, t0);
1826 else
1827 rshr_u(r1, t2, t0);
1828 zero = beqi(_jit->pc.w, t3, 0);
1829 over = beqi(_jit->pc.w, t3, __WORDSIZE);
1830 done = fallback_jmpi(_jit->pc.w);
1831 patch_at(over, _jit->pc.w);
1832 /* overflow */
1833 movi(r0, 0);
1834 done_over = fallback_jmpi(_jit->pc.w);
1835 /* zero */
1836 patch_at(zero, _jit->pc.w);
1837 if (sign)
1838 rshi(r1, t2, __WORDSIZE - 1);
1839 else
1840 movi(r1, 0);
1841 patch_at(done, _jit->pc.w);
1842 patch_at(done_over, _jit->pc.w);
1843 jit_unget_reg(s0);
1844 if (t2 != r2)
1845 jit_unget_reg(s2);
1846 if (t3 != r3)
1847 jit_unget_reg(s3);
1848}
1849
1850static void
1851_xlshi(jit_state_t *_jit, jit_bool_t sign,
1852 jit_int32_t r0, jit_int32_t r1, jit_int32_t r2, jit_word_t i0)
1853{
1854 if (i0 == 0) {
1855 movr(r0, r2);
1856 if (sign)
1857 rshi(r1, r2, __WORDSIZE - 1);
1858 else
1859 movi(r1, 0);
1860 }
1861 else if (i0 == __WORDSIZE) {
1862 movr(r1, r2);
1863 movi(r0, 0);
1864 }
1865 else {
1866 assert((jit_uword_t)i0 <= __WORDSIZE);
1867 if (sign)
1868 rshi(r1, r2, __WORDSIZE - i0);
1869 else
1870 rshi_u(r1, r2, __WORDSIZE - i0);
1871 lshi(r0, r2, i0);
1872 }
1873}
1874
4a71579b
PC		 1875static void
1876_rshi(jit_state_t *_jit, jit_int32_t r0, jit_int32_t r1, jit_word_t i0)
1877{
1878 if (i0 == 0)
1879 movr(r0, r1);
1880 else {
1881# if __WORDSIZE == 32
1882 SRAWI(r0, r1, i0);
1883# else
1884 SRADI(r0, r1, i0);
1885# endif
1886 }
1887}
1888
1889static void
1890_rshi_u(jit_state_t *_jit, jit_int32_t r0, jit_int32_t r1, jit_word_t i0)
1891{
1892 if (i0 == 0)
1893 movr(r0, r1);
1894 else {
1895# if __WORDSIZE == 32
1896 SRWI(r0, r1, i0);
1897# else
1898 SRDI(r0, r1, i0);
1899# endif
1900 }
1901}
1902
ba86ff93
PC		 1903static void
1904_xrshr(jit_state_t *_jit, jit_bool_t sign,
1905 jit_int32_t r0, jit_int32_t r1, jit_int32_t r2, jit_int32_t r3)
1906{
1907 jit_int32_t t0, s0, t2, s2, t3, s3;
1908 jit_word_t over, zero, done, done_over;
1909 s0 = jit_get_reg(jit_class_gpr);
1910 t0 = rn(s0);
1911 if (r0 == r2 || r1 == r2) {
1912 s2 = jit_get_reg(jit_class_gpr);
1913 t2 = rn(s2);
1914 movr(t2, r2);
1915 }
1916 else
1917 t2 = r2;
1918 if (r0 == r3 || r1 == r3) {
1919 s3 = jit_get_reg(jit_class_gpr);
1920 t3 = rn(s3);
1921 movr(t3, r3);
1922 }
1923 else
1924 t3 = r3;
1925 rsbi(t0, t3, __WORDSIZE);
1926 if (sign)
1927 rshr(r0, t2, t3);
1928 else
1929 rshr_u(r0, t2, t3);
1930 lshr(r1, t2, t0);
1931 zero = beqi(_jit->pc.w, t3, 0);
1932 over = beqi(_jit->pc.w, t3, __WORDSIZE);
1933 done = jmpi(_jit->pc.w);
1934 patch_at(over, _jit->pc.w);
1935 /* underflow */
1936 if (sign)
1937 rshi(r0, t2, __WORDSIZE - 1);
1938 else
1939 movi(r0, 0);
1940 done_over = jmpi(_jit->pc.w);
1941 /* zero */
1942 patch_at(zero, _jit->pc.w);
1943 if (sign)
1944 rshi(r1, t2, __WORDSIZE - 1);
1945 else
1946 movi(r1, 0);
1947 patch_at(done, _jit->pc.w);
1948 patch_at(done_over, _jit->pc.w);
1949 jit_unget_reg(s0);
1950 if (t2 != r2)
1951 jit_unget_reg(s2);
1952 if (t3 != r3)
1953 jit_unget_reg(s3);
1954}
1955
1956static void
1957_xrshi(jit_state_t *_jit, jit_bool_t sign,
1958 jit_int32_t r0, jit_int32_t r1, jit_int32_t r2, jit_word_t i0)
1959{
1960 if (i0 == 0) {
1961 movr(r0, r2);
1962 if (sign)
1963 rshi(r1, r2, __WORDSIZE - 1);
1964 else
1965 movi(r1, 0);
1966 }
1967 else if (i0 == __WORDSIZE) {
1968 movr(r1, r2);
1969 if (sign)
1970 rshi(r0, r2, __WORDSIZE - 1);
1971 else
1972 movi(r0, 0);
1973 }
1974 else {
1975 assert((jit_uword_t)i0 <= __WORDSIZE);
1976 lshi(r1, r2, __WORDSIZE - i0);
1977 if (sign)
1978 rshi(r0, r2, i0);
1979 else
1980 rshi_u(r0, r2, i0);
1981 }
1982}
1983
1984static void
1985_lroti(jit_state_t *_jit, jit_int32_t r0, jit_int32_t r1, jit_word_t i0)
1986{
1987 if (i0 == 0)
1988 movr(r0, r1);
1989 else {
1990# if __WORDSIZE == 32
1991 ROTLWI(r0, r1, i0);
1992# else
1993 RLDICL(r0, r1, i0, 0);
1994# endif
1995 }
1996}
1997
1998static void
1999_rrotr(jit_state_t *_jit, jit_int32_t r0, jit_int32_t r1, jit_int32_t r2)
2000{
2001 jit_int32_t reg;
2002 if (r0 != r1 && r0 != r2) {
2003 rsbi(r0, r2, __WORDSIZE);
2004 lrotr(r0, r1, r0);
2005 }
2006 else {
2007 reg = jit_get_reg(jit_class_gpr);
2008 rsbi(rn(reg), r2, __WORDSIZE);
2009 lrotr(r0, r1, rn(reg));
2010 jit_unget_reg(reg);
2011 }
2012}
2013
2014static void
2015_rroti(jit_state_t *_jit, jit_int32_t r0, jit_int32_t r1, jit_word_t i0)
2016{
2017 if (i0 == 0)
2018 movr(r0, r1);
2019 else {
2020# if __WORDSIZE == 32
2021 ROTRWI(r0, r1, i0);
2022# else
2023 RLDICL(r0, r1, 64 - i0, 0);
2024# endif
2025 }
2026}
2027
4a71579b
PC		 2028static void
2029_ltr(jit_state_t *_jit, jit_int32_t r0, jit_int32_t r1, jit_int32_t r2)
2030{
79bfeef6 2031 CMPX(r1, r2);
4a71579b
PC		 2032 MFCR(r0);
2033 EXTRWI(r0, r0, 1, CR_LT);
2034}
2035
2036static void
2037_lti(jit_state_t *_jit, jit_int32_t r0, jit_int32_t r1, jit_word_t i0)
2038{
2039 jit_int32_t reg;
2040 if (can_sign_extend_short_p(i0))
79bfeef6 2041 CMPXI(r1, i0);
4a71579b
PC		 2042 else {
2043 reg = jit_get_reg(jit_class_gpr);
2044 movi(rn(reg), i0);
79bfeef6 2045 CMPX(r1, rn(reg));
4a71579b
PC		 2046 jit_unget_reg(reg);
2047 }
2048 MFCR(r0);
2049 EXTRWI(r0, r0, 1, CR_LT);
2050}
2051
2052static void
2053_ltr_u(jit_state_t *_jit, jit_int32_t r0, jit_int32_t r1, jit_int32_t r2)
2054{
2055 CMPLW(r1, r2);
2056 MFCR(r0);
2057 EXTRWI(r0, r0, 1, CR_LT);
2058}
2059
2060static void
2061_lti_u(jit_state_t *_jit, jit_int32_t r0, jit_int32_t r1, jit_word_t i0)
2062{
2063 jit_int32_t reg;
2064 if (can_zero_extend_short_p(i0))
2065 CMPLWI(r1, i0);
2066 else {
2067 reg = jit_get_reg(jit_class_gpr);
2068 movi(rn(reg), i0);
2069 CMPLW(r1, rn(reg));
2070 jit_unget_reg(reg);
2071 }
2072 MFCR(r0);
2073 EXTRWI(r0, r0, 1, CR_LT);
2074}
2075
2076static void
2077_ler(jit_state_t *_jit, jit_int32_t r0, jit_int32_t r1, jit_int32_t r2)
2078{
79bfeef6 2079 CMPX(r1, r2);
4a71579b
PC		 2080 CRNOT(CR_GT, CR_GT);
2081 MFCR(r0);
2082 EXTRWI(r0, r0, 1, CR_GT);
2083}
2084
2085static void
2086_lei(jit_state_t *_jit, jit_int32_t r0, jit_int32_t r1, jit_word_t i0)
2087{
2088 jit_int32_t reg;
2089 if (can_sign_extend_short_p(i0))
79bfeef6 2090 CMPXI(r1, i0);
4a71579b
PC		 2091 else {
2092 reg = jit_get_reg(jit_class_gpr);
2093 movi(rn(reg), i0);
79bfeef6 2094 CMPX(r1, rn(reg));
4a71579b
PC		 2095 jit_unget_reg(reg);
2096 }
2097 CRNOT(CR_GT, CR_GT);
2098 MFCR(r0);
2099 EXTRWI(r0, r0, 1, CR_GT);
2100}
2101
2102static void
2103_ler_u(jit_state_t *_jit, jit_int32_t r0, jit_int32_t r1, jit_int32_t r2)
2104{
2105 CMPLW(r1, r2);
2106 CRNOT(CR_GT, CR_GT);
2107 MFCR(r0);
2108 EXTRWI(r0, r0, 1, CR_GT);
2109}
2110
2111static void
2112_lei_u(jit_state_t *_jit, jit_int32_t r0, jit_int32_t r1, jit_word_t i0)
2113{
2114 jit_int32_t reg;
2115 if (can_zero_extend_short_p(i0))
2116 CMPLWI(r1, i0);
2117 else {
2118 reg = jit_get_reg(jit_class_gpr);
2119 movi(rn(reg), i0);
2120 CMPLW(r1, rn(reg));
2121 jit_unget_reg(reg);
2122 }
2123 CRNOT(CR_GT, CR_GT);
2124 MFCR(r0);
2125 EXTRWI(r0, r0, 1, CR_GT);
2126}
2127
2128static void
2129_eqr(jit_state_t *_jit, jit_int32_t r0, jit_int32_t r1, jit_int32_t r2)
2130{
79bfeef6 2131 CMPX(r1, r2);
4a71579b
PC		 2132 MFCR(r0);
2133 EXTRWI(r0, r0, 1, CR_EQ);
2134}
2135
2136static void
2137_eqi(jit_state_t *_jit, jit_int32_t r0, jit_int32_t r1, jit_word_t i0)
2138{
2139 jit_int32_t reg;
2140 if (can_sign_extend_short_p(i0))
79bfeef6 2141 CMPXI(r1, i0);
4a71579b
PC		 2142 else if (can_zero_extend_short_p(i0))
2143 CMPLWI(r1, i0);
2144 else {
2145 reg = jit_get_reg(jit_class_gpr);
2146 movi(rn(reg), i0);
79bfeef6 2147 CMPX(r1, rn(reg));
4a71579b
PC		 2148 jit_unget_reg(reg);
2149 }
2150 MFCR(r0);
2151 EXTRWI(r0, r0, 1, CR_EQ);
2152}
2153
2154static void
2155_ger(jit_state_t *_jit, jit_int32_t r0, jit_int32_t r1, jit_int32_t r2)
2156{
79bfeef6 2157 CMPX(r1, r2);
4a71579b
PC		 2158 CRNOT(CR_LT, CR_LT);
2159 MFCR(r0);
2160 EXTRWI(r0, r0, 1, CR_LT);
2161}
2162
2163static void
2164_gei(jit_state_t *_jit, jit_int32_t r0, jit_int32_t r1, jit_word_t i0)
2165{
2166 jit_int32_t reg;
2167 if (can_sign_extend_short_p(i0))
79bfeef6 2168 CMPXI(r1, i0);
4a71579b
PC		 2169 else {
2170 reg = jit_get_reg(jit_class_gpr);
2171 movi(rn(reg), i0);
79bfeef6 2172 CMPX(r1, rn(reg));
4a71579b
PC		 2173 jit_unget_reg(reg);
2174 }
2175 CRNOT(CR_LT, CR_LT);
2176 MFCR(r0);
2177 EXTRWI(r0, r0, 1, CR_LT);
2178}
2179
2180static void
2181_ger_u(jit_state_t *_jit, jit_int32_t r0, jit_int32_t r1, jit_int32_t r2)
2182{
2183 CMPLW(r1, r2);
2184 CRNOT(CR_LT, CR_LT);
2185 MFCR(r0);
2186 EXTRWI(r0, r0, 1, CR_LT);
2187}
2188
2189static void
2190_gei_u(jit_state_t *_jit, jit_int32_t r0, jit_int32_t r1, jit_word_t i0)
2191{
2192 jit_int32_t reg;
2193 if (can_zero_extend_short_p(i0))
2194 CMPLWI(r1, i0);
2195 else {
2196 reg = jit_get_reg(jit_class_gpr);
2197 movi(rn(reg), i0);
2198 CMPLW(r1, rn(reg));
2199 jit_unget_reg(reg);
2200 }
2201 CRNOT(CR_LT, CR_LT);
2202 MFCR(r0);
2203 EXTRWI(r0, r0, 1, CR_LT);
2204}
2205
2206static void
2207_gtr(jit_state_t *_jit, jit_int32_t r0, jit_int32_t r1, jit_int32_t r2)
2208{
79bfeef6 2209 CMPX(r1, r2);
4a71579b
PC		 2210 MFCR(r0);
2211 EXTRWI(r0, r0, 1, CR_GT);
2212}
2213
2214static void
2215_gti(jit_state_t *_jit, jit_int32_t r0, jit_int32_t r1, jit_word_t i0)
2216{
2217 jit_int32_t reg;
2218 if (can_sign_extend_short_p(i0))
79bfeef6 2219 CMPXI(r1, i0);
4a71579b
PC		 2220 else {
2221 reg = jit_get_reg(jit_class_gpr);
2222 movi(rn(reg), i0);
79bfeef6 2223 CMPX(r1, rn(reg));
4a71579b
PC		 2224 jit_unget_reg(reg);
2225 }
2226 MFCR(r0);
2227 EXTRWI(r0, r0, 1, CR_GT);
2228}
2229
2230static void
2231_gtr_u(jit_state_t *_jit, jit_int32_t r0, jit_int32_t r1, jit_int32_t r2)
2232{
2233 CMPLW(r1, r2);
2234 MFCR(r0);
2235 EXTRWI(r0, r0, 1, CR_GT);
2236}
2237
2238static void
2239_gti_u(jit_state_t *_jit, jit_int32_t r0, jit_int32_t r1, jit_word_t i0)
2240{
2241 jit_int32_t reg;
2242 if (can_zero_extend_short_p(i0))
2243 CMPLWI(r1, i0);
2244 else {
2245 reg = jit_get_reg(jit_class_gpr);
2246 movi(rn(reg), i0);
2247 CMPLW(r1, rn(reg));
2248 jit_unget_reg(reg);
2249 }
2250 MFCR(r0);
2251 EXTRWI(r0, r0, 1, CR_GT);
2252}
2253
2254static void
2255_ner(jit_state_t *_jit, jit_int32_t r0, jit_int32_t r1, jit_int32_t r2)
2256{
79bfeef6 2257 CMPX(r1, r2);
4a71579b
PC		 2258 CRNOT(CR_EQ, CR_EQ);
2259 MFCR(r0);
2260 EXTRWI(r0, r0, 1, CR_EQ);
2261}
2262
2263static void
2264_nei(jit_state_t *_jit, jit_int32_t r0, jit_int32_t r1, jit_word_t i0)
2265{
2266 jit_int32_t reg;
2267 if (can_sign_extend_short_p(i0))
79bfeef6 2268 CMPXI(r1, i0);
4a71579b
PC		 2269 else if (can_zero_extend_short_p(i0))
2270 CMPLWI(r1, i0);
2271 else {
2272 reg = jit_get_reg(jit_class_gpr);
2273 movi(rn(reg), i0);
79bfeef6 2274 CMPX(r1, rn(reg));
4a71579b
PC		 2275 jit_unget_reg(reg);
2276 }
2277 CRNOT(CR_EQ, CR_EQ);
2278 MFCR(r0);
2279 EXTRWI(r0, r0, 1, CR_EQ);
2280}
2281
2282static jit_word_t
2283_bltr(jit_state_t *_jit, jit_word_t i0, jit_int32_t r0, jit_int32_t r1)
2284{
2285 jit_word_t d, w;
79bfeef6 2286 CMPX(r0, r1);
4a71579b
PC		 2287 w = _jit->pc.w;
2288 d = (i0 - w) & ~3;
2289 BLT(d);
2290 return (w);
2291}
2292
2293static jit_word_t
2294_blti(jit_state_t *_jit, jit_word_t i0, jit_int32_t r0, jit_word_t i1)
2295{
2296 jit_int32_t reg;
2297 jit_word_t d, w;
2298 if (can_sign_extend_short_p(i1))
79bfeef6 2299 CMPXI(r0, i1);
4a71579b
PC		 2300 else {
2301 reg = jit_get_reg(jit_class_gpr);
2302 movi(rn(reg), i1);
79bfeef6 2303 CMPX(r0, rn(reg));
4a71579b
PC		 2304 jit_unget_reg(reg);
2305 }
2306 w = _jit->pc.w;
2307 d = (i0 - w) & ~3;
2308 BLT(d);
2309 return (w);
2310}
2311
2312static jit_word_t
2313_bltr_u(jit_state_t *_jit, jit_word_t i0, jit_int32_t r0, jit_int32_t r1)
2314{
2315 jit_word_t d, w;
2316 CMPLW(r0, r1);
2317 w = _jit->pc.w;
2318 d = (i0 - w) & ~3;
2319 BLT(d);
2320 return (w);
2321}
2322
2323static jit_word_t
2324_blti_u(jit_state_t *_jit, jit_word_t i0, jit_int32_t r0, jit_word_t i1)
2325{
2326 jit_int32_t reg;
2327 jit_word_t d, w;
2328 if (can_zero_extend_short_p(i1))
2329 CMPLWI(r0, i1);
2330 else {
2331 reg = jit_get_reg(jit_class_gpr);
2332 movi(rn(reg), i1);
2333 CMPLW(r0, rn(reg));
2334 jit_unget_reg(reg);
2335 }
2336 w = _jit->pc.w;
2337 d = (i0 - w) & ~3;
2338 BLT(d);
2339 return (w);
2340}
2341
2342static jit_word_t
2343_bler(jit_state_t *_jit, jit_word_t i0, jit_int32_t r0, jit_int32_t r1)
2344{
2345 jit_word_t d, w;
79bfeef6 2346 CMPX(r0, r1);
4a71579b
PC		 2347 w = _jit->pc.w;
2348 d = (i0 - w) & ~3;
2349 BLE(d);
2350 return (w);
2351}
2352
2353static jit_word_t
2354_blei(jit_state_t *_jit, jit_word_t i0, jit_int32_t r0, jit_word_t i1)
2355{
2356 jit_int32_t reg;
2357 jit_word_t d, w;
2358 if (can_sign_extend_short_p(i1))
79bfeef6 2359 CMPXI(r0, i1);
4a71579b
PC		 2360 else {
2361 reg = jit_get_reg(jit_class_gpr);
2362 movi(rn(reg), i1);
79bfeef6 2363 CMPX(r0, rn(reg));
4a71579b
PC		 2364 jit_unget_reg(reg);
2365 }
2366 w = _jit->pc.w;
2367 d = (i0 - w) & ~3;
2368 BLE(d);
2369 return (w);
2370}
2371
2372static jit_word_t
2373_bler_u(jit_state_t *_jit, jit_word_t i0, jit_int32_t r0, jit_int32_t r1)
2374{
2375 jit_word_t d, w;
2376 CMPLW(r0, r1);
2377 w = _jit->pc.w;
2378 d = (i0 - w) & ~3;
2379 BLE(d);
2380 return (w);
2381}
2382
2383static jit_word_t
2384_blei_u(jit_state_t *_jit, jit_word_t i0, jit_int32_t r0, jit_word_t i1)
2385{
2386 jit_int32_t reg;
2387 jit_word_t d, w;
2388 if (can_zero_extend_short_p(i1))
2389 CMPLWI(r0, i1);
2390 else {
2391 reg = jit_get_reg(jit_class_gpr);
2392 movi(rn(reg), i1);
2393 CMPLW(r0, rn(reg));
2394 jit_unget_reg(reg);
2395 }
2396 w = _jit->pc.w;
2397 d = (i0 - w) & ~3;
2398 BLE(d);
2399 return (w);
2400}
2401
2402static jit_word_t
2403_beqr(jit_state_t *_jit, jit_word_t i0, jit_int32_t r0, jit_int32_t r1)
2404{
2405 jit_word_t d, w;
79bfeef6 2406 CMPX(r0, r1);
4a71579b
PC		 2407 w = _jit->pc.w;
2408 d = (i0 - w) & ~3;
2409 BEQ(d);
2410 return (w);
2411}
2412
2413static jit_word_t
2414_beqi(jit_state_t *_jit, jit_word_t i0, jit_int32_t r0, jit_word_t i1)
2415{
2416 jit_int32_t reg;
2417 jit_word_t d, w;
2418 if (can_sign_extend_short_p(i1))
79bfeef6 2419 CMPXI(r0, i1);
4a71579b
PC		 2420 else if (can_zero_extend_short_p(i1))
2421 CMPLWI(r0, i1);
2422 else {
2423 reg = jit_get_reg(jit_class_gpr);
2424 movi(rn(reg), i1);
79bfeef6 2425 CMPX(r0, rn(reg));
4a71579b
PC		 2426 jit_unget_reg(reg);
2427 }
2428 w = _jit->pc.w;
2429 d = (i0 - w) & ~3;
2430 BEQ(d);
2431 return (w);
2432}
2433
2434static jit_word_t
2435_bger(jit_state_t *_jit, jit_word_t i0, jit_int32_t r0, jit_int32_t r1)
2436{
2437 jit_word_t d, w;
79bfeef6 2438 CMPX(r0, r1);
4a71579b
PC		 2439 w = _jit->pc.w;
2440 d = (i0 - w) & ~3;
2441 BGE(d);
2442 return (w);
2443}
2444
2445static jit_word_t
2446_bgei(jit_state_t *_jit, jit_word_t i0, jit_int32_t r0, jit_word_t i1)
2447{
2448 jit_int32_t reg;
2449 jit_word_t d, w;
2450 if (can_sign_extend_short_p(i1))
79bfeef6 2451 CMPXI(r0, i1);
4a71579b
PC		 2452 else {
2453 reg = jit_get_reg(jit_class_gpr);
2454 movi(rn(reg), i1);
79bfeef6 2455 CMPX(r0, rn(reg));
4a71579b
PC		 2456 jit_unget_reg(reg);
2457 }
2458 w = _jit->pc.w;
2459 d = (i0 - w) & ~3;
2460 BGE(d);
2461 return (w);
2462}
2463
2464static jit_word_t
2465_bger_u(jit_state_t *_jit, jit_word_t i0, jit_int32_t r0, jit_int32_t r1)
2466{
2467 jit_word_t d, w;
2468 CMPLW(r0, r1);
2469 w = _jit->pc.w;
2470 d = (i0 - w) & ~3;
2471 BGE(d);
2472 return (w);
2473}
2474
2475static jit_word_t
2476_bgei_u(jit_state_t *_jit, jit_word_t i0, jit_int32_t r0, jit_word_t i1)
2477{
2478 jit_int32_t reg;
2479 jit_word_t d, w;
2480 if (can_zero_extend_short_p(i1))
2481 CMPLWI(r0, i1);
2482 else {
2483 reg = jit_get_reg(jit_class_gpr);
2484 movi(rn(reg), i1);
2485 CMPLW(r0, rn(reg));
2486 jit_unget_reg(reg);
2487 }
2488 w = _jit->pc.w;
2489 d = (i0 - w) & ~3;
2490 BGE(d);
2491 return (w);
2492}
2493
2494static jit_word_t
2495_bgtr(jit_state_t *_jit, jit_word_t i0, jit_int32_t r0, jit_int32_t r1)
2496{
2497 jit_word_t d, w;
79bfeef6 2498 CMPX(r0, r1);
4a71579b
PC		 2499 w = _jit->pc.w;
2500 d = (i0 - w) & ~3;
2501 BGT(d);
2502 return (w);
2503}
2504
2505static jit_word_t
2506_bgti(jit_state_t *_jit, jit_word_t i0, jit_int32_t r0, jit_word_t i1)
2507{
2508 jit_int32_t reg;
2509 jit_word_t d, w;
2510 if (can_sign_extend_short_p(i1))
79bfeef6 2511 CMPXI(r0, i1);
4a71579b
PC		 2512 else {
2513 reg = jit_get_reg(jit_class_gpr);
2514 movi(rn(reg), i1);
79bfeef6 2515 CMPX(r0, rn(reg));
4a71579b
PC		 2516 jit_unget_reg(reg);
2517 }
2518 w = _jit->pc.w;
2519 d = (i0 - w) & ~3;
2520 BGT(d);
2521 return (w);
2522}
2523
2524static jit_word_t
2525_bgtr_u(jit_state_t *_jit, jit_word_t i0, jit_int32_t r0, jit_int32_t r1)
2526{
2527 jit_word_t d, w;
2528 CMPLW(r0, r1);
2529 w = _jit->pc.w;
2530 d = (i0 - w) & ~3;
2531 BGT(d);
2532 return (w);
2533}
2534
2535static jit_word_t
2536_bgti_u(jit_state_t *_jit, jit_word_t i0, jit_int32_t r0, jit_word_t i1)
2537{
2538 jit_int32_t reg;
2539 jit_word_t d, w;
2540 if (can_zero_extend_short_p(i1))
2541 CMPLWI(r0, i1);
2542 else {
2543 reg = jit_get_reg(jit_class_gpr);
2544 movi(rn(reg), i1);
2545 CMPLW(r0, rn(reg));
2546 jit_unget_reg(reg);
2547 }
2548 w = _jit->pc.w;
2549 d = (i0 - w) & ~3;
2550 BGT(d);
2551 return (w);
2552}
2553
2554static jit_word_t
2555_bner(jit_state_t *_jit, jit_word_t i0, jit_int32_t r0, jit_int32_t r1)
2556{
2557 jit_word_t d, w;
79bfeef6 2558 CMPX(r0, r1);
4a71579b
PC		 2559 w = _jit->pc.w;
2560 d = (i0 - w) & ~3;
2561 BNE(d);
2562 return (w);
2563}
2564
2565static jit_word_t
2566_bnei(jit_state_t *_jit, jit_word_t i0, jit_int32_t r0, jit_word_t i1)
2567{
2568 jit_int32_t reg;
2569 jit_word_t d, w;
2570 if (can_sign_extend_short_p(i1))
79bfeef6 2571 CMPXI(r0, i1);
4a71579b
PC		 2572 else if (can_zero_extend_short_p(i1))
2573 CMPLWI(r0, i1);
2574 else {
2575 reg = jit_get_reg(jit_class_gpr);
2576 movi(rn(reg), i1);
79bfeef6 2577 CMPX(r0, rn(reg));
4a71579b
PC		 2578 jit_unget_reg(reg);
2579 }
2580 w = _jit->pc.w;
2581 d = (i0 - w) & ~3;
2582 BNE(d);
2583 return (w);
2584}
2585
2586static jit_word_t
2587_bmsr(jit_state_t *_jit, jit_word_t i0, jit_int32_t r0, jit_int32_t r1)
2588{
2589 jit_word_t w;
2590 jit_int32_t reg;
2591 reg = jit_get_reg(jit_class_gpr|jit_class_nospill);
2592 andr(rn(reg), r0, r1);
2593 w = bnei(i0, rn(reg), 0);
2594 jit_unget_reg(reg);
2595 return (w);
2596}
2597
2598static jit_word_t
2599_bmsi(jit_state_t *_jit, jit_word_t i0, jit_int32_t r0, jit_word_t i1)
2600{
2601 jit_word_t w;
2602 jit_int32_t reg;
2603 reg = jit_get_reg(jit_class_gpr|jit_class_nospill);
2604 andi(rn(reg), r0, i1);
2605 w = bnei(i0, rn(reg), 0);
2606 jit_unget_reg(reg);
2607 return (w);
2608}
2609
2610static jit_word_t
2611_bmcr(jit_state_t *_jit, jit_word_t i0, jit_int32_t r0, jit_int32_t r1)
2612{
2613 jit_word_t w;
2614 jit_int32_t reg;
2615 reg = jit_get_reg(jit_class_gpr|jit_class_nospill);
2616 andr(rn(reg), r0, r1);
2617 w = beqi(i0, rn(reg), 0);
2618 jit_unget_reg(reg);
2619 return (w);
2620}
2621
2622static jit_word_t
2623_bmci(jit_state_t *_jit, jit_word_t i0, jit_int32_t r0, jit_word_t i1)
2624{
2625 jit_word_t w;
2626 jit_int32_t reg;
2627 reg = jit_get_reg(jit_class_gpr|jit_class_nospill);
2628 andi(rn(reg), r0, i1);
2629 w = beqi(i0, rn(reg), 0);
2630 jit_unget_reg(reg);
2631 return (w);
2632}
2633
2634static jit_word_t
2635_boaddr(jit_state_t *_jit, jit_word_t i0, jit_int32_t r0, jit_int32_t r1)
2636{
2637 jit_word_t d, w;
2638 ADDO(r0, r0, r1);
2639 MCRXR(CR_0);
2640 w = _jit->pc.w;
2641 d = (i0 - w) & ~3;
2642 BGT(d); /* GT = bit 1 of XER = OV */
2643 return (w);
2644}
2645
2646static jit_word_t
2647_boaddi(jit_state_t *_jit, jit_word_t i0, jit_int32_t r0, jit_word_t i1)
2648{
2649 jit_word_t w;
2650 jit_int32_t reg;
2651 reg = jit_get_reg(jit_class_gpr|jit_class_nospill);
2652 movi(rn(reg), i1);
2653 w = boaddr(i0, r0, rn(reg));
2654 jit_unget_reg(reg);
2655 return (w);
2656}
2657
2658static jit_word_t
2659_bxaddr(jit_state_t *_jit, jit_word_t i0, jit_int32_t r0, jit_int32_t r1)
2660{
2661 jit_word_t d, w;
2662 ADDO(r0, r0, r1);
2663 MCRXR(CR_0);
2664 w = _jit->pc.w;
2665 d = (i0 - w) & ~3;
2666 BLE(d);
2667 return (w);
2668}
2669
2670static jit_word_t
2671_bxaddi(jit_state_t *_jit, jit_word_t i0, jit_int32_t r0, jit_word_t i1)
2672{
2673 jit_word_t w;
2674 jit_int32_t reg;
2675 reg = jit_get_reg(jit_class_gpr|jit_class_nospill);
2676 movi(rn(reg), i1);
2677 w = bxaddr(i0, r0, rn(reg));
2678 jit_unget_reg(reg);
2679 return (w);
2680}
2681
2682static jit_word_t
2683_bosubr(jit_state_t *_jit, jit_word_t i0, jit_int32_t r0, jit_int32_t r1)
2684{
2685 jit_word_t d, w;
2686 SUBO(r0, r0, r1);
2687 MCRXR(CR_0);
2688 w = _jit->pc.w;
2689 d = (i0 - w) & ~3;
2690 BGT(d);
2691 return (w);
2692}
2693
2694static jit_word_t
2695_bosubi(jit_state_t *_jit, jit_word_t i0, jit_int32_t r0, jit_word_t i1)
2696{
2697 jit_word_t w;
2698 jit_int32_t reg;
2699 reg = jit_get_reg(jit_class_gpr|jit_class_nospill);
2700 movi(rn(reg), i1);
2701 w = bosubr(i0, r0, rn(reg));
2702 jit_unget_reg(reg);
2703 return (w);
2704}
2705
2706static jit_word_t
2707_bxsubr(jit_state_t *_jit, jit_word_t i0, jit_int32_t r0, jit_int32_t r1)
2708{
2709 jit_word_t d, w;
2710 SUBO(r0, r0, r1);
2711 MCRXR(CR_0);
2712 w = _jit->pc.w;
2713 d = (i0 - w) & ~3;
2714 BLE(d);
2715 return (w);
2716}
2717
2718static jit_word_t
2719_bxsubi(jit_state_t *_jit, jit_word_t i0, jit_int32_t r0, jit_word_t i1)
2720{
2721 jit_word_t w;
2722 jit_int32_t reg;
2723 reg = jit_get_reg(jit_class_gpr|jit_class_nospill);
2724 movi(rn(reg), i1);
2725 w = bxsubr(i0, r0, rn(reg));
2726 jit_unget_reg(reg);
2727 return (w);
2728}
2729
2730static jit_word_t
2731_boaddr_u(jit_state_t *_jit, jit_word_t i0, jit_int32_t r0, jit_int32_t r1)
2732{
2733 jit_word_t d, w;
2734 ADDC(r0, r0, r1);
2735 MCRXR(CR_0);
2736 w = _jit->pc.w;
2737 d = (i0 - w) & ~3;
2738 BEQ(d); /* EQ = bit 2 of XER = CA */
2739 return (w);
2740}
2741
2742static jit_word_t
2743_boaddi_u(jit_state_t *_jit, jit_word_t i0, jit_int32_t r0, jit_word_t i1)
2744{
2745 jit_int32_t reg;
2746 jit_word_t d, w;
2747 if (can_sign_extend_short_p(i1)) {
2748 ADDIC(r0, r0, i1);
2749 MCRXR(CR_0);
2750 w = _jit->pc.w;
2751 d = (i0 - w) & ~3;
2752 BEQ(d);
2753 return (w);
2754 }
2755 reg = jit_get_reg(jit_class_gpr|jit_class_nospill);
2756 movi(rn(reg), i1);
2757 w = boaddr_u(i0, r0, rn(reg));
2758 jit_unget_reg(reg);
2759 return (w);
2760}
2761
2762static jit_word_t
2763_bxaddr_u(jit_state_t *_jit, jit_word_t i0, jit_int32_t r0, jit_int32_t r1)
2764{
2765 jit_word_t d, w;
2766 ADDC(r0, r0, r1);
2767 MCRXR(CR_0);
2768 w = _jit->pc.w;
2769 d = (i0 - w) & ~3;
2770 BNE(d);
2771 return (w);
2772}
2773
2774static jit_word_t
2775_bxaddi_u(jit_state_t *_jit, jit_word_t i0, jit_int32_t r0, jit_word_t i1)
2776{
2777 jit_int32_t reg;
2778 jit_word_t d, w;
2779 if (can_sign_extend_short_p(i1)) {
2780 ADDIC(r0, r0, i1);
2781 MCRXR(CR_0);
2782 w = _jit->pc.w;
2783 d = (i0 - w) & ~3;
2784 BNE(d);
2785 return (w);
2786 }
2787 reg = jit_get_reg(jit_class_gpr|jit_class_nospill);
2788 movi(rn(reg), i1);
2789 w = bxaddr_u(i0, r0, rn(reg));
2790 jit_unget_reg(reg);
2791 return (w);
2792}
2793
2794static jit_word_t
2795_bosubr_u(jit_state_t *_jit, jit_word_t i0, jit_int32_t r0, jit_int32_t r1)
2796{
2797 jit_word_t d, w;
2798 SUBC(r0, r0, r1);
2799 MCRXR(CR_0);
2800 w = _jit->pc.w;
2801 d = (i0 - w) & ~3;
2802 BNE(d); /* PPC uses "carry" not "borrow" */
2803 return (w);
2804}
2805
2806static jit_word_t
2807_bosubi_u(jit_state_t *_jit, jit_word_t i0, jit_int32_t r0, jit_word_t i1)
2808{
2809 jit_word_t w;
2810 jit_int32_t reg;
2811 reg = jit_get_reg(jit_class_gpr|jit_class_nospill);
2812 movi(rn(reg), i1);
2813 w = bosubr_u(i0, r0, rn(reg));
2814 jit_unget_reg(reg);
2815 return (w);
2816}
2817
2818static jit_word_t
2819_bxsubr_u(jit_state_t *_jit, jit_word_t i0, jit_int32_t r0, jit_int32_t r1)
2820{
2821 jit_word_t d, w;
2822 SUBC(r0, r0, r1);
2823 MCRXR(CR_0);
2824 w = _jit->pc.w;
2825 d = (i0 - w) & ~3;
2826 BEQ(d);
2827 return (w);
2828}
2829
2830static jit_word_t
2831_bxsubi_u(jit_state_t *_jit, jit_word_t i0, jit_int32_t r0, jit_word_t i1)
2832{
2833 jit_word_t w;
2834 jit_int32_t reg;
2835 reg = jit_get_reg(jit_class_gpr|jit_class_nospill);
2836 movi(rn(reg), i1);
2837 w = bxsubr_u(i0, r0, rn(reg));
2838 jit_unget_reg(reg);
2839 return (w);
2840}
2841
2842static void
2843_ldr_c(jit_state_t *_jit, jit_int32_t r0, jit_int32_t r1)
2844{
2845 ldr_uc(r0, r1);
2846 extr_c(r0, r0);
2847}
2848
2849static void
2850_ldi_c(jit_state_t *_jit, jit_int32_t r0, jit_word_t i0)
2851{
2852 ldi_uc(r0, i0);
2853 extr_c(r0, r0);
2854}
2855
2856static void
2857_ldxr_c(jit_state_t *_jit, jit_int32_t r0, jit_int32_t r1, jit_int32_t r2)
2858{
2859 ldxr_uc(r0, r1, r2);
2860 extr_c(r0, r0);
2861}
2862
2863static void
2864_ldxi_c(jit_state_t *_jit, jit_int32_t r0, jit_int32_t r1, jit_word_t i0)
2865{
2866 ldxi_uc(r0, r1, i0);
2867 extr_c(r0, r0);
2868}
2869
2870static void
2871_ldi_uc(jit_state_t *_jit, jit_int32_t r0, jit_word_t i0)
2872{
2873 jit_bool_t inv;
2874 jit_int32_t reg;
2875 jit_word_t lo, hi;
2876 if (can_sign_extend_short_p(i0))
2877 LBZ(r0, _R0_REGNO, i0);
2878 else if (can_sign_extend_int_p(i0)) {
2879 hi = (jit_int16_t)((i0 >> 16) + ((jit_uint16_t)i0 >> 15));
2880 lo = (jit_int16_t)(i0 - (hi << 16));
2881 reg = jit_get_reg(jit_class_gpr);
2882 if ((inv = reg == _R0)) reg = jit_get_reg(jit_class_gpr);
2883 LIS(rn(reg), hi);
2884 LBZ(r0, rn(reg), lo);
2885 jit_unget_reg(reg);
2886 if (inv) jit_unget_reg(_R0);
2887 }
2888 else {
2889 reg = jit_get_reg(jit_class_gpr);
2890 movi(rn(reg), i0);
2891 ldr_uc(r0, rn(reg));
2892 jit_unget_reg(reg);
2893 }
2894}
2895
2896static void
2897_ldxr_uc(jit_state_t *_jit, jit_int32_t r0, jit_int32_t r1, jit_int32_t r2)
2898{
2899 jit_int32_t reg;
2900 if (r1 == _R0_REGNO) {
2901 if (r2 != _R0_REGNO)
2902 LBZX(r0, r2, r1);
2903 else {
2904 reg = jit_get_reg(jit_class_gpr);
2905 movr(rn(reg), r1);
2906 LBZX(r0, rn(reg), r2);
2907 jit_unget_reg(reg);
2908 }
2909 }
2910 else
2911 LBZX(r0, r1, r2);
2912}
2913
2914static void
2915_ldxi_uc(jit_state_t *_jit, jit_int32_t r0, jit_int32_t r1, jit_word_t i0)
2916{
2917 jit_int32_t reg;
2918 if (i0 == 0)
2919 ldr_uc(r0, r1);
2920 else if (can_sign_extend_short_p(i0)) {
2921 if (r1 == _R0_REGNO) {
2922 reg = jit_get_reg(jit_class_gpr);
2923 movr(rn(reg), r1);
2924 LBZ(r0, rn(reg), i0);
2925 jit_unget_reg(reg);
2926 }
2927 else
2928 LBZ(r0, r1, i0);
2929 }
2930 else {
2931 reg = jit_get_reg(jit_class_gpr);
2932 movi(rn(reg), i0);
2933 ldxr_uc(r0, r1, rn(reg));
2934 jit_unget_reg(reg);
2935 }
2936}
2937
2938static void
2939_ldi_s(jit_state_t *_jit, jit_int32_t r0, jit_word_t i0)
2940{
2941 jit_bool_t inv;
2942 jit_int32_t reg;
2943 jit_word_t lo, hi;
2944 if (can_sign_extend_short_p(i0))
2945 LHA(r0, _R0_REGNO, i0);
2946 else if (can_sign_extend_int_p(i0)) {
2947 hi = (jit_int16_t)((i0 >> 16) + ((jit_uint16_t)i0 >> 15));
2948 lo = (jit_int16_t)(i0 - (hi << 16));
2949 reg = jit_get_reg(jit_class_gpr);
2950 if ((inv = reg == _R0)) reg = jit_get_reg(jit_class_gpr);
2951 LIS(rn(reg), hi);
2952 LHA(r0, rn(reg), lo);
2953 jit_unget_reg(reg);
2954 if (inv) jit_unget_reg(_R0);
2955 }
2956 else {
2957 reg = jit_get_reg(jit_class_gpr);
2958 movi(rn(reg), i0);
2959 ldr_s(r0, rn(reg));
2960 jit_unget_reg(reg);
2961 }
2962}
2963
2964static void
2965_ldxr_s(jit_state_t *_jit, jit_int32_t r0, jit_int32_t r1, jit_int32_t r2)
2966{
2967 jit_int32_t reg;
2968 if (r1 == _R0_REGNO) {
2969 if (r2 != _R0_REGNO)
2970 LHAX(r0, r2, r1);
2971 else {
2972 reg = jit_get_reg(jit_class_gpr);
2973 movr(rn(reg), r1);
2974 LHAX(r0, rn(reg), r2);
2975 jit_unget_reg(reg);
2976 }
2977 }
2978 else
2979 LHAX(r0, r1, r2);
2980}
2981
2982static void
2983_ldxi_s(jit_state_t *_jit, jit_int32_t r0, jit_int32_t r1, jit_word_t i0)
2984{
2985 jit_int32_t reg;
2986 if (i0 == 0)
2987 ldr_s(r0, r1);
2988 else if (can_sign_extend_short_p(i0)) {
2989 if (r1 == _R0_REGNO) {
2990 reg = jit_get_reg(jit_class_gpr);
2991 movr(rn(reg), r1);
2992 LHA(r0, rn(reg), i0);
2993 jit_unget_reg(reg);
2994 }
2995 else
2996 LHA(r0, r1, i0);
2997 }
2998 else {
2999 reg = jit_get_reg(jit_class_gpr);
3000 movi(rn(reg), i0);
3001 ldxr_s(r0, r1, rn(reg));
3002 jit_unget_reg(reg);
3003 }
3004}
3005
3006static void
3007_ldi_us(jit_state_t *_jit, jit_int32_t r0, jit_word_t i0)
3008{
3009 jit_bool_t inv;
3010 jit_int32_t reg;
3011 jit_word_t lo, hi;
3012 if (can_sign_extend_short_p(i0))
3013 LHZ(r0, _R0_REGNO, i0);
3014 else if (can_sign_extend_int_p(i0)) {
3015 hi = (jit_int16_t)((i0 >> 16) + ((jit_uint16_t)i0 >> 15));
3016 lo = (jit_int16_t)(i0 - (hi << 16));
3017 reg = jit_get_reg(jit_class_gpr);
3018 if ((inv = reg == _R0)) reg = jit_get_reg(jit_class_gpr);
3019 LIS(rn(reg), hi);
3020 LHZ(r0, rn(reg), lo);
3021 jit_unget_reg(reg);
3022 if (inv) jit_unget_reg(_R0);
3023 }
3024 else {
3025 reg = jit_get_reg(jit_class_gpr);
3026 movi(rn(reg), i0);
3027 ldr_us(r0, rn(reg));
3028 jit_unget_reg(reg);
3029 }
3030}
3031
3032static void
3033_ldxr_us(jit_state_t *_jit, jit_int32_t r0, jit_int32_t r1, jit_int32_t r2)
3034{
3035 jit_int32_t reg;
3036 if (r1 == _R0_REGNO) {
3037 if (r2 != _R0_REGNO)
3038 LHZX(r0, r2, r1);
3039 else {
3040 reg = jit_get_reg(jit_class_gpr);
3041 movr(rn(reg), r1);
3042 LHZX(r0, rn(reg), r2);
3043 jit_unget_reg(reg);
3044 }
3045 }
3046 else
3047 LHZX(r0, r1, r2);
3048}
3049
3050static void
3051_ldxi_us(jit_state_t *_jit, jit_int32_t r0, jit_int32_t r1, jit_word_t i0)
3052{
3053 jit_int32_t reg;
3054 if (i0 == 0)
3055 ldr_us(r0, r1);
3056 else if (can_sign_extend_short_p(i0)) {
3057 if (r1 == _R0_REGNO) {
3058 reg = jit_get_reg(jit_class_gpr);
3059 movr(rn(reg), r1);
3060 LHZ(r0, rn(reg), i0);
3061 jit_unget_reg(reg);
3062 }
3063 else
3064 LHZ(r0, r1, i0);
3065 }
3066 else {
3067 reg = jit_get_reg(jit_class_gpr);
3068 movi(rn(reg), i0);
3069 ldxr_us(r0, r1, rn(reg));
3070 jit_unget_reg(reg);
3071 }
3072}
3073
3074# if __WORDSIZE == 32
3075static void
3076_ldi_i(jit_state_t *_jit, jit_int32_t r0, jit_word_t i0)
3077{
3078 jit_bool_t inv;
3079 jit_int32_t reg;
3080 jit_word_t lo, hi;
3081 if (can_sign_extend_short_p(i0))
3082 LWZ(r0, _R0_REGNO, i0);
3083 else if (can_sign_extend_int_p(i0)) {
3084 hi = (jit_int16_t)((i0 >> 16) + ((jit_uint16_t)i0 >> 15));
3085 lo = (jit_int16_t)(i0 - (hi << 16));
3086 reg = jit_get_reg(jit_class_gpr);
3087 if ((inv = reg == _R0)) reg = jit_get_reg(jit_class_gpr);
3088 LIS(rn(reg), hi);
3089 LWZ(r0, rn(reg), lo);
3090 jit_unget_reg(reg);
3091 if (inv) jit_unget_reg(_R0);
3092 }
3093 else {
3094 reg = jit_get_reg(jit_class_gpr);
3095 movi(rn(reg), i0);
3096 ldr_i(r0, rn(reg));
3097 jit_unget_reg(reg);
3098 }
3099}
3100
3101static void
3102_ldxr_i(jit_state_t *_jit, jit_int32_t r0, jit_int32_t r1, jit_int32_t r2)
3103{
3104 jit_int32_t reg;
3105 if (r1 == _R0_REGNO) {
3106 if (r2 != _R0_REGNO)
3107 LWZX(r0, r2, r1);
3108 else {
3109 reg = jit_get_reg(jit_class_gpr);
3110 movr(rn(reg), r1);
3111 LWZX(r0, rn(reg), r2);
3112 jit_unget_reg(reg);
3113 }
3114 }
3115 else
3116 LWZX(r0, r1, r2);
3117}
3118
3119static void
3120_ldxi_i(jit_state_t *_jit, jit_int32_t r0, jit_int32_t r1, jit_word_t i0)
3121{
3122 jit_int32_t reg;
3123 if (i0 == 0)
3124 ldr_i(r0, r1);
3125 else if (can_sign_extend_short_p(i0)) {
3126 if (r1 == _R0_REGNO) {
3127 reg = jit_get_reg(jit_class_gpr);
3128 movr(rn(reg), r1);
3129 LWZ(r0, rn(reg), i0);
3130 jit_unget_reg(reg);
3131 }
3132 else
3133 LWZ(r0, r1, i0);
3134 }
3135 else {
3136 reg = jit_get_reg(jit_class_gpr);
3137 movi(rn(reg), i0);
3138 ldxr_i(r0, r1, rn(reg));
3139 jit_unget_reg(reg);
3140 }
3141}
3142
3143# else
3144static void
3145_ldi_i(jit_state_t *_jit, jit_int32_t r0, jit_word_t i0)
3146{
3147 jit_bool_t inv;
3148 jit_int32_t reg;
3149 jit_word_t lo, hi;
ba86ff93 3150 if (can_sign_extend_short_p(i0) && !(i0 & 3))
4a71579b 3151 LWA(r0, _R0_REGNO, i0);
ba86ff93 3152 else if (can_sign_extend_int_p(i0) && !(i0 & 3)) {
4a71579b
PC		 3153 hi = (jit_int16_t)((i0 >> 16) + ((jit_uint16_t)i0 >> 15));
3154 lo = (jit_int16_t)(i0 - (hi << 16));
3155 reg = jit_get_reg(jit_class_gpr);
3156 if ((inv = reg == _R0)) reg = jit_get_reg(jit_class_gpr);
3157 LIS(rn(reg), hi);
3158 LWA(r0, rn(reg), lo);
3159 jit_unget_reg(reg);
3160 if (inv) jit_unget_reg(_R0);
3161 }
3162 else {
3163 reg = jit_get_reg(jit_class_gpr);
3164 movi(rn(reg), i0);
3165 ldr_i(r0, rn(reg));
3166 jit_unget_reg(reg);
3167 }
3168}
3169
3170static void
3171_ldxr_i(jit_state_t *_jit, jit_int32_t r0, jit_int32_t r1, jit_int32_t r2)
3172{
3173 jit_int32_t reg;
3174 if (r1 == _R0_REGNO) {
3175 if (r2 != _R0_REGNO)
79bfeef6 3176 LWAX(r0, r2, r1);
4a71579b
PC		 3177 else {
3178 reg = jit_get_reg(jit_class_gpr);
3179 movr(rn(reg), r1);
3180 LWAX(r0, rn(reg), r2);
3181 jit_unget_reg(reg);
3182 }
3183 }
3184 else
79bfeef6 3185 LWAX(r0, r1, r2);
4a71579b
PC		 3186}
3187
3188static void
3189_ldxi_i(jit_state_t *_jit, jit_int32_t r0, jit_int32_t r1, jit_word_t i0)
3190{
3191 jit_int32_t reg;
3192 if (i0 == 0)
3193 ldr_i(r0, r1);
ba86ff93 3194 else if (can_sign_extend_short_p(i0) && !(i0 & 3)) {
4a71579b
PC		 3195 if (r1 == _R0_REGNO) {
3196 reg = jit_get_reg(jit_class_gpr);
3197 movr(rn(reg), r1);
3198 LWA(r0, rn(reg), i0);
3199 jit_unget_reg(reg);
3200 }
3201 else
3202 LWA(r0, r1, i0);
3203 }
3204 else {
3205 reg = jit_get_reg(jit_class_gpr);
3206 movi(rn(reg), i0);
3207 ldxr_i(r0, r1, rn(reg));
3208 jit_unget_reg(reg);
3209 }
3210}
3211
3212static void
3213_ldi_ui(jit_state_t *_jit, jit_int32_t r0, jit_word_t i0)
3214{
3215 jit_bool_t inv;
3216 jit_int32_t reg;
3217 jit_word_t lo, hi;
3218 if (can_sign_extend_short_p(i0))
3219 LWZ(r0, _R0_REGNO, i0);
3220 else if (can_sign_extend_int_p(i0)) {
3221 hi = (jit_int16_t)((i0 >> 16) + ((jit_uint16_t)i0 >> 15));
3222 lo = (jit_int16_t)(i0 - (hi << 16));
3223 reg = jit_get_reg(jit_class_gpr);
3224 if ((inv = reg == _R0)) reg = jit_get_reg(jit_class_gpr);
3225 LIS(rn(reg), hi);
3226 LWZ(r0, rn(reg), lo);
3227 jit_unget_reg(reg);
3228 if (inv) jit_unget_reg(_R0);
3229 }
3230 else {
3231 reg = jit_get_reg(jit_class_gpr);
3232 movi(rn(reg), i0);
3233 ldr_ui(r0, rn(reg));
3234 jit_unget_reg(reg);
3235 }
3236}
3237
3238static void
3239_ldxr_ui(jit_state_t *_jit, jit_int32_t r0, jit_int32_t r1, jit_int32_t r2)
3240{
3241 jit_int32_t reg;
3242 if (r1 == _R0_REGNO) {
3243 if (r2 != _R0_REGNO)
3244 LWZX(r0, r2, r1);
3245 else {
3246 reg = jit_get_reg(jit_class_gpr);
3247 movr(rn(reg), r1);
3248 LWZX(r0, rn(reg), r2);
3249 jit_unget_reg(reg);
3250 }
3251 }
3252 else
3253 LWZX(r0, r1, r2);
3254}
3255
3256static void
3257_ldxi_ui(jit_state_t *_jit, jit_int32_t r0, jit_int32_t r1, jit_word_t i0)
3258{
3259 jit_int32_t reg;
3260 if (i0 == 0)
3261 ldr_i(r0, r1);
3262 else if (can_sign_extend_short_p(i0)) {
3263 if (r1 == _R0_REGNO) {
3264 reg = jit_get_reg(jit_class_gpr);
3265 movr(rn(reg), r1);
3266 LWZ(r0, rn(reg), i0);
3267 jit_unget_reg(reg);
3268 }
3269 else
3270 LWZ(r0, r1, i0);
3271 }
3272 else {
3273 reg = jit_get_reg(jit_class_gpr);
3274 movi(rn(reg), i0);
3275 ldxr_ui(r0, r1, rn(reg));
3276 jit_unget_reg(reg);
3277 }
3278}
3279
3280static void
3281_ldi_l(jit_state_t *_jit, jit_int32_t r0, jit_word_t i0)
3282{
3283 jit_bool_t inv;
3284 jit_int32_t reg;
3285 jit_word_t lo, hi;
ba86ff93 3286 if (can_sign_extend_short_p(i0) && !(i0 & 3))
4a71579b 3287 LD(r0, _R0_REGNO, i0);
ba86ff93 3288 else if (can_sign_extend_int_p(i0) && !(i0 & 3)) {
4a71579b
PC		 3289 hi = (jit_int16_t)((i0 >> 16) + ((jit_uint16_t)i0 >> 15));
3290 lo = (jit_int16_t)(i0 - (hi << 16));
3291 reg = jit_get_reg(jit_class_gpr);
3292 if ((inv = reg == _R0)) reg = jit_get_reg(jit_class_gpr);
3293 LIS(rn(reg), hi);
3294 LD(r0, rn(reg), lo);
3295 jit_unget_reg(reg);
3296 if (inv) jit_unget_reg(_R0);
3297 }
3298 else {
3299 reg = jit_get_reg(jit_class_gpr);
3300 movi(rn(reg), i0);
3301 ldr_l(r0, rn(reg));
3302 jit_unget_reg(reg);
3303 }
3304}
3305
3306static void
3307_ldxr_l(jit_state_t *_jit, jit_int32_t r0, jit_int32_t r1, jit_int32_t r2)
3308{
3309 jit_int32_t reg;
3310 if (r1 == _R0_REGNO) {
3311 if (r2 != _R0_REGNO)
3312 LDX(r0, r2, r1);
3313 else {
3314 reg = jit_get_reg(jit_class_gpr);
3315 movr(rn(reg), r1);
3316 LDX(r0, rn(reg), r2);
3317 jit_unget_reg(reg);
3318 }
3319 }
3320 else
3321 LDX(r0, r1, r2);
3322}
3323
3324static void
3325_ldxi_l(jit_state_t *_jit, jit_int32_t r0, jit_int32_t r1, jit_word_t i0)
3326{
3327 jit_int32_t reg;
3328 if (i0 == 0)
3329 ldr_l(r0, r1);
ba86ff93 3330 else if (can_sign_extend_short_p(i0) && !(i0 & 3)) {
4a71579b
PC		 3331 if (r1 == _R0_REGNO) {
3332 reg = jit_get_reg(jit_class_gpr);
3333 movr(rn(reg), r1);
3334 LD(r0, rn(reg), i0);
3335 jit_unget_reg(reg);
3336 }
3337 else
3338 LD(r0, r1, i0);
3339 }
3340 else {
3341 reg = jit_get_reg(jit_class_gpr);
3342 movi(rn(reg), i0);
3343 ldxr_l(r0, r1, rn(reg));
3344 jit_unget_reg(reg);
3345 }
3346}
3347# endif
3348
3349static void
3350_sti_c(jit_state_t *_jit, jit_word_t i0, jit_int32_t r0)
3351{
3352 jit_bool_t inv;
3353 jit_int32_t reg;
3354 jit_word_t lo, hi;
3355 if (can_sign_extend_short_p(i0))
3356 STB(r0, _R0_REGNO, i0);
3357 else if (can_sign_extend_int_p(i0)) {
3358 hi = (jit_int16_t)((i0 >> 16) + ((jit_uint16_t)i0 >> 15));
3359 lo = (jit_int16_t)(i0 - (hi << 16));
3360 reg = jit_get_reg(jit_class_gpr);
3361 if ((inv = reg == _R0)) reg = jit_get_reg(jit_class_gpr);
3362 LIS(rn(reg), hi);
3363 STB(r0, rn(reg), lo);
3364 jit_unget_reg(reg);
3365 if (inv) jit_unget_reg(_R0);
3366 }
3367 else {
3368 reg = jit_get_reg(jit_class_gpr);
3369 movi(rn(reg), i0);
3370 str_c(rn(reg), r0);
3371 jit_unget_reg(reg);
3372 }
3373}
3374
3375static void
3376_stxr_c(jit_state_t *_jit, jit_int32_t r0, jit_int32_t r1, jit_int32_t r2)
3377{
3378 jit_int32_t reg;
3379 if (r0 == _R0_REGNO) {
3380 if (r1 != _R0_REGNO)
3381 STBX(r2, r1, r0);
3382 else {
3383 reg = jit_get_reg(jit_class_gpr);
3384 movr(rn(reg), r0);
3385 STBX(r2, rn(reg), r1);
3386 jit_unget_reg(reg);
3387 }
3388 }
3389 else
3390 STBX(r2, r0, r1);
3391}
3392
3393static void
3394_stxi_c(jit_state_t *_jit, jit_word_t i0, jit_int32_t r0, jit_int32_t r1)
3395{
3396 jit_int32_t reg;
3397 if (i0 == 0)
3398 str_c(r0, r1);
3399 else if (can_sign_extend_short_p(i0)) {
3400 if (r0 == _R0_REGNO) {
3401 reg = jit_get_reg(jit_class_gpr);
3402 movr(rn(reg), i0);
3403 STB(r1, rn(reg), i0);
3404 jit_unget_reg(reg);
3405 }
3406 else
3407 STB(r1, r0, i0);
3408 }
3409 else {
3410 reg = jit_get_reg(jit_class_gpr);
3411 movi(rn(reg), i0);
3412 stxr_c(rn(reg), r0, r1);
3413 jit_unget_reg(reg);
3414 }
3415}
3416
3417static void
3418_sti_s(jit_state_t *_jit, jit_word_t i0, jit_int32_t r0)
3419{
3420 jit_bool_t inv;
3421 jit_int32_t reg;
3422 jit_word_t lo, hi;
3423 if (can_sign_extend_short_p(i0))
3424 STH(r0, _R0_REGNO, i0);
3425 else if (can_sign_extend_int_p(i0)) {
3426 hi = (jit_int16_t)((i0 >> 16) + ((jit_uint16_t)i0 >> 15));
3427 lo = (jit_int16_t)(i0 - (hi << 16));
3428 reg = jit_get_reg(jit_class_gpr);
3429 if ((inv = reg == _R0)) reg = jit_get_reg(jit_class_gpr);
3430 LIS(rn(reg), hi);
3431 STH(r0, rn(reg), lo);
3432 jit_unget_reg(reg);
3433 if (inv) jit_unget_reg(_R0);
3434 }
3435 else {
3436 reg = jit_get_reg(jit_class_gpr);
3437 movi(rn(reg), i0);
3438 str_s(rn(reg), r0);
3439 jit_unget_reg(reg);
3440 }
3441}
3442
3443static void
3444_stxr_s(jit_state_t *_jit, jit_int32_t r0, jit_int32_t r1, jit_int32_t r2)
3445{
3446 jit_int32_t reg;
3447 if (r0 == _R0_REGNO) {
3448 if (r1 != _R0_REGNO)
3449 STHX(r2, r1, r0);
3450 else {
3451 reg = jit_get_reg(jit_class_gpr);
3452 movr(rn(reg), r0);
3453 STHX(r2, rn(reg), r1);
3454 jit_unget_reg(reg);
3455 }
3456 }
3457 else
3458 STHX(r2, r0, r1);
3459}
3460
3461static void
3462_stxi_s(jit_state_t *_jit, jit_word_t i0, jit_int32_t r0, jit_int32_t r1)
3463{
3464 jit_int32_t reg;
3465 if (i0 == 0)
3466 str_s(r0, r1);
3467 else if (can_sign_extend_short_p(i0)) {
3468 if (r0 == _R0_REGNO) {
3469 reg = jit_get_reg(jit_class_gpr);
3470 movr(rn(reg), i0);
3471 STH(r1, rn(reg), i0);
3472 jit_unget_reg(reg);
3473 }
3474 else
3475 STH(r1, r0, i0);
3476 }
3477 else {
3478 reg = jit_get_reg(jit_class_gpr);
3479 movi(rn(reg), i0);
3480 stxr_s(rn(reg), r0, r1);
3481 jit_unget_reg(reg);
3482 }
3483}
3484
3485static void
3486_sti_i(jit_state_t *_jit, jit_word_t i0, jit_int32_t r0)
3487{
3488 jit_bool_t inv;
3489 jit_int32_t reg;
3490 jit_word_t lo, hi;
3491 if (can_sign_extend_short_p(i0))
3492 STW(r0, _R0_REGNO, i0);
3493 else if (can_sign_extend_int_p(i0)) {
3494 hi = (jit_int16_t)((i0 >> 16) + ((jit_uint16_t)i0 >> 15));
3495 lo = (jit_int16_t)(i0 - (hi << 16));
3496 reg = jit_get_reg(jit_class_gpr);
3497 if ((inv = reg == _R0)) reg = jit_get_reg(jit_class_gpr);
3498 LIS(rn(reg), hi);
3499 STW(r0, rn(reg), lo);
3500 jit_unget_reg(reg);
3501 if (inv) jit_unget_reg(_R0);
3502 }
3503 else {
3504 reg = jit_get_reg(jit_class_gpr);
3505 movi(rn(reg), i0);
3506 str_i(rn(reg), r0);
3507 jit_unget_reg(reg);
3508 }
3509}
3510
3511static void
3512_stxr_i(jit_state_t *_jit, jit_int32_t r0, jit_int32_t r1, jit_int32_t r2)
3513{
3514 jit_int32_t reg;
3515 if (r0 == _R0_REGNO) {
3516 if (r1 != _R0_REGNO)
3517 STWX(r2, r1, r0);
3518 else {
3519 reg = jit_get_reg(jit_class_gpr);
3520 movr(rn(reg), r0);
3521 STWX(r2, rn(reg), r1);
3522 jit_unget_reg(reg);
3523 }
3524 }
3525 else
3526 STWX(r2, r0, r1);
3527}
3528
3529static void
3530_stxi_i(jit_state_t *_jit, jit_word_t i0, jit_int32_t r0, jit_int32_t r1)
3531{
3532 jit_int32_t reg;
3533 if (i0 == 0)
3534 str_i(r0, r1);
3535 else if (can_sign_extend_short_p(i0)) {
3536 if (r0 == _R0_REGNO) {
3537 reg = jit_get_reg(jit_class_gpr);
3538 movr(rn(reg), i0);
3539 STW(r1, rn(reg), i0);
3540 jit_unget_reg(reg);
3541 }
3542 else
3543 STW(r1, r0, i0);
3544 }
3545 else {
3546 reg = jit_get_reg(jit_class_gpr);
3547 movi(rn(reg), i0);
3548 stxr_i(rn(reg), r0, r1);
3549 jit_unget_reg(reg);
3550 }
3551}
3552
3553# if __WORDSIZE == 64
3554static void
3555_sti_l(jit_state_t *_jit, jit_word_t i0, jit_int32_t r0)
3556{
3557 jit_bool_t inv;
3558 jit_int32_t reg;
3559 jit_word_t lo, hi;
ba86ff93 3560 if (can_sign_extend_short_p(i0) && !(i0 & 3))
4a71579b 3561 STD(r0, _R0_REGNO, i0);
ba86ff93 3562 else if (can_sign_extend_int_p(i0) && !(i0 & 3)) {
4a71579b
PC		 3563 hi = (jit_int16_t)((i0 >> 16) + ((jit_uint16_t)i0 >> 15));
3564 lo = (jit_int16_t)(i0 - (hi << 16));
3565 reg = jit_get_reg(jit_class_gpr);
3566 if ((inv = reg == _R0)) reg = jit_get_reg(jit_class_gpr);
3567 LIS(rn(reg), hi);
3568 STD(r0, rn(reg), lo);
3569 jit_unget_reg(reg);
3570 if (inv) jit_unget_reg(_R0);
3571 }
3572 else {
3573 reg = jit_get_reg(jit_class_gpr);
3574 movi(rn(reg), i0);
3575 str_l(rn(reg), r0);
3576 jit_unget_reg(reg);
3577 }
3578}
3579
3580static void
3581_stxr_l(jit_state_t *_jit, jit_int32_t r0, jit_int32_t r1, jit_int32_t r2)
3582{
3583 jit_int32_t reg;
3584 if (r0 == _R0_REGNO) {
3585 if (r1 != _R0_REGNO)
3586 STDX(r2, r1, r0);
3587 else {
3588 reg = jit_get_reg(jit_class_gpr);
3589 movr(rn(reg), r0);
3590 STDX(r2, rn(reg), r1);
3591 jit_unget_reg(reg);
3592 }
3593 }
3594 else
3595 STDX(r2, r0, r1);
3596}
3597
3598static void
3599_stxi_l(jit_state_t *_jit, jit_word_t i0, jit_int32_t r0, jit_int32_t r1)
3600{
3601 jit_int32_t reg;
3602 if (i0 == 0)
3603 str_l(r0, r1);
ba86ff93 3604 else if (can_sign_extend_short_p(i0) && !(i0 & 3)) {
4a71579b
PC		 3605 if (r0 == _R0_REGNO) {
3606 reg = jit_get_reg(jit_class_gpr);
3607 movr(rn(reg), i0);
3608 STD(r1, rn(reg), i0);
3609 jit_unget_reg(reg);
3610 }
3611 else
3612 STD(r1, r0, i0);
3613 }
3614 else {
3615 reg = jit_get_reg(jit_class_gpr);
3616 movi(rn(reg), i0);
3617 stxr_l(rn(reg), r0, r1);
3618 jit_unget_reg(reg);
3619 }
3620}
3621# endif
3622
3623static void
3624_jmpr(jit_state_t *_jit, jit_int32_t r0)
3625{
3626#if 0
3627 MTLR(r0);
3628 BLR();
3629#else
3630 MTCTR(r0);
3631 BCTR();
3632#endif
3633}
3634
3635/* pc relative jump */
3636static jit_word_t
3637_jmpi(jit_state_t *_jit, jit_word_t i0)
3638{
3639 jit_int32_t reg;
3640 jit_word_t w, d;
3641 w = _jit->pc.w;
3642 d = (i0 - w) & ~3;
3643 if (can_sign_extend_jump_p(d))
3644 B(d);
3645 else {
3646 reg = jit_get_reg(jit_class_gpr|jit_class_nospill);
3647 w = movi_p(rn(reg), i0);
3648 jmpr(rn(reg));
3649 jit_unget_reg(reg);
3650 }
3651 return (w);
3652}
3653
3654/* absolute jump */
3655static jit_word_t
3656_jmpi_p(jit_state_t *_jit, jit_word_t i0)
3657{
3658 jit_word_t w;
3659 jit_int32_t reg;
3660 reg = jit_get_reg(jit_class_gpr|jit_class_nospill);
3661 w = movi_p(rn(reg), i0);
3662 jmpr(rn(reg));
3663 jit_unget_reg(reg);
3664 return (w);
3665}
3666
3667static void
3668_callr(jit_state_t *_jit, jit_int32_t r0
3669# if _CALL_SYSV
3670 , jit_int32_t varargs
3671# endif
3672 )
3673{
3674# if _CALL_AIXDESC
3675 stxi(sizeof(void*) * 5, _SP_REGNO, _R2_REGNO);
3676 /* FIXME Pretend to not know about r11? */
3677 if (r0 == _R0_REGNO) {
3678 movr(_R11_REGNO, _R0_REGNO);
3679 ldxi(_R2_REGNO, _R11_REGNO, sizeof(void*));
3680 ldxi(_R11_REGNO, _R11_REGNO, sizeof(void*) * 2);
3681 }
3682 else {
3683 ldxi(_R2_REGNO, r0, sizeof(void*));
3684 ldxi(_R11_REGNO, r0, sizeof(void*) * 2);
3685 }
3686 ldr(r0, r0);
3687# else
3688# if _CALL_SYSV
3689 /* Tell double arguments were passed in registers. */
3690 if (varargs)
3691 CREQV(6, 6, 6);
3692# endif
3693 movr(_R12_REGNO, r0);
3694# endif
3695
3696 MTCTR(r0);
3697 BCTRL();
3698
3699# if _CALL_AIXDESC
3700 ldxi(_R2_REGNO, _SP_REGNO, sizeof(void*) * 5);
3701# endif
3702}
3703
3704/* assume fixed address or reachable address */
79bfeef6 3705static jit_word_t
4a71579b
PC		 3706_calli(jit_state_t *_jit, jit_word_t i0
3707# if _CALL_SYSV
3708 , jit_int32_t varargs
3709# endif
3710 )
3711{
79bfeef6 3712 jit_word_t w;
4a71579b
PC		 3713# if _CALL_SYSV
3714 jit_word_t d;
ba3814c1
PC		 3715 d = (i0 - _jit->pc.w - !!varargs * 4) & ~3;
3716 if (can_sign_extend_jump_p(d)) {
79bfeef6
PC		 3717 /* Tell double arguments were passed in registers. */
3718 if (varargs)
3719 CREQV(6, 6, 6);
3720 w = _jit->pc.w;
3721 BL(d);
3722 }
3723 else
4a71579b
PC		 3724# endif
3725 {
79bfeef6 3726 w = _jit->pc.w;
4a71579b
PC		 3727 movi(_R12_REGNO, i0);
3728 callr(_R12_REGNO
3729# if _CALL_SYSV
3730 , varargs
3731# endif
3732 );
3733 }
79bfeef6 3734 return (w);
4a71579b
PC		 3735}
3736
3737/* absolute jump */
3738static jit_word_t
3739_calli_p(jit_state_t *_jit, jit_word_t i0
3740# if _CALL_SYSV
3741 , jit_int32_t varargs
3742# endif
3743 )
3744{
3745 jit_word_t w;
3746 w = movi_p(_R12_REGNO, i0);
3747 callr(_R12_REGNO
3748# if _CALL_SYSV
3749 , varargs
3750# endif
3751 );
3752 return (w);
3753}
3754
4a71579b
PC		 3755static void
3756_prolog(jit_state_t *_jit, jit_node_t *node)
3757{
ba86ff93 3758 jit_int32_t regno;
4a71579b
PC		 3759 jit_word_t offset;
3760
3761 if (_jitc->function->define_frame || _jitc->function->assume_frame) {
3762 jit_int32_t frame = -_jitc->function->frame;
ba86ff93 3763 jit_check_frame();
4a71579b
PC		 3764 assert(_jitc->function->self.aoff >= frame);
3765 if (_jitc->function->assume_frame)
3766 return;
3767 _jitc->function->self.aoff = frame;
3768 }
3769 if (_jitc->function->allocar) {
3770 _jitc->function->self.aoff -= 2 * sizeof(jit_word_t);
3771 _jitc->function->self.aoff &= -16;
3772 }
3773 _jitc->function->stack = ((_jitc->function->self.alen +
3774 _jitc->function->self.size -
3775 _jitc->function->self.aoff) + 15) & -16;
3776
ba86ff93
PC		 3777 if (_jitc->function->need_frame)
3778 _jitc->function->need_stack = _jitc->function->need_return = 1;
3779
3780 if (!_jitc->function->need_stack) {
3781 for (regno = 0; regno < jit_size(iregs); regno++) {
3782 if (jit_regset_tstbit(&_jitc->function->regset, iregs[regno])) {
3783 _jitc->function->need_stack =
3784 _jitc->function->need_return = 1;
3785 break;
3786 }
3787 }
3788 if (!_jitc->function->need_stack) {
3789 for (offset = 0; offset < jit_size(fregs); offset++) {
3790 if (jit_regset_tstbit(&_jitc->function->regset, fregs[offset])) {
3791 _jitc->function->need_stack =
3792 _jitc->function->need_return = 1;
3793 break;
3794 }
3795 }
3796 }
3797 }
4a71579b
PC		 3798
3799 /* params >= %r31+params_offset+(8*sizeof(jit_word_t))
3800 * alloca < %r31-80 */
3801
ba86ff93
PC		 3802 /* return address */
3803 if (_jitc->function->need_return) {
3804 MFLR(_R0_REGNO);
4a71579b 3805#if _CALL_SYSV
ba86ff93 3806 stxi(sizeof(jit_word_t), _SP_REGNO, _R0_REGNO);
4a71579b 3807#else
ba86ff93 3808 stxi(sizeof(void*) * 2, _SP_REGNO, _R0_REGNO);
4a71579b 3809#endif
4a71579b
PC		 3810 }
3811
ba86ff93
PC		 3812 if (_jitc->function->need_stack) {
3813 offset = -gpr_save_area;
3814 for (regno = 0; regno < jit_size(iregs);
3815 regno++, offset += sizeof(void*)) {
3816 if (jit_regset_tstbit(&_jitc->function->regset, iregs[regno]))
3817 stxi(offset, _SP_REGNO, rn(iregs[regno]));
3818 }
3819 for (offset = 0; offset < jit_size(fregs); offset++) {
3820 if (jit_regset_tstbit(&_jitc->function->regset, fregs[offset]))
3821 stxi_d(-(gpr_save_area + 8 + offset * 8),
3822 _SP_REGNO, rn(fregs[offset]));
3823 }
3824 }
4a71579b 3825
ba86ff93
PC		 3826 if (_jitc->function->need_frame) {
3827 stxi(-(sizeof(void*)), _SP_REGNO, _FP_REGNO);
3828 movr(_FP_REGNO, _SP_REGNO);
3829 }
3830 if (_jitc->function->need_stack) {
4a71579b 3831#if __WORDSIZE == 32
ba86ff93 3832 STWU(_SP_REGNO, _SP_REGNO, -_jitc->function->stack);
4a71579b 3833#else
ba86ff93 3834 STDU(_SP_REGNO, _SP_REGNO, -_jitc->function->stack);
4a71579b 3835#endif
ba86ff93 3836 }
4a71579b
PC		 3837
3838 if (_jitc->function->allocar) {
3839 regno = jit_get_reg(jit_class_gpr);
3840 movi(rn(regno), _jitc->function->self.aoff);
3841 stxi_i(_jitc->function->aoffoff, _FP_REGNO, rn(regno));
3842 jit_unget_reg(regno);
3843 }
3844
3845#if !_CALL_SYSV
3846 if (_jitc->function->self.call & jit_call_varargs) {
3847 for (regno = _jitc->function->vagp; jit_arg_reg_p(regno); ++regno)
3848 stxi(params_offset + regno * sizeof(jit_word_t),
3849 _FP_REGNO, rn(JIT_RA0 - regno));
3850 }
3851#else
3852 if (_jitc->function->self.call & jit_call_varargs) {
3853 for (regno = _jitc->function->vagp; jit_arg_reg_p(regno); ++regno)
3854 stxi(_jitc->function->vaoff + first_gp_offset +
3855 regno * sizeof(jit_word_t), _FP_REGNO, rn(JIT_RA0 - regno));
3856 for (regno = _jitc->function->vafp; jit_arg_f_reg_p(regno); ++regno)
3857 stxi_d(_jitc->function->vaoff + first_fp_offset +
3858 regno * va_fp_increment, _FP_REGNO,
3859 rn(JIT_FA0 - regno));
3860 }
3861#endif
3862}
3863
3864static void
3865_epilog(jit_state_t *_jit, jit_node_t *node)
3866{
ba86ff93 3867 jit_int32_t regno;
4a71579b
PC		 3868 jit_word_t offset;
3869
3870 if (_jitc->function->assume_frame)
3871 return;
ba86ff93
PC		 3872 if (_jitc->function->need_stack) {
3873 if (_jitc->function->allocar)
3874 ldr(_SP_REGNO, _SP_REGNO);
3875 else
3876 addi(_SP_REGNO, _SP_REGNO, _jitc->function->stack);
3877 }
3878
3879 if (_jitc->function->need_return) {
4a71579b 3880#if _CALL_SYSV
ba86ff93 3881 ldxi(_R0_REGNO, _SP_REGNO, sizeof(jit_word_t));
4a71579b 3882#else
ba86ff93 3883 ldxi(_R0_REGNO, _SP_REGNO, sizeof(void*) * 2);
4a71579b 3884#endif
4a71579b 3885 }
ba86ff93
PC		 3886
3887 if (_jitc->function->need_stack) {
3888 offset = -gpr_save_area;
3889 for (regno = 0; regno < jit_size(iregs);
3890 regno++, offset += sizeof(void*)) {
3891 if (jit_regset_tstbit(&_jitc->function->regset, iregs[regno]))
3892 ldxi(rn(iregs[regno]), _SP_REGNO, offset);
3893 }
3894 for (offset = 0; offset < 8; offset++) {
3895 if (jit_regset_tstbit(&_jitc->function->regset, fregs[offset]))
3896 ldxi_d(rn(fregs[offset]), _SP_REGNO,
3897 -(gpr_save_area + 8 + offset * 8));
3898 }
4a71579b
PC		 3899 }
3900
ba86ff93
PC		 3901 if (_jitc->function->need_return)
3902 MTLR(_R0_REGNO);
3903 if (_jitc->function->need_frame)
3904 ldxi(_FP_REGNO, _SP_REGNO, -(sizeof(void*)));
4a71579b
PC		 3905
3906 BLR();
3907}
3908
3909static void
3910_vastart(jit_state_t *_jit, jit_int32_t r0)
3911{
3912#if !_CALL_SYSV
3913 assert(_jitc->function->self.call & jit_call_varargs);
3914 /* Initialize stack pointer to the first stack argument. */
3915 addi(r0, _FP_REGNO, _jitc->function->self.size);
3916#else
3917 jit_int32_t reg;
3918 assert(_jitc->function->self.call & jit_call_varargs);
3919
3920 /* Return jit_va_list_t in the register argument */
3921 addi(r0, _FP_REGNO, _jitc->function->vaoff);
3922 reg = jit_get_reg(jit_class_gpr);
3923
3924 /* Initialize the gp counter. */
3925 movi(rn(reg), _jitc->function->vagp);
3926 stxi_c(offsetof(jit_va_list_t, ngpr), r0, rn(reg));
3927
3928 /* Initialize the fp counter. */
3929 movi(rn(reg), _jitc->function->vafp);
3930 stxi_c(offsetof(jit_va_list_t, nfpr), r0, rn(reg));
3931
3932 /* Initialize overflow pointer to the first stack argument. */
3933 addi(rn(reg), _FP_REGNO, _jitc->function->self.size);
3934 stxi(offsetof(jit_va_list_t, over), r0, rn(reg));
3935
3936 /* Initialize register save area pointer. */
3937 addi(rn(reg), r0, first_gp_offset);
3938 stxi(offsetof(jit_va_list_t, save), r0, rn(reg));
3939
3940 jit_unget_reg(reg);
3941#endif
3942}
3943
3944static void
3945_vaarg(jit_state_t *_jit, jit_int32_t r0, jit_int32_t r1)
3946{
3947#if !_CALL_SYSV
3948 assert(_jitc->function->self.call & jit_call_varargs);
3949 /* Load argument. */
3950 ldr(r0, r1);
3951 /* Update va_list. */
3952 addi(r1, r1, sizeof(jit_word_t));
3953#else
3954 jit_int32_t rg0;
3955 jit_int32_t rg1;
3956 jit_word_t ge_code;
3957 jit_word_t lt_code;
3958
3959 assert(_jitc->function->self.call & jit_call_varargs);
3960
3961 rg0 = jit_get_reg(jit_class_gpr);
3962 rg1 = jit_get_reg(jit_class_gpr);
3963
3964 /* Load the gp offset in save area in the first temporary. */
3965 ldxi_uc(rn(rg0), r1, offsetof(jit_va_list_t, ngpr));
3966
3967 /* Jump over if there are no remaining arguments in the save area. */
3968 ge_code = bgei(_jit->pc.w, rn(rg0), 8);
3969
3970 /* Update the gp counter. */
3971 addi(rn(rg1), rn(rg0), 1);
3972 stxi_c(offsetof(jit_va_list_t, ngpr), r1, rn(rg1));
3973
3974 /* Load the save area pointer in the second temporary. */
3975 ldxi(rn(rg1), r1, offsetof(jit_va_list_t, save));
3976
3977 /* Load the vararg argument in the first argument. */
3978 lshi(rn(rg0), rn(rg0), va_gp_shift);
3979 ldxr(r0, rn(rg1), rn(rg0));
3980
3981 /* Will only need one temporary register below. */
3982 jit_unget_reg(rg1);
3983
3984 /* Jump over overflow code. */
3985 lt_code = _jit->pc.w;
3986 B(0);
3987
3988 /* Where to land if argument is in overflow area. */
3989 patch_at(ge_code, _jit->pc.w);
3990
3991 /* Load overflow pointer. */
3992 ldxi(rn(rg0), r1, offsetof(jit_va_list_t, over));
3993
3994 /* Load argument. */
3995 ldr(r0, rn(rg0));
3996
3997 /* Update overflow pointer. */
3998 addi(rn(rg0), rn(rg0), va_gp_increment);
3999 stxi(offsetof(jit_va_list_t, over), r1, rn(rg0));
4000
4001 /* Where to land if argument is in save area. */
4002 patch_at(lt_code, _jit->pc.w);
4003
4004 jit_unget_reg(rg0);
4005#endif
4006}
4007
4008static void
4009_vaarg_d(jit_state_t *_jit, jit_int32_t r0, jit_int32_t r1)
4010{
4011#if !_CALL_SYSV
4012 assert(_jitc->function->self.call & jit_call_varargs);
4013 /* Load argument. */
4014 ldr_d(r0, r1);
4015 /* Update va_list. */
4016 addi(r1, r1, sizeof(jit_float64_t));
4017#else
4018 jit_int32_t rg0;
4019 jit_int32_t rg1;
4020 jit_word_t ge_code;
4021 jit_word_t lt_code;
4022
4023 assert(_jitc->function->self.call & jit_call_varargs);
4024
4025 rg0 = jit_get_reg(jit_class_gpr);
4026 rg1 = jit_get_reg(jit_class_gpr);
4027
4028 /* Load the fp offset in save area in the first temporary. */
4029 ldxi_uc(rn(rg0), r1, offsetof(jit_va_list_t, nfpr));
4030
4031 /* Jump over if there are no remaining arguments in the save area. */
4032 ge_code = bgei(_jit->pc.w, rn(rg0), 8);
4033
4034 /* Update the fp counter. */
4035 addi(rn(rg1), rn(rg0), 1);
4036 stxi_c(offsetof(jit_va_list_t, nfpr), r1, rn(rg1));
4037
4038 /* Load the save area pointer in the second temporary. */
4039 ldxi(rn(rg1), r1, offsetof(jit_va_list_t, save));
4040
4041 /* Load the vararg argument in the first argument. */
4042 lshi(rn(rg0), rn(rg0), 3);
4043 addi(rn(rg0), rn(rg0), offsetof(jit_va_list_t, first_fp_argument) -
4044 offsetof(jit_va_list_t, first_gp_argument));
4045 ldxr_d(r0, rn(rg1), rn(rg0));
4046
4047 /* Jump over overflow code. */
4048 lt_code = _jit->pc.w;
4049 B(0);
4050
4051 /* Where to land if argument is in overflow area. */
4052 patch_at(ge_code, _jit->pc.w);
4053
4054 /* Load overflow pointer. */
4055 ldxi(rn(rg0), r1, offsetof(jit_va_list_t, over));
4056
4057# if __WORDSIZE == 32
4058 /* Align if required. */
4059 andi(rn(rg1), rn(rg0), 7);
4060 addr(rn(rg0), rn(rg0), rn(rg1));
4061# endif
4062
4063 /* Load argument. */
4064 ldr_d(r0, rn(rg0));
4065
4066 /* Update overflow pointer. */
4067 addi(rn(rg0), rn(rg0), va_fp_increment);
4068 stxi(offsetof(jit_va_list_t, over), r1, rn(rg0));
4069
4070 /* Where to land if argument is in save area. */
4071 patch_at(lt_code, _jit->pc.w);
4072
4073 jit_unget_reg(rg0);
4074 jit_unget_reg(rg1);
4075#endif
4076}
4077
4078static void
4079_patch_at(jit_state_t *_jit, jit_word_t instr, jit_word_t label)
4080{
4081 jit_word_t d;
4082 union {
4083 jit_int32_t *i;
4084 jit_word_t w;
4085 } u;
4086 u.w = instr;
4087 switch ((u.i[0] & 0xfc000000) >> 26) {
4088 case 16: /* BCx */
4089 d = label - instr;
4090 assert(!(d & 3));
4091 if (!can_sign_extend_short_p(d)) {
4092 /* use absolute address */
4093 assert(can_sign_extend_short_p(label));
79bfeef6 4094 d = label | 2;
4a71579b
PC		 4095 }
4096 u.i[0] = (u.i[0] & ~0xfffd) | (d & 0xfffe);
4097 break;
4098 case 18: /* Bx */
4099#if _CALL_AIXDESC
4100 if (_jitc->jump && (!(u.i[0] & 1))) { /* jmpi label */
4101 /* zero is used for toc and env, so, quick check
4102 * if this is a "jmpi main" like initial jit
4103 * instruction */
4104 if (((long *)label)[1] == 0 && ((long *)label)[2] == 0) {
4105 for (d = 0; d < _jitc->prolog.offset; d++) {
4106 /* not so pretty, but hides powerpc
4107 * specific abi intrinsics and/or
4108 * implementation from user */
4109 if (_jitc->prolog.ptr[d] == label) {
4110 label += sizeof(void*) * 3;
4111 break;
4112 }
4113 }
4114 }
4115 }
4116#endif
4117 d = label - instr;
4118 assert(!(d & 3));
4119 if (!can_sign_extend_jump_p(d)) {
4120 /* use absolute address */
4121 assert(can_sign_extend_jump_p(label));
79bfeef6 4122 d = label | 2;
4a71579b 4123 }
79bfeef6 4124 u.i[0] = (u.i[0] & ~0x3fffffc) | (d & 0x3fffffd);
4a71579b
PC		 4125 break;
4126 case 15: /* LI */
4127#if __WORDSIZE == 32
4128# define MTCTR_OFF 2
4129# define BCTR_OFF 3
4130#else
4131# define MTCTR_OFF 6
4132# define BCTR_OFF 7
4133#endif
4134#if _CALL_AIXDESC
4135 /* movi reg label; jmpr reg */
4136 if (_jitc->jump &&
4137#if 0
4138 /* check for MLTR(reg) */
4139 (u.i[MTCTR_OFF] >> 26) == 31 &&
4140 ((u.i[MTCTR_OFF] >> 16) & 0x3ff) == 8 &&
4141 ((u.i[MTCTR_OFF] >> 1) & 0x3ff) == 467 &&
4142 /* check for BLR */
4143 u.i[BCTR_OFF] == 0x4e800020) {
4144#else
4145 /* check for MTCTR(reg) */
4146 (u.i[MTCTR_OFF] >> 26) == 31 &&
4147 ((u.i[MTCTR_OFF] >> 16) & 0x3ff) == 9 &&
4148 ((u.i[MTCTR_OFF] >> 1) & 0x3ff) == 467 &&
4149 /* check for BCTR */
4150 u.i[BCTR_OFF] == 0x4e800420) {
4151#endif
4152 /* zero is used for toc and env, so, quick check
4153 * if this is a "jmpi main" like initial jit
4154 * instruction */
4155 if (((long *)label)[1] == 0 && ((long *)label)[2] == 0) {
4156 for (d = 0; d < _jitc->prolog.offset; d++) {
4157 /* not so pretty, but hides powerpc
4158 * specific abi intrinsics and/or
4159 * implementation from user */
4160 if (_jitc->prolog.ptr[d] == label) {
4161 label += sizeof(void*) * 3;
4162 break;
4163 }
4164 }
4165 }
4166 }
4167#endif
4168#undef BCTR_OFF
4169#undef MTCTR_OFF
4170#if __WORDSIZE == 32
4171 assert(!(u.i[0] & 0x1f0000));
4172 u.i[0] = (u.i[0] & ~0xffff) | ((label >> 16) & 0xffff);
4173 assert((u.i[1] & 0xfc000000) >> 26 == 24); /* ORI */
4174 assert(((u.i[1] >> 16) & 0x1f) == ((u.i[1] >> 21) & 0x1f));
4175 u.i[1] = (u.i[1] & ~0xffff) | (label & 0xffff);
4176#else
4177 assert(!(u.i[0] & 0x1f0000));
4178 u.i[0] = (u.i[0] & ~0xffff) | ((label >> 48) & 0xffff);
4179 assert((u.i[1] & 0xfc000000) >> 26 == 24); /* ORI */
4180 assert(((u.i[1] >> 16) & 0x1f) == ((u.i[1] >> 21) & 0x1f));
4181 u.i[1] = (u.i[1] & ~0xffff) | ((label >> 32) & 0xffff);
4182 /* not fully validating SLDI */
4183 assert((u.i[2] & 0xfc000000) >> 26 == 30); /* SLDI */
4184 assert(((u.i[2] >> 16) & 0x1f) == ((u.i[2] >> 21) & 0x1f));
4185 assert((u.i[3] & 0xfc000000) >> 26 == 24); /* ORI */
4186 assert(((u.i[3] >> 16) & 0x1f) == ((u.i[3] >> 21) & 0x1f));
4187 u.i[3] = (u.i[3] & ~0xffff) | ((label >> 16) & 0xffff);
4188 /* not fully validating SLDI */
4189 assert((u.i[4] & 0xfc000000) >> 26 == 30); /* SLDI */
4190 assert(((u.i[4] >> 16) & 0x1f) == ((u.i[4] >> 21) & 0x1f));
4191 assert((u.i[5] & 0xfc000000) >> 26 == 24); /* ORI */
4192 assert(((u.i[5] >> 16) & 0x1f) == ((u.i[5] >> 21) & 0x1f));
4193 u.i[5] = (u.i[5] & ~0xffff) | (label & 0xffff);
4194#endif
4195 break;
4196 default:
4197 assert(!"unhandled branch opcode");
4198 }
4199}
4200#endif
ARM optimized PCSX fork