notaz.gp2x.de / pcsx_rearmed.git / blame
? search:
summary | shortlog | log | commit | commitdiff | tree
blob | blame (incremental) | history | HEAD
git subrepo pull (merge) --force deps/lightning
[pcsx_rearmed.git] / deps / lightning / lib / jit_sparc-cpu.c
CommitLineData
4a71579b 1/*
79bfeef6 2 * Copyright (C) 2013-2023 Free Software Foundation, Inc.
4a71579b
PC		 3 *
4 * This file is part of GNU lightning.
5 *
6 * GNU lightning is free software; you can redistribute it and/or modify it
7 * under the terms of the GNU Lesser General Public License as published
8 * by the Free Software Foundation; either version 3, or (at your option)
9 * any later version.
10 *
11 * GNU lightning is distributed in the hope that it will be useful, but
12 * WITHOUT ANY WARRANTY; without even the implied warranty of MERCHANTABILITY
13 * or FITNESS FOR A PARTICULAR PURPOSE. See the GNU Lesser General Public
14 * License for more details.
15 *
16 * Authors:
17 * Paulo Cesar Pereira de Andrade
18 */
19
20#if PROTO
21# define _G2_REGNO 0x02
22# define _G3_REGNO 0x03
23# define _G4_REGNO 0x04
24# define _O0_REGNO 0x08
25# define _O1_REGNO 0x09
26# define _SP_REGNO 0x0e
27# define _FP_REGNO 0x1e
28# define _O7_REGNO 0x0f
29# define _L0_REGNO 0x10
30# define _L1_REGNO 0x11
31# define _L2_REGNO 0x12
32# define _L3_REGNO 0x13
33# define _L4_REGNO 0x14
34# define _L5_REGNO 0x15
35# define _L6_REGNO 0x16
36# define _L7_REGNO 0x17
37# define _I7_REGNO 0x1f
38/*
39 * - previous stack frame
40 * fp ----
41 * fp- local variables (in lightning, 8 bytes space for float conversion)
42 * fp- alloca
43 * sp+ stack arguments
44 * sp+ 6 words to save register arguments
45 * sp+ 1 word for hidden address of aggregate return value (32 bits only)
46 * sp+ 16 words for in and local registers
47 * sp ----
48 * decreasing memory address - next stack frame (not yet allocated)
49 */
50# define stack_framesize ((16 + (__WORDSIZE == 32) + 6) * sizeof(jit_word_t))
51typedef union {
52 struct { jit_uint32_t b: 2; } op;
53 struct { jit_uint32_t _: 2; jit_uint32_t b: 1; } a;
54 struct { jit_uint32_t _: 2; jit_uint32_t b: 5; } rd;
55 struct { jit_uint32_t _: 2; jit_uint32_t b: 30; } disp30;
56 struct { jit_uint32_t _: 3; jit_uint32_t b: 4; } cond;
57 struct { jit_uint32_t _: 7; jit_uint32_t b: 3; } op2;
58 struct { jit_uint32_t _: 7; jit_uint32_t b: 6; } op3;
59 struct { jit_uint32_t _: 10; jit_uint32_t b: 1; } cc1;
60 struct { jit_uint32_t _: 10; jit_uint32_t b: 22; } imm22;
61 struct { jit_uint32_t _: 10; jit_uint32_t b: 22; } disp22;
62 struct { jit_uint32_t _: 11; jit_uint32_t b: 1; } cc0;
63 struct { jit_uint32_t _: 12; jit_uint32_t b: 1; } p;
64 struct { jit_uint32_t _: 13; jit_uint32_t b: 19; } disp19;
65 struct { jit_uint32_t _: 13; jit_uint32_t b: 5; } rs1;
66 struct { jit_uint32_t _: 18; jit_uint32_t b: 1; } i;
67 struct { jit_uint32_t _: 18; jit_uint32_t b: 9; } opf;
ba86ff93 68 struct { jit_uint32_t _: 18; jit_uint32_t b: 5; } rs3;
4a71579b
PC		 69 struct { jit_uint32_t _: 19; jit_uint32_t b: 1; } x;
70 struct { jit_uint32_t _: 19; jit_uint32_t b: 8; } asi;
71 struct { jit_uint32_t _: 19; jit_uint32_t b: 6; } res;
72 struct { jit_uint32_t _: 19; jit_uint32_t b: 13; } simm13;
73 struct { jit_uint32_t _: 20; jit_uint32_t b: 7; } asix;
74 struct { jit_uint32_t _: 20; jit_uint32_t b: 6; } asis;
ba86ff93 75 struct { jit_uint32_t _: 23; jit_uint32_t b: 4; } op5;
4a71579b 76 struct { jit_uint32_t _: 25; jit_uint32_t b: 7; } imm7;
ba86ff93 77 struct { jit_uint32_t _: 26; jit_uint32_t b: 6; } shim;
4a71579b
PC		 78 struct { jit_uint32_t _: 27; jit_uint32_t b: 5; } rs2;
79 jit_int32_t v;
80} jit_instr_t;
81# define ii(i) *_jit->pc.ui++ = i
82# define s7_p(imm) ((imm) <= 63 && (imm) >= -64)
83# define s13_p(imm) ((imm) <= 4095 && (imm) >= -4096)
84# define s19_p(imm) ((imm) <= 262143 && (imm) >= -262144)
85# define s22_p(imm) ((imm) <= 2097151 && (imm) >= -20971512)
86# define s30_p(imm) ((imm) <= 536870911 && (imm) >= -536870912)
87# define f1(op, disp30) _f1(_jit, op, disp30)
88static void _f1(jit_state_t*,jit_int32_t,jit_int32_t);
89# define f2r(op, rd, op2, imm22) _f2r(_jit, op, rd, op2, imm22)
90static void _f2r(jit_state_t*,jit_int32_t,jit_int32_t,jit_int32_t,jit_int32_t);
91# define f2b(op, a, cond, op2, disp22) _f2b(_jit, op, a, cond, op2, disp22)
92static void
93_f2b(jit_state_t*,jit_int32_t,jit_int32_t,jit_int32_t,jit_int32_t,jit_int32_t);
94# if __WORDSIZE == 64
95# define f2bp(op,a,cond,op2,cc1,cc0,p,disp19) \
96 _f2bp(_jit,op,a,cond,op2,cc1,cc0,p,disp19)
97static void
98_f2bp(jit_state_t*,jit_int32_t,jit_int32_t,jit_int32_t,jit_int32_t,jit_int32_t,
99 jit_int32_t,jit_int32_t,jit_int32_t);
100# endif
101# define f3r(op, rd, op3, rs1, rs2) _f3r(_jit, op, rd, op3, rs1, rs2)
102static void _f3r(jit_state_t*,
103 jit_int32_t,jit_int32_t,jit_int32_t,jit_int32_t,jit_int32_t);
ba86ff93 104# if __WORDSIZE == 64 || CHECK_LZCNT
c0c16242
PC		 105# define f3ri(op, rd, op3, rs1, rs2) _f3ri(_jit, op, rd, op3, rs1, rs2)
106static void _f3ri(jit_state_t*,
107 jit_int32_t,jit_int32_t,jit_int32_t,jit_int32_t,jit_int32_t);
4a71579b
PC		 108# define f3rx(op, rd, op3, rs1, rs2) _f3rx(_jit, op, rd, op3, rs1, rs2)
109static void _f3rx(jit_state_t*,
110 jit_int32_t,jit_int32_t,jit_int32_t,jit_int32_t,jit_int32_t);
111# endif
112# define f3i(op, rd, op3, rs1, simm13) _f3i(_jit, op, rd, op3, rs1, simm13)
113static void _f3i(jit_state_t*,
114 jit_int32_t,jit_int32_t,jit_int32_t,jit_int32_t,jit_int32_t);
115# define f3s(op, rd, op3, rs1, simm13) _f3s(_jit, op, rd, op3, rs1, simm13)
116static void _f3s(jit_state_t*,
117 jit_int32_t,jit_int32_t,jit_int32_t,jit_int32_t,jit_int32_t);
118# define f3t(cond, rs1, i, ri) _f3t(_jit, cond, rs1, i, ri)
119static void _f3t(jit_state_t*,jit_int32_t,jit_int32_t,jit_int32_t,jit_int32_t)
120 maybe_unused;
c0c16242 121# define f3a(op,rd,op3,rs1,asi,rs2) _f3a(_jit, op, rd, op3, rs1, asi, rs2)
4a71579b
PC		 122static void _f3a(jit_state_t*,jit_int32_t,
123 jit_int32_t,jit_int32_t,jit_int32_t,jit_int32_t,jit_int32_t)
124 maybe_unused;
79bfeef6
PC		 125# define f2c1(op,rd,op3,rs1,opf,rs2) _f2c1(_jit,op,rd,op3,rs1,opf,rs2)
126static void
127_f2c1(jit_state_t*,jit_int32_t, jit_int32_t,
128 jit_int32_t,jit_int32_t,jit_int32_t,jit_int32_t)
129 maybe_unused;
4a71579b
PC		 130# define LDSB(rs1, rs2, rd) f3r(3, rd, 9, rs1, rs2)
131# define LDSBI(rs1, imm, rd) f3i(3, rd, 9, rs1, imm)
132# define LDSH(rs1, rs2, rd) f3r(3, rd, 10, rs1, rs2)
133# define LDSHI(rs1, imm, rd) f3i(3, rd, 10, rs1, imm)
134# define LDUB(rs1, rs2, rd) f3r(3, rd, 1, rs1, rs2)
135# define LDUBI(rs1, imm, rd) f3i(3, rd, 1, rs1, imm)
136# define LDUH(rs1, rs2, rd) f3r(3, rd, 2, rs1, rs2)
137# define LDUHI(rs1, imm, rd) f3i(3, rd, 2, rs1, imm)
138# if __WORDSIZE == 32
139# define LD(rs1, rs2, rd) f3r(3, rd, 0, rs1, rs2)
140# define LDI(rs1, imm, rd) f3i(3, rd, 0, rs1, imm)
141# define LDD(rs1, rs2, rd) f3r(3, rd, 3, rs1, rs2)
142# define LDDI(rs1, imm, rd) f3i(3, rd, 3, rs1, imm)
143# else
144# define LDSW(rs1, rs2, rd) f3r(3, rd, 8, rs1, rs2)
145# define LDSWI(rs1, imm, rd) f3i(3, rd, 8, rs1, imm)
146# define LDUW(rs1, rs2, rd) f3r(3, rd, 0, rs1, rs2)
147# define LDUWI(rs1, imm, rd) f3i(3, rd, 0, rs1, imm)
148# define LDX(rs1, rs2, rd) f3r(3, rd, 11, rs1, rs2)
149# define LDXI(rs1, imm, rd) f3i(3, rd, 11, rs1, imm)
150# endif
151# define LDSBA(rs1, rs2, asi, rd) f3a(3, rd, 25, rs1, asi, rs2)
152# define LDSHA(rs1, rs2, asi, rd) f3a(3, rd, 26, rs1, asi, rs2)
153# define LDUBA(rs1, rs2, asi, rd) f3a(3, rd, 17, rs1, asi, rs2)
154# define LDUHA(rs1, rs2, asi, rd) f3a(3, rd, 18, rs1, asi, rs2)
155# if __WORDSIZE == 32
156# define LDA(rs1, rs2, asi, rd) f3a(3, rd, 16, rs1, asi, rs2)
157# define LDDA(rs1, rs2, asi, rd) f3a(3, rd, 19, rs1, asi, rs2)
158# else
159# define LDSWA(rs1, rs2, asi, rd) f3a(3, rd, 24, rs1, asi, rs2)
160# define LDUWA(rs1, rs2, asi, rd) f3a(3, rd, 16, rs1, asi, rs2)
161# define LDXA(rs1, rs2, asi, rd) f3a(3, rd, 27, rs1, asi, rs2)
162# endif
163# define LDC(rs1, rs2, rd) f3r(3, rd, 48, rs1, rs2)
164# define LDCI(rs1, imm, rd) f3i(3, rd, 48, rs1, imm)
165# define LDDC(rs1, rs2, rd) f3r(3, rd, 51, rs1, rs2)
166# define LDDCI(rs1, imm, rd) f3i(3, rd, 51, rs1, imm)
167# define LDCSR(rs1, rs2, rd) f3r(3, rd, 49, rs1, rs2)
168# define LDCSRI(rs1, imm, rd) f3i(3, rd, 49, rs1, imm)
169# define STB(rd, rs1, rs2) f3r(3, rd, 5, rs1, rs2)
170# define STBI(rd, rs1, imm) f3i(3, rd, 5, rs1, imm)
171# define STH(rd, rs1, rs2) f3r(3, rd, 6, rs1, rs2)
172# define STHI(rd, rs1, imm) f3i(3, rd, 6, rs1, imm)
173# if __WORDSIZE == 32
174# define ST(rd, rs1, rs2) f3r(3, rd, 4, rs1, rs2)
175# define STI(rd, rs1, imm) f3i(3, rd, 4, rs1, imm)
176# define STD(rrd, s1, rs2) f3r(3, rd, 7, rs1, rs2)
177# define STDI(rd, rs1, imm) f3i(3, rd, 7, rs1, imm)
178# else
179# define STW(rd, rs1, rs2) f3r(3, rd, 4, rs1, rs2)
180# define STWI(rd, rs1, imm) f3i(3, rd, 4, rs1, imm)
181# define STX(rd, rs1, rs2) f3r(3, rd, 14, rs1, rs2)
182# define STXI(rd, rs1, imm) f3i(3, rd, 14, rs1, imm)
183# endif
184# define STBA(rd, rs1, rs2) f3a(3, rd, 21, rs1, asi, rs2)
185# define STHA(rd, rs1, rs2) f3a(3, rd, 22, rs1, asi, rs2)
186# if __WORDSIZE == 32
187# define STA(rd, rs1, rs2) f3a(3, rd, 20, rs1, asi, rs2)
188# define STDA(rd, rs1, rs2) f3a(3, rd, 23, rs1, asi, rs2)
189# else
190# define STWA(rd, rs1, rs2) f3a(3, rd, 20, rs1, asi, rs2)
191# define STXA(rd, rs1, rs2) f3a(3, rd, 30, rs1, asi, rs2)
192# endif
193# define STC(rd, rs1, rs2) f3r(3, rd, 52, rs1, rs2)
194# define STCI(rd, rs1, imm) f3i(3, rd, 52, rs1, imm)
195# define STDC(rd, rs1, rs2) f3r(3, rd, 55, rs1, rs2)
196# define STDCI(rd, rs1, imm) f3i(3, rd, 55, rs1, imm)
197# define STCSR(rd, rs1, rs2) f3r(3, rd, 53, rs1, rs2)
198# define STCSRI(rd, rs1, imm) f3i(3, rd, 53, rs1, imm)
199# define STDCQ(rd, rs1, rs2) f3r(3, rd, 54, rs1, rs2)
200# define STDCQI(rd, rs1, imm) f3i(3, rd, 54, rs1, imm)
201# define LDSTUB(rs1, rs2, rd) f3r(3, rd, 13, rs1, rs2)
202# define LDSTUBI(rs1, imm, rd) f3r(3, rd, 13, rs1, imm)
203# define LDSTUBA(rs1, rs2, asi, rd) f3a(3, rd, 21, rs1, asi, rs2)
204# define SWAP(rs1, rs2, rd) f3r(3, rd, 15, rs1, rs2)
205# define SWAPI(rs1, imm, rd) f3r(3, rd, 15, rs1, imm)
206# define SWAPA(rs1, rs2, asi, rd) f3a(3, rd, 23, rs1, asi, rs2)
c0c16242
PC		 207/* Sparc v9 deprecates SWAP* in favor of CAS*A */
208# define CASA(rs1, rs2, rd) f3a(3, rd, 60, rs1, 128, rs2)
209# if __WORDSIZE == 64
210# define CASXA(rs1, rs2, rd) f3a(3, rd, 62, rs1, 128, rs2)
211# endif
4a71579b
PC		 212# define NOP() SETHI(0, 0)
213# define HI(im) ((im) >> 10)
214# define LO(im) ((im) & 0x3ff)
215# define SETHI(im, rd) f2r(0, rd, 4, im)
216# define AND(rs1, rs2, rd) f3r(2, rd, 1, rs1, rs2)
217# define ANDI(rs1, imm, rd) f3i(2, rd, 1, rs1, imm)
218# define ANDcc(rs1, rs2, rd) f3r(2, rd, 17, rs1, rs2)
219# define ANDIcc(rs1, imm, rd) f3i(2, rd, 17, rs1, imm)
220# define BTST(rs1, rs2) ANDcc(rs1, rs2, 0)
221# define BTSTI(rs1, imm) ANDIcc(rs1, imm, 0)
222# define ANDN(rs1, rs2, rd) f3r(2, rd, 5, rs1, rs2)
223# define ANDNI(rs1, imm, rd) f3i(2, rd, 5, rs1, imm)
224# define ANDNcc(rs1, rs2, rd) f3r(2, rd, 21, rs1, rs2)
225# define ANDNIcc(rs1, imm, rd) f3i(2, rd, 21, rs1, imm)
226# define OR(rs1, rs2, rd) f3r(2, rd, 2, rs1, rs2)
227# define ORI(rs1, imm, rd) f3i(2, rd, 2, rs1, imm)
228# define ORcc(rs1, rs2, rd) f3r(2, rd, 18, rs1, rs2)
229# define ORIcc(rs1, imm, rd) f3i(2, rd, 18, rs1, imm)
230# define ORN(rs1, rs2, rd) f3r(2, rd, 6, rs1, rs2)
231# define ORNI(rs1, imm, rd) f3i(2, rd, 6, rs1, imm)
232# define ORNcc(rs1, rs2, rd) f3r(2, rd, 22, rs1, rs2)
233# define ORNIcc(rs1, imm, rd) f3i(2, rd, 22, rs1, imm)
234# define XOR(rs1, rs2, rd) f3r(2, rd, 3, rs1, rs2)
235# define XORI(rs1, imm, rd) f3i(2, rd, 3, rs1, imm)
236# define XORcc(rs1, rs2, rd) f3r(2, rd, 19, rs1, rs2)
237# define XORIcc(rs1, imm, rd) f3i(2, rd, 19, rs1, imm)
238# define XNOR(rs1, rs2, rd) f3r(2, rd, 7, rs1, rs2)
239# define XNORI(rs1, imm, rd) f3i(2, rd, 7, rs1, imm)
240# define XNORcc(rs1, rs2, rd) f3r(2, rd, 23, rs1, rs2)
241# define XNORIcc(rs1, imm, rd) f3i(2, rd, 23, rs1, imm)
242# define SLL(rs1, rs2, rd) f3r(2, rd, 37, rs1, rs2)
243# define SLLI(rs1, imm, rd) f3i(2, rd, 37, rs1, imm)
244# define SRL(rs1, rs2, rd) f3r(2, rd, 38, rs1, rs2)
245# define SRLI(rs1, imm, rd) f3i(2, rd, 38, rs1, imm)
246# define SRA(rs1, rs2, rd) f3r(2, rd, 39, rs1, rs2)
247# define SRAI(rs1, imm, rd) f3i(2, rd, 39, rs1, imm)
ba86ff93 248# if __WORDSIZE == 64 || CHECK_LZCNT
4a71579b
PC		 249# define SLLX(rs1, rs2, rd) f3rx(2, rd, 37, rs1, rs2)
250# define SLLXI(rs1, imm, rd) f3s(2, rd, 37, rs1, imm)
251# define SRLX(rs1, rs2, rd) f3rx(2, rd, 38, rs1, rs2)
252# define SRLXI(rs1, imm, rd) f3s(2, rd, 38, rs1, imm)
253# define SRAX(rs1, rs2, rd) f3rx(2, rd, 39, rs1, rs2)
254# define SRAXI(rs1, imm, rd) f3s(2, rd, 39, rs1, imm)
255# endif
256# define ADD(rs1, rs2, rd) f3r(2, rd, 0, rs1, rs2)
257# define ADDI(rs1, imm, rd) f3i(2, rd, 0, rs1, imm)
258# define ADDcc(rs1, rs2, rd) f3r(2, rd, 16, rs1, rs2)
259# define ADDIcc(rs1, imm, rd) f3i(2, rd, 16, rs1, imm)
260# define ADDX(rs1, rs2, rd) f3r(2, rd, 8, rs1, rs2)
261# define ADDXI(rs1, imm, rd) f3i(2, rd, 8, rs1, imm)
262# define ADDXcc(rs1, rs2, rd) f3r(2, rd, 24, rs1, rs2)
263# define ADDXIcc(rs1, imm, rd) f3i(2, rd, 24, rs1, imm)
264# define TADDcc(rs1, rs2, rd) f3r(2, rd, 32, rs1, rs2)
265# define TADDIcc(rs1, imm, rd) f3i(2, rd, 32, rs1, imm)
266# define TADDccTV(rs1, rs2, rd) f3r(2, rd, 34, rs1, rs2)
267# define TADDIccTV(rs1, imm, rd) f3i(2, rd, 34, rs1, imm)
268# define SUB(rs1, rs2, rd) f3r(2, rd, 4, rs1, rs2)
269# define NEG(rs1, rd) SUB(0, rs1, rd)
270# define SUBI(rs1, imm, rd) f3i(2, rd, 4, rs1, imm)
271# define SUBcc(rs1, rs2, rd) f3r(2, rd, 20, rs1, rs2)
272# define SUBIcc(rs1, imm, rd) f3i(2, rd, 20, rs1, imm)
273# define CMP(rs1, rs2) SUBcc(rs1, rs2, 0)
274# define CMPI(rs1, imm) SUBIcc(rs1, imm, 0)
275# define SUBX(rs1, rs2, rd) f3r(2, rd, 12, rs1, rs2)
276# define SUBXI(rs1, imm, rd) f3i(2, rd, 12, rs1, imm)
277# define SUBXcc(rs1, rs2, rd) f3r(2, rd, 28, rs1, rs2)
278# define SUBXIcc(rs1, imm, rd) f3i(2, rd, 28, rs1, imm)
279# define TSUBcc(rs1, rs2, rd) f3r(2, rd, 33, rs1, rs2)
280# define TDADDIcc(rs1, imm, rd) f3i(2, rd, 33, rs1, imm)
281# define TSUBccTV(rs1, rs2, rd) f3r(2, rd, 35, rs1, rs2)
282# define TSUBIccTV(rs1, imm, rd) f3i(2, rd, 35, rs1, imm)
283# define MULScc(rs1, rs2, rd) f3r(2, rd, 36, rs1, rs2)
284# define MULSIcc(rs1, imm, rd) f3i(2, rd, 36, rs1, imm)
285# define UMUL(rs1, rs2, rd) f3r(2, rd, 10, rs1, rs2)
286# define UMULI(rs1, imm, rd) f3i(2, rd, 10, rs1, imm)
287# define SMUL(rs1, rs2, rd) f3r(2, rd, 11, rs1, rs2)
288# define SMULI(rs1, imm, rd) f3i(2, rd, 11, rs1, imm)
289# define UMULcc(rs1, rs2, rd) f3r(2, rd, 26, rs1, rs2)
290# define UMULIcc(rs1, imm, rd) f3i(2, rd, 26, rs1, imm)
291# define SMULcc(rs1, rs2, rd) f3r(2, rd, 27, rs1, rs2)
292# define SMULIcc(rs1, imm, rd) f3i(2, rd, 27, rs1, imm)
293# define UDIV(rs1, rs2, rd) f3r(2, rd, 14, rs1, rs2)
294# define UDIVI(rs1, imm, rd) f3i(2, rd, 14, rs1, imm)
295# define SDIV(rs1, rs2, rd) f3r(2, rd, 15, rs1, rs2)
296# define SDIVI(rs1, imm, rd) f3i(2, rd, 15, rs1, imm)
297# define UDIVcc(rs1, rs2, rd) f3r(2, rd, 30, rs1, rs2)
298# define UDIVIcc(rs1, imm, rd) f3i(2, rd, 30, rs1, imm)
299# define SDIVcc(rs1, rs2, rd) f3r(2, rd, 31, rs1, rs2)
300# define SDIVIcc(rs1, imm, rd) f3i(2, rd, 31, rs1, imm)
301# if __WORDSIZE == 64
302# define MULX(rs1, rs2, rd) f3r(2, rd, 9, rs1, rs2)
303# define MULXI(rs1, imm, rd) f3i(2, rd, 9, rs1, imm)
304# define SDIVX(rs1, rs2, rd) f3r(2, rd, 45, rs1, rs2)
305# define SDIVXI(rs1, imm, rd) f3i(2, rd, 45, rs1, imm)
306# define UDIVX(rs1, rs2, rd) f3r(2, rd, 13, rs1, rs2)
307# define UDIVXI(rs1, imm, rd) f3i(2, rd, 13, rs1, imm)
308# endif
309# define SAVE(rs1, rs2, rd) f3r(2, rd, 60, rs1, rs2)
310# define SAVEI(rs1, imm, rd) f3i(2, rd, 60, rs1, imm)
311# define RESTORE(rs1, rs2, rd) f3r(2, rd, 61, rs1, rs2)
312# define RESTOREI(rs1, imm, rd) f3i(2, rd, 61, rs1, imm)
313# define SPARC_BA 8 /* always */
314# define SPARC_BN 0 /* never */
315# define SPARC_BNE 9 /* not equal - not Z */
316# define SPARC_BNZ SPARC_BNE
317# define SPARC_BE 1 /* equal - Z */
318# define SPARC_BZ SPARC_BE
319# define SPARC_BG 10 /* greater - not (Z or (N xor V)) */
320# define SPARC_BLE 2 /* less or equal - Z or (N xor V) */
321# define SPARC_BGE 11 /* greater or equal - not (N xor V) */
322# define SPARC_BL 3 /* less - N xor V */
323# define SPARC_BGU 12 /* greater unsigned - not (C or Z) */
324# define SPARC_BLEU 4 /* less or equal unsigned - C or Z */
325# define SPARC_BCC 13 /* carry clear - not C */
326# define SPARC_BGEU SPARC_BCC
327# define SPARC_BCS 5 /* carry set - C */
328# define SPARC_BLU SPARC_BCS
329# define SPARC_BPOS 14 /* positive - not N */
330# define SPARC_BNEG 6 /* negative - N */
331# define SPARC_BVC 15 /* overflow clear - not V */
332# define SPARC_BVS 7 /* overflow set - V */
333/* Preferred BPcc integer branch opcodes */
334# if __WORDSIZE == 64
335# define SPARC_BPA 8 /* always - 1 */
336# define SPARC_BPN 0 /* never - 0 */
337# define SPARC_BPNE 9 /* not equal - not Z */
338# define SPARC_BPE 1 /* equal - Z */
339# define SPARC_BPG 10 /* greater - not (Z or (N xor V)) */
340# define SPARC_BPLE 2 /* less or equal - Z or (N xor V) */
341# define SPARC_BPGE 11 /* greater or equal - not (N xor V) */
342# define SPARC_BPL 3 /* less - N xor V */
343# define SPARC_BPGU 12 /* greater unsigned - not (C or V) */
344# define SPARC_BPLEU 4 /* less or equal unsigned - C or Z */
345# define SPARC_BPCC 13 /* carry clear (greater than or equal, unsigned) - not C */
346# define SPARC_BPCS 5 /* carry set (less than, unsigned) - C */
347# define SPARC_BPPOS 14 /* positive - not N */
348# define SPARC_BPNEG 6 /* negative - N */
349# define SPARC_BPVC 15 /* overflow clear - not V */
350# define SPARC_BPVS 7 /* overflow set - V */
351# endif
352# define B(cc, imm) f2b(0, 0, cc, 2, imm)
353# define Ba(cc, imm) f2b(0, 1, cc, 2, imm)
354# define BA(imm) B(SPARC_BA, imm)
355# define BAa(imm) Ba(SPARC_BA, imm)
356# define BN(imm) B(SPARC_BN, imm)
357# define BNa(imm) Ba(SPARC_BN, imm)
358# define BNE(imm) B(SPARC_BNE, imm)
359# define BNEa(imm) Ba(SPARC_BNE, imm)
360# define BNZ(imm) BNE(imm)
361# define BNZa(imm) BNEa(imm)
362# define BE(imm) B(SPARC_BE, imm)
363# define BEa(imm) Ba(SPARC_BE, imm)
364# define BZ(imm) BE(imm)
365# define BZa(imm) BEa(imm)
366# define BG(imm) B(SPARC_BG, imm)
367# define BGa(imm) Ba(SPARC_BG, imm)
368# define BLE(imm) B(SPARC_BLE, imm)
369# define BLEa(imm) Ba(SPARC_BLE, imm)
370# define BGE(imm) B(SPARC_BGE, imm)
371# define BGEa(imm) Ba(SPARC_BGE, imm)
372# define BL(imm) B(SPARC_BL, imm)
373# define BLa(imm) Ba(SPARC_BL, imm)
374# define BGU(imm) B(SPARC_BGU, imm)
375# define BGUa(imm) Ba(SPARC_BGU, imm)
376# define BLEU(imm) B(SPARC_BLEU, imm)
377# define BLEUa(imm) Ba(SPARC_BLEU, imm)
378# define BCC(imm) B(SPARC_BCC, imm)
379# define BCCa(imm) Ba(SPARC_BCC, imm)
380# define BGEU(imm) BCC(imm)
381# define BGEUa(imm) BCCa(imm)
382# define BCS(imm) B(SPARC_BCS, imm)
383# define BCSa(imm) Ba(SPARC_BCS, imm)
384# define BLU(imm) BCS(imm)
385# define BLUa(imm) BCSa(imm)
386# define BPOS(imm) B(SPARC_BPOS, imm)
387# define BPOSa(imm) Ba(SPARC_BPOS, imm)
388# define BNEG(imm) B(SPARC_BNEG, imm)
389# define BNEGa(imm) Ba(SPARC_BNEG, imm)
390# define BVC(imm) B(SPARC_BVC, imm)
391# define BVCa(imm) Ba(SPARC_BVC, imm)
392# define BVS(imm) B(SPARC_BVS, imm)
393# define BVSa(imm) Ba(SPARC_BVS, imm)
394# if __WORDSIZE == 64
395# define BPccap(cc,a,cc1, cc2,p,imm) f2bp(0, a, cc, 1, cc1, cc0, p, imm)
396# define BPap(cc, imm) f2bp(0, 1, cc, 1, 1, 0, p, imm)
397# define BPa(cc, imm) f2bp(0, 1, cc, 1, 1, 0, 1, imm)
398# define BP(cc, imm) f2bp(0, 0, cc, 1, 1, 0, 1, imm)
399# define BPA(imm) BP(SPARC_BPA, imm)
400# define BPN(imm) BP(SPARC_BPN, imm)
401# define BNPE(imm) BP(SPARC_BPNE, imm)
402# define BPE(imm) BP(SPARC_BPE, imm)
403# define BPG(imm) BP(SPARC_BPG, imm)
404# define BPLE(imm) BP(SPARC_BPLE, imm)
405# define BPGE(imm) BP(SPARC_BPGE, imm)
406# define BPL(imm) BP(SPARC_BPL, imm)
407# define BPGU(imm) BP(SPARC_BPGU, imm)
408# define BPLEU(imm) BP(SPARC_BPLEU, imm)
409# define BPCC(imm) BP(SPARC_BPCC, imm)
410# define BPCS(imm) BP(SPARC_BPCS, imm)
411# define BPPOS(imm) BP(SPARC_BPPOS, imm)
412# define BPNEG(imm) BP(SPARC_BPNEG, imm)
413# define BPVC(imm) BP(SPARC_BPVC, imm)
414# define BPVS(imm) BP(SPARC_BPVS, imm)
415# endif
416# define SPARC_CBA 8 /* always */
417# define SPARC_CBN 0 /* never */
418# define SPARC_CB3 7 /* 3 */
419# define SPARC_CB2 6 /* 2 */
420# define SPARC_CB23 5 /* 2 or 3 */
421# define SPARC_CB1 4 /* 1 */
422# define SPARC_CB13 3 /* 1 or 3 */
423# define SPARC_CB12 2 /* 1 or 2 */
424# define SPARC_CB123 1 /* 1 or 2 or 3 */
425# define SPARC_CB0 9 /* 0 */
426# define SPARC_CB03 10 /* 0 or 3 */
427# define SPARC_CB02 11 /* 0 or 2 */
428# define SPARC_CB023 12 /* 0 or 2 or 3 */
429# define SPARC_CB01 13 /* 0 or 1 */
430# define SPARC_CB013 14 /* 0 or 1 or 3 */
431# define SPARC_CB012 15 /* 0 or 1 or 2 */
432# define CB(cc, imm) f2b(0, 0, cc, 7, imm)
433# define CBa(cc, imm) f2b(0, 1, cc, 7, imm)
434# define CBA(imm) CB(SPARC_CBA, imm)
435# define CBAa(imm) CBa(SPARC_CBA, imm)
436# define CBN(imm) CB(SPARC_CBN, imm)
437# define CBNa(imm) CBa(SPARC_CBN, imm)
438# define CB3(imm) CB(SPARC_CB3, imm)
439# define CB3a(imm) CBa(SPARC_CB3, imm)
440# define CB2(imm) CB(SPARC_CB2, imm)
441# define CB2a(imm) CBa(SPARC_CB2, imm)
442# define CB23(imm) CB(SPARC_CB23, imm)
443# define CB23a(imm) CBa(SPARC_CB23, imm)
444# define CB1(imm) CB(SPARC_CB1, imm)
445# define CB1a(imm) CBa(SPARC_CB1, imm)
446# define CB13(imm) CB(SPARC_CB13, imm)
447# define CB13a(imm) CBa(SPARC_CB13, imm)
448# define CB12(imm) CB(SPARC_CB12, imm)
449# define CB12a(imm) CBa(SPARC_CB12, imm)
450# define CB123(imm) CB(SPARC_CB123, imm)
451# define CB123a(imm) CBa(SPARC_CB123, imm)
452# define CB0(imm) CB(SPARC_CB0, imm)
453# define CB0a(imm) CBa(SPARC_CB0, imm)
454# define CB03(imm) CB(SPARC_CB03, imm)
455# define CB03a(imm) CBa(SPARC_CB03, imm)
456# define CB02(imm) CB(SPARC_CB02, imm)
457# define CB02a(imm) CBa(SPARC_CB02, imm)
458# define CB023(imm) CB(SPARC_CB103, imm)
459# define CB023a(imm) CBa(SPARC_CB023, imm)
460# define CB01(imm) CB(SPARC_CB01, imm)
461# define CB01a(imm) CBa(SPARC_CB01, imm)
462# define CB013(imm) CB(SPARC_CB013, imm)
463# define CB013a(imm) CBa(SPARC_CB013, imm)
464# define CB012(imm) CB(SPARC_CB012, imm)
465# define CB012a(imm) CBa(SPARC_CB012, imm)
466# define CALLI(imm) f1(1, imm)
467# define CALL(r0) JMPL(_O7_REGNO, r0, 0)
468# define RETL() JMPLI(0, _O7_REGNO, 8)
469# define RET() JMPLI(0, _I7_REGNO, 8)
470# define JMPL(rd, rs1, rs2) f3r(2, rd, 56, rs1, rs2)
471# define JMPLI(rd, rs1, imm) f3i(2, rd, 56, rs1, imm)
472# define RETT(rs1, rs2) f3r(2, 0, 57, rs1, rs2)
473# define RETTI(rs1, imm) f3i(2, 0, 57, rs1, imm)
474# define SPARC_TA 8 /* always */
475# define SPARC_TN 0 /* never */
476# define SPARC_TNE 9 /* not equal - not Z */
477# define SPARC_TNZ SPARC_BNE
478# define SPARC_TE 1 /* equal - Z */
479# define SPARC_TZ SPARC_BE
480# define SPARC_TG 10 /* greater - not (Z or (N xor V)) */
481# define SPARC_TLE 2 /* less or equal - Z or (N xor V) */
482# define SPARC_TGE 11 /* greater or equal - not (N xor V) */
483# define SPARC_TL 3 /* less - N xor V */
484# define SPARC_TGU 12 /* greater unsigned - not (C or Z) */
485# define SPARC_TLEU 4 /* less or equal unsigned - C or Z */
486# define SPARC_TCC 13 /* carry clear - not C */
487# define SPARC_TGEU SPARC_BCC
488# define SPARC_TCS 5 /* carry set - C */
489# define SPARC_TLU SPARC_BCS
490# define SPARC_TPOS 14 /* positive - not N */
491# define SPARC_TNEG 6 /* negative - N */
492# define SPARC_TVC 15 /* overflow clear - not V */
493# define SPARC_TVS 7 /* overflow set - V */
494# define T(cc, rs1, rs2) f3t(cc, rs1, 0, rs2)
495# define TI(cc, rs1, imm) f3t(cc, rs1, 1, imm)
496# define TA(rs1, rs2) T(SPARC_TA, rs1, rs2)
497# define TAI(rs1, imm) TI(SPARC_TA, rs1, imm)
498# define TN(rs1, rs2) T(SPARC_TN, rs1, rs2)
499# define TNI(rs1, imm) TI(SPARC_TN, rs1, imm)
500# define TNE(rs1, rs2) T(SPARC_TNE, rs1, rs2)
501# define TNEI(rs1, imm) TI(SPARC_TNE, rs1, imm)
502# define TNZ(rs1, rs2) TNE(rs1, rs2)
503# define TNZI(rs1, imm) TNEI(rs1, imm)
504# define TE(rs1, rs2) T(SPARC_TE, rs1, rs2)
505# define TEI(rs1, imm) TI(SPARC_TE, rs1, imm)
506# define TZ(rs1, rs2) TE(rs1, rs2)
507# define TZI(rs1, imm) TEI(rs1, imm)
508# define TG(rs1, rs2) T(SPARC_TG, rs1, rs2)
509# define TGI(rs1, imm) TI(SPARC_TG, rs1, imm)
510# define TLE(rs1, rs2) T(SPARC_TLE, rs1, rs2)
511# define TLEI(rs1, imm) TI(SPARC_TLE, rs1, imm)
512# define TGE(rs1, rs2) T(SPARC_TGE, rs1, rs2)
513# define TGEI(rs1, imm) TI(SPARC_TGE, rs1, imm)
514# define TL(rs1, rs2) T(SPARC_TL, rs1, rs2)
515# define TLI(rs1, imm) TI(SPARC_TL, rs1, imm)
516# define TGU(rs1, rs2) T(SPARC_TGU, rs1, rs2)
517# define TGUI(rs1, imm) TI(SPARC_TGU, rs1, imm)
518# define TLEU(rs1, rs2) T(SPARC_TLEU, rs1, rs2)
519# define TLEUI(rs1, imm) TI(SPARC_TLEU, rs1, imm)
520# define TCC(rs1, rs2) T(SPARC_TCC, rs1, rs2)
521# define TCCI(rs1, imm) TI(SPARC_TCC, rs1, imm)
522# define TGEU(rs1, rs2) TCC(rs1, rs2)
523# define TGEUI(rs1, imm) TCCI(rs1, imm)
524# define TCS(rs1, rs2) T(SPARC_TCC, rs1, rs2)
525# define TCSI(rs1, imm) TI(SPARC_TCC, rs1, imm)
526# define TLU(rs1, rs2) TCS(rs1, rs2)
527# define TLUI(rs1, imm) TCSI(rs1, imm)
528# define TPOS(rs1, rs2) T(SPARC_TPOS, rs1, rs2)
529# define TPOSI(rs1, imm) TI(SPARC_TPOS, rs1, imm)
530# define TNEG(rs1, rs2) T(SPARC_TNEG, rs1, rs2)
531# define TNEGI(rs1, imm) TI(SPARC_TNEG, rs1, imm)
532# define TVC(rs1, rs2) T(SPARC_TVC, rs1, rs2)
533# define TVCI(rs1, imm) TI(SPARC_TVC, rs1, imm)
534# define TVS(rs1, rs2) T(SPARC_TVS, rs1, rs2)
535# define TVSI(rs1, imm) TI(SPARC_TVS, rs1, imm)
536# define RDY(rd) f3r(2, rd, 40, 0, 0)
537# define RDASR(rs1, rd) f3r(2, rd, 40, rs1, 0)
538# define RDPSR(rd) f3r(2, rd, 41, 0, 0)
539# define RDWIM(rd) f3r(2, rd, 42, 0, 0)
540# define RDTBR(rd) f3r(2, rd, 43, 0, 0)
541# define WRY(rs1, rs2) f3r(2, 0, 48, rs1, rs2)
542# define WRYI(rs1, imm) f3i(2, 0, 48, rs1, imm)
543# define WRASR(rs1, rs2, rd) f3r(2, rd, 48, rs1, rs2)
544# define WRASRI(rs1, imm, rd) f3i(2, rd, 48, rs1, imm)
545# define WRPSR(rs1, rs2, rd) f3r(2, rd, 49, rs1, rs2)
546# define WRPSRI(rs1, imm, rd) f3i(2, rd, 49, rs1, imm)
547# define WRWIM(rs1, rs2, rd) f3r(2, rd, 50, rs1, rs2)
548# define WRWIMI(rs1, imm, rd) f3i(2, rd, 50, rs1, imm)
549# define WRTBR(rs1, rs2, rd) f3r(2, rd, 51, rs1, rs2)
550# define WRTBRI(rs1, imm, rd) f3i(2, rd, 51, rs1, imm)
551# define STBAR() f3i(2, 0, 40, 15, 0)
552# define UNIMP(imm) f2r(0, 0, 0, imm)
553# define FLUSH(rs1, rs2) f3r(2, 0, 59, rs1, rs2)
554# define FLUSHI(rs1, im) f3i(2, 0, 59, rs1, imm)
79bfeef6 555# define LZCNT(rs2, rd) f2c1(2, rd, 54, 0, 23, rs2)
4a71579b
PC		 556# define nop(i0) _nop(_jit, i0)
557static void _nop(jit_state_t*, jit_int32_t);
558# define movr(r0, r1) _movr(_jit, r0, r1)
559static void _movr(jit_state_t*, jit_int32_t, jit_int32_t);
560# define movi(r0, i0) _movi(_jit, r0, i0)
561static void _movi(jit_state_t*, jit_int32_t, jit_word_t);
562# define movi_p(r0, i0) _movi_p(_jit, r0, i0)
563static jit_word_t _movi_p(jit_state_t*, jit_int32_t, jit_word_t);
40a44dcb
PC		 564# define bswapr_us(r0, r1) generic_bswapr_us(_jit, r0, r1)
565# define bswapr_ui(r0, r1) generic_bswapr_ui(_jit, r0, r1)
566# define bswapr_ul(r0, r1) generic_bswapr_ul(_jit, r0, r1)
567# define movnr(r0,r1,r2) _movnr(_jit,r0,r1,r2)
568static void _movnr(jit_state_t*,jit_int32_t,jit_int32_t,jit_int32_t);
569# define movzr(r0,r1,r2) _movzr(_jit,r0,r1,r2)
570static void _movzr(jit_state_t*,jit_int32_t,jit_int32_t,jit_int32_t);
ba3814c1
PC		 571# define casx(r0, r1, r2, r3, i0) _casx(_jit, r0, r1, r2, r3, i0)
572static void _casx(jit_state_t *_jit,jit_int32_t,jit_int32_t,
573 jit_int32_t,jit_int32_t,jit_word_t);
574#define casr(r0, r1, r2, r3) casx(r0, r1, r2, r3, 0)
575#define casi(r0, i0, r1, r2) casx(r0, _NOREG, r1, r2, i0)
4a71579b
PC		 576# define comr(r0, r1) XNOR(r1, 0, r0)
577# define negr(r0, r1) NEG(r1, r0)
79bfeef6
PC		 578# define clor(r0, r1) _clor(_jit, r0, r1)
579static void _clor(jit_state_t*, jit_int32_t, jit_int32_t);
580# define clzr(r0, r1) _clzr(_jit, r0, r1)
581static void _clzr(jit_state_t*, jit_int32_t, jit_int32_t);
582# define ctor(r0, r1) _ctor(_jit, r0, r1)
583static void _ctor(jit_state_t*, jit_int32_t, jit_int32_t);
584# define ctzr(r0, r1) _ctzr(_jit, r0, r1)
585static void _ctzr(jit_state_t*, jit_int32_t, jit_int32_t);
4a71579b
PC		 586# define addr(r0, r1, r2) ADD(r1, r2, r0)
587# define addi(r0, r1, i0) _addi(_jit, r0, r1, i0)
588static void _addi(jit_state_t*, jit_int32_t, jit_int32_t, jit_word_t);
589# if __WORDSIZE == 32
590# define addcr(r0, r1, r2) ADDcc(r1, r2, r0)
591# else
592# define addcr(r0, r1, r2) _addcr(_jit, r0, r1, r2)
593static void _addcr(jit_state_t*, jit_int32_t, jit_int32_t, jit_int32_t);
594# endif
595# define addci(r0, r1, i0) _addci(_jit, r0, r1, i0)
596static void _addci(jit_state_t*, jit_int32_t, jit_int32_t, jit_word_t);
597# if __WORDSIZE == 32
598# define addxr(r0, r1, r2) ADDXcc(r1, r2, r0)
599# else
600# define addxr(r0, r1, r2) _addxr(_jit, r0, r1, r2)
601static void _addxr(jit_state_t*, jit_int32_t, jit_int32_t, jit_int32_t);
602# endif
603# define addxi(r0, r1, i0) _addxi(_jit, r0, r1, i0)
604static void _addxi(jit_state_t*, jit_int32_t, jit_int32_t, jit_word_t);
605# define subr(r0, r1, r2) SUB(r1, r2, r0)
606# define subi(r0, r1, i0) _subi(_jit, r0, r1, i0)
607static void _subi(jit_state_t*, jit_int32_t, jit_int32_t, jit_word_t);
608# if __WORDSIZE == 32
609# define subcr(r0, r1, r2) SUBcc(r1, r2, r0)
610# else
611# define subcr(r0, r1, r2) _subcr(_jit, r0, r1, r2)
612static void _subcr(jit_state_t*, jit_int32_t, jit_int32_t, jit_int32_t);
613# endif
614# define subci(r0, r1, i0) _subci(_jit, r0, r1, i0)
615static void _subci(jit_state_t*, jit_int32_t, jit_int32_t, jit_word_t);
616# if __WORDSIZE == 32
617# define subxr(r0, r1, r2) SUBXcc(r1, r2, r0)
618# else
619# define subxr(r0, r1, r2) _subxr(_jit, r0, r1, r2)
620static void _subxr(jit_state_t*, jit_int32_t, jit_int32_t, jit_int32_t);
621# endif
622# define subxi(r0, r1, i0) _subxi(_jit, r0, r1, i0)
623static void _subxi(jit_state_t*, jit_int32_t, jit_int32_t, jit_word_t);
624# define rsbi(r0, r1, i0) _rsbi(_jit, r0, r1, i0)
625static void _rsbi(jit_state_t*,jit_int32_t,jit_int32_t,jit_word_t);
626# if __WORDSIZE == 32
627# define mulr(r0, r1, r2) UMUL(r1, r2, r0)
628# else
629# define mulr(r0, r1, r2) MULX(r1, r2, r0)
630# endif
631# define muli(r0, r1, i0) _muli(_jit, r0, r1, i0)
632static void _muli(jit_state_t*, jit_int32_t, jit_int32_t, jit_word_t);
ba86ff93
PC		 633# define hmulr(r0, r1, r2) qmulr(JIT_NOREG, r0, r1, r2)
634# define hmuli(r0, r1, i0) qmuli(JIT_NOREG, r0, r1, i0)
635# define hmulr_u(r0, r1, r2) qmulr_u(JIT_NOREG, r0, r1, r2)
636# define hmuli_u(r0, r1, i0) qmuli_u(JIT_NOREG, r0, r1, i0)
4a71579b
PC		 637# if __WORDSIZE == 32
638# define qmulr(r0,r1,r2,r3) iqmulr(r0,r1,r2,r3,1)
639# define qmulr_u(r0,r1,r2,r3) iqmulr(r0,r1,r2,r3,0)
640# define iqmulr(r0,r1,r2,r3,cc) _iqmulr(_jit,r0,r1,r2,r3,cc)
641static void _iqmulr(jit_state_t*,jit_int32_t,jit_int32_t,
642 jit_int32_t,jit_int32_t,jit_bool_t);
643# define qmuli(r0,r1,r2,i0) iqmuli(r0,r1,r2,i0,1)
644# define qmuli_u(r0,r1,r2,i0) iqmuli(r0,r1,r2,i0,0)
645# define iqmuli(r0,r1,r2,i0,cc) _iqmuli(_jit,r0,r1,r2,i0,cc)
646static void _iqmuli(jit_state_t*,jit_int32_t,jit_int32_t,
647 jit_int32_t,jit_word_t,jit_bool_t);
648# else
649# define qmulr(r0,r1,r2,r3) _qmulr(_jit,r0,r1,r2,r3)
650static void _qmulr(jit_state_t*,jit_int32_t,jit_int32_t,
651 jit_int32_t,jit_int32_t);
652# define qmuli(r0,r1,r2,i0) _qmuli(_jit,r0,r1,r2,i0)
653static void _qmuli(jit_state_t*,jit_int32_t,jit_int32_t,
654 jit_int32_t,jit_word_t);
655# define qmulr_u(r0,r1,r2,r3) _qmulr_u(_jit,r0,r1,r2,r3)
656static void _qmulr_u(jit_state_t*,jit_int32_t,jit_int32_t,
657 jit_int32_t,jit_int32_t);
658# define qmuli_u(r0,r1,r2,i0) _qmuli_u(_jit,r0,r1,r2,i0)
659static void _qmuli_u(jit_state_t*,jit_int32_t,jit_int32_t,
660 jit_int32_t,jit_word_t);
661# endif
662# define divr(r0, r1, r2) _divr(_jit, r0, r1, r2)
663static void _divr(jit_state_t*, jit_int32_t, jit_int32_t, jit_int32_t);
664# define divi(r0, r1, i0) _divi(_jit, r0, r1, i0)
665static void _divi(jit_state_t*, jit_int32_t, jit_int32_t, jit_word_t);
666# define divr_u(r0, r1, r2) _divr_u(_jit, r0, r1, r2)
667static void _divr_u(jit_state_t*, jit_int32_t, jit_int32_t, jit_int32_t);
668# define divi_u(r0, r1, i0) _divi_u(_jit, r0, r1, i0)
669static void _divi_u(jit_state_t*, jit_int32_t, jit_int32_t, jit_word_t);
670# define qdivr(r0,r1,r2,r3) iqdivr(r0,r1,r2,r3,1)
671# define qdivr_u(r0,r1,r2,r3) iqdivr(r0,r1,r2,r3,0)
672# define iqdivr(r0,r1,r2,r3,cc) _iqdivr(_jit,r0,r1,r2,r3,cc)
673static void _iqdivr(jit_state_t*,jit_int32_t,jit_int32_t,
674 jit_int32_t,jit_int32_t,jit_bool_t);
675# define qdivi(r0,r1,r2,i0) iqdivi(r0,r1,r2,i0,1)
676# define qdivi_u(r0,r1,r2,i0) iqdivi(r0,r1,r2,i0,0)
677# define iqdivi(r0,r1,r2,i0,cc) _iqdivi(_jit,r0,r1,r2,i0,cc)
678static void _iqdivi(jit_state_t*,jit_int32_t,jit_int32_t,
679 jit_int32_t,jit_word_t,jit_bool_t);
680# define remr(r0, r1, r2) _remr(_jit, r0, r1, r2)
681static void _remr(jit_state_t*, jit_int32_t, jit_int32_t, jit_int32_t);
682# define remi(r0, r1, i0) _remi(_jit, r0, r1, i0)
683static void _remi(jit_state_t*, jit_int32_t, jit_int32_t, jit_word_t);
684# define remr_u(r0, r1, r2) _remr_u(_jit, r0, r1, r2)
685static void _remr_u(jit_state_t*, jit_int32_t, jit_int32_t, jit_int32_t);
686# define remi_u(r0, r1, i0) _remi_u(_jit, r0, r1, i0)
687static void _remi_u(jit_state_t*, jit_int32_t, jit_int32_t, jit_word_t);
688# define andr(r0, r1, r2) AND(r1, r2, r0)
689# define andi(r0, r1, i0) _andi(_jit, r0, r1, i0)
690static void _andi(jit_state_t*, jit_int32_t, jit_int32_t, jit_word_t);
691# define orr(r0, r1, r2) OR(r1, r2, r0)
692# define ori(r0, r1, i0) _ori(_jit, r0, r1, i0)
693static void _ori(jit_state_t*, jit_int32_t, jit_int32_t, jit_word_t);
694# define xorr(r0, r1, r2) XOR(r1, r2, r0)
695# define xori(r0, r1, i0) _xori(_jit, r0, r1, i0)
696static void _xori(jit_state_t*, jit_int32_t, jit_int32_t, jit_word_t);
697# if __WORDSIZE == 32
698# define lshr(r0, r1, r2) SLL(r1, r2, r0)
699# define lshi(r0, r1, i0) SLLI(r1, i0, r0)
700# define rshr(r0, r1, r2) SRA(r1, r2, r0)
701# define rshi(r0, r1, i0) SRAI(r1, i0, r0)
702# define rshr_u(r0, r1, r2) SRL(r1, r2, r0)
703# define rshi_u(r0, r1, i0) SRLI(r1, i0, r0)
704# else
705# define lshr(r0, r1, r2) SLLX(r1, r2, r0)
706# define lshi(r0, r1, i0) SLLXI(r1, i0, r0)
707# define rshr(r0, r1, r2) SRAX(r1, r2, r0)
708# define rshi(r0, r1, i0) SRAXI(r1, i0, r0)
709# define rshr_u(r0, r1, r2) SRLX(r1, r2, r0)
710# define rshi_u(r0, r1, i0) SRLXI(r1, i0, r0)
711# endif
4a71579b
PC		 712# define extr_c(r0,r1) _extr_c(_jit,r0,r1)
713static void _extr_c(jit_state_t*,jit_int32_t,jit_int32_t);
714# define extr_uc(r0,r1) andi(r0, r1, 0xff)
715# define extr_s(r0,r1) _extr_s(_jit,r0,r1)
716static void _extr_s(jit_state_t*,jit_int32_t,jit_int32_t);
717# define extr_us(r0,r1) _extr_us(_jit,r0,r1)
718static void _extr_us(jit_state_t*,jit_int32_t,jit_int32_t);
40a44dcb 719# if __WORDSIZE == 64
4a71579b
PC		 720# define extr_i(r0,r1) _extr_i(_jit,r0,r1)
721static void _extr_i(jit_state_t*,jit_int32_t,jit_int32_t);
722# define extr_ui(r0,r1) _extr_ui(_jit,r0,r1)
723static void _extr_ui(jit_state_t*,jit_int32_t,jit_int32_t);
724# endif
725# define cr(cc, r0, r1, r2) _cr(_jit, cc, r0, r1, r2)
726static void _cr(jit_state_t*,jit_int32_t,jit_int32_t,jit_int32_t,jit_int32_t);
727# define cw(cc, r0, r1, i0) _cw(_jit, cc, r0, r1, i0)
728static void _cw(jit_state_t*,jit_int32_t,jit_int32_t,jit_int32_t,jit_word_t);
729# if __WORDSIZE == 32
730# define ltr(r0, r1, r2) cr(SPARC_BL, r0, r1, r2)
731# define lti(r0, r1, i0) cw(SPARC_BL, r0, r1, i0)
732# define ltr_u(r0, r1, r2) cr(SPARC_BLU, r0, r1, r2)
733# define lti_u(r0, r1, i0) cw(SPARC_BLU, r0, r1, i0)
734# define ler(r0, r1, r2) cr(SPARC_BLE, r0, r1, r2)
735# define lei(r0, r1, i0) cw(SPARC_BLE, r0, r1, i0)
736# define ler_u(r0, r1, r2) cr(SPARC_BLEU, r0, r1, r2)
737# define lei_u(r0, r1, i0) cw(SPARC_BLEU, r0, r1, i0)
738# define eqr(r0, r1, r2) cr(SPARC_BE, r0, r1, r2)
739# define eqi(r0, r1, i0) cw(SPARC_BE, r0, r1, i0)
740# define ger(r0, r1, r2) cr(SPARC_BGE, r0, r1, r2)
741# define gei(r0, r1, i0) cw(SPARC_BGE, r0, r1, i0)
742# define ger_u(r0, r1, r2) cr(SPARC_BGEU, r0, r1, r2)
743# define gei_u(r0, r1, i0) cw(SPARC_BGEU, r0, r1, i0)
744# define gtr(r0, r1, r2) cr(SPARC_BG, r0, r1, r2)
745# define gti(r0, r1, i0) cw(SPARC_BG, r0, r1, i0)
746# define gtr_u(r0, r1, r2) cr(SPARC_BGU, r0, r1, r2)
747# define gti_u(r0, r1, i0) cw(SPARC_BGU, r0, r1, i0)
748# define ner(r0, r1, r2) cr(SPARC_BNE, r0, r1, r2)
749# define nei(r0, r1, i0) cw(SPARC_BNE, r0, r1, i0)
750# else
751# define ltr(r0, r1, r2) cr(SPARC_BPL, r0, r1, r2)
752# define lti(r0, r1, i0) cw(SPARC_BPL, r0, r1, i0)
753# define ltr_u(r0, r1, r2) cr(SPARC_BPCS, r0, r1, r2)
754# define lti_u(r0, r1, i0) cw(SPARC_BPCS, r0, r1, i0)
755# define ler(r0, r1, r2) cr(SPARC_BPLE, r0, r1, r2)
756# define lei(r0, r1, i0) cw(SPARC_BPLE, r0, r1, i0)
757# define ler_u(r0, r1, r2) cr(SPARC_BPLEU, r0, r1, r2)
758# define lei_u(r0, r1, i0) cw(SPARC_BPLEU, r0, r1, i0)
759# define eqr(r0, r1, r2) cr(SPARC_BPE, r0, r1, r2)
760# define eqi(r0, r1, i0) cw(SPARC_BPE, r0, r1, i0)
761# define ger(r0, r1, r2) cr(SPARC_BPGE, r0, r1, r2)
762# define gei(r0, r1, i0) cw(SPARC_BPGE, r0, r1, i0)
763# define ger_u(r0, r1, r2) cr(SPARC_BPCC, r0, r1, r2)
764# define gei_u(r0, r1, i0) cw(SPARC_BPCC, r0, r1, i0)
765# define gtr(r0, r1, r2) cr(SPARC_BPG, r0, r1, r2)
766# define gti(r0, r1, i0) cw(SPARC_BPG, r0, r1, i0)
767# define gtr_u(r0, r1, r2) cr(SPARC_BPGU, r0, r1, r2)
768# define gti_u(r0, r1, i0) cw(SPARC_BPGU, r0, r1, i0)
769# define ner(r0, r1, r2) cr(SPARC_BPNE, r0, r1, r2)
770# define nei(r0, r1, i0) cw(SPARC_BPNE, r0, r1, i0)
771# endif
772# define ldr_c(r0, r1) LDSB(r1, 0, r0)
773# define ldi_c(r0, i0) _ldi_c(_jit, r0, i0)
774static void _ldi_c(jit_state_t*,jit_int32_t,jit_word_t);
775# define ldr_uc(r0, r1) LDUB(r1, 0, r0)
776# define ldi_uc(r0, i0) _ldi_uc(_jit, r0, i0)
777static void _ldi_uc(jit_state_t*,jit_int32_t,jit_word_t);
778# define ldr_s(r0, r1) LDSH(r1, 0, r0)
779# define ldi_s(r0, i0) _ldi_s(_jit, r0, i0)
780static void _ldi_s(jit_state_t*,jit_int32_t,jit_word_t);
781# define ldr_us(r0, r1) LDUH(r1, 0, r0)
782# define ldi_us(r0, i0) _ldi_us(_jit, r0, i0)
783static void _ldi_us(jit_state_t*,jit_int32_t,jit_word_t);
784# if __WORDSIZE == 32
785# define ldr_i(r0, r1) LD(r1, 0, r0)
786# define ldr(u, v) ldr_i(u, v)
787# define ldi(u, v) ldi_i(u, v)
788# else
789# define ldr_i(r0, r1) LDSW(r1, 0, r0)
790# define ldr_ui(r0, r1) LDUW(r1, 0, r0)
791# define ldr_l(r0, r1) LDX(r1, 0, r0)
792# define ldr(u, v) ldr_l(u, v)
793# define ldi(u, v) ldi_l(u, v)
794# endif
795# define ldi_i(r0, i0) _ldi_i(_jit, r0, i0)
796static void _ldi_i(jit_state_t*,jit_int32_t,jit_word_t);
797# if __WORDSIZE == 64
798# define ldi_ui(r0, i0) _ldi_ui(_jit, r0, i0)
799static void _ldi_ui(jit_state_t*,jit_int32_t,jit_word_t);
800# define ldi_l(r0, i0) _ldi_l(_jit, r0, i0)
801static void _ldi_l(jit_state_t*,jit_int32_t,jit_word_t);
802# endif
803# define ldxr_c(r0, r1, r2) LDSB(r1, r2, r0)
804# define ldxi_c(r0, r1, i0) _ldxi_c(_jit, r0, r1, i0)
805static void _ldxi_c(jit_state_t*,jit_int32_t,jit_int32_t,jit_word_t);
806# define ldxr_uc(r0, r1, r2) LDUB(r1, r2, r0)
807# define ldxi_uc(r0, r1, i0) _ldxi_uc(_jit, r0, r1, i0)
808static void _ldxi_uc(jit_state_t*,jit_int32_t,jit_int32_t,jit_word_t);
809# define ldxr_s(r0, r1, r2) LDSH(r1, r2, r0)
810# define ldxi_s(r0, r1, i0) _ldxi_s(_jit, r0, r1, i0)
811static void _ldxi_s(jit_state_t*,jit_int32_t,jit_int32_t,jit_word_t);
812# define ldxr_us(r0, r1, r2) LDUH(r1, r2, r0)
813# define ldxi_us(r0, r1, i0) _ldxi_us(_jit, r0, r1, i0)
814static void _ldxi_us(jit_state_t*,jit_int32_t,jit_int32_t,jit_word_t);
815# if __WORDSIZE == 32
816# define ldxr(u, v, w) ldxr_i(u, v, w)
817# define ldxr_i(r0, r1, r2) LD(r1, r2, r0)
818# define ldxi(u, v, w) ldxi_i(u, v, w)
819# else
820# define ldxr(u, v, w) ldxr_l(u, v, w)
821# define ldxr_i(r0, r1, r2) LDSW(r1, r2, r0)
822# define ldxr_ui(r0, r1, r2) LDUW(r1, r2, r0)
823# define ldxr_l(r0, r1, r2) LDX(r1, r2, r0)
824# define ldxi(u, v, w) ldxi_l(u, v, w)
825# endif
826# define ldxi_i(r0, r1, i0) _ldxi_i(_jit, r0, r1, i0)
827static void _ldxi_i(jit_state_t*,jit_int32_t,jit_int32_t,jit_word_t);
828# if __WORDSIZE == 64
829# define ldxi_ui(r0, r1, i0) _ldxi_ui(_jit, r0, r1, i0)
830static void _ldxi_ui(jit_state_t*,jit_int32_t,jit_int32_t,jit_word_t);
831# define ldxi_l(r0, r1, i0) _ldxi_l(_jit, r0, r1, i0)
832static void _ldxi_l(jit_state_t*,jit_int32_t,jit_int32_t,jit_word_t);
833# endif
834# define str_c(r0, r1) STB(r1, r0, 0)
835# define sti_c(i0, r0) _sti_c(_jit, i0, r0)
836static void _sti_c(jit_state_t*,jit_word_t,jit_int32_t);
837# define str_s(r0, r1) STH(r1, r0, 0)
838# define sti_s(i0, r0) _sti_s(_jit, i0, r0)
839static void _sti_s(jit_state_t*,jit_word_t,jit_int32_t);
840# if __WORDSIZE == 32
841# define str(u, v) str_i(u, v)
842# define str_i(r0, r1) STI(r1, r0, 0)
843# define sti(u, v) sti_i(u, v)
844# else
845# define str(u, v) str_l(u, v)
846# define str_i(r0, r1) STW(r1, r0, 0)
847# define str_l(r0, r1) STX(r1, r0, 0)
848# define sti(u, v) sti_l(u, v)
849# endif
850# define sti_i(i0, r0) _sti_i(_jit, i0, r0)
851static void _sti_i(jit_state_t*,jit_word_t,jit_int32_t);
852# if __WORDSIZE == 64
853# define sti_l(i0, r0) _sti_l(_jit, i0, r0)
854static void _sti_l(jit_state_t*,jit_word_t,jit_int32_t);
855# endif
856# define stxr_c(r0, r1, r2) STB(r2, r1, r0)
857# define stxi_c(i0, r0, r1) _stxi_c(_jit, i0, r0, r1)
858static void _stxi_c(jit_state_t*,jit_word_t,jit_int32_t,jit_int32_t);
859# define stxr_s(r0, r1, r2) STH(r2, r1, r0)
860# define stxi_s(i0, r0, r1) _stxi_s(_jit, i0, r0, r1)
861static void _stxi_s(jit_state_t*,jit_word_t,jit_int32_t,jit_int32_t);
862# if __WORDSIZE == 32
863# define stxr(u, v, w) stxr_i(u, v, w)
864# define stxr_i(r0, r1, r2) ST(r2, r1, r0)
865# define stxi(u, v, w) stxi_i(u, v, w)
866# else
867# define stxr(u, v, w) stxr_l(u, v, w)
868# define stxr_i(r0, r1, r2) STW(r2, r1, r0)
869# define stxi(u, v, w) stxi_l(u, v, w)
870# define stxr_l(r0, r1, r2) STX(r2, r1, r0)
871# endif
872# define stxi_i(i0, r0, r1) _stxi_i(_jit, i0, r0, r1)
873static void _stxi_i(jit_state_t*,jit_word_t,jit_int32_t,jit_int32_t);
874# if __WORDSIZE == 64
875# define stxi_l(i0, r0, r1) _stxi_l(_jit, i0, r0, r1)
876static void _stxi_l(jit_state_t*,jit_word_t,jit_int32_t,jit_int32_t);
877# endif
878# define br(cc, i0, r0, r1) _br(_jit, cc, i0, r0, r1)
879static jit_word_t
880_br(jit_state_t*,jit_int32_t,jit_word_t,jit_int32_t,jit_int32_t);
881# define bw(cc, i0, r0, i1) _bw(_jit, cc, i0, r0, i1)
882static jit_word_t
883_bw(jit_state_t*,jit_int32_t,jit_word_t,jit_int32_t,jit_word_t);
884# if __WORDSIZE == 32
885# define bltr(i0, r0, r1) br(SPARC_BL, i0, r0, r1)
886# define blti(i0, r0, i1) bw(SPARC_BL, i0, r0, i1)
887# define bltr_u(i0, r0, r1) br(SPARC_BLU, i0, r0, r1)
888# define blti_u(i0, r0, i1) bw(SPARC_BLU, i0, r0, i1)
889# define bler(i0, r0, r1) br(SPARC_BLE, i0, r0, r1)
890# define blei(i0, r0, i1) bw(SPARC_BLE, i0, r0, i1)
891# define bler_u(i0, r0, r1) br(SPARC_BLEU, i0, r0, r1)
892# define blei_u(i0, r0, i1) bw(SPARC_BLEU, i0, r0, i1)
893# define beqr(i0, r0, r1) br(SPARC_BE, i0, r0, r1)
894# define beqi(i0, r0, i1) bw(SPARC_BE, i0, r0, i1)
895# define bger(i0, r0, r1) br(SPARC_BGE, i0, r0, r1)
896# define bgei(i0, r0, i1) bw(SPARC_BGE, i0, r0, i1)
897# define bger_u(i0, r0, r1) br(SPARC_BGEU, i0, r0, r1)
898# define bgei_u(i0, r0, i1) bw(SPARC_BGEU, i0, r0, i1)
899# define bgtr(i0, r0, r1) br(SPARC_BG, i0, r0, r1)
900# define bgti(i0, r0, i1) bw(SPARC_BG, i0, r0, i1)
901# define bgtr_u(i0, r0, r1) br(SPARC_BGU, i0, r0, r1)
902# define bgti_u(i0, r0, i1) bw(SPARC_BGU, i0, r0, i1)
903# define bner(i0, r0, r1) br(SPARC_BNE, i0, r0, r1)
904# define bnei(i0, r0, i1) bw(SPARC_BNE, i0, r0, i1)
905# else
906# define bltr(i0, r0, r1) br(SPARC_BPL, i0, r0, r1)
907# define blti(i0, r0, i1) bw(SPARC_BPL, i0, r0, i1)
908# define bltr_u(i0, r0, r1) br(SPARC_BPCS, i0, r0, r1)
909# define blti_u(i0, r0, i1) bw(SPARC_BPCS, i0, r0, i1)
910# define bler(i0, r0, r1) br(SPARC_BPLE, i0, r0, r1)
911# define blei(i0, r0, i1) bw(SPARC_BPLE, i0, r0, i1)
912# define bler_u(i0, r0, r1) br(SPARC_BPLEU, i0, r0, r1)
913# define blei_u(i0, r0, i1) bw(SPARC_BPLEU, i0, r0, i1)
914# define beqr(i0, r0, r1) br(SPARC_BPE, i0, r0, r1)
915# define beqi(i0, r0, i1) bw(SPARC_BPE, i0, r0, i1)
916# define bger(i0, r0, r1) br(SPARC_BPGE, i0, r0, r1)
917# define bgei(i0, r0, i1) bw(SPARC_BPGE, i0, r0, i1)
918# define bger_u(i0, r0, r1) br(SPARC_BPCC, i0, r0, r1)
919# define bgei_u(i0, r0, i1) bw(SPARC_BPCC, i0, r0, i1)
920# define bgtr(i0, r0, r1) br(SPARC_BPG, i0, r0, r1)
921# define bgti(i0, r0, i1) bw(SPARC_BPG, i0, r0, i1)
922# define bgtr_u(i0, r0, r1) br(SPARC_BPGU, i0, r0, r1)
923# define bgti_u(i0, r0, i1) bw(SPARC_BPGU, i0, r0, i1)
924# define bner(i0, r0, r1) br(SPARC_BPNE, i0, r0, r1)
925# define bnei(i0, r0, i1) bw(SPARC_BPNE, i0, r0, i1)
926# endif
927# define b_asr(jif,add,sgn,i0,r0,r1) _b_asr(_jit,jif,add,sgn,i0,r0,r1)
928static jit_word_t
929_b_asr(jit_state_t*,jit_bool_t,jit_bool_t,jit_bool_t,
930 jit_word_t,jit_int32_t,jit_int32_t);
931# define b_asw(jif,add,sgn,i0,r0,i1) _b_asw(_jit,jif,add,sgn,i0,r0,i1)
932static jit_word_t
933_b_asw(jit_state_t*,jit_bool_t,jit_bool_t,jit_bool_t,
934 jit_word_t,jit_int32_t,jit_word_t);
935# define boaddr(i0, r0, r1) b_asr(1, 1, 1, i0, r0, r1)
936# define boaddi(i0, r0, i1) b_asw(1, 1, 1, i0, r0, i1)
937# define boaddr_u(i0, r0, r1) b_asr(1, 1, 0, i0, r0, r1)
938# define boaddi_u(i0, r0, i1) b_asw(1, 1, 0, i0, r0, i1)
939# define bxaddr(i0, r0, r1) b_asr(0, 1, 1, i0, r0, r1)
940# define bxaddi(i0, r0, i1) b_asw(0, 1, 1, i0, r0, i1)
941# define bxaddr_u(i0, r0, r1) b_asr(0, 1, 0, i0, r0, r1)
942# define bxaddi_u(i0, r0, i1) b_asw(0, 1, 0, i0, r0, i1)
943# define bosubr(i0, r0, r1) b_asr(1, 0, 1, i0, r0, r1)
944# define bosubi(i0, r0, i1) b_asw(1, 0, 1, i0, r0, i1)
945# define bosubr_u(i0, r0, r1) b_asr(1, 0, 0, i0, r0, r1)
946# define bosubi_u(i0, r0, i1) b_asw(1, 0, 0, i0, r0, i1)
947# define bxsubr(i0, r0, r1) b_asr(0, 0, 1, i0, r0, r1)
948# define bxsubi(i0, r0, i1) b_asw(0, 0, 1, i0, r0, i1)
949# define bxsubr_u(i0, r0, r1) b_asr(0, 0, 0, i0, r0, r1)
950# define bxsubi_u(i0, r0, i1) b_asw(0, 0, 0, i0, r0, i1)
951# define bm_r(set, i0, r0, r1) _bm_r(_jit,set,i0,r0,r1)
952static jit_word_t
953_bm_r(jit_state_t*,jit_bool_t,jit_word_t,jit_int32_t,jit_int32_t);
954# define bm_w(set,i0,r0,i1) _bm_w(_jit,set,i0,r0,i1)
955static jit_word_t
956_bm_w(jit_state_t*,jit_bool_t,jit_word_t,jit_int32_t,jit_word_t);
957# define bmsr(i0, r0, r1) bm_r(1, i0, r0, r1)
958# define bmsi(i0, r0, i1) bm_w(1, i0, r0, i1)
959# define bmcr(i0, r0, r1) bm_r(0, i0, r0, r1)
960# define bmci(i0, r0, i1) bm_w(0, i0, r0, i1)
961# define jmpr(r0) _jmpr(_jit, r0)
962static void _jmpr(jit_state_t*,jit_int32_t);
963# define jmpi(i0) _jmpi(_jit, i0)
79bfeef6 964static jit_word_t _jmpi(jit_state_t*,jit_word_t);
4a71579b
PC		 965# define jmpi_p(i0) _jmpi_p(_jit, i0)
966static jit_word_t _jmpi_p(jit_state_t*,jit_word_t);
967# define callr(r0) _callr(_jit, r0)
968static void _callr(jit_state_t*,jit_int32_t);
969# define calli(i0) _calli(_jit, i0)
79bfeef6 970static jit_word_t _calli(jit_state_t*,jit_word_t);
4a71579b
PC		 971# define calli_p(i0) _calli_p(_jit, i0)
972static jit_word_t _calli_p(jit_state_t*,jit_word_t);
973# define prolog(node) _prolog(_jit, node)
974static void _prolog(jit_state_t*,jit_node_t*);
975# define epilog(node) _epilog(_jit, node)
976static void _epilog(jit_state_t*,jit_node_t*);
977#define vastart(r0) _vastart(_jit, r0)
978static void _vastart(jit_state_t*, jit_int32_t);
979#define vaarg(r0, r1) _vaarg(_jit, r0, r1)
980static void _vaarg(jit_state_t*, jit_int32_t, jit_int32_t);
981#define patch_at(jump, label) _patch_at(_jit, jump, label)
982static void _patch_at(jit_state_t*,jit_word_t,jit_word_t);
983#endif
984
985#if CODE
986static void
987_f2r(jit_state_t *_jit,
988 jit_int32_t op, jit_int32_t rd, jit_int32_t op2, jit_int32_t imm22)
989{
990 jit_instr_t v;
991 assert(!(op & 0xfffffffc));
992 assert(!(rd & 0xffffffe0));
993 assert(!(op2 & 0xfffffff8));
994 assert(s22_p(imm22));
995 v.op.b = op;
996 v.rd.b = rd;
997 v.op2.b = op2;
998 v.imm22.b = imm22;
999 ii(v.v);
1000}
1001
1002static void
1003_f2b(jit_state_t *_jit,
1004 jit_int32_t op, jit_int32_t a, jit_int32_t cond, jit_int32_t op2,
1005 jit_int32_t disp22)
1006{
1007 jit_instr_t v;
1008 assert(!(op & 0xfffffffc));
1009 assert(!(a & 0xfffffffe));
1010 assert(!(cond & 0xfffffff0));
1011 assert(!(op2 & 0xfffffff8));
1012 assert(s22_p(disp22));
1013 v.op.b = op;
1014 v.a.b = a;
1015 v.cond.b = cond;
1016 v.op2.b = op2;
1017 v.disp22.b = disp22;
1018 ii(v.v);
1019}
1020
1021# if __WORDSIZE == 64
1022static void
1023_f2bp(jit_state_t *_jit,
1024 jit_int32_t op, jit_int32_t a, jit_int32_t cond, jit_int32_t op2,
1025 jit_int32_t cc1, jit_int32_t cc0, jit_int32_t p, jit_int32_t disp19)
1026{
1027 jit_instr_t v;
1028 assert(!(op & 0xfffffffc));
1029 assert(!(a & 0xfffffffe));
1030 assert(!(cond & 0xfffffff0));
1031 assert(!(op2 & 0xfffffff8));
1032 assert(s19_p(disp19));
1033 v.op.b = op;
1034 v.a.b = a;
1035 v.cond.b = cond;
1036 v.op2.b = op2;
1037 v.cc1.b = cc1;
1038 v.cc0.b = cc0;
1039 v.p.b = p;
1040 v.disp19.b = disp19;
1041 ii(v.v);
1042}
1043# endif
1044
1045static void
1046_f3r(jit_state_t *_jit, jit_int32_t op, jit_int32_t rd,
1047 jit_int32_t op3, jit_int32_t rs1, jit_int32_t rs2)
1048{
1049 jit_instr_t v;
1050 assert(!(op & 0xfffffffc));
1051 assert(!(rd & 0xffffffe0));
1052 assert(!(op3 & 0xffffffc0));
1053 assert(!(rs1 & 0xffffffe0));
1054 assert(!(rs2 & 0xffffffe0));
1055 v.op.b = op;
1056 v.rd.b = rd;
1057 v.op3.b = op3;
1058 v.rs1.b = rs1;
1059 v.i.b = 0;
1060 v.asi.b = 0;
1061 v.rs2.b = rs2;
1062 ii(v.v);
1063}
1064
1065# if __WORDSIZE == 64
c0c16242
PC		 1066static void
1067_f3ri(jit_state_t *_jit, jit_int32_t op, jit_int32_t rd,
1068 jit_int32_t op3, jit_int32_t rs1, jit_int32_t rs2)
1069{
1070 jit_instr_t v;
1071 assert(!(op & 0xfffffffc));
1072 assert(!(rd & 0xffffffe0));
1073 assert(!(op3 & 0xffffffc0));
1074 assert(!(rs1 & 0xffffffe0));
1075 assert(!(rs2 & 0xffffffe0));
1076 v.op.b = op;
1077 v.rd.b = rd;
1078 v.op3.b = op3;
1079 v.rs1.b = rs1;
1080 v.i.b = 1;
1081 v.asi.b = 0;
1082 v.rs2.b = rs2;
1083 ii(v.v);
1084}
1085
4a71579b
PC		 1086static void
1087_f3rx(jit_state_t *_jit, jit_int32_t op, jit_int32_t rd,
1088 jit_int32_t op3, jit_int32_t rs1, jit_int32_t rs2)
1089{
1090 jit_instr_t v;
1091 assert(!(op & 0xfffffffc));
1092 assert(!(rd & 0xffffffe0));
1093 assert(!(op3 & 0xffffffc0));
1094 assert(!(rs1 & 0xffffffe0));
1095 assert(!(rs2 & 0xffffffe0));
1096 v.op.b = op;
1097 v.rd.b = rd;
1098 v.op3.b = op3;
1099 v.rs1.b = rs1;
1100 v.i.b = 0;
1101 v.x.b = 1;
1102 v.asix.b = 0;
1103 v.rs2.b = rs2;
1104 ii(v.v);
1105}
1106
1107static void
1108_f3s(jit_state_t *_jit, jit_int32_t op, jit_int32_t rd,
1109 jit_int32_t op3, jit_int32_t rs1, jit_int32_t shim)
1110{
1111 jit_instr_t v;
1112 assert(!(op & 0xfffffffc));
1113 assert(!(rd & 0xffffffe0));
1114 assert(!(op3 & 0xffffffc0));
1115 assert(!(rs1 & 0xffffffe0));
1116 assert(!(shim & 0xffffffc0));
1117 v.op.b = op;
1118 v.rd.b = rd;
1119 v.op3.b = op3;
1120 v.rs1.b = rs1;
1121 v.i.b = 1;
1122 v.x.b = 1;
1123 v.asis.b = 0;
1124 v.shim.b = shim;
1125 ii(v.v);
1126}
1127# endif
1128
1129static void
1130_f3i(jit_state_t *_jit, jit_int32_t op, jit_int32_t rd,
1131 jit_int32_t op3, jit_int32_t rs1, jit_int32_t simm13)
1132{
1133 jit_instr_t v;
1134 assert(!(op & 0xfffffffc));
1135 assert(!(rd & 0xffffffe0));
1136 assert(!(op3 & 0xffffffc0));
1137 assert(!(rs1 & 0xffffffe0));
1138 assert(s13_p(simm13));
1139 v.op.b = op;
1140 v.rd.b = rd;
1141 v.op3.b = op3;
1142 v.rs1.b = rs1;
1143 v.i.b = 1;
1144 v.simm13.b = simm13;
1145 ii(v.v);
1146}
1147
1148static void
1149_f3t(jit_state_t *_jit, jit_int32_t cond,
1150 jit_int32_t rs1, jit_int32_t i, jit_int32_t rs2_imm7)
1151{
1152 jit_instr_t v;
1153 assert(!(cond & 0xfffffff0));
1154 assert(!(i & 0xfffffffe));
1155 assert(!(rs1 & 0xffffffe0));
1156 v.op.b = 2;
1157 v.rd.b = cond;
1158 v.op3.b = 58;
1159 v.i.b = i;
1160 if (i) {
1161 assert(s7_p(rs2_imm7));
1162 v.res.b = 0;
1163 v.imm7.b = rs2_imm7;
1164 }
1165 else {
1166 assert(!(rs2_imm7 & 0xffffffe0));
1167 v.asi.b = 0;
1168 v.rs2.b = rs2_imm7;
1169 }
1170 ii(v.v);
1171}
1172
1173static void
1174_f3a(jit_state_t *_jit, jit_int32_t op, jit_int32_t rd,
1175 jit_int32_t op3, jit_int32_t rs1, jit_int32_t asi, jit_int32_t rs2)
1176{
1177 jit_instr_t v;
1178 assert(!(op & 0xfffffffc));
1179 assert(!(rd & 0xffffffe0));
1180 assert(!(op3 & 0xffffffc0));
1181 assert(!(rs1 & 0xffffffe0));
1182 assert(!(asi & 0xffffff00));
1183 assert(!(rs2 & 0xffffffe0));
1184 v.op.b = op;
1185 v.rd.b = rd;
1186 v.op3.b = op3;
1187 v.rs1.b = rs1;
1188 v.i.b = 0;
1189 v.asi.b = asi;
1190 v.rs2.b = rs2;
1191 ii(v.v);
1192}
1193
1194static void
1195_f1(jit_state_t *_jit, jit_int32_t op, jit_int32_t disp30)
1196{
1197 jit_instr_t v;
1198 assert(!(op & 0xfffffffc));
1199 assert(s30_p(disp30));
1200 v.op.b = op;
1201 v.disp30.b = disp30;
1202 ii(v.v);
1203}
1204
79bfeef6
PC		 1205static void
1206_f2c1(jit_state_t *_jit, jit_int32_t op, jit_int32_t rd,
1207 jit_int32_t op3, jit_int32_t rs1, jit_int32_t opf, jit_int32_t rs2)
1208{
1209 jit_instr_t v;
1210 assert(!(op & 0xfffffffc));
1211 assert(!(rd & 0xffffffe0));
ba86ff93 1212 assert(!(op3 & 0xffffffc0));
79bfeef6
PC		 1213 assert(!(rs1 & 0xffffffe0));
1214 assert(!(opf & 0xfffffe00));
1215 assert(!(rs2 & 0xfffffe00));
1216 v.op.b = op;
1217 v.rd.b = rd;
1218 v.op3.b = op3;
1219 v.rs1.b = rs1;
1220 v.opf.b = opf;
1221 v.rs2.b = rs2;
1222 ii(v.v);
1223}
1224
4a71579b
PC		 1225static void
1226_nop(jit_state_t *_jit, jit_int32_t i0)
1227{
1228 for (; i0 > 0; i0 -= 4)
1229 NOP();
1230 assert(i0 == 0);
1231}
1232
1233static void
1234_movr(jit_state_t *_jit, jit_int32_t r0, jit_int32_t r1)
1235{
1236 if (r0 != r1)
1237 ORI(r1, 0, r0);
1238}
1239
1240static void
1241_movi(jit_state_t *_jit, jit_int32_t r0, jit_word_t i0)
1242{
1243 if (s13_p(i0))
1244 ORI(0, i0, r0);
1245 else {
1246# if __WORDSIZE == 64
1247 if (i0 & 0xffffffff00000000) {
1248 jit_int32_t reg = jit_get_reg(jit_class_gpr);
1249 movi(rn(reg), (i0 >> 32) & 0xffffffff);
1250 movi(r0, i0 & 0xffffffff);
1251 lshi(rn(reg), rn(reg), 32);
1252 OR(rn(reg), r0, r0);
1253 jit_unget_reg(reg);
1254 }
1255 else {
1256# endif
1257 SETHI(HI((int)i0), r0);
1258 if (LO(i0))
1259 ORI(r0, LO(i0), r0);
1260# if __WORDSIZE == 64
1261 }
1262# endif
1263 }
1264}
1265
1266static jit_word_t
1267_movi_p(jit_state_t *_jit, jit_int32_t r0, jit_word_t i0)
1268{
1269 jit_word_t w;
1270# if __WORDSIZE == 64
1271 jit_int32_t reg;
1272# endif
1273 w = _jit->pc.w;
1274# if __WORDSIZE == 64
1275 reg = jit_get_reg(jit_class_gpr);
1276 SETHI(HI((int)i0), r0);
1277 ORI(r0, LO(i0), r0);
1278 i0 = (int)(i0 >> 32);
1279 SETHI(HI(i0), rn(reg));
1280 ORI(rn(reg), LO(i0), rn(reg));
1281 SLLXI(rn(reg), 32, rn(reg));
1282 OR(rn(reg), r0, r0);
1283 jit_unget_reg(reg);
1284# else
1285 SETHI(HI(i0), r0);
1286 ORI(r0, LO(i0), r0);
1287# endif
1288 return (w);
1289}
1290
40a44dcb
PC		 1291static void
1292_movnr(jit_state_t *_jit, jit_int32_t r0, jit_int32_t r1, jit_int32_t r2)
1293{
1294 jit_word_t w;
1295 w = beqi(_jit->pc.w, r2, 0);
1296 ORI(r1, 0, r0);
1297 patch_at(w, _jit->pc.w);
1298}
1299
1300static void
1301_movzr(jit_state_t *_jit, jit_int32_t r0, jit_int32_t r1, jit_int32_t r2)
1302{
1303 jit_word_t w;
1304 w = bnei(_jit->pc.w, r2, 0);
1305 ORI(r1, 0, r0);
1306 patch_at(w, _jit->pc.w);
1307}
1308
ba3814c1
PC		 1309static void
1310_casx(jit_state_t *_jit, jit_int32_t r0, jit_int32_t r1,
1311 jit_int32_t r2, jit_int32_t r3, jit_word_t i0)
1312{
c0c16242
PC		 1313 jit_int32_t iscasi, r1_reg;
1314 if ((iscasi = (r1 == _NOREG))) {
1315 r1_reg = jit_get_reg(jit_class_gpr);
1316 r1 = rn(r1_reg);
1317 movi(r1, i0);
1318 }
1319 /* Do not clobber r2 */
1320 movr(r0, r3);
1321 /* The CASXA instruction compares the value in register r[rs2] with
1322 * the doubleword in memory pointed to by the doubleword address in
1323 * r[rs1]. If the values are equal, the value in r[rd] is swapped
1324 * with the doubleword pointed to by the doubleword address in r[rs1].
1325 * If the values are not equal, the contents of the doubleword pointed
1326 * to by r[rs1] replaces the value in r[rd], but the memory location
1327 * remains unchanged.
1328 */
1329# if __WORDSIZE == 32
1330 CASA(r1, r2, r0);
1331# else
1332 CASXA(r1, r2, r0);
1333# endif
1334 eqr(r0, r0, r2);
1335 if (iscasi)
1336 jit_unget_reg(r1_reg);
ba3814c1
PC		 1337}
1338
79bfeef6
PC		 1339static void
1340_clor(jit_state_t *_jit, jit_int32_t r0, jit_int32_t r1)
1341{
1342 if (jit_cpu.lzcnt) {
1343 comr(r0, r1);
1344 clzr(r0, r0);
1345 }
1346 else
1347 fallback_clo(r0, r1);
1348}
1349
1350static void
1351_clzr(jit_state_t *_jit, jit_int32_t r0, jit_int32_t r1)
1352{
ba86ff93 1353# if CHECK_LZCNT
79bfeef6 1354 if (jit_cpu.lzcnt) {
ba86ff93 1355# if __WORDSIZE == 32
79bfeef6
PC		 1356 jit_word_t w;
1357 SLLXI(r1, 32, r0);
1358 LZCNT(r0, r0);
79bfeef6
PC		 1359 w = blei(_jit->pc.w, r0, 31);
1360 rshi(r0, r0, 1); /* r0 is 64 */
1361 patch_at(w, _jit->pc.w);
ba86ff93 1362# else
79bfeef6 1363 LZCNT(r1, r0);
ba86ff93 1364# endif
79bfeef6
PC		 1365 }
1366 else
ba86ff93 1367# endif
79bfeef6
PC		 1368 fallback_clz(r0, r1);
1369}
1370
1371static void
1372_ctor(jit_state_t *_jit, jit_int32_t r0, jit_int32_t r1)
1373{
1374 if (jit_cpu.lzcnt) {
ba86ff93
PC		 1375 comr(r0, r1);
1376 ctzr(r0, r0);
79bfeef6
PC		 1377 }
1378 else
1379 fallback_cto(r0, r1);
1380}
1381
1382static void
1383_ctzr(jit_state_t *_jit, jit_int32_t r0, jit_int32_t r1)
1384{
ba86ff93 1385 jit_int32_t t0, t1;
79bfeef6 1386 if (jit_cpu.lzcnt) {
ba86ff93
PC		 1387 t0 = jit_get_reg(jit_class_gpr);
1388 t1 = jit_get_reg(jit_class_gpr);
1389 negr(rn(t0), r1);
1390 andr(rn(t0), rn(t0), r1);
1391 clzr(r0, rn(t0));
1392 xori(rn(t1), r0, __WORDSIZE - 1);
1393 movnr(r0, rn(t1), rn(t0));
1394 jit_unget_reg(t0);
1395 jit_unget_reg(t1);
79bfeef6
PC		 1396 }
1397 else
1398 fallback_ctz(r0, r1);
1399}
1400
4a71579b
PC		 1401static void
1402_addi(jit_state_t *_jit, jit_int32_t r0, jit_int32_t r1, jit_word_t i0)
1403{
1404 jit_int32_t reg;
1405 if (s13_p(i0))
1406 ADDI(r1, i0, r0);
1407 else {
1408 reg = jit_get_reg(jit_class_gpr);
1409 movi(rn(reg), i0);
1410 addr(r0, r1, rn(reg));
1411 jit_unget_reg(reg);
1412 }
1413}
1414
1415# if __WORDSIZE == 64
1416static void
1417_addcr(jit_state_t *_jit, jit_int32_t r0, jit_int32_t r1, jit_int32_t r2)
1418{
1419 jit_int32_t reg;
1420 if (jit_carry == _NOREG)
1421 jit_carry = jit_get_reg(jit_class_gpr);
1422 if (r0 == r1) {
1423 reg = jit_get_reg(jit_class_gpr);
1424 addr(rn(reg), r1, r2);
1425 ltr_u(rn(jit_carry), rn(reg), r1);
1426 movr(r0, rn(reg));
1427 jit_unget_reg(reg);
1428 }
1429 else {
1430 addr(r0, r1, r2);
1431 ltr_u(rn(jit_carry), r0, r1);
1432 }
1433}
1434# endif
1435
1436static void
1437_addci(jit_state_t *_jit, jit_int32_t r0, jit_int32_t r1, jit_word_t i0)
1438{
1439# if __WORDSIZE == 32
1440 jit_int32_t reg;
1441 if (s13_p(i0))
1442 ADDIcc(r1, i0, r0);
1443 else {
1444 reg = jit_get_reg(jit_class_gpr);
1445 movi(rn(reg), i0);
1446 addcr(r0, r1, rn(reg));
1447 jit_unget_reg(reg);
1448 }
1449# else
1450 jit_int32_t reg;
1451 if (jit_carry == _NOREG)
1452 jit_carry = jit_get_reg(jit_class_gpr);
1453 if (r0 == r1) {
1454 reg = jit_get_reg(jit_class_gpr);
1455 addi(rn(reg), r1, i0);
1456 ltr_u(rn(jit_carry), rn(reg), r1);
1457 movr(r0, rn(reg));
1458 jit_unget_reg(reg);
1459 }
1460 else {
1461 addi(r0, r1, i0);
1462 ltr_u(rn(jit_carry), r0, r1);
1463 }
1464# endif
1465}
1466
1467# if __WORDSIZE == 64
1468static void
1469_addxr(jit_state_t *_jit, jit_int32_t r0, jit_int32_t r1, jit_int32_t r2)
1470{
1471 jit_int32_t reg;
1472 assert(jit_carry != _NOREG);
1473 reg = jit_get_reg(jit_class_gpr);
1474 movr(rn(reg), rn(jit_carry));
1475 addcr(r0, r1, r2);
1476 addcr(r0, r0, rn(reg));
1477 jit_unget_reg(reg);
1478}
1479# endif
1480
1481static void
1482_addxi(jit_state_t *_jit, jit_int32_t r0, jit_int32_t r1, jit_word_t i0)
1483{
1484# if __WORDSIZE == 32
1485 jit_int32_t reg;
1486 if (s13_p(i0))
1487 ADDXIcc(r1, i0, r0);
1488 else {
1489 reg = jit_get_reg(jit_class_gpr);
1490 movi(rn(reg), i0);
1491 addxr(r0, r1, rn(reg));
1492 jit_unget_reg(reg);
1493 }
1494# else
1495 jit_int32_t reg;
1496 assert(jit_carry != _NOREG);
1497 reg = jit_get_reg(jit_class_gpr);
1498 movr(rn(reg), rn(jit_carry));
1499 addci(r0, r1, i0);
1500 addcr(r0, r0, rn(reg));
1501 jit_unget_reg(reg);
1502# endif
1503}
1504
1505static void
1506_subi(jit_state_t *_jit, jit_int32_t r0, jit_int32_t r1, jit_word_t i0)
1507{
1508 jit_int32_t reg;
1509 if (s13_p(i0))
1510 SUBI(r1, i0, r0);
1511 else {
1512 reg = jit_get_reg(jit_class_gpr);
1513 movi(rn(reg), i0);
1514 subr(r0, r1, rn(reg));
1515 jit_unget_reg(reg);
1516 }
1517}
1518
1519# if __WORDSIZE == 64
1520static void
1521_subcr(jit_state_t *_jit, jit_int32_t r0, jit_int32_t r1, jit_int32_t r2)
1522{
1523 jit_int32_t reg;
1524 if (jit_carry == _NOREG)
1525 jit_carry = jit_get_reg(jit_class_gpr);
1526 if (r0 == r1) {
1527 reg = jit_get_reg(jit_class_gpr);
1528 subr(rn(reg), r1, r2);
1529 ltr_u(rn(jit_carry), r1, rn(reg));
1530 movr(r0, rn(reg));
1531 jit_unget_reg(reg);
1532 }
1533 else {
1534 subr(r0, r1, r2);
1535 ltr_u(rn(jit_carry), r1, r0);
1536 }
1537}
1538# endif
1539
1540static void
1541_subci(jit_state_t *_jit, jit_int32_t r0, jit_int32_t r1, jit_word_t i0)
1542{
1543# if __WORDSIZE == 32
1544 jit_int32_t reg;
1545 if (s13_p(i0))
1546 SUBIcc(r1, i0, r0);
1547 else {
1548 reg = jit_get_reg(jit_class_gpr);
1549 movi(rn(reg), i0);
1550 subcr(r0, r1, rn(reg));
1551 jit_unget_reg(reg);
1552 }
1553# else
1554 jit_int32_t reg;
1555 if (jit_carry == _NOREG)
1556 jit_carry = jit_get_reg(jit_class_gpr);
1557 if (r0 == r1) {
1558 reg = jit_get_reg(jit_class_gpr);
1559 addi(rn(reg), r1, -i0);
1560 ltr_u(rn(jit_carry), r1, rn(reg));
1561 movr(r0, rn(reg));
1562 jit_unget_reg(reg);
1563 }
1564 else {
1565 addi(r0, r1, -i0);
1566 ltr_u(rn(jit_carry), r1, r0);
1567 }
1568# endif
1569}
1570
1571# if __WORDSIZE == 64
1572static void
1573_subxr(jit_state_t *_jit, jit_int32_t r0, jit_int32_t r1, jit_int32_t r2)
1574{
1575 jit_int32_t reg;
1576 assert(jit_carry != _NOREG);
1577 reg = jit_get_reg(jit_class_gpr);
1578 movr(rn(reg), rn(jit_carry));
1579 subcr(r0, r1, r2);
1580 subcr(r0, r0, rn(reg));
1581 jit_unget_reg(reg);
1582}
1583#endif
1584
1585static void
1586_subxi(jit_state_t *_jit, jit_int32_t r0, jit_int32_t r1, jit_word_t i0)
1587{
1588# if __WORDSIZE == 32
1589 jit_int32_t reg;
1590 if (s13_p(i0))
1591 SUBXIcc(r1, i0, r0);
1592 else {
1593 reg = jit_get_reg(jit_class_gpr);
1594 movi(rn(reg), i0);
1595 subxr(r0, r1, rn(reg));
1596 jit_unget_reg(reg);
1597 }
1598# else
1599 jit_int32_t reg;
1600 assert(jit_carry != _NOREG);
1601 reg = jit_get_reg(jit_class_gpr);
1602 movr(rn(reg), rn(jit_carry));
1603 subci(r0, r1, i0);
1604 subcr(r0, r0, rn(reg));
1605 jit_unget_reg(reg);
1606# endif
1607}
1608
1609static void
1610_rsbi(jit_state_t *_jit, jit_int32_t r0, jit_int32_t r1, jit_word_t i0)
1611{
1612 subi(r0, r1, i0);
1613 negr(r0, r0);
1614}
1615
1616static void
1617_muli(jit_state_t *_jit, jit_int32_t r0, jit_int32_t r1, jit_word_t i0)
1618{
1619 jit_int32_t reg;
1620 if (s13_p(i0)) {
1621# if __WORDSIZE == 32
1622 UMULI(r1, i0, r0);
1623# else
1624 MULXI(r1, i0, r0);
1625# endif
1626 }
1627 else {
1628 reg = jit_get_reg(jit_class_gpr);
1629 movi(rn(reg), i0);
1630 mulr(r0, r1, rn(reg));
1631 jit_unget_reg(reg);
1632 }
1633}
1634
1635# if __WORDSIZE == 32
1636static void
1637_iqmulr(jit_state_t *_jit, jit_int32_t r0, jit_int32_t r1,
1638 jit_int32_t r2, jit_int32_t r3, jit_bool_t sign)
1639{
ba86ff93
PC		 1640 if (r0 == JIT_NOREG)
1641 r0 = r1;
4a71579b
PC		 1642 if (sign)
1643 SMUL(r2, r3, r0);
1644 else
1645 UMUL(r2, r3, r0);
1646 RDY(r1);
1647}
1648
1649static void
1650_iqmuli(jit_state_t *_jit, jit_int32_t r0, jit_int32_t r1,
1651 jit_int32_t r2, jit_word_t i0, jit_bool_t sign)
1652{
1653 jit_int32_t reg;
1654 if (s13_p(i0)) {
ba86ff93
PC		 1655 if (r0 == JIT_NOREG)
1656 r0 = r1;
4a71579b
PC		 1657 if (sign)
1658 SMULI(r2, i0, r0);
1659 else
1660 UMULI(r2, i0, r0);
1661 RDY(r1);
1662 }
1663 else {
1664 reg = jit_get_reg(jit_class_gpr);
1665 movi(rn(reg), i0);
1666 iqmulr(r0, r1, r2, rn(reg), sign);
1667 jit_unget_reg(reg);
1668 }
1669}
1670
1671# else
1672static __int128_t __llmul(jit_word_t a, jit_word_t b)
1673{
1674 return (__int128_t)a * (__int128_t)b;
1675}
1676
1677# define QMUL_PROLOG() \
1678 do { \
1679 (void)jit_get_reg(_O0|jit_class_gpr|jit_class_named); \
1680 (void)jit_get_reg(_O1|jit_class_gpr|jit_class_named); \
1681 if (r0 != _G2_REGNO && r1 != _G2_REGNO) \
1682 stxi(BIAS(-8), _FP_REGNO, _G2_REGNO); \
1683 if (r0 != _G3_REGNO && r1 != _G3_REGNO) \
1684 stxi(BIAS(-16), _FP_REGNO, _G3_REGNO); \
1685 if (r0 != _G4_REGNO && r1 != _G4_REGNO) \
1686 stxi(BIAS(-24), _FP_REGNO, _G4_REGNO); \
1687 } while (0)
1688
1689# define QMUL_EPILOG() \
1690 do { \
1691 if (r0 != _G2_REGNO && r1 != _G2_REGNO) \
1692 ldxi(_G2_REGNO, _FP_REGNO, BIAS(-8)); \
1693 if (r0 != _G3_REGNO && r1 != _G3_REGNO) \
1694 ldxi(_G3_REGNO, _FP_REGNO, BIAS(-16)); \
1695 if (r0 != _G4_REGNO && r1 != _G4_REGNO) \
1696 ldxi(_G4_REGNO, _FP_REGNO, BIAS(-24)); \
1697 (void)jit_unget_reg(_O0); \
1698 (void)jit_unget_reg(_O1); \
1699 } while (0)
1700
1701static void
1702_qmulr(jit_state_t *_jit, jit_int32_t r0, jit_int32_t r1,
1703 jit_int32_t r2, jit_int32_t r3)
1704{
1705 QMUL_PROLOG();
1706 movr(_O0_REGNO, r3);
1707 movr(_O1_REGNO, r2);
1708 calli((jit_word_t)__llmul);
ba86ff93
PC		 1709 if (r0 != JIT_NOREG)
1710 movr(r0, _O1_REGNO);
4a71579b
PC		 1711 movr(r1, _O0_REGNO);
1712 QMUL_EPILOG();
1713}
1714
1715static void
1716_qmuli(jit_state_t *_jit, jit_int32_t r0, jit_int32_t r1,
1717 jit_int32_t r2, jit_word_t i0)
1718{
1719 QMUL_PROLOG();
1720 movi(_O0_REGNO, i0);
1721 movr(_O1_REGNO, r2);
1722 calli((jit_word_t)__llmul);
ba86ff93
PC		 1723 if (r0 != JIT_NOREG)
1724 movr(r0, _O1_REGNO);
4a71579b
PC		 1725 movr(r1, _O0_REGNO);
1726 QMUL_EPILOG();
1727}
1728
1729static __uint128_t __ullmul(jit_uword_t a, jit_uword_t b)
1730{
1731 return (__uint128_t)a * (__uint128_t)b;
1732}
1733
1734static void
1735_qmulr_u(jit_state_t *_jit, jit_int32_t r0, jit_int32_t r1,
1736 jit_int32_t r2, jit_int32_t r3)
1737{
1738 QMUL_PROLOG();
1739 movr(_O0_REGNO, r3);
1740 movr(_O1_REGNO, r2);
1741 calli((jit_word_t)__ullmul);
ba86ff93
PC		 1742 if (r0 != JIT_NOREG)
1743 movr(r0, _O1_REGNO);
4a71579b
PC		 1744 movr(r1, _O0_REGNO);
1745 QMUL_EPILOG();
1746}
1747
1748static void
1749_qmuli_u(jit_state_t *_jit, jit_int32_t r0, jit_int32_t r1,
1750 jit_int32_t r2, jit_word_t i0)
1751{
1752 QMUL_PROLOG();
1753 movi(_O0_REGNO, i0);
1754 movr(_O1_REGNO, r2);
1755 calli((jit_word_t)__ullmul);
ba86ff93
PC		 1756 if (r0 != JIT_NOREG)
1757 movr(r0, _O1_REGNO);
4a71579b
PC		 1758 movr(r1, _O0_REGNO);
1759 QMUL_EPILOG();
1760}
1761# endif
1762
1763static void
1764_divr(jit_state_t *_jit, jit_int32_t r0, jit_int32_t r1, jit_int32_t r2)
1765{
1766# if __WORDSIZE == 32
1767 jit_int32_t reg;
1768 reg = jit_get_reg(jit_class_gpr);
1769 rshi(rn(reg), r1, 31);
1770 WRY(rn(reg), 0);
1771 SDIV(r1, r2, r0);
1772 jit_unget_reg(reg);
1773# else
1774 SDIVX(r1, r2, r0);
1775# endif
1776}
1777
1778static void
1779_divi(jit_state_t *_jit, jit_int32_t r0, jit_int32_t r1, jit_word_t i0)
1780{
1781 jit_int32_t reg;
1782# if __WORDSIZE == 32
1783 reg = jit_get_reg(jit_class_gpr);
1784# endif
1785 if (s13_p(i0)) {
1786# if __WORDSIZE == 32
1787 rshi(rn(reg), r1, 31);
1788 WRY(rn(reg), 0);
1789 SDIVI(r1, i0, r0);
1790# else
1791 SDIVXI(r1, i0, r0);
1792# endif
1793 }
1794 else {
1795# if __WORDSIZE == 64
1796 reg = jit_get_reg(jit_class_gpr);
1797# endif
1798 movi(rn(reg), i0);
1799 divr(r0, r1, rn(reg));
1800# if __WORDSIZE == 64
1801 jit_unget_reg(reg);
1802# endif
1803 }
1804# if __WORDSIZE == 32
1805 jit_unget_reg(reg);
1806# endif
1807}
1808
1809static void
1810_divr_u(jit_state_t *_jit, jit_int32_t r0, jit_int32_t r1, jit_int32_t r2)
1811{
1812# if __WORDSIZE == 32
1813 WRYI(0, 0);
1814 UDIV(r1, r2, r0);
1815# else
1816 UDIVX(r1, r2, r0);
1817# endif
1818}
1819
1820static void
1821_divi_u(jit_state_t *_jit, jit_int32_t r0, jit_int32_t r1, jit_word_t i0)
1822{
1823 jit_int32_t reg;
1824 if (s13_p(i0)) {
1825# if __WORDSIZE == 32
1826 WRYI(0, 0);
1827 UDIVI(r1, i0, r0);
1828# else
1829 UDIVXI(r1, i0, r0);
1830# endif
1831 }
1832 else {
1833 reg = jit_get_reg(jit_class_gpr);
1834 movi(rn(reg), i0);
1835 divr_u(r0, r1, rn(reg));
1836 jit_unget_reg(reg);
1837 }
1838}
1839
1840static void
1841_iqdivr(jit_state_t *_jit, jit_int32_t r0, jit_int32_t r1,
1842 jit_int32_t r2, jit_int32_t r3, jit_bool_t sign)
1843{
1844 jit_int32_t sv0, rg0;
1845 jit_int32_t sv1, rg1;
1846
1847 if (r0 == r2 || r0 == r3) {
1848 sv0 = jit_get_reg(jit_class_gpr);
1849 rg0 = rn(sv0);
1850 }
1851 else
1852 rg0 = r0;
1853 if (r1 == r2 || r1 == r3) {
1854 sv1 = jit_get_reg(jit_class_gpr);
1855 rg1 = rn(sv1);
1856 }
1857 else
1858 rg1 = r1;
1859
1860 if (sign)
1861 divr(rg0, r2, r3);
1862 else
1863 divr_u(rg0, r2, r3);
1864 mulr(rg1, r3, rg0);
1865 subr(rg1, r2, rg1);
1866 if (rg0 != r0) {
1867 movr(r0, rg0);
1868 jit_unget_reg(sv0);
1869 }
1870 if (rg1 != r1) {
1871 movr(r1, rg1);
1872 jit_unget_reg(sv1);
1873 }
1874}
1875
1876static void
1877_iqdivi(jit_state_t *_jit, jit_int32_t r0, jit_int32_t r1,
1878 jit_int32_t r2, jit_word_t i0, jit_bool_t sign)
1879{
1880 jit_int32_t reg;
1881 reg = jit_get_reg(jit_class_gpr);
1882 movi(rn(reg), i0);
1883 iqdivr(r0, r1, r2, rn(reg), sign);
1884 jit_unget_reg(reg);
1885}
1886
1887static void
1888_remr(jit_state_t *_jit, jit_int32_t r0, jit_int32_t r1, jit_int32_t r2)
1889{
1890 jit_int32_t reg;
1891 if (r0 == r1 || r0 == r2) {
1892 reg = jit_get_reg(jit_class_gpr);
1893 divr(rn(reg), r1, r2);
1894 mulr(rn(reg), r2, rn(reg));
1895 subr(r0, r1, rn(reg));
1896 jit_unget_reg(reg);
1897 }
1898 else {
1899 divr(r0, r1, r2);
1900 mulr(r0, r2, r0);
1901 subr(r0, r1, r0);
1902 }
1903}
1904
1905static void
1906_remi(jit_state_t *_jit, jit_int32_t r0, jit_int32_t r1, jit_word_t i0)
1907{
1908 jit_int32_t reg;
1909 reg = jit_get_reg(jit_class_gpr);
1910 movi(rn(reg), i0);
1911 remr(r0, r1, rn(reg));
1912 jit_unget_reg(reg);
1913}
1914
1915static void
1916_remr_u(jit_state_t *_jit, jit_int32_t r0, jit_int32_t r1, jit_int32_t r2)
1917{
1918 jit_int32_t reg;
1919 if (r0 == r1 || r0 == r2) {
1920 reg = jit_get_reg(jit_class_gpr);
1921 divr_u(rn(reg), r1, r2);
1922 mulr(rn(reg), r2, rn(reg));
1923 subr(r0, r1, rn(reg));
1924 jit_unget_reg(reg);
1925 }
1926 else {
1927 divr_u(r0, r1, r2);
1928 mulr(r0, r2, r0);
1929 subr(r0, r1, r0);
1930 }
1931}
1932
1933static void
1934_remi_u(jit_state_t *_jit, jit_int32_t r0, jit_int32_t r1, jit_word_t i0)
1935{
1936 jit_int32_t reg;
1937 reg = jit_get_reg(jit_class_gpr);
1938 movi(rn(reg), i0);
1939 remr_u(r0, r1, rn(reg));
1940 jit_unget_reg(reg);
1941}
1942
1943static void
1944_andi(jit_state_t *_jit, jit_int32_t r0, jit_int32_t r1, jit_word_t i0)
1945{
1946 jit_int32_t reg;
1947 if (s13_p(i0))
1948 ANDI(r1, i0, r0);
1949 else {
1950 reg = jit_get_reg(jit_class_gpr);
1951 movi(rn(reg), i0);
1952 andr(r0, r1, rn(reg));
1953 jit_unget_reg(reg);
1954 }
1955}
1956
1957static void
1958_ori(jit_state_t *_jit, jit_int32_t r0, jit_int32_t r1, jit_word_t i0)
1959{
1960 jit_int32_t reg;
1961 if (s13_p(i0))
1962 ORI(r1, i0, r0);
1963 else {
1964 reg = jit_get_reg(jit_class_gpr);
1965 movi(rn(reg), i0);
1966 orr(r0, r1, rn(reg));
1967 jit_unget_reg(reg);
1968 }
1969}
1970
1971static void
1972_xori(jit_state_t *_jit, jit_int32_t r0, jit_int32_t r1, jit_word_t i0)
1973{
1974 jit_int32_t reg;
1975 if (s13_p(i0))
1976 XORI(r1, i0, r0);
1977 else {
1978 reg = jit_get_reg(jit_class_gpr);
1979 movi(rn(reg), i0);
1980 xorr(r0, r1, rn(reg));
1981 jit_unget_reg(reg);
1982 }
1983}
1984
1985static void
1986_extr_c(jit_state_t *_jit, jit_int32_t r0, jit_int32_t r1)
1987{
1988 lshi(r0, r1, __WORDSIZE - 8);
1989 rshi(r0, r0, __WORDSIZE - 8);
1990}
1991
1992static void
1993_extr_s(jit_state_t *_jit, jit_int32_t r0, jit_int32_t r1)
1994{
1995 lshi(r0, r1, __WORDSIZE - 16);
1996 rshi(r0, r0, __WORDSIZE - 16);
1997}
1998
1999static void
2000_extr_us(jit_state_t *_jit, jit_int32_t r0, jit_int32_t r1)
2001{
2002 lshi(r0, r1, __WORDSIZE - 16);
2003 rshi_u(r0, r0, __WORDSIZE - 16);
2004}
2005
2006#if __WORDSIZE == 64
2007static void
2008_extr_i(jit_state_t *_jit, jit_int32_t r0, jit_int32_t r1)
2009{
2010 lshi(r0, r1, __WORDSIZE - 32);
2011 rshi(r0, r0, __WORDSIZE - 32);
2012}
2013
2014static void
2015_extr_ui(jit_state_t *_jit, jit_int32_t r0, jit_int32_t r1)
2016{
2017 lshi(r0, r1, __WORDSIZE - 32);
2018 rshi_u(r0, r0, __WORDSIZE - 32);
2019}
2020#endif
2021
2022static void
2023_cr(jit_state_t *_jit, jit_int32_t cc,
2024 jit_int32_t r0, jit_int32_t r1, jit_int32_t r2)
2025{
2026 CMP(r1, r2);
2027# if __WORDSIZE == 32
2028 Ba(cc, 3);
2029# else
2030 BPa(cc, 3);
2031# endif
2032 movi(r0, 1);
2033 movi(r0, 0);
2034}
2035
2036static void
2037_cw(jit_state_t *_jit, jit_int32_t cc,
2038 jit_int32_t r0, jit_int32_t r1, jit_word_t i0)
2039{
2040 jit_int32_t reg;
2041 if (s13_p(i0)) {
2042 CMPI(r1, i0);
2043# if __WORDSIZE == 32
2044 Ba(cc, 3);
2045# else
2046 BPa(cc, 3);
2047# endif
2048 movi(r0, 1);
2049 movi(r0, 0);
2050 }
2051 else {
2052 reg = jit_get_reg(jit_class_gpr);
2053 movi(rn(reg), i0);
2054 cr(cc, r0, r1, rn(reg));
2055 jit_unget_reg(reg);
2056 }
2057}
2058
2059static void
2060_ldi_c(jit_state_t *_jit, jit_int32_t r0, jit_word_t i0)
2061{
2062 jit_int32_t reg;
2063 if (s13_p(i0))
2064 LDSBI(0, i0, r0);
2065 else {
2066 reg = jit_get_reg(jit_class_gpr);
2067 movi(rn(reg), i0);
2068 ldr_c(r0, rn(reg));
2069 jit_unget_reg(reg);
2070 }
2071}
2072
2073static void
2074_ldi_uc(jit_state_t *_jit, jit_int32_t r0, jit_word_t i0)
2075{
2076 jit_int32_t reg;
2077 if (s13_p(i0))
2078 LDUBI(0, i0, r0);
2079 else {
2080 reg = jit_get_reg(jit_class_gpr);
2081 movi(rn(reg), i0);
2082 ldr_uc(r0, rn(reg));
2083 jit_unget_reg(reg);
2084 }
2085}
2086
2087static void
2088_ldi_s(jit_state_t *_jit, jit_int32_t r0, jit_word_t i0)
2089{
2090 jit_int32_t reg;
2091 if (s13_p(i0))
2092 LDSHI(0, i0, r0);
2093 else {
2094 reg = jit_get_reg(jit_class_gpr);
2095 movi(rn(reg), i0);
2096 ldr_s(r0, rn(reg));
2097 jit_unget_reg(reg);
2098 }
2099}
2100
2101static void
2102_ldi_us(jit_state_t *_jit, jit_int32_t r0, jit_word_t i0)
2103{
2104 jit_int32_t reg;
2105 if (s13_p(i0))
2106 LDUHI(0, i0, r0);
2107 else {
2108 reg = jit_get_reg(jit_class_gpr);
2109 movi(rn(reg), i0);
2110 ldr_us(r0, rn(reg));
2111 jit_unget_reg(reg);
2112 }
2113}
2114
2115static void
2116_ldi_i(jit_state_t *_jit, jit_int32_t r0, jit_word_t i0)
2117{
2118 jit_int32_t reg;
2119 if (s13_p(i0)) {
2120# if __WORDSIZE == 32
2121 LDI(0, i0, r0);
2122# else
2123 LDSWI(0, i0, r0);
2124# endif
2125 }
2126 else {
2127 reg = jit_get_reg(jit_class_gpr);
2128 movi(rn(reg), i0);
2129 ldr_i(r0, rn(reg));
2130 jit_unget_reg(reg);
2131 }
2132}
2133
2134# if __WORDSIZE == 64
2135static void
2136_ldi_ui(jit_state_t *_jit, jit_int32_t r0, jit_word_t i0)
2137{
2138 jit_int32_t reg;
2139 if (s13_p(i0))
2140 LDUWI(0, i0, r0);
2141 else {
2142 reg = jit_get_reg(jit_class_gpr);
2143 movi(rn(reg), i0);
2144 ldr_ui(r0, rn(reg));
2145 jit_unget_reg(reg);
2146 }
2147}
2148
2149static void
2150_ldi_l(jit_state_t *_jit, jit_int32_t r0, jit_word_t i0)
2151{
2152 jit_int32_t reg;
2153 if (s13_p(i0))
2154 LDXI(0, i0, r0);
2155 else {
2156 reg = jit_get_reg(jit_class_gpr);
2157 movi(rn(reg), i0);
2158 ldr_l(r0, rn(reg));
2159 jit_unget_reg(reg);
2160 }
2161}
2162# endif
2163
2164static void
2165_ldxi_c(jit_state_t *_jit, jit_int32_t r0, jit_int32_t r1, jit_word_t i0)
2166{
2167 jit_int32_t reg;
2168 if (s13_p(i0))
2169 LDSBI(r1, i0, r0);
2170 else {
2171 reg = jit_get_reg(jit_class_gpr);
2172 movi(rn(reg), i0);
2173 ldxr_c(r0, r1, rn(reg));
2174 jit_unget_reg(reg);
2175 }
2176}
2177
2178static void
2179_ldxi_uc(jit_state_t *_jit, jit_int32_t r0, jit_int32_t r1, jit_word_t i0)
2180{
2181 jit_int32_t reg;
2182 if (s13_p(i0))
2183 LDUBI(r1, i0, r0);
2184 else {
2185 reg = jit_get_reg(jit_class_gpr);
2186 movi(rn(reg), i0);
2187 ldxr_uc(r0, r1, rn(reg));
2188 jit_unget_reg(reg);
2189 }
2190}
2191
2192static void
2193_ldxi_s(jit_state_t *_jit, jit_int32_t r0, jit_int32_t r1, jit_word_t i0)
2194{
2195 jit_int32_t reg;
2196 if (s13_p(i0))
2197 LDSHI(r1, i0, r0);
2198 else {
2199 reg = jit_get_reg(jit_class_gpr);
2200 movi(rn(reg), i0);
2201 ldxr_s(r0, r1, rn(reg));
2202 jit_unget_reg(reg);
2203 }
2204}
2205
2206static void
2207_ldxi_us(jit_state_t *_jit, jit_int32_t r0, jit_int32_t r1, jit_word_t i0)
2208{
2209 jit_int32_t reg;
2210 if (s13_p(i0))
2211 LDUHI(r1, i0, r0);
2212 else {
2213 reg = jit_get_reg(jit_class_gpr);
2214 movi(rn(reg), i0);
2215 ldxr_us(r0, r1, rn(reg));
2216 jit_unget_reg(reg);
2217 }
2218}
2219
2220static void
2221_ldxi_i(jit_state_t *_jit, jit_int32_t r0, jit_int32_t r1, jit_word_t i0)
2222{
2223 jit_int32_t reg;
2224 if (s13_p(i0)) {
2225# if __WORDSIZE == 32
2226 LDI(r1, i0, r0);
2227# else
2228 LDSWI(r1, i0, r0);
2229# endif
2230 }
2231 else {
2232 reg = jit_get_reg(jit_class_gpr);
2233 movi(rn(reg), i0);
2234 ldxr_i(r0, r1, rn(reg));
2235 jit_unget_reg(reg);
2236 }
2237}
2238
2239# if __WORDSIZE == 64
2240static void
2241_ldxi_ui(jit_state_t *_jit, jit_int32_t r0, jit_int32_t r1, jit_word_t i0)
2242{
2243 jit_int32_t reg;
2244 if (s13_p(i0))
2245 LDUWI(r1, i0, r0);
2246 else {
2247 reg = jit_get_reg(jit_class_gpr);
2248 movi(rn(reg), i0);
2249 ldxr_ui(r0, r1, rn(reg));
2250 jit_unget_reg(reg);
2251 }
2252}
2253
2254static void
2255_ldxi_l(jit_state_t *_jit, jit_int32_t r0, jit_int32_t r1, jit_word_t i0)
2256{
2257 jit_int32_t reg;
2258 if (s13_p(i0))
2259 LDXI(r1, i0, r0);
2260 else {
2261 reg = jit_get_reg(jit_class_gpr);
2262 movi(rn(reg), i0);
2263 ldxr_l(r0, r1, rn(reg));
2264 jit_unget_reg(reg);
2265 }
2266}
2267# endif
2268
2269static void
2270_sti_c(jit_state_t *_jit, jit_word_t i0, jit_int32_t r0)
2271{
2272 jit_int32_t reg;
2273 if (s13_p(i0))
2274 STBI(r0, 0, i0);
2275 else {
2276 reg = jit_get_reg(jit_class_gpr);
2277 movi(rn(reg), i0);
2278 str_c(rn(reg), r0);
2279 jit_unget_reg(reg);
2280 }
2281}
2282
2283static void
2284_sti_s(jit_state_t *_jit, jit_word_t i0, jit_int32_t r0)
2285{
2286 jit_int32_t reg;
2287 if (s13_p(i0))
2288 STHI(r0, 0, i0);
2289 else {
2290 reg = jit_get_reg(jit_class_gpr);
2291 movi(rn(reg), i0);
2292 str_s(rn(reg), r0);
2293 jit_unget_reg(reg);
2294 }
2295}
2296
2297static void
2298_sti_i(jit_state_t *_jit, jit_word_t i0, jit_int32_t r0)
2299{
2300 jit_int32_t reg;
2301 if (s13_p(i0)) {
2302# if __WORDSIZE == 32
2303 STI(r0, 0, i0);
2304# else
2305 STWI(r0, 0, i0);
2306# endif
2307 }
2308 else {
2309 reg = jit_get_reg(jit_class_gpr);
2310 movi(rn(reg), i0);
2311 str_i(rn(reg), r0);
2312 jit_unget_reg(reg);
2313 }
2314}
2315
2316# if __WORDSIZE == 64
2317static void
2318_sti_l(jit_state_t *_jit, jit_word_t i0, jit_int32_t r0)
2319{
2320 jit_int32_t reg;
2321 if (s13_p(i0))
2322 STXI(r0, 0, i0);
2323 else {
2324 reg = jit_get_reg(jit_class_gpr);
2325 movi(rn(reg), i0);
2326 str_l(rn(reg), r0);
2327 jit_unget_reg(reg);
2328 }
2329}
2330# endif
2331
2332static void
2333_stxi_c(jit_state_t *_jit, jit_word_t i0, jit_int32_t r0, jit_int32_t r1)
2334{
2335 jit_int32_t reg;
2336 if (s13_p(i0))
2337 STBI(r1, r0, i0);
2338 else {
2339 reg = jit_get_reg(jit_class_gpr);
2340 movi(rn(reg), i0);
2341 stxr_c(r0, rn(reg), r1);
2342 jit_unget_reg(reg);
2343 }
2344}
2345
2346static void
2347_stxi_s(jit_state_t *_jit, jit_word_t i0, jit_int32_t r0, jit_int32_t r1)
2348{
2349 jit_int32_t reg;
2350 if (s13_p(i0))
2351 STHI(r1, r0, i0);
2352 else {
2353 reg = jit_get_reg(jit_class_gpr);
2354 movi(rn(reg), i0);
2355 stxr_s(r0, rn(reg), r1);
2356 jit_unget_reg(reg);
2357 }
2358}
2359
2360static void
2361_stxi_i(jit_state_t *_jit, jit_word_t i0, jit_int32_t r0, jit_int32_t r1)
2362{
2363 jit_int32_t reg;
2364 if (s13_p(i0)) {
2365# if __WORDSIZE == 32
2366 STI(r1, r0, i0);
2367# else
2368 STWI(r1, r0, i0);
2369# endif
2370 }
2371 else {
2372 reg = jit_get_reg(jit_class_gpr);
2373 movi(rn(reg), i0);
2374 stxr_i(r0, rn(reg), r1);
2375 jit_unget_reg(reg);
2376 }
2377}
2378
2379# if __WORDSIZE == 64
2380static void
2381_stxi_l(jit_state_t *_jit, jit_word_t i0, jit_int32_t r0, jit_int32_t r1)
2382{
2383 jit_int32_t reg;
2384 if (s13_p(i0))
2385 STXI(r1, r0, i0);
2386 else {
2387 reg = jit_get_reg(jit_class_gpr);
2388 movi(rn(reg), i0);
2389 stxr_l(r0, rn(reg), r1);
2390 jit_unget_reg(reg);
2391 }
2392}
2393# endif
2394
2395static jit_word_t
2396_br(jit_state_t *_jit, jit_int32_t cc,
2397 jit_word_t i0, jit_int32_t r0, jit_int32_t r1)
2398{
2399 jit_word_t w;
2400 CMP(r0, r1);
2401 w = _jit->pc.w;
2402# if __WORDSIZE == 32
2403 B(cc, (i0 - w) >> 2);
2404# else
2405 BP(cc, (i0 - w) >> 2);
2406# endif
2407 NOP();
2408 return (w);
2409}
2410
2411static jit_word_t
2412_bw(jit_state_t *_jit, jit_int32_t cc,
2413 jit_word_t i0, jit_int32_t r0, jit_word_t i1)
2414{
2415 jit_word_t w;
2416 jit_int32_t reg;
2417 if (s13_p(i1)) {
2418 CMPI(r0, i1);
2419 w = _jit->pc.w;
2420# if __WORDSIZE == 32
2421 B(cc, (i0 - w) >> 2);
2422# else
79bfeef6 2423 BP(cc, (i0 - w) >> 2);
4a71579b
PC		 2424# endif
2425 NOP();
2426 }
2427 else {
2428 reg = jit_get_reg(jit_class_gpr|jit_class_nospill);
2429 movi(rn(reg), i1);
2430 w = br(cc, i0, r0, rn(reg));
2431 jit_unget_reg(reg);
2432 }
2433 return (w);
2434}
2435
2436static jit_word_t
2437_b_asr(jit_state_t *_jit, jit_bool_t jif, jit_bool_t add, jit_bool_t sgn,
2438 jit_word_t i0, jit_int32_t r0, jit_int32_t r1)
2439{
2440 jit_word_t w;
2441 if (add)
2442 ADDcc(r0, r1, r0);
2443 else
2444 SUBcc(r0, r1, r0);
2445 w = _jit->pc.w;
2446# if __WORDSIZE == 32
2447 B(sgn ?
2448 (jif ? SPARC_BVS : SPARC_BVC) :
2449 (jif ? SPARC_BCS : SPARC_BCC),
2450 (i0 - w) >> 2);
2451# else
2452 BP(sgn ?
2453 (jif ? SPARC_BPVS : SPARC_BPVC) :
2454 (jif ? SPARC_BPCS : SPARC_BPCC),
2455 (i0 - w) >> 2);
2456# endif
2457 NOP();
2458 return (w);
2459}
2460
2461static jit_word_t
2462_b_asw(jit_state_t *_jit, jit_bool_t jif, jit_bool_t add, jit_bool_t sgn,
2463 jit_word_t i0, jit_int32_t r0, jit_word_t i1)
2464{
2465 jit_word_t w;
2466 jit_int32_t reg;
2467 if (s13_p(i1)) {
2468 if (add)
2469 ADDIcc(r0, i1, r0);
2470 else
2471 SUBIcc(r0, i1, r0);
2472 w = _jit->pc.w;
2473# if __WORDSIZE == 32
2474 B(sgn ?
2475 (jif ? SPARC_BVS : SPARC_BVC) :
2476 (jif ? SPARC_BCS : SPARC_BCC),
2477 (i0 - w) >> 2);
2478# else
2479 BP(sgn ?
2480 (jif ? SPARC_BPVS : SPARC_BPVC) :
2481 (jif ? SPARC_BPCS : SPARC_BPCC),
2482 (i0 - w) >> 2);
2483# endif
2484 NOP();
2485 }
2486 else {
2487 reg = jit_get_reg(jit_class_gpr|jit_class_nospill);
2488 movi(rn(reg), i1);
2489 w = b_asr(jif, add, sgn, i0, r0, rn(reg));
2490 jit_unget_reg(reg);
2491 }
2492 return (w);
2493}
2494
2495static jit_word_t
2496_bm_r(jit_state_t *_jit, jit_bool_t set,
2497 jit_word_t i0, jit_int32_t r0, jit_int32_t r1)
2498{
2499 jit_word_t w;
2500 BTST(r0, r1);
2501 w = _jit->pc.w;
2502# if __WORDSIZE == 32
2503 B(set ? SPARC_BNZ : SPARC_BZ, (i0 - w) >> 2);
2504# else
2505 BP(set ? SPARC_BPNE : SPARC_BPE, (i0 - w) >> 2);
2506# endif
2507 NOP();
2508 return (w);
2509}
2510
2511static jit_word_t
2512_bm_w(jit_state_t *_jit, jit_bool_t set,
2513 jit_word_t i0, jit_int32_t r0, jit_word_t i1)
2514{
2515 jit_word_t w;
2516 jit_int32_t reg;
2517 if (s13_p(i1)) {
2518 BTSTI(r0, i1);
2519 w = _jit->pc.w;
2520# if __WORDSIZE == 32
2521 B(set ? SPARC_BNZ : SPARC_BZ, (i0 - w) >> 2);
2522# else
2523 BP(set ? SPARC_BPNE : SPARC_BPE, (i0 - w) >> 2);
2524# endif
2525 NOP();
2526 }
2527 else {
2528 reg = jit_get_reg(jit_class_gpr|jit_class_nospill);
2529 movi(rn(reg), i1);
2530 w = bm_r(set, i0, r0, rn(reg));
2531 jit_unget_reg(reg);
2532 }
2533 return (w);
2534}
2535
2536static void
2537_jmpr(jit_state_t *_jit, jit_int32_t r0)
2538{
2539 JMPL(0, r0, 0);
2540 NOP();
2541}
2542
79bfeef6 2543static jit_word_t
4a71579b
PC		 2544_jmpi(jit_state_t *_jit, jit_word_t i0)
2545{
4a71579b 2546 jit_int32_t reg;
79bfeef6
PC		 2547 jit_word_t d, w;
2548 w = _jit->pc.w;
2549 d = (i0 - w) >> 2;
2550 if (s22_p(d)) {
2551 BA(d);
4a71579b
PC		 2552 NOP();
2553 }
2554 else {
2555 reg = jit_get_reg(jit_class_gpr|jit_class_nospill);
2556 movi(rn(reg), i0);
2557 jmpr(rn(reg));
2558 jit_unget_reg(reg);
2559 }
79bfeef6 2560 return (w);
4a71579b
PC		 2561}
2562
2563static jit_word_t
2564_jmpi_p(jit_state_t *_jit, jit_word_t i0)
2565{
2566 jit_word_t w;
2567 jit_int32_t reg;
2568 reg = jit_get_reg(jit_class_gpr|jit_class_nospill);
2569 w = movi_p(rn(reg), i0);
2570 jmpr(rn(reg));
2571 jit_unget_reg(reg);
2572 return (w);
2573}
2574
2575static void
2576_callr(jit_state_t *_jit, jit_int32_t r0)
2577{
2578 CALL(r0);
2579 NOP();
2580}
2581
79bfeef6 2582static jit_word_t
4a71579b
PC		 2583_calli(jit_state_t *_jit, jit_word_t i0)
2584{
79bfeef6
PC		 2585 jit_word_t d, w;
2586 w = _jit->pc.w;
2587 d = (i0 - w) >> 2;
2588 if (s30_p(d)) {
2589 CALLI(d);
c0c16242
PC		 2590 NOP();
2591 }
2592 else
79bfeef6
PC		 2593 w = calli_p(i0);
2594 return (w);
4a71579b
PC		 2595}
2596
2597static jit_word_t
2598_calli_p(jit_state_t *_jit, jit_word_t i0)
2599{
2600 jit_word_t w;
2601 jit_int32_t reg;
2602 reg = jit_get_reg(jit_class_gpr);
2603 w = movi_p(rn(reg), i0);
2604 callr(rn(reg));
2605 jit_unget_reg(reg);
2606 return (w);
2607}
2608
2609#define OFF(n) BIAS(((n) * sizeof(jit_word_t)))
2610static void
2611_prolog(jit_state_t *_jit, jit_node_t *node)
2612{
2613 jit_int32_t reg;
2614 if (_jitc->function->define_frame || _jitc->function->assume_frame) {
2615 jit_int32_t frame = -_jitc->function->frame;
2616 assert(_jitc->function->self.aoff >= frame);
2617 if (_jitc->function->assume_frame)
2618 return;
2619 _jitc->function->self.aoff = frame;
2620 }
2621 if (_jitc->function->allocar)
2622 _jitc->function->self.aoff &= -16;
2623 /* align at 16 bytes boundary */
2624 _jitc->function->stack = ((stack_framesize +
2625 _jitc->function->self.alen -
2626 _jitc->function->self.aoff) + 15) & -16;
2627 SAVEI(_SP_REGNO, -_jitc->function->stack, _SP_REGNO);
2628
2629 /* (most) other backends do not save incoming arguments, so,
2630 * only save locals here */
2631 if (jit_regset_tstbit(&_jitc->function->regset, _L0))
2632 stxi(OFF(0), _SP_REGNO, _L0_REGNO);
2633 if (jit_regset_tstbit(&_jitc->function->regset, _L1))
2634 stxi(OFF(1), _SP_REGNO, _L1_REGNO);
2635 if (jit_regset_tstbit(&_jitc->function->regset, _L2))
2636 stxi(OFF(2), _SP_REGNO, _L2_REGNO);
2637 if (jit_regset_tstbit(&_jitc->function->regset, _L3))
2638 stxi(OFF(3), _SP_REGNO, _L3_REGNO);
2639 if (jit_regset_tstbit(&_jitc->function->regset, _L4))
2640 stxi(OFF(4), _SP_REGNO, _L4_REGNO);
2641 if (jit_regset_tstbit(&_jitc->function->regset, _L5))
2642 stxi(OFF(5), _SP_REGNO, _L5_REGNO);
2643 if (jit_regset_tstbit(&_jitc->function->regset, _L6))
2644 stxi(OFF(6), _SP_REGNO, _L6_REGNO);
2645 if (jit_regset_tstbit(&_jitc->function->regset, _L7))
2646 stxi(OFF(7), _SP_REGNO, _L7_REGNO);
2647
2648 if (_jitc->function->allocar) {
2649 reg = jit_get_reg(jit_class_gpr);
2650 movi(rn(reg), BIAS(_jitc->function->self.aoff));
2651 /* Already "biased" by allocai */
2652 stxi_i(_jitc->function->aoffoff, _FP_REGNO, rn(reg));
2653 jit_unget_reg(reg);
2654 }
2655
2656 if (_jitc->function->self.call & jit_call_varargs) {
2657 for (reg = _jitc->function->vagp; jit_arg_reg_p(reg); ++reg)
2658 stxi(BIAS((16 + (__WORDSIZE == 32)) * sizeof(jit_word_t) +
2659 reg * sizeof(jit_word_t)), _FP_REGNO, rn(_I0 + reg));
2660 }
2661}
2662
2663static void
2664_epilog(jit_state_t *_jit, jit_node_t *node)
2665{
2666 if (_jitc->function->assume_frame)
2667 return;
79bfeef6
PC		 2668 if (_jitc->function->allocar)
2669 subi(_SP_REGNO, _FP_REGNO, _jitc->function->stack);
4a71579b 2670 if (jit_regset_tstbit(&_jitc->function->regset, _L0))
79bfeef6 2671 ldxi(_L0_REGNO, _SP_REGNO, _jitc->function->stack + OFF(0));
4a71579b 2672 if (jit_regset_tstbit(&_jitc->function->regset, _L1))
79bfeef6 2673 ldxi(_L1_REGNO, _SP_REGNO, _jitc->function->stack + OFF(1));
4a71579b 2674 if (jit_regset_tstbit(&_jitc->function->regset, _L2))
79bfeef6 2675 ldxi(_L2_REGNO, _SP_REGNO, _jitc->function->stack + OFF(2));
4a71579b 2676 if (jit_regset_tstbit(&_jitc->function->regset, _L3))
79bfeef6 2677 ldxi(_L3_REGNO, _SP_REGNO, _jitc->function->stack + OFF(3));
4a71579b 2678 if (jit_regset_tstbit(&_jitc->function->regset, _L4))
79bfeef6 2679 ldxi(_L4_REGNO, _SP_REGNO, _jitc->function->stack + OFF(4));
4a71579b 2680 if (jit_regset_tstbit(&_jitc->function->regset, _L5))
79bfeef6 2681 ldxi(_L5_REGNO, _SP_REGNO, _jitc->function->stack + OFF(5));
4a71579b 2682 if (jit_regset_tstbit(&_jitc->function->regset, _L6))
79bfeef6 2683 ldxi(_L6_REGNO, _SP_REGNO, _jitc->function->stack + OFF(6));
4a71579b 2684 if (jit_regset_tstbit(&_jitc->function->regset, _L7))
79bfeef6 2685 ldxi(_L7_REGNO, _SP_REGNO, _jitc->function->stack + OFF(7));
4a71579b
PC		 2686 RESTOREI(0, 0, 0);
2687 RETL();
2688 NOP();
2689}
2690
2691static void
2692_vastart(jit_state_t *_jit, jit_int32_t r0)
2693{
2694 /* Initialize stack pointer to the first stack argument. */
2695 if (jit_arg_reg_p(_jitc->function->vagp))
2696 addi(r0, _FP_REGNO, BIAS((16 + (__WORDSIZE == 32) +
2697 _jitc->function->vagp) *
2698 sizeof(jit_word_t)));
2699 else
2700 addi(r0, _FP_REGNO, BIAS(_jitc->function->self.size));
2701}
2702
2703static void
2704_vaarg(jit_state_t *_jit, jit_int32_t r0, jit_int32_t r1)
2705{
2706 assert(_jitc->function->self.call & jit_call_varargs);
2707
2708 /* Load argument. */
2709 ldr(r0, r1);
2710
2711 /* Update vararg stack pointer. */
2712 addi(r1, r1, sizeof(jit_word_t));
2713}
2714
2715static void
2716_patch_at(jit_state_t *_jit, jit_word_t instr, jit_word_t label)
2717{
2718 jit_instr_t i;
2719 union {
2720 jit_int32_t *i;
2721 jit_word_t w;
2722 } u;
2723
2724 u.w = instr;
2725 i.v = u.i[0];
2726
2727 if (i.op.b == 0) { /* conditional branch */
2728 if (i.op2.b == 2 || i.op2.b == 6) { /* int or float condition */
2729 i.disp22.b = (label - instr) >> 2;
2730 u.i[0] = i.v;
2731 }
2732# if __WORDSIZE == 64
2733 else if (i.op2.b == 1) {
2734 i.disp19.b = (label - instr) >> 2;
2735 u.i[0] = i.v;
2736 }
2737# endif
2738 else if (i.op2.b == 4) { /* movi_p */
2739 /* SETHI */
2740 i.imm22.b = HI((int)label);
2741 u.i[0] = i.v;
2742 i.v = u.i[1];
2743 if (i.op.b == 2 && i.op3.b == 2) {
2744 /* ORI */
2745 i.simm13.b = LO(label);
2746 u.i[1] = i.v;
2747# if __WORDSIZE == 64
2748 i.v = u.i[2];
2749 assert(i.op2.b == 4);
2750 label = (label >> 32) & 0xffffffff;
2751 i.imm22.b = HI((int)label);
2752 u.i[2] = i.v;
2753 i.v = u.i[3];
2754 assert(i.op.b == 2 && i.op3.b == 2);
2755 /* ORI */
2756 i.simm13.b = LO(label);
2757 u.i[3] = i.v;
2758# endif
2759 }
2760 else
2761 abort();
2762 }
2763 else
2764 abort();
2765 }
79bfeef6
PC		 2766 else if (i.op.b == 1) {
2767 assert(s30_p((label - instr) >> 2));
2768 i.disp30.b = (label - instr) >> 2;
2769 u.i[0] = i.v;
2770 }
4a71579b
PC		 2771 else
2772 abort();
2773}
2774#endif
ARM optimized PCSX fork