notaz.gp2x.de / pcsx_rearmed.git / blame
? search:
summary | shortlog | log | commit | commitdiff | tree
blob | blame (incremental) | history | HEAD
Update lightrec 20220910 (#686)
[pcsx_rearmed.git] / deps / lightning / lib / jit_aarch64-cpu.c
CommitLineData
4a71579b
PC		 1/*
2 * Copyright (C) 2013-2019 Free Software Foundation, Inc.
3 *
4 * This file is part of GNU lightning.
5 *
6 * GNU lightning is free software; you can redistribute it and/or modify it
7 * under the terms of the GNU Lesser General Public License as published
8 * by the Free Software Foundation; either version 3, or (at your option)
9 * any later version.
10 *
11 * GNU lightning is distributed in the hope that it will be useful, but
12 * WITHOUT ANY WARRANTY; without even the implied warranty of MERCHANTABILITY
13 * or FITNESS FOR A PARTICULAR PURPOSE. See the GNU Lesser General Public
14 * License for more details.
15 *
16 * Authors:
17 * Paulo Cesar Pereira de Andrade
18 */
19
20#if PROTO
21typedef union {
22/* aarch64-opc.c */
23# define ui jit_uint32_t
24# if __BYTE_ORDER == __LITTLE_ENDIAN
25 /* cond2: condition in truly conditional-executed inst. */
26 struct { ui b: 4; } cond2;
27 /* nzcv: flag bit specifier, encoded in the "nzcv" field. */
28 struct { ui b: 4; } nzcv;
29 /* defgh: d:e:f:g:h bits in AdvSIMD modified immediate. */
30 struct { ui _: 5; ui b: 5; } defgh;
31 /* abc: a:b:c bits in AdvSIMD modified immediate. */
32 struct { ui _: 16; ui b: 3; } abc;
33 /* imm19: e.g. in CBZ. */
34 struct { ui _: 5; ui b: 19; } imm19;
35 /* immhi: e.g. in ADRP. */
36 struct { ui _: 5; ui b: 19; } immhi;
37 /* immlo: e.g. in ADRP. */
38 struct { ui _: 29; ui b: 2; } immlo;
39 /* size: in most AdvSIMD and floating-point instructions. */
40 struct { ui _: 22; ui b: 2; } size;
41 /* vldst_size: size field in the AdvSIMD load/store inst. */
42 struct { ui _: 10; ui b: 2; } vldst_size;
43 /* op: in AdvSIMD modified immediate instructions. */
44 struct { ui _: 29; ui b: 1; } op;
45 /* Q: in most AdvSIMD instructions. */
46 struct { ui _: 30; ui b: 1; } Q;
47 /* Rt: in load/store instructions. */
48 struct { ui b: 5; } Rt;
49 /* Rd: in many integer instructions. */
50 struct { ui b: 5; } Rd;
51 /* Rn: in many integer instructions. */
52 struct { ui _: 5; ui b: 5; } Rn;
53 /* Rt2: in load/store pair instructions. */
54 struct { ui _: 10; ui b: 5; } Rt2;
55 /* Ra: in fp instructions. */
56 struct { ui _: 10; ui b: 5; } Ra;
57 /* op2: in the system instructions. */
58 struct { ui _: 5; ui b: 3; } op2;
59 /* CRm: in the system instructions. */
60 struct { ui _: 8; ui b: 4; } CRm;
61 /* CRn: in the system instructions. */
62 struct { ui _: 12; ui b: 4; } CRn;
63 /* op1: in the system instructions. */
64 struct { ui _: 16; ui b: 3; } op1;
65 /* op0: in the system instructions. */
66 struct { ui _: 19; ui b: 2; } op0;
67 /* imm3: in add/sub extended reg instructions. */
68 struct { ui _: 10; ui b: 3; } imm3;
69 /* cond: condition flags as a source operand. */
70 struct { ui _: 12; ui b: 4; } cond;
71 /* opcode: in advsimd load/store instructions. */
72 struct { ui _: 12; ui b: 4; } opcode;
73 /* cmode: in advsimd modified immediate instructions. */
74 struct { ui _: 12; ui b: 4; } cmode;
75 /* asisdlso_opcode: opcode in advsimd ld/st single element. */
76 struct { ui _: 13; ui b: 3; } asisdlso_opcode;
77 /* len: in advsimd tbl/tbx instructions. */
78 struct { ui _: 13; ui b: 2; } len;
79 /* Rm: in ld/st reg offset and some integer inst. */
80 struct { ui _: 16; ui b: 5; } Rm;
81 /* Rs: in load/store exclusive instructions. */
82 struct { ui _: 16; ui b: 5; } Rs;
83 /* option: in ld/st reg offset + add/sub extended reg inst. */
84 struct { ui _: 13; ui b: 3; } option;
85 /* S: in load/store reg offset instructions. */
86 struct { ui _: 12; ui b: 1; } S;
87 /* hw: in move wide constant instructions. */
88 struct { ui _: 21; ui b: 2; } hw;
89 /* opc: in load/store reg offset instructions. */
90 struct { ui _: 22; ui b: 2; } opc;
91 /* opc1: in load/store reg offset instructions. */
92 struct { ui _: 23; ui b: 1; } opc1;
93 /* shift: in add/sub reg/imm shifted instructions. */
94 struct { ui _: 22; ui b: 2; } shift;
95 /* type: floating point type field in fp data inst. */
96 struct { ui _: 22; ui b: 2; } type;
97 /* ldst_size: size field in ld/st reg offset inst. */
98 struct { ui _: 30; ui b: 2; } ldst_size;
99 /* imm6: in add/sub reg shifted instructions. */
100 struct { ui _: 10; ui b: 6; } imm6;
101 /* imm4: in advsimd ext and advsimd ins instructions. */
102 struct { ui _: 11; ui b: 4; } imm4;
103 /* imm5: in conditional compare (immediate) instructions. */
104 struct { ui _: 16; ui b: 5; } imm5;
105 /* imm7: in load/store pair pre/post index instructions. */
106 struct { ui _: 15; ui b: 7; } imm7;
107 /* imm8: in floating-point scalar move immediate inst. */
108 struct { ui _: 13; ui b: 8; } imm8;
109 /* imm9: in load/store pre/post index instructions. */
110 struct { ui _: 12; ui b: 9; } imm9;
111 /* imm12: in ld/st unsigned imm or add/sub shifted inst. */
112 struct { ui _: 10; ui b: 12; } imm12;
113 /* imm14: in test bit and branch instructions. */
114 struct { ui _: 5; ui b: 14; } imm14;
115 /* imm16: in exception instructions. */
116 struct { ui _: 5; ui b: 16; } imm16;
117 /* imm26: in unconditional branch instructions. */
118 struct { ui b: 26; } imm26;
119 /* imms: in bitfield and logical immediate instructions. */
120 struct { ui _: 10; ui b: 6; } imms;
121 /* immr: in bitfield and logical immediate instructions. */
122 struct { ui _: 16; ui b: 6; } immr;
123 /* immb: in advsimd shift by immediate instructions. */
124 struct { ui _: 16; ui b: 3; } immb;
125 /* immh: in advsimd shift by immediate instructions. */
126 struct { ui _: 19; ui b: 4; } immh;
127 /* N: in logical (immediate) instructions. */
128 struct { ui _: 22; ui b: 1; } N;
129 /* index: in ld/st inst deciding the pre/post-index. */
130 struct { ui _: 11; ui b: 1; } index;
131 /* index2: in ld/st pair inst deciding the pre/post-index. */
132 struct { ui _: 24; ui b: 1; } index2;
133 /* sf: in integer data processing instructions. */
134 struct { ui _: 31; ui b: 1; } sf;
135 /* H: in advsimd scalar x indexed element instructions. */
136 struct { ui _: 11; ui b: 1; } H;
137 /* L: in advsimd scalar x indexed element instructions. */
138 struct { ui _: 21; ui b: 1; } L;
139 /* M: in advsimd scalar x indexed element instructions. */
140 struct { ui _: 20; ui b: 1; } M;
141 /* b5: in the test bit and branch instructions. */
142 struct { ui _: 31; ui b: 1; } b5;
143 /* b40: in the test bit and branch instructions. */
144 struct { ui _: 19; ui b: 5; } b40;
145 /* scale: in the fixed-point scalar to fp converting inst. */
146 struct { ui _: 10; ui b: 6; } scale;
147# else
148 struct { ui _: 28; ui b: 4; } cond2;
149 struct { ui _: 28; ui b: 4; } nzcv;
150 struct { ui _: 22; ui b: 5; } defgh;
151 struct { ui _: 13; ui b: 3; } abc;
152 struct { ui _: 8; ui b: 19; } imm19;
153 struct { ui _: 8; ui b: 19; } immhi;
154 struct { ui _: 1; ui b: 29; } immlo;
155 struct { ui _: 8; ui b: 2; } size;
156 struct { ui _: 20; ui b: 2; } vldst_size;
157 struct { ui _: 2; ui b: 1; } op;
158 struct { ui _: 1; ui b: 1; } Q;
159 struct { ui _: 27; ui b: 1; } Rt;
160 struct { ui _: 27; ui b: 1; } Rd;
161 struct { ui _: 22; ui b: 5; } Rn;
162 struct { ui _: 17; ui b: 5; } Rt2;
163 struct { ui _: 17; ui b: 5; } Ra;
164 struct { ui _: 24; ui b: 3; } op2;
165 struct { ui _: 20; ui b: 4; } CRm;
166 struct { ui _: 16; ui b: 4; } CRn;
167 struct { ui _: 13; ui b: 3; } op1;
168 struct { ui _: 11; ui b: 2; } op0;
169 struct { ui _: 19; ui b: 3; } imm3;
170 struct { ui _: 16; ui b: 4; } cond;
171 struct { ui _: 16; ui b: 4; } opcode;
172 struct { ui _: 16; ui b: 4; } cmode;
173 struct { ui _: 16; ui b: 3; } asisdlso_opcode;
174 struct { ui _: 17; ui b: 2; } len;
175 struct { ui _: 11; ui b: 5; } Rm;
176 struct { ui _: 11; ui b: 5; } Rs;
177 struct { ui _: 16; ui b: 3; } option;
178 struct { ui _: 19; ui b: 1; } S;
179 struct { ui _: 9; ui b: 2; } hw;
180 struct { ui _: 8; ui b: 2; } opc;
181 struct { ui _: 8; ui b: 1; } opc1;
182 struct { ui _: 8; ui b: 2; } shift;
183 struct { ui _: 8; ui b: 2; } type;
184 struct { ui b: 2; } ldst_size;
185 struct { ui _: 16; ui b: 6; } imm6;
186 struct { ui _: 17; ui b: 4; } imm4;
187 struct { ui _: 11; ui b: 5; } imm5;
188 struct { ui _: 10; ui b: 7; } imm7;
189 struct { ui _: 11; ui b: 8; } imm8;
190 struct { ui _: 11; ui b: 9; } imm9;
191 struct { ui _: 10; ui b: 12; } imm12;
192 struct { ui _: 13; ui b: 14; } imm14;
193 struct { ui _: 11; ui b: 16; } imm16;
194 struct { ui _: 6; ui b: 26; } imm26;
195 struct { ui _: 16; ui b: 6; } imms;
196 struct { ui _: 10; ui b: 6; } immr;
197 struct { ui _: 13; ui b: 3; } immb;
198 struct { ui _: 9; ui b: 4; } immh;
199 struct { ui _: 9; ui b: 1; } N;
200 struct { ui _: 20; ui b: 1; } index;
201 struct { ui _: 7; ui b: 1; } index2;
202 struct { ui b: 1; } sf;
203 struct { ui _: 20; ui b: 1; } H;
204 struct { ui _: 10; ui b: 1; } L;
205 struct { ui _: 11; ui b: 1; } M;
206 struct { ui b: 1; } b5;
207 struct { ui _: 8; ui b: 5; } b40;
208 struct { ui _: 16; ui b: 6; } scale;
209# endif
210 jit_int32_t w;
211# undef ui
212} instr_t;
213# define stack_framesize 160
214# define ii(i) *_jit->pc.ui++ = i
215# define ldr(r0,r1) ldr_l(r0,r1)
216# define ldxr(r0,r1,r2) ldxr_l(r0,r1,r2)
217# define ldxi(r0,r1,i0) ldxi_l(r0,r1,i0)
218# define stxi(i0,r0,r1) stxi_l(i0,r0,r1)
219# define FP_REGNO 0x1d
220# define LR_REGNO 0x1e
221# define SP_REGNO 0x1f
222# define XZR_REGNO 0x1f
223# define WZR_REGNO XZR_REGNO
224# define LSL_12 0x00400000
225# define MOVI_LSL_16 0x00200000
226# define MOVI_LSL_32 0x00400000
227# define MOVI_LSL_48 0x00600000
228# define XS 0x80000000 /* Wn -> Xn */
229# define DS 0x00400000 /* Sn -> Dn */
230# define CC_NE 0x0
231# define CC_EQ 0x1
232# define CC_CC 0x2
233# define CC_LO CC_CC
234# define CC_CS 0x3
235# define CC_HS CC_CS
236# define CC_PL 0x4
237# define CC_MI 0x5
238# define CC_VC 0x6
239# define CC_VS 0x7
240# define CC_LS 0x8
241# define CC_HI 0x9
242# define CC_LT 0xa
243# define CC_GE 0xb
244# define CC_LE 0xc
245# define CC_GT 0xd
246# define CC_NV 0xe
247# define CC_AL 0xf
248/* Branches need inverted condition */
249# define BCC_EQ 0x0
250# define BCC_NE 0x1
251# define BCC_CS 0x2
252# define BCC_HS BCC_CS
253# define BCC_CC 0x3
254# define BCC_LO BCC_CC
255# define BCC_MI 0x4
256# define BCC_PL 0x5
257# define BCC_VS 0x6
258# define BCC_VC 0x7
259# define BCC_HI 0x8
260# define BCC_LS 0x9
261# define BCC_GE 0xa
262# define BCC_LT 0xb
263# define BCC_GT 0xc
264# define BCC_LE 0xd
265# define BCC_AL 0xe
266# define BCC_NV 0xf
267/* adapted and cut down to only tested and required by lightning,
268 * from data in binutils/aarch64-tbl.h */
269# define A64_ADCS 0x3a000000
270# define A64_SBCS 0x7a000000
271# define A64_ADDI 0x11000000
272# define A64_ADDSI 0xb1000000
273# define A64_SUBI 0x51000000
274# define A64_SUBSI 0x71000000
275# define A64_ADD 0x0b000000
276# define A64_ADDS 0x2b000000
277# define A64_SUB 0x4b000000
278# define A64_NEG 0x4b0003e0
279# define A64_SUBS 0x6b000000
280# define A64_CMP 0x6b00001f
281# define A64_SBFM 0x93400000
282# define A64_UBFM 0x53400000
283# define A64_UBFX 0x53000000
284# define A64_B 0x14000000
285# define A64_BL 0x94000000
286# define A64_BR 0xd61f0000
287# define A64_BLR 0xd63f0000
288# define A64_RET 0xd65f0000
289# define A64_CBZ 0x34000000
290# define A64_CBNZ 0x35000000
291# define A64_B_C 0x54000000
292# define A64_CSINC 0x1a800400
e0659411 293# define A64_CSSEL 0x1a800000
4a71579b
PC		 294# define A64_REV 0xdac00c00
295# define A64_UDIV 0x1ac00800
296# define A64_SDIV 0x1ac00c00
297# define A64_LSL 0x1ac02000
298# define A64_LSR 0x1ac02400
299# define A64_ASR 0x1ac02800
300# define A64_MUL 0x1b007c00
301# define A64_SMULL 0x9b207c00
302# define A64_SMULH 0x9b407c00
303# define A64_UMULL 0x9ba07c00
304# define A64_UMULH 0x9bc07c00
305# define A64_STRBI 0x39000000
306# define A64_LDRBI 0x39400000
307# define A64_LDRSBI 0x39800000
308# define A64_STRI 0xf9000000
309# define A64_LDRI 0xf9400000
310# define A64_STRHI 0x79000000
311# define A64_LDRHI 0x79400000
312# define A64_LDRSHI 0x79800000
313# define A64_STRWI 0xb9000000
314# define A64_LDRWI 0xb9400000
315# define A64_LDRSWI 0xb9800000
316# define A64_STRB 0x38206800
317# define A64_LDRB 0x38606800
318# define A64_LDRSB 0x38e06800
319# define A64_STR 0xf8206800
320# define A64_LDR 0xf8606800
ba3814c1
PC		 321# define A64_LDAXR 0xc85ffc00
322# define A64_STLXR 0xc800fc00
4a71579b
PC		 323# define A64_STRH 0x78206800
324# define A64_LDRH 0x78606800
325# define A64_LDRSH 0x78a06800
326# define A64_STRW 0xb8206800
327# define A64_LDRW 0xb8606800
328# define A64_LDRSW 0xb8a06800
329# define A64_STURB 0x38000000
330# define A64_LDURB 0x38400000
331# define A64_LDURSB 0x38800000
332# define A64_STUR 0xf8000000
333# define A64_LDUR 0xf8400000
334# define A64_STURH 0x78000000
335# define A64_LDURH 0x78400000
336# define A64_LDURSH 0x78800000
337# define A64_STURW 0xb8000000
338# define A64_LDURW 0xb8400000
339# define A64_LDURSW 0xb8800000
340# define A64_STP 0x29000000
341# define A64_LDP 0x29400000
342# define A64_STP_POS 0x29800000
343# define A64_LDP_PRE 0x28c00000
344# define A64_ANDI 0x12400000
345# define A64_ORRI 0x32400000
346# define A64_EORI 0x52400000
347# define A64_ANDSI 0x72000000
348# define A64_AND 0x0a000000
349# define A64_ORR 0x2a000000
350# define A64_MOV 0x2a0003e0 /* AKA orr Rd,xzr,Rm */
351# define A64_MVN 0x2a2003e0
352# define A64_UXTW 0x2a0003e0 /* AKA MOV */
353# define A64_EOR 0x4a000000
354# define A64_ANDS 0x6a000000
355# define A64_MOVN 0x12800000
356# define A64_MOVZ 0x52800000
357# define A64_MOVK 0x72800000
358# define SBFM(Rd,Rn,ImmR,ImmS) oxxrs(A64_SBFM|XS,Rd,Rn,ImmR,ImmS)
359# define UBFM(Rd,Rn,ImmR,ImmS) oxxrs(A64_UBFM|XS,Rd,Rn,ImmR,ImmS)
360# define UBFX(Rd,Rn,ImmR,ImmS) oxxrs(A64_UBFX,Rd,Rn,ImmR,ImmS)
361# define CMP(Rn,Rm) oxx_(A64_CMP|XS,Rn,Rm)
362# define CMPI(Rn,Imm12) oxxi(A64_SUBSI|XS,XZR_REGNO,Rn,Imm12)
363# define CMPI_12(Rn,Imm12) oxxi(A64_SUBSI|XS|LSL_12,XZR_REGNO,Rn,Imm12)
364# define CMNI(Rn,Imm12) oxxi(A64_ADDSI|XS,XZR_REGNO,Rn,Imm12)
365# define CMNI_12(Rn,Imm12) oxxi(A64_ADDSI|XS|LSL_12,XZR_REGNO,Rn,Imm12)
366# define CSINC(Rd,Rn,Rm,Cc) oxxxc(A64_CSINC|XS,Rd,Rn,Rm,Cc)
367# define TST(Rn,Rm) oxxx(A64_ANDS|XS,XZR_REGNO,Rn,Rm)
368/* actually should use oxxrs but logical_immediate returns proper encoding */
369# define TSTI(Rn,Imm12) oxxi(A64_ANDSI,XZR_REGNO,Rn,Imm12)
370# define MOV(Rd,Rm) ox_x(A64_MOV|XS,Rd,Rm)
371# define MVN(Rd,Rm) ox_x(A64_MVN|XS,Rd,Rm)
372# define NEG(Rd,Rm) ox_x(A64_NEG|XS,Rd,Rm)
373# define MOVN(Rd,Imm16) ox_h(A64_MOVN|XS,Rd,Imm16)
374# define MOVN_16(Rd,Imm16) ox_h(A64_MOVN|XS|MOVI_LSL_16,Rd,Imm16)
375# define MOVN_32(Rd,Imm16) ox_h(A64_MOVN|XS|MOVI_LSL_32,Rd,Imm16)
376# define MOVN_48(Rd,Imm16) ox_h(A64_MOVN|XS|MOVI_LSL_48,Rd,Imm16)
377# define MOVZ(Rd,Imm16) ox_h(A64_MOVZ|XS,Rd,Imm16)
378# define MOVZ_16(Rd,Imm16) ox_h(A64_MOVZ|XS|MOVI_LSL_16,Rd,Imm16)
379# define MOVZ_32(Rd,Imm16) ox_h(A64_MOVZ|XS|MOVI_LSL_32,Rd,Imm16)
380# define MOVZ_48(Rd,Imm16) ox_h(A64_MOVZ|XS|MOVI_LSL_48,Rd,Imm16)
381# define MOVK(Rd,Imm16) ox_h(A64_MOVK|XS,Rd,Imm16)
382# define MOVK_16(Rd,Imm16) ox_h(A64_MOVK|XS|MOVI_LSL_16,Rd,Imm16)
383# define MOVK_32(Rd,Imm16) ox_h(A64_MOVK|XS|MOVI_LSL_32,Rd,Imm16)
384# define MOVK_48(Rd,Imm16) ox_h(A64_MOVK|XS|MOVI_LSL_48,Rd,Imm16)
385# define ADD(Rd,Rn,Rm) oxxx(A64_ADD|XS,Rd,Rn,Rm)
386# define ADDI(Rd,Rn,Imm12) oxxi(A64_ADDI|XS,Rd,Rn,Imm12)
387# define ADDI_12(Rd,Rn,Imm12) oxxi(A64_ADDI|XS|LSL_12,Rd,Rn,Imm12)
388# define MOV_XSP(Rd,Rn) ADDI(Rd,Rn,0)
389# define ADDS(Rd,Rn,Rm) oxxx(A64_ADDS|XS,Rd,Rn,Rm)
390# define ADDSI(Rd,Rn,Imm12) oxxi(A64_ADDSI|XS,Rd,Rn,Imm12)
391# define ADDSI_12(Rd,Rn,Imm12) oxxi(A64_ADDSI|XS|LSL_12,Rd,Rn,Imm12)
392# define ADCS(Rd,Rn,Rm) oxxx(A64_ADCS|XS,Rd,Rn,Rm)
393# define SUB(Rd,Rn,Rm) oxxx(A64_SUB|XS,Rd,Rn,Rm)
394# define SUBI(Rd,Rn,Imm12) oxxi(A64_SUBI|XS,Rd,Rn,Imm12)
395# define SUBI_12(Rd,Rn,Imm12) oxxi(A64_SUBI|XS|LSL_12,Rd,Rn,Imm12)
396# define SUBS(Rd,Rn,Rm) oxxx(A64_SUBS|XS,Rd,Rn,Rm)
397# define SUBSI(Rd,Rn,Imm12) oxxi(A64_SUBSI|XS,Rd,Rn,Imm12)
398# define SUBSI_12(Rd,Rn,Imm12) oxxi(A64_SUBSI|XS|LSL_12,Rd,Rn,Imm12)
399# define SBCS(Rd,Rn,Rm) oxxx(A64_SBCS|XS,Rd,Rn,Rm)
400# define MUL(Rd,Rn,Rm) oxxx(A64_MUL|XS,Rd,Rn,Rm)
401# define SMULL(Rd,Rn,Rm) oxxx(A64_SMULL,Rd,Rn,Rm)
402# define SMULH(Rd,Rn,Rm) oxxx(A64_SMULH,Rd,Rn,Rm)
403# define UMULL(Rd,Rn,Rm) oxxx(A64_UMULL,Rd,Rn,Rm)
404# define UMULH(Rd,Rn,Rm) oxxx(A64_UMULH,Rd,Rn,Rm)
405# define SDIV(Rd,Rn,Rm) oxxx(A64_SDIV|XS,Rd,Rn,Rm)
406# define UDIV(Rd,Rn,Rm) oxxx(A64_UDIV|XS,Rd,Rn,Rm)
407# define LSL(Rd,Rn,Rm) oxxx(A64_LSL|XS,Rd,Rn,Rm)
408# define LSLI(r0,r1,i0) UBFM(r0,r1,(64-i0)&63,63-i0)
409# define ASR(Rd,Rn,Rm) oxxx(A64_ASR|XS,Rd,Rn,Rm)
410# define ASRI(r0,r1,i0) SBFM(r0,r1,i0,63)
411# define LSR(Rd,Rn,Rm) oxxx(A64_LSR|XS,Rd,Rn,Rm)
412# define LSRI(r0,r1,i0) UBFM(r0,r1,i0,63)
413# define AND(Rd,Rn,Rm) oxxx(A64_AND|XS,Rd,Rn,Rm)
414/* actually should use oxxrs but logical_immediate returns proper encoding */
415# define ANDI(Rd,Rn,Imm12) oxxi(A64_ANDI|XS,Rd,Rn,Imm12)
416# define ORR(Rd,Rn,Rm) oxxx(A64_ORR|XS,Rd,Rn,Rm)
417/* actually should use oxxrs but logical_immediate returns proper encoding */
418# define ORRI(Rd,Rn,Imm12) oxxi(A64_ORRI|XS,Rd,Rn,Imm12)
419# define EOR(Rd,Rn,Rm) oxxx(A64_EOR|XS,Rd,Rn,Rm)
420/* actually should use oxxrs but logical_immediate returns proper encoding */
421# define EORI(Rd,Rn,Imm12) oxxi(A64_EORI|XS,Rd,Rn,Imm12)
422# define SXTB(Rd,Rn) SBFM(Rd,Rn,0,7)
423# define SXTH(Rd,Rn) SBFM(Rd,Rn,0,15)
424# define SXTW(Rd,Rn) SBFM(Rd,Rn,0,31)
425# define UXTB(Rd,Rn) UBFX(Rd,Rn,0,7)
426# define UXTH(Rd,Rn) UBFX(Rd,Rn,0,15)
427# define UXTW(Rd,Rm) ox_x(A64_UXTW,Rd,Rm)
428# define REV(Rd,Rn) o_xx(A64_REV,Rd,Rn)
429# define LDRSB(Rt,Rn,Rm) oxxx(A64_LDRSB,Rt,Rn,Rm)
430# define LDRSBI(Rt,Rn,Imm12) oxxi(A64_LDRSBI,Rt,Rn,Imm12)
431# define LDURSB(Rt,Rn,Imm9) oxx9(A64_LDURSB,Rt,Rn,Imm9)
432# define LDRB(Rt,Rn,Rm) oxxx(A64_LDRB,Rt,Rn,Rm)
433# define LDRBI(Rt,Rn,Imm12) oxxi(A64_LDRBI,Rt,Rn,Imm12)
434# define LDURB(Rt,Rn,Imm9) oxx9(A64_LDURB,Rt,Rn,Imm9)
435# define LDRSH(Rt,Rn,Rm) oxxx(A64_LDRSH,Rt,Rn,Rm)
436# define LDRSHI(Rt,Rn,Imm12) oxxi(A64_LDRSHI,Rt,Rn,Imm12)
437# define LDURSH(Rt,Rn,Imm9) oxx9(A64_LDURSH,Rt,Rn,Imm9)
438# define LDRH(Rt,Rn,Rm) oxxx(A64_LDRH,Rt,Rn,Rm)
439# define LDRHI(Rt,Rn,Imm12) oxxi(A64_LDRHI,Rt,Rn,Imm12)
440# define LDURH(Rt,Rn,Imm9) oxx9(A64_LDURH,Rt,Rn,Imm9)
441# define LDRSW(Rt,Rn,Rm) oxxx(A64_LDRSW,Rt,Rn,Rm)
442# define LDRSWI(Rt,Rn,Imm12) oxxi(A64_LDRSWI,Rt,Rn,Imm12)
443# define LDURSW(Rt,Rn,Imm9) oxx9(A64_LDURSW,Rt,Rn,Imm9)
444# define LDRW(Rt,Rn,Rm) oxxx(A64_LDRW,Rt,Rn,Rm)
445# define LDRWI(Rt,Rn,Imm12) oxxi(A64_LDRWI,Rt,Rn,Imm12)
446# define LDURW(Rt,Rn,Imm9) oxx9(A64_LDURW,Rt,Rn,Imm9)
447# define LDR(Rt,Rn,Rm) oxxx(A64_LDR,Rt,Rn,Rm)
448# define LDRI(Rt,Rn,Imm12) oxxi(A64_LDRI,Rt,Rn,Imm12)
449# define LDUR(Rt,Rn,Imm9) oxx9(A64_LDUR,Rt,Rn,Imm9)
ba3814c1
PC		 450# define LDAXR(Rt,Rn) o_xx(A64_LDAXR,Rt,Rn)
451# define STLXR(Rs,Rt,Rn) oxxx(A64_STLXR,Rs,Rn,Rt)
4a71579b
PC		 452# define STRB(Rt,Rn,Rm) oxxx(A64_STRB,Rt,Rn,Rm)
453# define STRBI(Rt,Rn,Imm12) oxxi(A64_STRBI,Rt,Rn,Imm12)
454# define STURB(Rt,Rn,Imm9) oxx9(A64_STURB,Rt,Rn,Imm9)
455# define STRH(Rt,Rn,Rm) oxxx(A64_STRH,Rt,Rn,Rm)
456# define STRHI(Rt,Rn,Imm12) oxxi(A64_STRHI,Rt,Rn,Imm12)
457# define STURH(Rt,Rn,Imm9) oxx9(A64_STURH,Rt,Rn,Imm9)
458# define STRW(Rt,Rn,Rm) oxxx(A64_STRW,Rt,Rn,Rm)
459# define STRWI(Rt,Rn,Imm12) oxxi(A64_STRWI,Rt,Rn,Imm12)
460# define STURW(Rt,Rn,Imm9) oxx9(A64_STURW,Rt,Rn,Imm9)
461# define STR(Rt,Rn,Rm) oxxx(A64_STR,Rt,Rn,Rm)
462# define STRI(Rt,Rn,Imm12) oxxi(A64_STRI,Rt,Rn,Imm12)
463# define STUR(Rt,Rn,Imm9) oxx9(A64_STUR,Rt,Rn,Imm9)
464# define LDPI(Rt,Rt2,Rn,Simm7) oxxx7(A64_LDP|XS,Rt,Rt2,Rn,Simm7)
465# define STPI(Rt,Rt2,Rn,Simm7) oxxx7(A64_STP|XS,Rt,Rt2,Rn,Simm7)
466# define LDPI_PRE(Rt,Rt2,Rn,Simm7) oxxx7(A64_LDP_PRE|XS,Rt,Rt2,Rn,Simm7)
467# define STPI_POS(Rt,Rt2,Rn,Simm7) oxxx7(A64_STP_POS|XS,Rt,Rt2,Rn,Simm7)
468# define CSET(Rd,Cc) CSINC(Rd,XZR_REGNO,XZR_REGNO,Cc)
e0659411 469# define CSEL(Rd,Rn,Rm,Cc) oxxxc(A64_CSSEL|XS,Rd,Rn,Rm,Cc)
4a71579b
PC		 470# define B(Simm26) o26(A64_B,Simm26)
471# define BL(Simm26) o26(A64_BL,Simm26)
472# define BR(Rn) o_x_(A64_BR,Rn)
473# define BLR(Rn) o_x_(A64_BLR,Rn)
474# define RET() o_x_(A64_RET,LR_REGNO)
475# define B_C(Cc,Simm19) oc19(A64_B_C,Cc,Simm19)
476# define CBZ(Rd,Simm19) ox19(A64_CBZ|XS,Rd,Simm19)
477# define CBNZ(Rd,Simm19) ox19(A64_CBNZ|XS,Rd,Simm19)
478# define NOP() ii(0xd503201f)
479static jit_int32_t logical_immediate(jit_word_t);
480# define oxxx(Op,Rd,Rn,Rm) _oxxx(_jit,Op,Rd,Rn,Rm)
481static void _oxxx(jit_state_t*,jit_int32_t,jit_int32_t,jit_int32_t,jit_int32_t);
482# define oxxi(Op,Rd,Rn,Imm12) _oxxi(_jit,Op,Rd,Rn,Imm12)
483static void _oxxi(jit_state_t*,jit_int32_t,jit_int32_t,jit_int32_t,jit_int32_t);
484# define oxx9(Op,Rd,Rn,Imm9) _oxx9(_jit,Op,Rd,Rn,Imm9)
485static void _oxx9(jit_state_t*,jit_int32_t,jit_int32_t,jit_int32_t,jit_int32_t);
486# define ox19(Op,Rd,Simm19) _ox19(_jit,Op,Rd,Simm19)
487static void _ox19(jit_state_t*,jit_int32_t,jit_int32_t,jit_int32_t);
488# define oc19(Op,Cc,Simm19) _oc19(_jit,Op,Cc,Simm19)
489static void _oc19(jit_state_t*,jit_int32_t,jit_int32_t,jit_int32_t);
490# define o26(Op,Simm26) _o26(_jit,Op,Simm26)
491static void _oc26(jit_state_t*,jit_int32_t,jit_int32_t);
492# define ox_x(Op,Rd,Rn) _ox_x(_jit,Op,Rd,Rn)
493static void _ox_x(jit_state_t*,jit_int32_t,jit_int32_t,jit_int32_t);
494# define o_xx(Op,Rd,Rn) _o_xx(_jit,Op,Rd,Rn)
495static void _o_xx(jit_state_t*,jit_int32_t,jit_int32_t,jit_int32_t);
496# define oxx_(Op,Rn,Rm) _oxx_(_jit,Op,Rn,Rm)
497static void _oxx_(jit_state_t*,jit_int32_t,jit_int32_t,jit_int32_t);
498# define o_x_(Op,Rn) _o_x_(_jit,Op,Rn)
499static void _o_x_(jit_state_t*,jit_int32_t,jit_int32_t);
500# define ox_h(Op,Rd,Imm16) _ox_h(_jit,Op,Rd,Imm16)
501static void _ox_h(jit_state_t*,jit_int32_t,jit_int32_t,jit_int32_t);
502# define oxxrs(Op,Rd,Rn,R,S) _oxxrs(_jit,Op,Rd,Rn,R,S)
503static void _oxxrs(jit_state_t*,jit_int32_t,jit_int32_t,
504 jit_int32_t,jit_int32_t,jit_int32_t);
505# define oxxxc(Op,Rd,Rn,Rm,Cc) _oxxxc(_jit,Op,Rd,Rn,Rm,Cc)
506static void _oxxxc(jit_state_t*,jit_int32_t,jit_int32_t,
507 jit_int32_t,jit_int32_t,jit_int32_t);
508# define oxxx7(Op,Rt,Rt2,Rn,Simm7) _oxxx7(_jit,Op,Rt,Rt2,Rn,Simm7)
509static void _oxxx7(jit_state_t*,jit_int32_t,
510 jit_int32_t,jit_int32_t,jit_int32_t,jit_int32_t);
511# define nop(i0) _nop(_jit,i0)
512static void _nop(jit_state_t*,jit_int32_t);
513# define addr(r0,r1,r2) ADD(r0,r1,r2)
514# define addi(r0,r1,i0) _addi(_jit,r0,r1,i0)
515static void _addi(jit_state_t*,jit_int32_t,jit_int32_t,jit_word_t);
516# define addcr(r0,r1,r2) ADDS(r0,r1,r2)
517# define addci(r0,r1,i0) _addci(_jit,r0,r1,i0)
518static void _addci(jit_state_t*,jit_int32_t,jit_int32_t,jit_word_t);
519# define addxr(r0,r1,r2) ADCS(r0,r1,r2)
520# define addxi(r0,r1,i0) _addxi(_jit,r0,r1,i0)
521static void _addxi(jit_state_t*,jit_int32_t,jit_int32_t,jit_word_t);
522# define subr(r0,r1,r2) SUB(r0,r1,r2)
523# define subi(r0,r1,i0) _subi(_jit,r0,r1,i0)
524static void _subi(jit_state_t*,jit_int32_t,jit_int32_t,jit_word_t);
525# define subcr(r0,r1,r2) SUBS(r0,r1,r2)
526# define subci(r0,r1,i0) _subci(_jit,r0,r1,i0)
527static void _subci(jit_state_t*,jit_int32_t,jit_int32_t,jit_word_t);
528# define subxr(r0,r1,r2) SBCS(r0,r1,r2)
529# define subxi(r0,r1,i0) _subxi(_jit,r0,r1,i0)
530static void _subxi(jit_state_t*,jit_int32_t,jit_int32_t,jit_word_t);
531# define rsbi(r0, r1, i0) _rsbi(_jit, r0, r1, i0)
532static void _rsbi(jit_state_t*,jit_int32_t,jit_int32_t,jit_word_t);
533# define mulr(r0,r1,r2) MUL(r0,r1,r2)
534# define muli(r0,r1,i0) _muli(_jit,r0,r1,i0)
535static void _muli(jit_state_t*,jit_int32_t,jit_int32_t,jit_word_t);
536# define qmulr(r0,r1,r2,r3) _qmulr(_jit,r0,r1,r2,r3)
537static void _qmulr(jit_state_t*,jit_int32_t,
538 jit_int32_t,jit_int32_t,jit_int32_t);
539# define qmuli(r0,r1,r2,i0) _qmuli(_jit,r0,r1,r2,i0)
540static void _qmuli(jit_state_t*,jit_int32_t,
541 jit_int32_t,jit_int32_t,jit_word_t);
542# define qmulr_u(r0,r1,r2,r3) _qmulr_u(_jit,r0,r1,r2,r3)
543static void _qmulr_u(jit_state_t*,jit_int32_t,
544 jit_int32_t,jit_int32_t,jit_int32_t);
545# define qmuli_u(r0,r1,r2,i0) _qmuli_u(_jit,r0,r1,r2,i0)
546static void _qmuli_u(jit_state_t*,jit_int32_t,
547 jit_int32_t,jit_int32_t,jit_word_t);
548# define divr(r0,r1,r2) SDIV(r0,r1,r2)
549# define divi(r0,r1,i0) _divi(_jit,r0,r1,i0)
550static void _divi(jit_state_t*,jit_int32_t,jit_int32_t,jit_word_t);
551# define divr_u(r0,r1,r2) UDIV(r0,r1,r2)
552# define divi_u(r0,r1,i0) _divi_u(_jit,r0,r1,i0)
553static void _divi_u(jit_state_t*,jit_int32_t,jit_int32_t,jit_word_t);
554# define qdivr(r0,r1,r2,r3) _iqdivr(_jit,1,r0,r1,r2,r3)
555# define qdivr_u(r0,r1,r2,r3) _iqdivr(_jit,0,r0,r1,r2,r3)
556static void _iqdivr(jit_state_t*,jit_bool_t,
557 jit_int32_t,jit_int32_t,jit_int32_t,jit_int32_t);
558# define qdivi(r0,r1,r2,i0) _qdivi(_jit,r0,r1,r2,i0)
559static void _qdivi(jit_state_t*,jit_int32_t,
560 jit_int32_t,jit_int32_t,jit_word_t);
561# define qdivi_u(r0,r1,r2,i0) _qdivi_u(_jit,r0,r1,r2,i0)
562static void _qdivi_u(jit_state_t*,jit_int32_t,
563 jit_int32_t,jit_int32_t,jit_word_t);
564# define remr(r0,r1,r2) _remr(_jit,r0,r1,r2)
565static void _remr(jit_state_t*,jit_int32_t,jit_int32_t,jit_int32_t);
566# define remi(r0,r1,i0) _remi(_jit,r0,r1,i0)
567static void _remi(jit_state_t*,jit_int32_t,jit_int32_t,jit_word_t);
568# define remr_u(r0,r1,r2) _remr_u(_jit,r0,r1,r2)
569static void _remr_u(jit_state_t*,jit_int32_t,jit_int32_t,jit_int32_t);
570# define remi_u(r0,r1,i0) _remi_u(_jit,r0,r1,i0)
571static void _remi_u(jit_state_t*,jit_int32_t,jit_int32_t,jit_word_t);
572# define lshr(r0,r1,r2) LSL(r0,r1,r2)
573# define lshi(r0,r1,i0) _lshi(_jit,r0,r1,i0)
574static void _lshi(jit_state_t*,jit_int32_t,jit_int32_t,jit_word_t);
575# define rshr(r0,r1,r2) ASR(r0,r1,r2)
576# define rshi(r0,r1,i0) _rshi(_jit,r0,r1,i0)
577static void _rshi(jit_state_t*,jit_int32_t,jit_int32_t,jit_word_t);
578# define rshr_u(r0,r1,r2) LSR(r0,r1,r2)
579# define rshi_u(r0,r1,i0) _rshi_u(_jit,r0,r1,i0)
580static void _rshi_u(jit_state_t*,jit_int32_t,jit_int32_t,jit_word_t);
e0659411
PC		 581# define movnr(r0,r1,r2) _movnr(_jit,r0,r1,r2)
582static void _movnr(jit_state_t*,jit_int32_t,jit_int32_t,jit_int32_t);
583# define movzr(r0,r1,r2) _movzr(_jit,r0,r1,r2)
584static void _movzr(jit_state_t*,jit_int32_t,jit_int32_t,jit_int32_t);
4a71579b
PC		 585# define negr(r0,r1) NEG(r0,r1)
586# define comr(r0,r1) MVN(r0,r1)
587# define andr(r0,r1,r2) AND(r0,r1,r2)
588# define andi(r0,r1,i0) _andi(_jit,r0,r1,i0)
589static void _andi(jit_state_t*,jit_int32_t,jit_int32_t,jit_word_t);
590# define orr(r0,r1,r2) ORR(r0,r1,r2)
591# define ori(r0,r1,i0) _ori(_jit,r0,r1,i0)
592static void _ori(jit_state_t*,jit_int32_t,jit_int32_t,jit_word_t);
593# define xorr(r0,r1,r2) EOR(r0,r1,r2)
594# define xori(r0,r1,i0) _xori(_jit,r0,r1,i0)
595static void _xori(jit_state_t*,jit_int32_t,jit_int32_t,jit_word_t);
596# define ldr_c(r0,r1) LDRSBI(r0,r1,0)
597# define ldi_c(r0,i0) _ldi_c(_jit,r0,i0)
598static void _ldi_c(jit_state_t*,jit_int32_t,jit_word_t);
599# define ldr_uc(r0,r1) _ldr_uc(_jit,r0,r1)
600static void _ldr_uc(jit_state_t*,jit_int32_t,jit_int32_t);
601# define ldi_uc(r0,i0) _ldi_uc(_jit,r0,i0)
602static void _ldi_uc(jit_state_t*,jit_int32_t,jit_word_t);
603# define ldr_s(r0,r1) LDRSHI(r0,r1,0)
604# define ldi_s(r0,i0) _ldi_s(_jit,r0,i0)
605static void _ldi_s(jit_state_t*,jit_int32_t,jit_word_t);
606# define ldr_us(r0,r1) _ldr_us(_jit,r0,r1)
607static void _ldr_us(jit_state_t*,jit_int32_t,jit_int32_t);
608# define ldi_us(r0,i0) _ldi_us(_jit,r0,i0)
609static void _ldi_us(jit_state_t*,jit_int32_t,jit_word_t);
610# define ldr_i(r0,r1) LDRSWI(r0,r1,0)
611# define ldi_i(r0,i0) _ldi_i(_jit,r0,i0)
612static void _ldi_i(jit_state_t*,jit_int32_t,jit_word_t);
613# define ldr_ui(r0,r1) _ldr_ui(_jit,r0,r1)
614static void _ldr_ui(jit_state_t*,jit_int32_t,jit_int32_t);
615# define ldi_ui(r0,i0) _ldi_ui(_jit,r0,i0)
616static void _ldi_ui(jit_state_t*,jit_int32_t,jit_word_t);
617# define ldr_l(r0,r1) LDRI(r0,r1,0)
618static void _ldr_l(jit_state_t*,jit_int32_t,jit_int32_t);
619# define ldi_l(r0,i0) _ldi_l(_jit,r0,i0)
620static void _ldi_l(jit_state_t*,jit_int32_t,jit_word_t);
621# define ldxr_c(r0,r1,r2) _ldxr_c(_jit,r0,r1,r2)
622static void _ldxr_c(jit_state_t*,jit_int32_t,jit_int32_t,jit_int32_t);
623# define ldxi_c(r0,r1,i0) _ldxi_c(_jit,r0,r1,i0)
624static void _ldxi_c(jit_state_t*,jit_int32_t,jit_int32_t,jit_word_t);
625# define ldxr_uc(r0,r1,r2) _ldxr_uc(_jit,r0,r1,r2)
626static void _ldxr_uc(jit_state_t*,jit_int32_t,jit_int32_t,jit_int32_t);
627# define ldxi_uc(r0,r1,i0) _ldxi_uc(_jit,r0,r1,i0)
628static void _ldxi_uc(jit_state_t*,jit_int32_t,jit_int32_t,jit_word_t);
629# define ldxr_s(r0,r1,r2) LDRSH(r0,r1,r2)
630# define ldxi_s(r0,r1,i0) _ldxi_s(_jit,r0,r1,i0)
631static void _ldxi_s(jit_state_t*,jit_int32_t,jit_int32_t,jit_word_t);
632# define ldxr_us(r0,r1,r2) _ldxr_us(_jit,r0,r1,r2)
633static void _ldxr_us(jit_state_t*,jit_int32_t,jit_int32_t,jit_int32_t);
634# define ldxi_us(r0,r1,i0) _ldxi_us(_jit,r0,r1,i0)
635static void _ldxi_us(jit_state_t*,jit_int32_t,jit_int32_t,jit_word_t);
636# define ldxr_i(r0,r1,r2) LDRSW(r0,r1,r2)
637# define ldxi_i(r0,r1,i0) _ldxi_i(_jit,r0,r1,i0)
638static void _ldxi_i(jit_state_t*,jit_int32_t,jit_int32_t,jit_word_t);
639# define ldxr_ui(r0,r1,r2) _ldxr_ui(_jit,r0,r1,r2)
640static void _ldxr_ui(jit_state_t*,jit_int32_t,jit_int32_t,jit_int32_t);
641# define ldxi_ui(r0,r1,i0) _ldxi_ui(_jit,r0,r1,i0)
642static void _ldxi_ui(jit_state_t*,jit_int32_t,jit_int32_t,jit_word_t);
643# define ldxr_l(r0,r1,r2) LDR(r0,r1,r2)
644# define ldxi_l(r0,r1,i0) _ldxi_l(_jit,r0,r1,i0)
645static void _ldxi_l(jit_state_t*,jit_int32_t,jit_int32_t,jit_word_t);
646# define str_c(r0,r1) STRBI(r1,r0,0)
647# define sti_c(i0,r0) _sti_c(_jit,i0,r0)
648static void _sti_c(jit_state_t*,jit_word_t,jit_int32_t);
649# define str_s(r0,r1) STRHI(r1,r0,0)
650# define sti_s(i0,r0) _sti_s(_jit,i0,r0)
651static void _sti_s(jit_state_t*,jit_word_t,jit_int32_t);
652# define str_i(r0,r1) STRWI(r1,r0,0)
653# define sti_i(i0,r0) _sti_i(_jit,i0,r0)
654static void _sti_i(jit_state_t*,jit_word_t,jit_int32_t);
655# define str_l(r0,r1) STRI(r1,r0,0)
656# define sti_l(i0,r0) _sti_l(_jit,i0,r0)
657static void _sti_l(jit_state_t*,jit_word_t,jit_int32_t);
658# define stxr_c(r0,r1,r2) STRB(r2,r1,r0)
659# define stxi_c(i0,r0,r1) _stxi_c(_jit,i0,r0,r1)
660static void _stxi_c(jit_state_t*,jit_word_t,jit_int32_t,jit_int32_t);
661# define stxr_s(r0,r1,r2) STRH(r2,r1,r0)
662# define stxi_s(i0,r0,r1) _stxi_s(_jit,i0,r0,r1)
663static void _stxi_s(jit_state_t*,jit_word_t,jit_int32_t,jit_int32_t);
664# define stxr_i(r0,r1,r2) STRW(r2,r1,r0)
665# define stxi_i(i0,r0,r1) _stxi_i(_jit,i0,r0,r1)
666static void _stxi_i(jit_state_t*,jit_word_t,jit_int32_t,jit_int32_t);
667# define stxr_l(r0,r1,r2) STR(r2,r1,r0)
668# define stxi_l(i0,r0,r1) _stxi_l(_jit,i0,r0,r1)
669static void _stxi_l(jit_state_t*,jit_word_t,jit_int32_t,jit_int32_t);
40a44dcb
PC		 670# define bswapr_us(r0,r1) _bswapr_us(_jit,r0,r1)
671static void _bswapr_us(jit_state_t*,jit_int32_t,jit_int32_t);
672# define bswapr_ui(r0,r1) _bswapr_ui(_jit,r0,r1)
673static void _bswapr_ui(jit_state_t*,jit_int32_t,jit_int32_t);
674# define bswapr_ul(r0,r1) REV(r0,r1)
4a71579b
PC		 675# define extr_c(r0,r1) SXTB(r0,r1)
676# define extr_uc(r0,r1) UXTB(r0,r1)
677# define extr_s(r0,r1) SXTH(r0,r1)
678# define extr_us(r0,r1) UXTH(r0,r1)
679# define extr_i(r0,r1) SXTW(r0,r1)
680# define extr_ui(r0,r1) UXTW(r0,r1)
ba3814c1
PC		 681# define casx(r0, r1, r2, r3, i0) _casx(_jit, r0, r1, r2, r3, i0)
682static void _casx(jit_state_t *_jit,jit_int32_t,jit_int32_t,
683 jit_int32_t,jit_int32_t,jit_word_t);
684#define casr(r0, r1, r2, r3) casx(r0, r1, r2, r3, 0)
685#define casi(r0, i0, r1, r2) casx(r0, _NOREG, r1, r2, i0)
4a71579b
PC		 686# define movr(r0,r1) _movr(_jit,r0,r1)
687static void _movr(jit_state_t*,jit_int32_t,jit_int32_t);
688# define movi(r0,i0) _movi(_jit,r0,i0)
689static void _movi(jit_state_t*,jit_int32_t,jit_word_t);
690# define movi_p(r0,i0) _movi_p(_jit,r0,i0)
691static jit_word_t _movi_p(jit_state_t*,jit_int32_t,jit_word_t);
692# define ccr(cc,r0,r1,r2) _ccr(_jit,cc,r0,r1,r2)
693static void _ccr(jit_state_t*,jit_int32_t,jit_int32_t,jit_int32_t,jit_int32_t);
694# define cci(cc,r0,r1,i0) _cci(_jit,cc,r0,r1,i0)
695static void _cci(jit_state_t*,jit_int32_t,jit_int32_t,jit_int32_t,jit_word_t);
696# define ltr(r0,r1,r2) ccr(CC_LT,r0,r1,r2)
697# define lti(r0,r1,i0) cci(CC_LT,r0,r1,i0)
698# define ltr_u(r0,r1,r2) ccr(CC_CC,r0,r1,r2)
699# define lti_u(r0,r1,i0) cci(CC_CC,r0,r1,i0)
700# define ler(r0,r1,r2) ccr(CC_LE,r0,r1,r2)
701# define lei(r0,r1,i0) cci(CC_LE,r0,r1,i0)
702# define ler_u(r0,r1,r2) ccr(CC_LS,r0,r1,r2)
703# define lei_u(r0,r1,i0) cci(CC_LS,r0,r1,i0)
704# define eqr(r0,r1,r2) ccr(CC_EQ,r0,r1,r2)
705# define eqi(r0,r1,i0) cci(CC_EQ,r0,r1,i0)
706# define ger(r0,r1,r2) ccr(CC_GE,r0,r1,r2)
707# define gei(r0,r1,i0) cci(CC_GE,r0,r1,i0)
708# define ger_u(r0,r1,r2) ccr(CC_CS,r0,r1,r2)
709# define gei_u(r0,r1,i0) cci(CC_CS,r0,r1,i0)
710# define gtr(r0,r1,r2) ccr(CC_GT,r0,r1,r2)
711# define gti(r0,r1,i0) cci(CC_GT,r0,r1,i0)
712# define gtr_u(r0,r1,r2) ccr(CC_HI,r0,r1,r2)
713# define gti_u(r0,r1,i0) cci(CC_HI,r0,r1,i0)
714# define ner(r0,r1,r2) ccr(CC_NE,r0,r1,r2)
715# define nei(r0,r1,i0) cci(CC_NE,r0,r1,i0)
716# define bccr(cc,i0,r0,r1) _bccr(_jit,cc,i0,r0,r1)
717static jit_word_t
718_bccr(jit_state_t*,jit_int32_t,jit_word_t,jit_int32_t,jit_int32_t);
719# define bcci(cc,i0,r0,i1) _bcci(_jit,cc,i0,r0,i1)
720static jit_word_t
721_bcci(jit_state_t*,jit_int32_t,jit_word_t,jit_int32_t,jit_word_t);
722# define bltr(i0,r0,r1) bccr(BCC_LT,i0,r0,r1)
723# define blti(i0,r0,i1) bcci(BCC_LT,i0,r0,i1)
724# define bltr_u(i0,r0,r1) bccr(BCC_CC,i0,r0,r1)
725# define blti_u(i0,r0,i1) bcci(BCC_CC,i0,r0,i1)
726# define bler(i0,r0,r1) bccr(BCC_LE,i0,r0,r1)
727# define blei(i0,r0,i1) bcci(BCC_LE,i0,r0,i1)
728# define bler_u(i0,r0,r1) bccr(BCC_LS,i0,r0,r1)
729# define blei_u(i0,r0,i1) bcci(BCC_LS,i0,r0,i1)
730# define beqr(i0,r0,r1) bccr(BCC_EQ,i0,r0,r1)
731# define beqi(i0,r0,i1) _beqi(_jit,i0,r0,i1)
732static jit_word_t _beqi(jit_state_t*,jit_word_t,jit_int32_t,jit_word_t);
733# define bger(i0,r0,r1) bccr(BCC_GE,i0,r0,r1)
734# define bgei(i0,r0,i1) bcci(BCC_GE,i0,r0,i1)
735# define bger_u(i0,r0,r1) bccr(BCC_CS,i0,r0,r1)
736# define bgei_u(i0,r0,i1) bcci(BCC_CS,i0,r0,i1)
737# define bgtr(i0,r0,r1) bccr(BCC_GT,i0,r0,r1)
738# define bgti(i0,r0,i1) bcci(BCC_GT,i0,r0,i1)
739# define bgtr_u(i0,r0,r1) bccr(BCC_HI,i0,r0,r1)
740# define bgti_u(i0,r0,i1) bcci(BCC_HI,i0,r0,i1)
741# define bner(i0,r0,r1) bccr(BCC_NE,i0,r0,r1)
742# define bnei(i0,r0,i1) _bnei(_jit,i0,r0,i1)
743static jit_word_t _bnei(jit_state_t*,jit_word_t,jit_int32_t,jit_word_t);
744# define baddr(cc,i0,r0,r1) _baddr(_jit,cc,i0,r0,r1)
745static jit_word_t
746_baddr(jit_state_t*,jit_int32_t,jit_word_t,jit_int32_t,jit_int32_t);
747# define baddi(cc,i0,r0,i1) _baddi(_jit,cc,i0,r0,i1)
748static jit_word_t
749_baddi(jit_state_t*,jit_int32_t,jit_word_t,jit_int32_t,jit_word_t);
750# define boaddr(i0,r0,r1) baddr(BCC_VS,i0,r0,r1)
751# define boaddi(i0,r0,i1) baddi(BCC_VS,i0,r0,i1)
752# define boaddr_u(i0,r0,r1) baddr(BCC_HS,i0,r0,r1)
753# define boaddi_u(i0,r0,i1) baddi(BCC_HS,i0,r0,i1)
754# define bxaddr(i0,r0,r1) baddr(BCC_VC,i0,r0,r1)
755# define bxaddi(i0,r0,i1) baddi(BCC_VC,i0,r0,i1)
756# define bxaddr_u(i0,r0,r1) baddr(BCC_LO,i0,r0,r1)
757# define bxaddi_u(i0,r0,i1) baddi(BCC_LO,i0,r0,i1)
758# define bsubr(cc,i0,r0,r1) _bsubr(_jit,cc,i0,r0,r1)
759static jit_word_t
760_bsubr(jit_state_t*,jit_int32_t,jit_word_t,jit_int32_t,jit_int32_t);
761# define bsubi(cc,i0,r0,i1) _bsubi(_jit,cc,i0,r0,i1)
762static jit_word_t
763_bsubi(jit_state_t*,jit_int32_t,jit_word_t,jit_int32_t,jit_word_t);
764# define bosubr(i0,r0,r1) bsubr(BCC_VS,i0,r0,r1)
765# define bosubi(i0,r0,i1) bsubi(BCC_VS,i0,r0,i1)
766# define bosubr_u(i0,r0,r1) bsubr(BCC_LO,i0,r0,r1)
767# define bosubi_u(i0,r0,i1) bsubi(BCC_LO,i0,r0,i1)
768# define bxsubr(i0,r0,r1) bsubr(BCC_VC,i0,r0,r1)
769# define bxsubi(i0,r0,i1) bsubi(BCC_VC,i0,r0,i1)
770# define bxsubr_u(i0,r0,r1) bsubr(BCC_HS,i0,r0,r1)
771# define bxsubi_u(i0,r0,i1) bsubi(BCC_HS,i0,r0,i1)
772# define bmxr(cc,i0,r0,r1) _bmxr(_jit,cc,i0,r0,r1)
773static jit_word_t
774_bmxr(jit_state_t*,jit_int32_t,jit_word_t,jit_int32_t,jit_int32_t);
775# define bmxi(cc,i0,r0,r1) _bmxi(_jit,cc,i0,r0,r1)
776static jit_word_t
777_bmxi(jit_state_t*,jit_int32_t,jit_word_t,jit_int32_t,jit_word_t);
778# define bmsr(i0,r0,r1) bmxr(BCC_NE,i0,r0,r1)
779# define bmsi(i0,r0,i1) bmxi(BCC_NE,i0,r0,i1)
780# define bmcr(i0,r0,r1) bmxr(BCC_EQ,i0,r0,r1)
781# define bmci(i0,r0,i1) bmxi(BCC_EQ,i0,r0,i1)
782# define jmpr(r0) BR(r0)
783# define jmpi(i0) _jmpi(_jit,i0)
784static void _jmpi(jit_state_t*,jit_word_t);
785# define jmpi_p(i0) _jmpi_p(_jit,i0)
786static jit_word_t _jmpi_p(jit_state_t*,jit_word_t);
787# define callr(r0) BLR(r0)
788# define calli(i0) _calli(_jit,i0)
789static void _calli(jit_state_t*,jit_word_t);
790# define calli_p(i0) _calli_p(_jit,i0)
791static jit_word_t _calli_p(jit_state_t*,jit_word_t);
792# define prolog(i0) _prolog(_jit,i0)
793static void _prolog(jit_state_t*,jit_node_t*);
794# define epilog(i0) _epilog(_jit,i0)
795static void _epilog(jit_state_t*,jit_node_t*);
796# define vastart(r0) _vastart(_jit, r0)
797static void _vastart(jit_state_t*, jit_int32_t);
798# define vaarg(r0, r1) _vaarg(_jit, r0, r1)
799static void _vaarg(jit_state_t*, jit_int32_t, jit_int32_t);
800# define patch_at(jump,label) _patch_at(_jit,jump,label)
801static void _patch_at(jit_state_t*,jit_word_t,jit_word_t);
802#endif
803
804#if CODE
805static jit_int32_t
806logical_immediate(jit_word_t imm)
807{
808 /* There are 5334 possible immediate values, but to avoid the
809 * need of either too complex code or large lookup tables,
810 * only check for (simply) encodable common/small values */
811 switch (imm) {
812 case -16: return (0xf3b);
813 case -15: return (0xf3c);
814 case -13: return (0xf3d);
815 case -9: return (0xf3e);
816 case -8: return (0xf7c);
817 case -7: return (0xf7d);
818 case -5: return (0xf7e);
819 case -4: return (0xfbd);
820 case -3: return (0xfbe);
821 case -2: return (0xffe);
822 case 1: return (0x000);
823 case 2: return (0xfc0);
824 case 3: return (0x001);
825 case 4: return (0xf80);
826 case 6: return (0xfc1);
827 case 7: return (0x002);
828 case 8: return (0xf40);
829 case 12: return (0xf81);
830 case 14: return (0xfc2);
831 case 15: return (0x003);
832 case 16: return (0xf00);
833 default: return (-1);
834 }
835}
836
837static void
838_oxxx(jit_state_t *_jit, jit_int32_t Op,
839 jit_int32_t Rd, jit_int32_t Rn, jit_int32_t Rm)
840{
841 instr_t i;
842 assert(!(Rd & ~0x1f));
843 assert(!(Rn & ~0x1f));
844 assert(!(Rm & ~0x1f));
845 assert(!(Op & ~0xffe0fc00));
846 i.w = Op;
847 i.Rd.b = Rd;
848 i.Rn.b = Rn;
849 i.Rm.b = Rm;
850 ii(i.w);
851}
852
853static void
854_oxxi(jit_state_t *_jit, jit_int32_t Op,
855 jit_int32_t Rd, jit_int32_t Rn, jit_int32_t Imm12)
856{
857 instr_t i;
858 assert(!(Rd & ~0x1f));
859 assert(!(Rn & ~0x1f));
860 assert(!(Imm12 & ~0xfff));
861 assert(!(Op & ~0xffe00000));
862 i.w = Op;
863 i.Rd.b = Rd;
864 i.Rn.b = Rn;
865 i.imm12.b = Imm12;
866 ii(i.w);
867}
868
869static void
870_oxx9(jit_state_t *_jit, jit_int32_t Op,
871 jit_int32_t Rd, jit_int32_t Rn, jit_int32_t Imm9)
872{
873 instr_t i;
874 assert(!(Rd & ~0x1f));
875 assert(!(Rn & ~0x1f));
876 assert(!(Imm9 & ~0x1ff));
877 assert(!(Op & ~0xffe00000));
878 i.w = Op;
879 i.Rd.b = Rd;
880 i.Rn.b = Rn;
881 i.imm9.b = Imm9;
882 ii(i.w);
883}
884
885static void
886_ox19(jit_state_t *_jit, jit_int32_t Op, jit_int32_t Rd, jit_int32_t Simm19)
887{
888 instr_t i;
889 assert(!(Rd & ~0x1f));
890 assert(Simm19 >= -262148 && Simm19 <= 262143);
891 assert(!(Op & ~0xff000000));
892 i.w = Op;
893 i.Rd.b = Rd;
894 i.imm19.b = Simm19;
895 ii(i.w);
896}
897
898static void
899_oc19(jit_state_t *_jit, jit_int32_t Op, jit_int32_t Cc, jit_int32_t Simm19)
900{
901 instr_t i;
902 assert(!(Cc & ~0xf));
903 assert(Simm19 >= -262148 && Simm19 <= 262143);
904 assert(!(Op & ~0xff000000));
905 i.w = Op;
906 i.cond2.b = Cc;
907 i.imm19.b = Simm19;
908 ii(i.w);
909}
910
911static void
912_o26(jit_state_t *_jit, jit_int32_t Op, jit_int32_t Simm26)
913{
914 instr_t i;
915 assert(Simm26 >= -33554432 && Simm26 <= 33554431);
916 assert(!(Op & ~0xfc000000));
917 i.w = Op;
918 i.imm26.b = Simm26;
919 ii(i.w);
920}
921
922static void
923_ox_x(jit_state_t *_jit, jit_int32_t Op, jit_int32_t Rd, jit_int32_t Rm)
924{
925 instr_t i;
926 assert(!(Rd & ~0x1f));
927 assert(!(Rm & ~0x1f));
928 assert(!(Op & ~0xffe0ffe0));
929 i.w = Op;
930 i.Rd.b = Rd;
931 i.Rm.b = Rm;
932 ii(i.w);
933}
934
935static void
936_o_xx(jit_state_t *_jit, jit_int32_t Op, jit_int32_t Rd, jit_int32_t Rn)
937{
938 instr_t i;
939 assert(!(Rd & ~0x1f));
940 assert(!(Rn & ~0x1f));
941 assert(!(Op & ~0xfffffc00));
942 i.w = Op;
943 i.Rd.b = Rd;
944 i.Rn.b = Rn;
945 ii(i.w);
946}
947
948static void
949_oxx_(jit_state_t *_jit, jit_int32_t Op, jit_int32_t Rn, jit_int32_t Rm)
950{
951 instr_t i;
952 assert(!(Rn & ~0x1f));
953 assert(!(Rm & ~0x1f));
954 assert(!(Op & ~0xffc0fc1f));
955 i.w = Op;
956 i.Rn.b = Rn;
957 i.Rm.b = Rm;
958 ii(i.w);
959}
960
961static void
962_o_x_(jit_state_t *_jit, jit_int32_t Op, jit_int32_t Rn)
963{
964 instr_t i;
965 assert(!(Rn & ~0x1f));
966 assert(!(Op & 0x3e0));
967 i.w = Op;
968 i.Rn.b = Rn;
969 ii(i.w);
970}
971
972static void
973_ox_h(jit_state_t *_jit, jit_int32_t Op, jit_int32_t Rd, jit_int32_t Imm16)
974{
975 instr_t i;
976 assert(!(Rd & ~0x1f));
977 assert(!(Imm16 & ~0xffff));
978 assert(!(Op & ~0xffe00000));
979 i.w = Op;
980 i.Rd.b = Rd;
981 i.imm16.b = Imm16;
982 ii(i.w);
983}
984
985static void
986_oxxrs(jit_state_t *_jit, jit_int32_t Op,
987 jit_int32_t Rd, jit_int32_t Rn, jit_int32_t R, jit_int32_t S)
988{
989 instr_t i;
990 assert(!(Rd & ~0x1f));
991 assert(!(Rn & ~0x1f));
992 assert(!(R & ~0x3f));
993 assert(!(S & ~0x3f));
994 assert(!(Op & ~0xffc00000));
995 i.w = Op;
996 i.Rd.b = Rd;
997 i.Rn.b = Rn;
998 i.immr.b = R;
999 i.imms.b = S;
1000 ii(i.w);
1001}
1002
1003static void
1004_oxxxc(jit_state_t *_jit, jit_int32_t Op,
1005 jit_int32_t Rd, jit_int32_t Rn, jit_int32_t Rm, jit_int32_t Cc)
1006{
1007 instr_t i;
1008 assert(!(Rd & ~0x1f));
1009 assert(!(Rn & ~0x1f));
1010 assert(!(Rm & ~0x1f));
1011 assert(!(Cc & ~0xf));
1012 assert(!(Op & ~0xffc00c00));
1013 i.w = Op;
1014 i.Rd.b = Rd;
1015 i.Rn.b = Rn;
1016 i.Rm.b = Rm;
1017 i.cond.b = Cc;
1018 ii(i.w);
1019}
1020
1021static void
1022_oxxx7(jit_state_t *_jit, jit_int32_t Op,
1023 jit_int32_t Rt, jit_int32_t Rt2, jit_int32_t Rn, jit_int32_t Simm7)
1024{
1025 instr_t i;
1026 assert(!(Rt & ~0x1f));
1027 assert(!(Rt2 & ~0x1f));
1028 assert(!(Rn & ~0x1f));
1029 assert(Simm7 >= -128 && Simm7 <= 127);
1030 assert(!(Op & ~0xffc003e0));
1031 i.w = Op;
1032 i.Rt.b = Rt;
1033 i.Rt2.b = Rt2;
1034 i.Rn.b = Rn;
1035 i.imm7.b = Simm7;
1036 ii(i.w);
1037}
1038
1039static void
1040_nop(jit_state_t *_jit, jit_int32_t i0)
1041{
1042 for (; i0 > 0; i0 -= 4)
1043 NOP();
1044 assert(i0 == 0);
1045}
1046
1047static void
1048_addi(jit_state_t *_jit, jit_int32_t r0, jit_int32_t r1, jit_word_t i0)
1049{
1050 jit_int32_t reg;
1051 jit_word_t is = i0 >> 12;
1052 jit_word_t in = -i0;
1053 jit_word_t iS = in >> 12;
1054 if ( i0 >= 0 && i0 <= 0xfff)
1055 ADDI (r0, r1, i0);
1056 else if ((is << 12) == i0 && is >= 0 && is <= 0xfff)
1057 ADDI_12(r0, r1, is);
1058 else if ( in >= 0 && in <= 0xfff)
1059 SUBI (r0, r1, in);
1060 else if ((iS << 12) == is && iS >= 0 && iS <= 0xfff)
1061 SUBI_12(r0, r1, iS);
1062 else {
1063 reg = jit_get_reg(jit_class_gpr);
1064 movi(rn(reg), i0);
1065 addr(r0, r1, rn(reg));
1066 jit_unget_reg(reg);
1067 }
1068}
1069
1070static void
1071_addci(jit_state_t *_jit, jit_int32_t r0, jit_int32_t r1, jit_word_t i0)
1072{
1073 jit_int32_t reg;
1074 jit_word_t is = i0 >> 12;
1075 jit_word_t in = -i0;
1076 jit_word_t iS = in >> 12;
1077 if ( i0 >= 0 && i0 <= 0xfff)
1078 ADDSI (r0, r1, i0);
1079 else if ((is << 12) == i0 && is >= 0 && is <= 0xfff)
1080 ADDSI_12(r0, r1, is);
1081 else if ( in >= 0 && in <= 0xfff)
1082 SUBSI (r0, r1, in);
1083 else if ((iS << 12) == is && iS >= 0 && iS <= 0xfff)
1084 SUBSI_12(r0, r1, iS);
1085 else {
1086 reg = jit_get_reg(jit_class_gpr);
1087 movi(rn(reg), i0);
1088 addcr(r0, r1, rn(reg));
1089 jit_unget_reg(reg);
1090 }
1091}
1092
1093static void
1094_addxi(jit_state_t *_jit, jit_int32_t r0, jit_int32_t r1, jit_word_t i0)
1095{
1096 jit_int32_t reg;
1097 reg = jit_get_reg(jit_class_gpr);
1098 movi(rn(reg), i0);
1099 addxr(r0, r1, rn(reg));
1100 jit_unget_reg(reg);
1101}
1102
1103static void
1104_subi(jit_state_t *_jit, jit_int32_t r0, jit_int32_t r1, jit_word_t i0)
1105{
1106 jit_int32_t reg;
1107 jit_word_t is = i0 >> 12;
1108 if ( i0 >= 0 && i0 <= 0xfff)
1109 SUBI (r0, r1, i0);
1110 else if ((is << 12) == i0 && is >= 0 && is <= 0xfff)
1111 SUBI_12(r0, r1, is);
1112 else {
1113 reg = jit_get_reg(jit_class_gpr);
1114 movi(rn(reg), i0);
1115 subr(r0, r1, rn(reg));
1116 jit_unget_reg(reg);
1117 }
1118}
1119
1120static void
1121_subci(jit_state_t *_jit, jit_int32_t r0, jit_int32_t r1, jit_word_t i0)
1122{
1123 jit_int32_t reg;
1124 jit_word_t is = i0 >> 12;
1125 if ( i0 >= 0 && i0 <= 0xfff)
1126 SUBSI (r0, r1, i0);
1127 else if ((is << 12) == i0 && is >= 0 && is <= 0xfff)
1128 SUBSI_12(r0, r1, is);
1129 else {
1130 reg = jit_get_reg(jit_class_gpr);
1131 movi(rn(reg), i0);
1132 subcr(r0, r1, rn(reg));
1133 jit_unget_reg(reg);
1134 }
1135}
1136
1137static void
1138_subxi(jit_state_t *_jit, jit_int32_t r0, jit_int32_t r1, jit_word_t i0)
1139{
1140 jit_int32_t reg;
1141 reg = jit_get_reg(jit_class_gpr);
1142 movi(rn(reg), i0);
1143 subxr(r0, r1, rn(reg));
1144 jit_unget_reg(reg);
1145}
1146
1147static void
1148_rsbi(jit_state_t *_jit, jit_int32_t r0, jit_int32_t r1, jit_word_t i0)
1149{
1150 subi(r0, r1, i0);
1151 negr(r0, r0);
1152}
1153
1154static void
1155_muli(jit_state_t *_jit, jit_int32_t r0, jit_int32_t r1, jit_word_t i0)
1156{
1157 jit_int32_t reg;
1158 reg = jit_get_reg(jit_class_gpr);
1159 movi(rn(reg), i0);
1160 mulr(r0, r1, rn(reg));
1161 jit_unget_reg(reg);
1162}
1163
1164static void
1165_qmulr(jit_state_t *_jit, jit_int32_t r0,
1166 jit_int32_t r1, jit_int32_t r2, jit_int32_t r3)
1167{
1168 jit_int32_t reg;
1169 if (r0 == r2 || r0 == r3) {
1170 reg = jit_get_reg(jit_class_gpr);
1171 mulr(rn(reg), r2, r3);
1172 }
1173 else
1174 mulr(r0, r2, r3);
1175 SMULH(r1, r2, r3);
1176 if (r0 == r2 || r0 == r3) {
1177 movr(r0, rn(reg));
1178 jit_unget_reg(reg);
1179 }
1180}
1181
1182static void
1183_qmuli(jit_state_t *_jit, jit_int32_t r0,
1184 jit_int32_t r1, jit_int32_t r2, jit_word_t i0)
1185{
1186 jit_int32_t reg;
1187 reg = jit_get_reg(jit_class_gpr);
1188 movi(rn(reg), i0);
1189 qmulr(r0, r1, r2, rn(reg));
1190 jit_unget_reg(reg);
1191}
1192
1193static void
1194_qmulr_u(jit_state_t *_jit, jit_int32_t r0,
1195 jit_int32_t r1, jit_int32_t r2, jit_int32_t r3)
1196{
1197 jit_int32_t reg;
1198 if (r0 == r2 || r0 == r3) {
1199 reg = jit_get_reg(jit_class_gpr);
1200 mulr(rn(reg), r2, r3);
1201 }
1202 else
1203 mulr(r0, r2, r3);
1204 UMULH(r1, r2, r3);
1205 if (r0 == r2 || r0 == r3) {
1206 movr(r0, rn(reg));
1207 jit_unget_reg(reg);
1208 }
1209}
1210
1211static void
1212_qmuli_u(jit_state_t *_jit, jit_int32_t r0,
1213 jit_int32_t r1, jit_int32_t r2, jit_word_t i0)
1214{
1215 jit_int32_t reg;
1216 reg = jit_get_reg(jit_class_gpr);
1217 movi(rn(reg), i0);
1218 qmulr_u(r0, r1, r2, rn(reg));
1219 jit_unget_reg(reg);
1220}
1221
1222static void
1223_divi(jit_state_t *_jit, jit_int32_t r0, jit_int32_t r1, jit_word_t i0)
1224{
1225 jit_int32_t reg;
1226 reg = jit_get_reg(jit_class_gpr);
1227 movi(rn(reg), i0);
1228 divr(r0, r1, rn(reg));
1229 jit_unget_reg(reg);
1230}
1231
1232static void
1233_divi_u(jit_state_t *_jit, jit_int32_t r0, jit_int32_t r1, jit_word_t i0)
1234{
1235 jit_int32_t reg;
1236 reg = jit_get_reg(jit_class_gpr);
1237 movi(rn(reg), i0);
1238 divr_u(r0, r1, rn(reg));
1239 jit_unget_reg(reg);
1240}
1241
1242static void
1243_iqdivr(jit_state_t *_jit, jit_bool_t sign,
1244 jit_int32_t r0, jit_int32_t r1, jit_int32_t r2, jit_int32_t r3)
1245{
1246 jit_int32_t sv0, rg0;
1247 jit_int32_t sv1, rg1;
1248 if (r0 == r2 || r0 == r3) {
1249 sv0 = jit_get_reg(jit_class_gpr);
1250 rg0 = rn(sv0);
1251 }
1252 else
1253 rg0 = r0;
1254 if (r1 == r2 || r1 == r3) {
1255 sv1 = jit_get_reg(jit_class_gpr);
1256 rg1 = rn(sv1);
1257 }
1258 else
1259 rg1 = r1;
1260 if (sign)
1261 divr(rg0, r2, r3);
1262 else
1263 divr_u(rg0, r2, r3);
1264 mulr(rg1, r3, rg0);
1265 subr(rg1, r2, rg1);
1266 if (rg0 != r0) {
1267 movr(r0, rg0);
1268 jit_unget_reg(sv0);
1269 }
1270 if (rg1 != r1) {
1271 movr(r1, rg1);
1272 jit_unget_reg(sv1);
1273 }
1274}
1275
1276static void
1277_qdivi(jit_state_t *_jit, jit_int32_t r0,
1278 jit_int32_t r1, jit_int32_t r2, jit_word_t i0)
1279{
1280 jit_int32_t reg;
1281 reg = jit_get_reg(jit_class_gpr);
1282 movi(rn(reg), i0);
1283 qdivr(r0, r1, r2, rn(reg));
1284 jit_unget_reg(reg);
1285}
1286
1287static void
1288_qdivi_u(jit_state_t *_jit, jit_int32_t r0,
1289 jit_int32_t r1, jit_int32_t r2, jit_word_t i0)
1290{
1291 jit_int32_t reg;
1292 reg = jit_get_reg(jit_class_gpr);
1293 movi(rn(reg), i0);
1294 qdivr_u(r0, r1, r2, rn(reg));
1295 jit_unget_reg(reg);
1296}
1297
1298static void
1299_remr(jit_state_t *_jit, jit_int32_t r0, jit_int32_t r1, jit_int32_t r2)
1300{
1301 jit_int32_t reg;
1302 if (r0 == r1 || r0 == r2) {
1303 reg = jit_get_reg(jit_class_gpr);
1304 divr(rn(reg), r1, r2);
1305 mulr(rn(reg), r2, rn(reg));
1306 subr(r0, r1, rn(reg));
1307 jit_unget_reg(reg);
1308 }
1309 else {
1310 divr(r0, r1, r2);
1311 mulr(r0, r2, r0);
1312 subr(r0, r1, r0);
1313 }
1314}
1315
1316static void
1317_remi(jit_state_t *_jit, jit_int32_t r0, jit_int32_t r1, jit_word_t i0)
1318{
1319 jit_int32_t reg;
1320 reg = jit_get_reg(jit_class_gpr);
1321 movi(rn(reg), i0);
1322 remr(r0, r1, rn(reg));
1323 jit_unget_reg(reg);
1324}
1325
1326static void
1327_remr_u(jit_state_t *_jit, jit_int32_t r0, jit_int32_t r1, jit_int32_t r2)
1328{
1329 jit_int32_t reg;
1330 if (r0 == r1 || r0 == r2) {
1331 reg = jit_get_reg(jit_class_gpr);
1332 divr_u(rn(reg), r1, r2);
1333 mulr(rn(reg), r2, rn(reg));
1334 subr(r0, r1, rn(reg));
1335 jit_unget_reg(reg);
1336 }
1337 else {
1338 divr_u(r0, r1, r2);
1339 mulr(r0, r2, r0);
1340 subr(r0, r1, r0);
1341 }
1342}
1343
1344static void
1345_remi_u(jit_state_t *_jit, jit_int32_t r0, jit_int32_t r1, jit_word_t i0)
1346{
1347 jit_int32_t reg;
1348 reg = jit_get_reg(jit_class_gpr);
1349 movi(rn(reg), i0);
1350 remr_u(r0, r1, rn(reg));
1351 jit_unget_reg(reg);
1352}
1353
1354static void
1355_lshi(jit_state_t *_jit, jit_int32_t r0, jit_int32_t r1, jit_word_t i0)
1356{
1357 if (i0 == 0)
1358 movr(r0, r1);
1359 else {
1360 assert(i0 > 0 && i0 < 64);
1361 LSLI(r0, r1, i0);
1362 }
1363}
1364
1365static void
1366_rshi(jit_state_t *_jit, jit_int32_t r0, jit_int32_t r1, jit_word_t i0)
1367{
1368 if (i0 == 0)
1369 movr(r0, r1);
1370 else {
1371 assert(i0 > 0 && i0 < 64);
1372 ASRI(r0, r1, i0);
1373 }
1374}
1375
1376static void
1377_rshi_u(jit_state_t *_jit, jit_int32_t r0, jit_int32_t r1, jit_word_t i0)
1378{
1379 if (i0 == 0)
1380 movr(r0, r1);
1381 else {
1382 assert(i0 > 0 && i0 < 64);
1383 LSRI(r0, r1, i0);
1384 }
1385}
1386
e0659411
PC		 1387static void
1388_movnr(jit_state_t *_jit, jit_int32_t r0, jit_int32_t r1, jit_int32_t r2)
1389{
1390 CMPI(r2, 0);
1391 CSEL(r0, r0, r1, CC_NE);
1392}
1393
1394static void
1395_movzr(jit_state_t *_jit, jit_int32_t r0, jit_int32_t r1, jit_int32_t r2)
1396{
1397 CMPI(r2, 0);
1398 CSEL(r0, r0, r1, CC_EQ);
1399}
1400
4a71579b
PC		 1401static void
1402_andi(jit_state_t *_jit, jit_int32_t r0, jit_int32_t r1, jit_word_t i0)
1403{
1404 jit_int32_t reg;
1405 jit_int32_t imm;
1406 if (i0 == 0)
1407 movi(r0, 0);
1408 else if (i0 == -1)
1409 movr(r0, r1);
1410 else {
1411 imm = logical_immediate(i0);
1412 if (imm != -1)
1413 ANDI(r0, r1, imm);
1414 else {
1415 reg = jit_get_reg(jit_class_gpr);
1416 movi(rn(reg), i0);
1417 andr(r0, r1, rn(reg));
1418 jit_unget_reg(reg);
1419 }
1420 }
1421}
1422
1423static void
1424_ori(jit_state_t *_jit, jit_int32_t r0, jit_int32_t r1, jit_word_t i0)
1425{
1426 jit_int32_t reg;
1427 jit_int32_t imm;
1428 if (i0 == 0)
1429 movr(r0, r1);
1430 else if (i0 == -1)
1431 movi(r0, -1);
1432 else {
1433 imm = logical_immediate(i0);
1434 if (imm != -1)
1435 ORRI(r0, r1, imm);
1436 else {
1437 reg = jit_get_reg(jit_class_gpr);
1438 movi(rn(reg), i0);
1439 orr(r0, r1, rn(reg));
1440 jit_unget_reg(reg);
1441 }
1442 }
1443}
1444
1445static void
1446_xori(jit_state_t *_jit, jit_int32_t r0, jit_int32_t r1, jit_word_t i0)
1447{
1448 jit_int32_t reg;
1449 jit_int32_t imm;
1450 if (i0 == 0)
1451 movr(r0, r1);
1452 else if (i0 == -1)
1453 comr(r0, r1);
1454 else {
1455 imm = logical_immediate(i0);
1456 if (imm != -1)
1457 EORI(r0, r1, imm);
1458 else {
1459 reg = jit_get_reg(jit_class_gpr);
1460 movi(rn(reg), i0);
1461 xorr(r0, r1, rn(reg));
1462 jit_unget_reg(reg);
1463 }
1464 }
1465}
1466
4a71579b 1467static void
40a44dcb 1468_bswapr_us(jit_state_t *_jit, jit_int32_t r0, jit_int32_t r1)
4a71579b 1469{
40a44dcb 1470 bswapr_ul(r0, r1);
4a71579b
PC		 1471 rshi_u(r0, r0, 48);
1472}
1473
1474static void
40a44dcb 1475_bswapr_ui(jit_state_t *_jit, jit_int32_t r0, jit_int32_t r1)
4a71579b 1476{
40a44dcb 1477 bswapr_ul(r0, r1);
4a71579b
PC		 1478 rshi_u(r0, r0, 32);
1479}
4a71579b
PC		 1480
1481static void
1482_ldi_c(jit_state_t *_jit, jit_int32_t r0, jit_word_t i0)
1483{
1484 jit_int32_t reg;
1485 reg = jit_get_reg(jit_class_gpr);
1486 movi(rn(reg), i0);
1487 ldr_c(r0, rn(reg));
1488 jit_unget_reg(reg);
1489}
1490
1491static void
1492_ldr_uc(jit_state_t *_jit, jit_int32_t r0, jit_int32_t r1)
1493{
1494 LDRBI(r0, r1, 0);
1495#if 0
1496 extr_uc(r0, r0);
1497#endif
1498}
1499
1500static void
1501_ldi_uc(jit_state_t *_jit, jit_int32_t r0, jit_word_t i0)
1502{
1503 jit_int32_t reg;
1504 reg = jit_get_reg(jit_class_gpr);
1505 movi(rn(reg), i0);
1506 ldr_uc(r0, rn(reg));
1507 jit_unget_reg(reg);
1508}
1509
1510static void
1511_ldi_s(jit_state_t *_jit, jit_int32_t r0, jit_word_t i0)
1512{
1513 jit_int32_t reg;
1514 reg = jit_get_reg(jit_class_gpr);
1515 movi(rn(reg), i0);
1516 ldr_s(r0, rn(reg));
1517 jit_unget_reg(reg);
1518}
1519
1520static void
1521_ldr_us(jit_state_t *_jit, jit_int32_t r0, jit_int32_t r1)
1522{
1523 LDRHI(r0, r1, 0);
1524#if 0
1525 extr_us(r0, r0);
1526#endif
1527}
1528
1529static void
1530_ldi_us(jit_state_t *_jit, jit_int32_t r0, jit_word_t i0)
1531{
1532 jit_int32_t reg;
1533 reg = jit_get_reg(jit_class_gpr);
1534 movi(rn(reg), i0);
1535 ldr_us(r0, rn(reg));
1536 jit_unget_reg(reg);
1537}
1538
1539static void
1540_ldi_i(jit_state_t *_jit, jit_int32_t r0, jit_word_t i0)
1541{
1542 jit_int32_t reg;
1543 reg = jit_get_reg(jit_class_gpr);
1544 movi(rn(reg), i0);
1545 ldr_i(r0, rn(reg));
1546 jit_unget_reg(reg);
1547}
1548
1549static void
1550_ldr_ui(jit_state_t *_jit, jit_int32_t r0, jit_int32_t r1)
1551{
1552 LDRWI(r0, r1, 0);
1553#if 0
1554 extr_ui(r0, r0);
1555#endif
1556}
1557
1558static void
1559_ldi_ui(jit_state_t *_jit, jit_int32_t r0, jit_word_t i0)
1560{
1561 jit_int32_t reg;
1562 reg = jit_get_reg(jit_class_gpr);
1563 movi(rn(reg), i0);
1564 ldr_ui(r0, rn(reg));
1565 jit_unget_reg(reg);
1566}
1567
1568static void
1569_ldi_l(jit_state_t *_jit, jit_int32_t r0, jit_word_t i0)
1570{
1571 jit_int32_t reg;
1572 reg = jit_get_reg(jit_class_gpr);
1573 movi(rn(reg), i0);
1574 ldr_l(r0, rn(reg));
1575 jit_unget_reg(reg);
1576}
1577
1578static void
1579_ldxr_c(jit_state_t *_jit, jit_int32_t r0, jit_int32_t r1, jit_int32_t r2)
1580{
1581 LDRSB(r0, r1, r2);
1582 extr_c(r0, r0);
1583}
1584
1585static void
1586_ldxi_c(jit_state_t *_jit, jit_int32_t r0, jit_int32_t r1, jit_word_t i0)
1587{
1588 jit_int32_t reg;
1589 if (i0 >= 0 && i0 <= 4095)
1590 LDRSBI(r0, r1, i0);
1591 else if (i0 > -256 && i0 < 0)
1592 LDURSB(r0, r1, i0 & 0x1ff);
1593 else {
1594 reg = jit_get_reg(jit_class_gpr);
1595 movi(rn(reg), i0);
1596 LDRSB(r0, r1, rn(reg));
1597 jit_unget_reg(reg);
1598 }
1599 extr_c(r0, r0);
1600}
1601
1602static void
1603_ldxr_uc(jit_state_t *_jit, jit_int32_t r0, jit_int32_t r1, jit_int32_t r2)
1604{
1605 LDRB(r0, r1, r2);
1606#if 0
1607 extr_uc(r0, r0);
1608#endif
1609}
1610
1611static void
1612_ldxi_uc(jit_state_t *_jit, jit_int32_t r0, jit_int32_t r1, jit_word_t i0)
1613{
1614 jit_int32_t reg;
1615 if (i0 >= 0 && i0 <= 4095)
1616 LDRBI(r0, r1, i0);
1617 else if (i0 > -256 && i0 < 0)
1618 LDURB(r0, r1, i0 & 0x1ff);
1619 else {
1620 reg = jit_get_reg(jit_class_gpr);
1621 addi(rn(reg), r1, i0);
1622 ldr_uc(r0, rn(reg));
1623 jit_unget_reg(reg);
1624 }
1625#if 0
1626 extr_uc(r0, r0);
1627#endif
1628}
1629
1630static void
1631_ldxi_s(jit_state_t *_jit, jit_int32_t r0, jit_int32_t r1, jit_word_t i0)
1632{
1633 jit_int32_t reg;
519a9ea1 1634 if (i0 >= 0 && i0 <= 8191 && !(i0 & 1))
4a71579b
PC		 1635 LDRSHI(r0, r1, i0 >> 1);
1636 else if (i0 > -256 && i0 < 0)
1637 LDURSH(r0, r1, i0 & 0x1ff);
1638 else {
1639 reg = jit_get_reg(jit_class_gpr);
1640 movi(rn(reg), i0);
1641 LDRSH(r0, r1, rn(reg));
1642 jit_unget_reg(reg);
1643 }
1644}
1645
1646static void
1647_ldxr_us(jit_state_t *_jit, jit_int32_t r0, jit_int32_t r1, jit_int32_t r2)
1648{
1649 LDRH(r0, r1, r2);
1650#if 0
1651 extr_us(r0, r0);
1652#endif
1653}
1654
1655static void
1656_ldxi_us(jit_state_t *_jit, jit_int32_t r0, jit_int32_t r1, jit_word_t i0)
1657{
1658 jit_int32_t reg;
519a9ea1 1659 if (i0 >= 0 && i0 <= 8191 && !(i0 & 1))
4a71579b
PC		 1660 LDRHI(r0, r1, i0 >> 1);
1661 else if (i0 > -256 && i0 < 0)
1662 LDURH(r0, r1, i0 & 0x1ff);
1663 else {
1664 reg = jit_get_reg(jit_class_gpr);
1665 movi(rn(reg), i0);
1666 LDRH(r0, r1, rn(reg));
1667 jit_unget_reg(reg);
1668 }
1669#if 0
1670 extr_us(r0, r0);
1671#endif
1672}
1673
1674static void
1675_ldxi_i(jit_state_t *_jit, jit_int32_t r0, jit_int32_t r1, jit_word_t i0)
1676{
1677 jit_int32_t reg;
519a9ea1 1678 if (i0 >= 0 && i0 <= 16383 && !(i0 & 3))
4a71579b
PC		 1679 LDRSWI(r0, r1, i0 >> 2);
1680 else if (i0 > -256 && i0 < 0)
1681 LDURSW(r0, r1, i0 & 0x1ff);
1682 else {
1683 reg = jit_get_reg(jit_class_gpr);
1684 addi(rn(reg), r1, i0);
1685 ldr_i(r0, rn(reg));
1686 jit_unget_reg(reg);
1687 }
1688}
1689
1690static void
1691_ldxr_ui(jit_state_t *_jit, jit_int32_t r0, jit_int32_t r1, jit_int32_t r2)
1692{
1693 LDRW(r0, r1, r2);
1694#if 0
1695 extr_ui(r0, r0);
1696#endif
1697}
1698
1699static void
1700_ldxi_ui(jit_state_t *_jit, jit_int32_t r0, jit_int32_t r1, jit_word_t i0)
1701{
1702 jit_int32_t reg;
519a9ea1 1703 if (i0 >= 0 && i0 <= 16383 && !(i0 & 3))
4a71579b
PC		 1704 LDRWI(r0, r1, i0 >> 2);
1705 else if (i0 > -256 && i0 < 0)
1706 LDURW(r0, r1, i0 & 0x1ff);
1707 else {
1708 reg = jit_get_reg(jit_class_gpr);
1709 movi(rn(reg), i0);
1710 LDRW(r0, r1, rn(reg));
1711 jit_unget_reg(reg);
1712 }
1713#if 0
1714 extr_ui(r0, r0);
1715#endif
1716}
1717
1718static void
1719_ldxi_l(jit_state_t *_jit, jit_int32_t r0, jit_int32_t r1, jit_word_t i0)
1720{
1721 jit_int32_t reg;
519a9ea1 1722 if (i0 >= 0 && i0 <= 32767 && !(i0 & 7))
4a71579b
PC		 1723 LDRI(r0, r1, i0 >> 3);
1724 else if (i0 > -256 && i0 < 0)
1725 LDUR(r0, r1, i0 & 0x1ff);
1726 else {
1727 reg = jit_get_reg(jit_class_gpr);
1728 addi(rn(reg), r1, i0);
1729 ldr_l(r0, rn(reg));
1730 jit_unget_reg(reg);
1731 }
1732}
1733
1734static void
1735_sti_c(jit_state_t *_jit, jit_word_t i0, jit_int32_t r0)
1736{
1737 jit_int32_t reg;
1738 reg = jit_get_reg(jit_class_gpr);
1739 movi(rn(reg), i0);
1740 str_c(rn(reg), r0);
1741 jit_unget_reg(reg);
1742}
1743
1744static void
1745_sti_s(jit_state_t *_jit, jit_word_t i0, jit_int32_t r0)
1746{
1747 jit_int32_t reg;
1748 reg = jit_get_reg(jit_class_gpr);
1749 movi(rn(reg), i0);
1750 str_s(rn(reg), r0);
1751 jit_unget_reg(reg);
1752}
1753
1754static void
1755_sti_i(jit_state_t *_jit, jit_word_t i0, jit_int32_t r0)
1756{
1757 jit_int32_t reg;
1758 reg = jit_get_reg(jit_class_gpr);
1759 movi(rn(reg), i0);
1760 str_i(rn(reg), r0);
1761 jit_unget_reg(reg);
1762}
1763
1764static void
1765_sti_l(jit_state_t *_jit, jit_word_t i0, jit_int32_t r0)
1766{
1767 jit_int32_t reg;
1768 reg = jit_get_reg(jit_class_gpr);
1769 movi(rn(reg), i0);
1770 str_l(rn(reg), r0);
1771 jit_unget_reg(reg);
1772}
1773
1774static void
1775_stxi_c(jit_state_t *_jit, jit_word_t i0, jit_int32_t r0, jit_int32_t r1)
1776{
1777 jit_int32_t reg;
1778 if (i0 >= 0 && i0 <= 4095)
1779 STRBI(r1, r0, i0);
1780 else if (i0 > -256 && i0 < 0)
1781 STURB(r1, r0, i0 & 0x1ff);
1782 else {
1783 reg = jit_get_reg(jit_class_gpr);
1784 addi(rn(reg), r0, i0);
1785 str_c(rn(reg), r1);
1786 jit_unget_reg(reg);
1787 }
1788}
1789
1790static void
1791_stxi_s(jit_state_t *_jit, jit_word_t i0, jit_int32_t r0, jit_int32_t r1)
1792{
1793 jit_int32_t reg;
519a9ea1 1794 if (i0 >= 0 && i0 <= 8191 && !(i0 & 1))
4a71579b
PC		 1795 STRHI(r1, r0, i0 >> 1);
1796 else if (i0 > -256 && i0 < 0)
1797 STURH(r1, r0, i0 & 0x1ff);
1798 else {
1799 reg = jit_get_reg(jit_class_gpr);
1800 addi(rn(reg), r0, i0);
1801 str_s(rn(reg), r1);
1802 jit_unget_reg(reg);
1803 }
1804}
1805
1806static void
1807_stxi_i(jit_state_t *_jit, jit_word_t i0, jit_int32_t r0, jit_int32_t r1)
1808{
1809 jit_int32_t reg;
519a9ea1 1810 if (i0 >= 0 && i0 <= 16383 && !(i0 & 3))
4a71579b
PC		 1811 STRWI(r1, r0, i0 >> 2);
1812 else if (i0 > -256 && i0 < 0)
1813 STURW(r1, r0, i0 & 0x1ff);
1814 else {
1815 reg = jit_get_reg(jit_class_gpr);
1816 addi(rn(reg), r0, i0);
1817 str_i(rn(reg), r1);
1818 jit_unget_reg(reg);
1819 }
1820}
1821
1822static void
1823_stxi_l(jit_state_t *_jit, jit_word_t i0, jit_int32_t r0, jit_int32_t r1)
1824{
1825 jit_int32_t reg;
519a9ea1 1826 if (i0 >= 0 && i0 <= 32767 && !(i0 & 7))
4a71579b
PC		 1827 STRI(r1, r0, i0 >> 3);
1828 else if (i0 > -256 && i0 < 0)
1829 STUR(r1, r0, i0 & 0x1ff);
1830 else {
1831 reg = jit_get_reg(jit_class_gpr);
1832 addi(rn(reg), r0, i0);
1833 str_l(rn(reg), r1);
1834 jit_unget_reg(reg);
1835 }
1836}
1837
ba3814c1
PC		 1838static void
1839_casx(jit_state_t *_jit, jit_int32_t r0, jit_int32_t r1,
1840 jit_int32_t r2, jit_int32_t r3, jit_word_t i0)
1841{
1842 jit_int32_t r1_reg, iscasi;
1843 jit_word_t retry, done, jump0, jump1;
1844 if ((iscasi = (r1 == _NOREG))) {
1845 r1_reg = jit_get_reg(jit_class_gpr);
1846 r1 = rn(r1_reg);
1847 movi(r1, i0);
1848 }
1849 /* retry: */
1850 retry = _jit->pc.w;
1851 LDAXR(r0, r1);
1852 jump0 = bner(_jit->pc.w, r0, r2); /* bne done r0 r2 */
1853 STLXR(r0, r3, r1);
1854 jump1 = bnei(_jit->pc.w, r0, 0); /* bnei retry r0 0 */
1855 /* done: */
1856 CSET(r0, CC_EQ);
1857 done = _jit->pc.w;
1858 patch_at(jump0, done);
1859 patch_at(jump1, retry);
1860 if (iscasi)
1861 jit_unget_reg(r1_reg);
1862}
1863
4a71579b
PC		 1864static void
1865_movr(jit_state_t *_jit, jit_int32_t r0, jit_int32_t r1)
1866{
1867 if (r0 != r1)
1868 MOV(r0, r1);
1869}
1870
1871static void
1872_movi(jit_state_t *_jit, jit_int32_t r0, jit_word_t i0)
1873{
1874 jit_word_t n0, ibit, nbit;
1875 n0 = ~i0;
1876 ibit = nbit = 0;
1877 if (i0 & 0x000000000000ffffL) ibit |= 1;
1878 if (i0 & 0x00000000ffff0000L) ibit |= 2;
1879 if (i0 & 0x0000ffff00000000L) ibit |= 4;
1880 if (i0 & 0xffff000000000000L) ibit |= 8;
1881 if (n0 & 0x000000000000ffffL) nbit |= 1;
1882 if (n0 & 0x00000000ffff0000L) nbit |= 2;
1883 if (n0 & 0x0000ffff00000000L) nbit |= 4;
1884 if (n0 & 0xffff000000000000L) nbit |= 8;
1885 switch (ibit) {
1886 case 0:
1887 MOVZ (r0, 0);
1888 break;
1889 case 1:
1890 MOVZ (r0, i0 & 0xffff);
1891 break;
1892 case 2:
1893 MOVZ_16(r0, (i0 >> 16) & 0xffff);
1894 break;
1895 case 3:
1896 MOVZ (r0, i0 & 0xffff);
1897 MOVK_16(r0, (i0 >> 16) & 0xffff);
1898 break;
1899 case 4:
1900 MOVZ_32(r0, (i0 >> 32) & 0xffff);
1901 break;
1902 case 5:
1903 MOVZ (r0, i0 & 0xffff);
1904 MOVK_32(r0, (i0 >> 32) & 0xffff);
1905 break;
1906 case 6:
1907 MOVZ_16(r0, (i0 >> 16) & 0xffff);
1908 MOVK_32(r0, (i0 >> 32) & 0xffff);
1909 break;
1910 case 7:
1911 if (nbit == 8)
1912 MOVN_48(r0, (n0 >> 48) & 0xffff);
1913 else {
1914 MOVZ (r0, i0 & 0xffff);
1915 MOVK_16(r0, (i0 >> 16) & 0xffff);
1916 MOVK_32(r0, (i0 >> 32) & 0xffff);
1917 }
1918 break;
1919 case 8:
1920 MOVZ_48(r0, (i0 >> 48) & 0xffff);
1921 break;
1922 case 9:
1923 MOVZ (r0, i0 & 0xffff);
1924 MOVK_48(r0, (i0 >> 48) & 0xffff);
1925 break;
1926 case 10:
1927 MOVZ_16(r0, (i0 >> 16) & 0xffff);
1928 MOVK_48(r0, (i0 >> 48) & 0xffff);
1929 break;
1930 case 11:
1931 if (nbit == 4)
1932 MOVN_32(r0, (n0 >> 32) & 0xffff);
1933 else {
1934 MOVZ (r0, i0 & 0xffff);
1935 MOVK_16(r0, (i0 >> 16) & 0xffff);
1936 MOVK_48(r0, (i0 >> 48) & 0xffff);
1937 }
1938 break;
1939 case 12:
1940 MOVZ_32(r0, (i0 >> 32) & 0xffff);
1941 MOVK_48(r0, (i0 >> 48) & 0xffff);
1942 break;
1943 case 13:
1944 if (nbit == 2)
1945 MOVN_16(r0, (n0 >> 16) & 0xffff);
1946 else {
1947 MOVZ (r0, i0 & 0xffff);
1948 MOVK_32(r0, (i0 >> 32) & 0xffff);
1949 MOVK_48(r0, (i0 >> 48) & 0xffff);
1950 }
1951 break;
1952 case 14:
1953 if (nbit == 1)
1954 MOVN (r0, (n0) & 0xffff);
1955 else {
1956 MOVZ_16(r0, (i0 >> 16) & 0xffff);
1957 MOVK_32(r0, (i0 >> 32) & 0xffff);
1958 MOVK_48(r0, (i0 >> 48) & 0xffff);
1959 }
1960 break;
1961 case 15:
1962 if (nbit == 0)
1963 MOVN (r0, 0);
1964 else if (nbit == 1)
1965 MOVN (r0, n0 & 0xffff);
1966 else if (nbit == 8)
1967 MOVN_48(r0, (n0 >> 48) & 0xffff);
1968 else {
1969 MOVZ (r0, i0 & 0xffff);
1970 MOVK_16(r0, (i0 >> 16) & 0xffff);
1971 MOVK_32(r0, (i0 >> 32) & 0xffff);
1972 MOVK_48(r0, (i0 >> 48) & 0xffff);
1973 }
1974 break;
1975 default:
1976 abort();
1977 }
1978}
1979
1980static jit_word_t
1981_movi_p(jit_state_t *_jit, jit_int32_t r0, jit_word_t i0)
1982{
1983 jit_word_t w;
1984 w = _jit->pc.w;
1985 MOVZ (r0, i0 & 0xffff);
1986 MOVK_16(r0, (i0 >> 16) & 0xffff);
1987 MOVK_32(r0, (i0 >> 32) & 0xffff);
1988 MOVK_48(r0, (i0 >> 48) & 0xffff);
1989 return (w);
1990}
1991
1992static void
1993_ccr(jit_state_t *_jit, jit_int32_t cc,
1994 jit_int32_t r0, jit_int32_t r1, jit_int32_t r2)
1995{
1996 CMP(r1, r2);
1997 CSET(r0, cc);
1998}
1999
2000static void
2001_cci(jit_state_t *_jit, jit_int32_t cc,
2002 jit_int32_t r0, jit_int32_t r1, jit_word_t i0)
2003{
2004 jit_int32_t reg;
2005 jit_word_t is = i0 >> 12;
2006 jit_word_t in = -i0;
2007 jit_word_t iS = in >> 12;
2008 if ( i0 >= 0 && i0 <= 0xfff)
2009 CMPI (r1, i0);
2010 else if ((is << 12) == i0 && is >= 0 && is <= 0xfff)
2011 CMPI_12(r1, is);
2012 else if ( in >= 0 && in <= 0xfff)
2013 CMNI (r1, in);
2014 else if ((iS << 12) == is && iS >= 0 && iS <= 0xfff)
2015 CMNI_12(r1, iS);
2016 else {
2017 reg = jit_get_reg(jit_class_gpr);
2018 movi(rn(reg), i0);
2019 CMP(r1, rn(reg));
2020 jit_unget_reg(reg);
2021 }
2022 CSET(r0, cc);
2023}
2024
2025static jit_word_t
2026_bccr(jit_state_t *_jit, jit_int32_t cc,
2027 jit_word_t i0, jit_int32_t r0, jit_int32_t r1)
2028{
2029 jit_word_t w, d;
2030 CMP(r0, r1);
2031 w = _jit->pc.w;
2032 d = (i0 - w) >> 2;
2033 B_C(cc, d);
2034 return (w);
2035}
2036
2037static jit_word_t
2038_bcci(jit_state_t *_jit, jit_int32_t cc,
2039 jit_word_t i0, jit_int32_t r0, jit_word_t i1)
2040{
2041 jit_int32_t reg;
2042 jit_word_t w, d;
2043 jit_word_t is = i1 >> 12;
2044 jit_word_t in = -i1;
2045 jit_word_t iS = in >> 12;
2046 if ( i1 >= 0 && i1 <= 0xfff)
2047 CMPI (r0, i1);
2048 else if ((is << 12) == i1 && is >= 0 && is <= 0xfff)
2049 CMPI_12(r0, is);
2050 else if ( in >= 0 && in <= 0xfff)
2051 CMNI (r0, in);
2052 else if ((iS << 12) == is && iS >= 0 && iS <= 0xfff)
2053 CMNI_12(r0, iS);
2054 else {
2055 reg = jit_get_reg(jit_class_gpr);
2056 movi(rn(reg), i1);
2057 CMP(r0, rn(reg));
2058 jit_unget_reg(reg);
2059 }
2060 w = _jit->pc.w;
2061 d = (i0 - w) >> 2;
2062 B_C(cc, d);
2063 return (w);
2064}
2065
2066static jit_word_t
2067_beqi(jit_state_t *_jit, jit_word_t i0, jit_int32_t r0, jit_word_t i1)
2068{
2069 jit_word_t w;
2070 if (i1 == 0) {
2071 w = _jit->pc.w;
2072 CBZ(r0, (i0 - w) >> 2);
2073 }
2074 else
2075 w = bcci(BCC_EQ, i0, r0, i1);
2076 return (w);
2077}
2078
2079static jit_word_t
2080_bnei(jit_state_t *_jit, jit_word_t i0, jit_int32_t r0, jit_word_t i1)
2081{
2082 jit_word_t w;
2083 if (i1 == 0) {
2084 w = _jit->pc.w;
2085 CBNZ(r0, (i0 - w) >> 2);
2086 }
2087 else
2088 w = bcci(BCC_NE, i0, r0, i1);
2089 return (w);
2090}
2091
2092static jit_word_t
2093_baddr(jit_state_t *_jit, jit_int32_t cc,
2094 jit_word_t i0, jit_int32_t r0, jit_int32_t r1)
2095{
2096 jit_word_t w;
2097 addcr(r0, r0, r1);
2098 w = _jit->pc.w;
2099 B_C(cc, (i0 - w) >> 2);
2100 return (w);
2101}
2102
2103static jit_word_t
2104_baddi(jit_state_t *_jit, jit_int32_t cc,
2105 jit_word_t i0, jit_int32_t r0, jit_word_t i1)
2106{
2107 jit_word_t w;
2108 addci(r0, r0, i1);
2109 w = _jit->pc.w;
2110 B_C(cc, (i0 - w) >> 2);
2111 return (w);
2112}
2113
2114static jit_word_t
2115_bsubr(jit_state_t *_jit, jit_int32_t cc,
2116 jit_word_t i0, jit_int32_t r0, jit_int32_t r1)
2117{
2118 jit_word_t w;
2119 subcr(r0, r0, r1);
2120 w = _jit->pc.w;
2121 B_C(cc, (i0 - w) >> 2);
2122 return (w);
2123}
2124
2125static jit_word_t
2126_bsubi(jit_state_t *_jit, jit_int32_t cc,
2127 jit_word_t i0, jit_int32_t r0, jit_word_t i1)
2128{
2129 jit_word_t w;
2130 subci(r0, r0, i1);
2131 w = _jit->pc.w;
2132 B_C(cc, (i0 - w) >> 2);
2133 return (w);
2134}
2135
2136static jit_word_t
2137_bmxr(jit_state_t *_jit, jit_int32_t cc,
2138 jit_word_t i0, jit_int32_t r0, jit_int32_t r1)
2139{
2140 jit_word_t w;
2141 TST(r0, r1);
2142 w = _jit->pc.w;
2143 B_C(cc, (i0 - w) >> 2);
2144 return (w);
2145}
2146
2147static jit_word_t
2148_bmxi(jit_state_t *_jit, jit_int32_t cc,
2149 jit_word_t i0, jit_int32_t r0, jit_word_t i1)
2150{
2151 jit_word_t w;
2152 jit_int32_t reg;
2153 jit_int32_t imm;
2154 imm = logical_immediate(i1);
2155 if (imm != -1)
2156 TSTI(r0, imm);
2157 else {
2158 reg = jit_get_reg(jit_class_gpr);
2159 movi(rn(reg), i1);
2160 TST(r0, rn(reg));
2161 jit_unget_reg(reg);
2162 }
2163 w = _jit->pc.w;
2164 B_C(cc, (i0 - w) >> 2);
2165 return (w);
2166}
2167
2168static void
2169_jmpi(jit_state_t *_jit, jit_word_t i0)
2170{
2171 jit_word_t w;
2172 jit_int32_t reg;
2173 w = (i0 - _jit->pc.w) >> 2;
2174 if (w >= -33554432 && w <= 33554431)
2175 B(w);
2176 else {
2177 reg = jit_get_reg(jit_class_gpr|jit_class_nospill);
2178 movi(rn(reg), i0);
2179 jmpr(rn(reg));
2180 jit_unget_reg(reg);
2181 }
2182}
2183
2184static jit_word_t
2185_jmpi_p(jit_state_t *_jit, jit_word_t i0)
2186{
2187 jit_word_t w;
2188 jit_int32_t reg;
2189 reg = jit_get_reg(jit_class_gpr|jit_class_nospill);
2190 w = movi_p(rn(reg), i0);
2191 jmpr(rn(reg));
2192 jit_unget_reg(reg);
2193 return (w);
2194}
2195
2196static void
2197_calli(jit_state_t *_jit, jit_word_t i0)
2198{
2199 jit_word_t w;
2200 jit_int32_t reg;
2201 w = (i0 - _jit->pc.w) >> 2;
2202 if (w >= -33554432 && w <= 33554431)
2203 BL(w);
2204 else {
2205 reg = jit_get_reg(jit_class_gpr);
2206 movi(rn(reg), i0);
2207 callr(rn(reg));
2208 jit_unget_reg(reg);
2209 }
2210}
2211
2212static jit_word_t
2213_calli_p(jit_state_t *_jit, jit_word_t i0)
2214{
2215 jit_word_t w;
2216 jit_int32_t reg;
2217 reg = jit_get_reg(jit_class_gpr);
2218 w = movi_p(rn(reg), i0);
2219 callr(rn(reg));
2220 jit_unget_reg(reg);
2221 return (w);
2222}
2223
2224/*
2225 * prolog and epilog not as "optimized" as one would like, but the
2226 * problem of overallocating stack space to save callee save registers
2227 * exists on all ports, and is still a todo to use a variable
2228 * stack_framesize
2229 * value, what would cause needing to patch some calls, most likely
2230 * the offset of jit_arg* of stack arguments.
2231 */
2232static void
2233_prolog(jit_state_t *_jit, jit_node_t *node)
2234{
2235 jit_int32_t reg;
2236 if (_jitc->function->define_frame || _jitc->function->assume_frame) {
2237 jit_int32_t frame = -_jitc->function->frame;
2238 assert(_jitc->function->self.aoff >= frame);
2239 if (_jitc->function->assume_frame)
2240 return;
2241 _jitc->function->self.aoff = frame;
2242 }
2243 if (_jitc->function->allocar)
2244 _jitc->function->self.aoff &= -16;
2245 _jitc->function->stack = ((_jitc->function->self.alen -
2246 /* align stack at 16 bytes */
2247 _jitc->function->self.aoff) + 15) & -16;
2248 STPI_POS(FP_REGNO, LR_REGNO, SP_REGNO, -(stack_framesize >> 3));
2249 MOV_XSP(FP_REGNO, SP_REGNO);
2250#define SPILL(L, R, O) \
2251 do { \
2252 if (jit_regset_tstbit(&_jitc->function->regset, _R##L)) { \
2253 if (jit_regset_tstbit(&_jitc->function->regset, _R##R)) \
2254 STPI(L, R, SP_REGNO, O); \
2255 else \
2256 STRI(L, SP_REGNO, O); \
2257 } \
2258 else if (jit_regset_tstbit(&_jitc->function->regset, _R##R)) \
2259 STRI(R, SP_REGNO, O + 1); \
2260 } while (0)
2261 SPILL(19, 20, 2);
2262 SPILL(21, 22, 4);
2263 SPILL(23, 24, 6);
2264 SPILL(25, 26, 8);
2265 SPILL(27, 28, 10);
2266#undef SPILL
2267#define SPILL(R, O) \
2268 do { \
2269 if (jit_regset_tstbit(&_jitc->function->regset, _V##R)) \
2270 stxi_d(O, SP_REGNO, R); \
2271 } while (0)
2272 SPILL( 8, 96);
2273 SPILL( 9, 104);
2274 SPILL(10, 112);
2275 SPILL(11, 120);
2276 SPILL(12, 128);
2277 SPILL(13, 136);
2278 SPILL(14, 144);
2279 SPILL(15, 152);
2280#undef SPILL
2281 if (_jitc->function->stack)
2282 subi(SP_REGNO, SP_REGNO, _jitc->function->stack);
2283 if (_jitc->function->allocar) {
2284 reg = jit_get_reg(jit_class_gpr);
2285 movi(rn(reg), _jitc->function->self.aoff);
2286 stxi_i(_jitc->function->aoffoff, FP_REGNO, rn(reg));
2287 jit_unget_reg(reg);
2288 }
2289
2290 if (_jitc->function->self.call & jit_call_varargs) {
2291 /* Save gp registers in the save area, if any is a vararg */
2292 for (reg = 8 - _jitc->function->vagp / -8;
2293 jit_arg_reg_p(reg); ++reg)
2294 stxi(_jitc->function->vaoff + offsetof(jit_va_list_t, x0) +
2295 reg * 8, FP_REGNO, rn(JIT_RA0 - reg));
2296
2297 for (reg = 8 - _jitc->function->vafp / -16;
2298 jit_arg_f_reg_p(reg); ++reg)
2299 /* Save fp registers in the save area, if any is a vararg */
2300 /* Note that the full 16 byte register is not saved, because
2301 * lightning only handles float and double, and, while
2302 * attempting to provide a va_list compatible pointer as
2303 * jit_va_start return, does not guarantee it (on all ports). */
2304 stxi_d(_jitc->function->vaoff + offsetof(jit_va_list_t, q0) +
2305 reg * 16 + offsetof(jit_qreg_t, l), FP_REGNO, rn(_V0 - reg));
2306 }
2307}
2308
2309static void
2310_epilog(jit_state_t *_jit, jit_node_t *node)
2311{
2312 if (_jitc->function->assume_frame)
2313 return;
2314 if (_jitc->function->stack)
2315 MOV_XSP(SP_REGNO, FP_REGNO);
2316#define LOAD(L, R, O) \
2317 do { \
2318 if (jit_regset_tstbit(&_jitc->function->regset, _R##L)) { \
2319 if (jit_regset_tstbit(&_jitc->function->regset, _R##R)) \
2320 LDPI(L, R, SP_REGNO, O); \
2321 else \
2322 LDRI(L, SP_REGNO, O); \
2323 } \
2324 else if (jit_regset_tstbit(&_jitc->function->regset, _R##R)) \
2325 LDRI(R, SP_REGNO, O + 1); \
2326 } while (0)
2327 LOAD(19, 20, 2);
2328 LOAD(21, 22, 4);
2329 LOAD(23, 24, 6);
2330 LOAD(25, 26, 8);
2331 LOAD(27, 28, 10);
2332#undef LOAD
2333#define LOAD(R, O) \
2334 do { \
2335 if (jit_regset_tstbit(&_jitc->function->regset, _V##R)) \
2336 ldxi_d(R, SP_REGNO, O); \
2337 } while (0)
2338 LOAD( 8, 96);
2339 LOAD( 9, 104);
2340 LOAD(10, 112);
2341 LOAD(11, 120);
2342 LOAD(12, 128);
2343 LOAD(13, 136);
2344 LOAD(14, 144);
2345 LOAD(15, 152);
2346#undef LOAD
2347 LDPI_PRE(FP_REGNO, LR_REGNO, SP_REGNO, stack_framesize >> 3);
2348 RET();
2349}
2350
2351static void
2352_vastart(jit_state_t *_jit, jit_int32_t r0)
2353{
2354 jit_int32_t reg;
2355
2356 assert(_jitc->function->self.call & jit_call_varargs);
2357
2358 /* Return jit_va_list_t in the register argument */
2359 addi(r0, FP_REGNO, _jitc->function->vaoff);
2360
2361 reg = jit_get_reg(jit_class_gpr);
2362
2363 /* Initialize stack pointer to the first stack argument. */
2364 addi(rn(reg), FP_REGNO, _jitc->function->self.size);
2365 stxi(offsetof(jit_va_list_t, stack), r0, rn(reg));
2366
2367 /* Initialize gp top pointer to the first stack argument. */
2368 addi(rn(reg), r0, va_gp_top_offset);
2369 stxi(offsetof(jit_va_list_t, gptop), r0, rn(reg));
2370
2371 /* Initialize fp top pointer to the first stack argument. */
2372 addi(rn(reg), r0, va_fp_top_offset);
2373 stxi(offsetof(jit_va_list_t, fptop), r0, rn(reg));
2374
2375 /* Initialize gp offset in the save area. */
2376 movi(rn(reg), _jitc->function->vagp);
2377 stxi_i(offsetof(jit_va_list_t, gpoff), r0, rn(reg));
2378
2379 /* Initialize fp offset in the save area. */
2380 movi(rn(reg), _jitc->function->vafp);
2381 stxi_i(offsetof(jit_va_list_t, fpoff), r0, rn(reg));
2382
2383 jit_unget_reg(reg);
2384}
2385
2386static void
2387_vaarg(jit_state_t *_jit, jit_int32_t r0, jit_int32_t r1)
2388{
2389 jit_word_t ge_code;
2390 jit_word_t lt_code;
2391 jit_int32_t rg0, rg1;
2392
2393 assert(_jitc->function->self.call & jit_call_varargs);
2394
2395 rg0 = jit_get_reg(jit_class_gpr);
2396 rg1 = jit_get_reg(jit_class_gpr);
2397
2398 /* Load the gp offset in save area in the first temporary. */
2399 ldxi_i(rn(rg0), r1, offsetof(jit_va_list_t, gpoff));
2400
2401 /* Jump over if there are no remaining arguments in the save area. */
2402 ge_code = bgei(_jit->pc.w, rn(rg0), 0);
2403
2404 /* Load the gp save pointer in the second temporary. */
2405 ldxi(rn(rg1), r1, offsetof(jit_va_list_t, gptop));
2406
2407 /* Load the vararg argument in the first argument. */
2408 ldxr(r0, rn(rg1), rn(rg0));
2409
2410 /* Update the gp offset. */
2411 addi(rn(rg0), rn(rg0), 8);
2412 stxi_i(offsetof(jit_va_list_t, gpoff), r1, rn(rg0));
2413
2414 /* Will only need one temporary register below. */
2415 jit_unget_reg(rg1);
2416
2417 /* Jump over overflow code. */
2418 lt_code = jmpi_p(_jit->pc.w);
2419
2420 /* Where to land if argument is in overflow area. */
2421 patch_at(ge_code, _jit->pc.w);
2422
2423 /* Load stack pointer. */
2424 ldxi(rn(rg0), r1, offsetof(jit_va_list_t, stack));
2425
2426 /* Load argument. */
2427 ldr(r0, rn(rg0));
2428
2429 /* Update stack pointer. */
2430 addi(rn(rg0), rn(rg0), 8);
2431 stxi(offsetof(jit_va_list_t, stack), r1, rn(rg0));
2432
2433 /* Where to land if argument is in gp save area. */
2434 patch_at(lt_code, _jit->pc.w);
2435
2436 jit_unget_reg(rg0);
2437}
2438
2439static void
2440_patch_at(jit_state_t *_jit, jit_word_t instr, jit_word_t label)
2441{
2442 instr_t i;
2443 jit_word_t d;
2444 jit_int32_t fc, ff, ffc;
2445 union {
2446 jit_int32_t *i;
2447 jit_word_t w;
2448 } u;
2449 u.w = instr;
2450 i.w = u.i[0];
2451 fc = i.w & 0xfc000000;
2452 ff = i.w & 0xff000000;
2453 ffc = i.w & 0xffc00000;
2454 if (fc == A64_B || fc == A64_BL) {
2455 d = (label - instr) >> 2;
2456 assert(d >= -33554432 && d <= 33554431);
2457 i.imm26.b = d;
2458 u.i[0] = i.w;
2459 }
2460 else if (ff == A64_B_C || ff == (A64_CBZ|XS) || ff == (A64_CBNZ|XS)) {
2461 d = (label - instr) >> 2;
2462 assert(d >= -262148 && d <= 262143);
2463 i.imm19.b = d;
2464 u.i[0] = i.w;
2465 }
2466 else if (ffc == (A64_MOVZ|XS)) {
2467 i.imm16.b = label;
2468 u.i[0] = i.w;
2469 i.w = u.i[1];
2470 assert((i.w & 0xffe00000) == (A64_MOVK|XS|MOVI_LSL_16));
2471 i.imm16.b = label >> 16;
2472 u.i[1] = i.w;
2473 i.w = u.i[2];
2474 assert((i.w & 0xffe00000) == (A64_MOVK|XS|MOVI_LSL_32));
2475 i.imm16.b = label >> 32;
2476 u.i[2] = i.w;
2477 i.w = u.i[3];
2478 assert((i.w & 0xffe00000) == (A64_MOVK|XS|MOVI_LSL_48));
2479 i.imm16.b = label >> 48;
2480 u.i[3] = i.w;
2481 }
2482 else
2483 abort();
2484}
2485#endif
ARM optimized PCSX fork