notaz.gp2x.de / pcsx_rearmed.git / blame
? search:
summary | shortlog | log | commit | commitdiff | tree
blob | blame (incremental) | history | HEAD
Merge pull request #724 from StormedBubbles/guncon_framing
[pcsx_rearmed.git] / deps / lightning / lib / jit_aarch64-cpu.c
CommitLineData
4a71579b 1/*
79bfeef6 2 * Copyright (C) 2013-2023 Free Software Foundation, Inc.
4a71579b
PC		 3 *
4 * This file is part of GNU lightning.
5 *
6 * GNU lightning is free software; you can redistribute it and/or modify it
7 * under the terms of the GNU Lesser General Public License as published
8 * by the Free Software Foundation; either version 3, or (at your option)
9 * any later version.
10 *
11 * GNU lightning is distributed in the hope that it will be useful, but
12 * WITHOUT ANY WARRANTY; without even the implied warranty of MERCHANTABILITY
13 * or FITNESS FOR A PARTICULAR PURPOSE. See the GNU Lesser General Public
14 * License for more details.
15 *
16 * Authors:
17 * Paulo Cesar Pereira de Andrade
18 */
19
20#if PROTO
21typedef union {
22/* aarch64-opc.c */
23# define ui jit_uint32_t
24# if __BYTE_ORDER == __LITTLE_ENDIAN
25 /* cond2: condition in truly conditional-executed inst. */
26 struct { ui b: 4; } cond2;
27 /* nzcv: flag bit specifier, encoded in the "nzcv" field. */
28 struct { ui b: 4; } nzcv;
29 /* defgh: d:e:f:g:h bits in AdvSIMD modified immediate. */
30 struct { ui _: 5; ui b: 5; } defgh;
31 /* abc: a:b:c bits in AdvSIMD modified immediate. */
32 struct { ui _: 16; ui b: 3; } abc;
33 /* imm19: e.g. in CBZ. */
34 struct { ui _: 5; ui b: 19; } imm19;
35 /* immhi: e.g. in ADRP. */
36 struct { ui _: 5; ui b: 19; } immhi;
37 /* immlo: e.g. in ADRP. */
38 struct { ui _: 29; ui b: 2; } immlo;
39 /* size: in most AdvSIMD and floating-point instructions. */
40 struct { ui _: 22; ui b: 2; } size;
41 /* vldst_size: size field in the AdvSIMD load/store inst. */
42 struct { ui _: 10; ui b: 2; } vldst_size;
43 /* op: in AdvSIMD modified immediate instructions. */
44 struct { ui _: 29; ui b: 1; } op;
45 /* Q: in most AdvSIMD instructions. */
46 struct { ui _: 30; ui b: 1; } Q;
47 /* Rt: in load/store instructions. */
48 struct { ui b: 5; } Rt;
49 /* Rd: in many integer instructions. */
50 struct { ui b: 5; } Rd;
51 /* Rn: in many integer instructions. */
52 struct { ui _: 5; ui b: 5; } Rn;
53 /* Rt2: in load/store pair instructions. */
54 struct { ui _: 10; ui b: 5; } Rt2;
55 /* Ra: in fp instructions. */
56 struct { ui _: 10; ui b: 5; } Ra;
57 /* op2: in the system instructions. */
58 struct { ui _: 5; ui b: 3; } op2;
59 /* CRm: in the system instructions. */
60 struct { ui _: 8; ui b: 4; } CRm;
61 /* CRn: in the system instructions. */
62 struct { ui _: 12; ui b: 4; } CRn;
63 /* op1: in the system instructions. */
64 struct { ui _: 16; ui b: 3; } op1;
65 /* op0: in the system instructions. */
66 struct { ui _: 19; ui b: 2; } op0;
67 /* imm3: in add/sub extended reg instructions. */
68 struct { ui _: 10; ui b: 3; } imm3;
69 /* cond: condition flags as a source operand. */
70 struct { ui _: 12; ui b: 4; } cond;
71 /* opcode: in advsimd load/store instructions. */
72 struct { ui _: 12; ui b: 4; } opcode;
73 /* cmode: in advsimd modified immediate instructions. */
74 struct { ui _: 12; ui b: 4; } cmode;
75 /* asisdlso_opcode: opcode in advsimd ld/st single element. */
76 struct { ui _: 13; ui b: 3; } asisdlso_opcode;
77 /* len: in advsimd tbl/tbx instructions. */
78 struct { ui _: 13; ui b: 2; } len;
79 /* Rm: in ld/st reg offset and some integer inst. */
80 struct { ui _: 16; ui b: 5; } Rm;
81 /* Rs: in load/store exclusive instructions. */
82 struct { ui _: 16; ui b: 5; } Rs;
83 /* option: in ld/st reg offset + add/sub extended reg inst. */
84 struct { ui _: 13; ui b: 3; } option;
85 /* S: in load/store reg offset instructions. */
86 struct { ui _: 12; ui b: 1; } S;
87 /* hw: in move wide constant instructions. */
88 struct { ui _: 21; ui b: 2; } hw;
89 /* opc: in load/store reg offset instructions. */
90 struct { ui _: 22; ui b: 2; } opc;
91 /* opc1: in load/store reg offset instructions. */
92 struct { ui _: 23; ui b: 1; } opc1;
93 /* shift: in add/sub reg/imm shifted instructions. */
94 struct { ui _: 22; ui b: 2; } shift;
95 /* type: floating point type field in fp data inst. */
96 struct { ui _: 22; ui b: 2; } type;
97 /* ldst_size: size field in ld/st reg offset inst. */
98 struct { ui _: 30; ui b: 2; } ldst_size;
99 /* imm6: in add/sub reg shifted instructions. */
100 struct { ui _: 10; ui b: 6; } imm6;
101 /* imm4: in advsimd ext and advsimd ins instructions. */
102 struct { ui _: 11; ui b: 4; } imm4;
103 /* imm5: in conditional compare (immediate) instructions. */
104 struct { ui _: 16; ui b: 5; } imm5;
105 /* imm7: in load/store pair pre/post index instructions. */
106 struct { ui _: 15; ui b: 7; } imm7;
107 /* imm8: in floating-point scalar move immediate inst. */
108 struct { ui _: 13; ui b: 8; } imm8;
109 /* imm9: in load/store pre/post index instructions. */
110 struct { ui _: 12; ui b: 9; } imm9;
111 /* imm12: in ld/st unsigned imm or add/sub shifted inst. */
112 struct { ui _: 10; ui b: 12; } imm12;
113 /* imm14: in test bit and branch instructions. */
114 struct { ui _: 5; ui b: 14; } imm14;
115 /* imm16: in exception instructions. */
116 struct { ui _: 5; ui b: 16; } imm16;
117 /* imm26: in unconditional branch instructions. */
118 struct { ui b: 26; } imm26;
119 /* imms: in bitfield and logical immediate instructions. */
120 struct { ui _: 10; ui b: 6; } imms;
121 /* immr: in bitfield and logical immediate instructions. */
122 struct { ui _: 16; ui b: 6; } immr;
123 /* immb: in advsimd shift by immediate instructions. */
124 struct { ui _: 16; ui b: 3; } immb;
125 /* immh: in advsimd shift by immediate instructions. */
126 struct { ui _: 19; ui b: 4; } immh;
127 /* N: in logical (immediate) instructions. */
128 struct { ui _: 22; ui b: 1; } N;
129 /* index: in ld/st inst deciding the pre/post-index. */
130 struct { ui _: 11; ui b: 1; } index;
131 /* index2: in ld/st pair inst deciding the pre/post-index. */
132 struct { ui _: 24; ui b: 1; } index2;
133 /* sf: in integer data processing instructions. */
134 struct { ui _: 31; ui b: 1; } sf;
135 /* H: in advsimd scalar x indexed element instructions. */
136 struct { ui _: 11; ui b: 1; } H;
137 /* L: in advsimd scalar x indexed element instructions. */
138 struct { ui _: 21; ui b: 1; } L;
139 /* M: in advsimd scalar x indexed element instructions. */
140 struct { ui _: 20; ui b: 1; } M;
141 /* b5: in the test bit and branch instructions. */
142 struct { ui _: 31; ui b: 1; } b5;
143 /* b40: in the test bit and branch instructions. */
144 struct { ui _: 19; ui b: 5; } b40;
145 /* scale: in the fixed-point scalar to fp converting inst. */
146 struct { ui _: 10; ui b: 6; } scale;
147# else
148 struct { ui _: 28; ui b: 4; } cond2;
149 struct { ui _: 28; ui b: 4; } nzcv;
150 struct { ui _: 22; ui b: 5; } defgh;
151 struct { ui _: 13; ui b: 3; } abc;
152 struct { ui _: 8; ui b: 19; } imm19;
153 struct { ui _: 8; ui b: 19; } immhi;
154 struct { ui _: 1; ui b: 29; } immlo;
155 struct { ui _: 8; ui b: 2; } size;
156 struct { ui _: 20; ui b: 2; } vldst_size;
157 struct { ui _: 2; ui b: 1; } op;
158 struct { ui _: 1; ui b: 1; } Q;
159 struct { ui _: 27; ui b: 1; } Rt;
160 struct { ui _: 27; ui b: 1; } Rd;
161 struct { ui _: 22; ui b: 5; } Rn;
162 struct { ui _: 17; ui b: 5; } Rt2;
163 struct { ui _: 17; ui b: 5; } Ra;
164 struct { ui _: 24; ui b: 3; } op2;
165 struct { ui _: 20; ui b: 4; } CRm;
166 struct { ui _: 16; ui b: 4; } CRn;
167 struct { ui _: 13; ui b: 3; } op1;
168 struct { ui _: 11; ui b: 2; } op0;
169 struct { ui _: 19; ui b: 3; } imm3;
170 struct { ui _: 16; ui b: 4; } cond;
171 struct { ui _: 16; ui b: 4; } opcode;
172 struct { ui _: 16; ui b: 4; } cmode;
173 struct { ui _: 16; ui b: 3; } asisdlso_opcode;
174 struct { ui _: 17; ui b: 2; } len;
175 struct { ui _: 11; ui b: 5; } Rm;
176 struct { ui _: 11; ui b: 5; } Rs;
177 struct { ui _: 16; ui b: 3; } option;
178 struct { ui _: 19; ui b: 1; } S;
179 struct { ui _: 9; ui b: 2; } hw;
180 struct { ui _: 8; ui b: 2; } opc;
181 struct { ui _: 8; ui b: 1; } opc1;
182 struct { ui _: 8; ui b: 2; } shift;
183 struct { ui _: 8; ui b: 2; } type;
184 struct { ui b: 2; } ldst_size;
185 struct { ui _: 16; ui b: 6; } imm6;
186 struct { ui _: 17; ui b: 4; } imm4;
187 struct { ui _: 11; ui b: 5; } imm5;
188 struct { ui _: 10; ui b: 7; } imm7;
189 struct { ui _: 11; ui b: 8; } imm8;
190 struct { ui _: 11; ui b: 9; } imm9;
191 struct { ui _: 10; ui b: 12; } imm12;
192 struct { ui _: 13; ui b: 14; } imm14;
193 struct { ui _: 11; ui b: 16; } imm16;
194 struct { ui _: 6; ui b: 26; } imm26;
195 struct { ui _: 16; ui b: 6; } imms;
196 struct { ui _: 10; ui b: 6; } immr;
197 struct { ui _: 13; ui b: 3; } immb;
198 struct { ui _: 9; ui b: 4; } immh;
199 struct { ui _: 9; ui b: 1; } N;
200 struct { ui _: 20; ui b: 1; } index;
201 struct { ui _: 7; ui b: 1; } index2;
202 struct { ui b: 1; } sf;
203 struct { ui _: 20; ui b: 1; } H;
204 struct { ui _: 10; ui b: 1; } L;
205 struct { ui _: 11; ui b: 1; } M;
206 struct { ui b: 1; } b5;
207 struct { ui _: 8; ui b: 5; } b40;
208 struct { ui _: 16; ui b: 6; } scale;
209# endif
210 jit_int32_t w;
211# undef ui
212} instr_t;
79bfeef6 213# define s26_p(d) ((d) >= -33554432 && (d) <= 33554431)
4a71579b
PC		 214# define ii(i) *_jit->pc.ui++ = i
215# define ldr(r0,r1) ldr_l(r0,r1)
216# define ldxr(r0,r1,r2) ldxr_l(r0,r1,r2)
217# define ldxi(r0,r1,i0) ldxi_l(r0,r1,i0)
218# define stxi(i0,r0,r1) stxi_l(i0,r0,r1)
219# define FP_REGNO 0x1d
220# define LR_REGNO 0x1e
221# define SP_REGNO 0x1f
222# define XZR_REGNO 0x1f
223# define WZR_REGNO XZR_REGNO
224# define LSL_12 0x00400000
225# define MOVI_LSL_16 0x00200000
226# define MOVI_LSL_32 0x00400000
227# define MOVI_LSL_48 0x00600000
228# define XS 0x80000000 /* Wn -> Xn */
229# define DS 0x00400000 /* Sn -> Dn */
230# define CC_NE 0x0
231# define CC_EQ 0x1
232# define CC_CC 0x2
233# define CC_LO CC_CC
234# define CC_CS 0x3
235# define CC_HS CC_CS
236# define CC_PL 0x4
237# define CC_MI 0x5
238# define CC_VC 0x6
239# define CC_VS 0x7
240# define CC_LS 0x8
241# define CC_HI 0x9
242# define CC_LT 0xa
243# define CC_GE 0xb
244# define CC_LE 0xc
245# define CC_GT 0xd
246# define CC_NV 0xe
247# define CC_AL 0xf
248/* Branches need inverted condition */
249# define BCC_EQ 0x0
250# define BCC_NE 0x1
251# define BCC_CS 0x2
252# define BCC_HS BCC_CS
253# define BCC_CC 0x3
254# define BCC_LO BCC_CC
255# define BCC_MI 0x4
256# define BCC_PL 0x5
257# define BCC_VS 0x6
258# define BCC_VC 0x7
259# define BCC_HI 0x8
260# define BCC_LS 0x9
261# define BCC_GE 0xa
262# define BCC_LT 0xb
263# define BCC_GT 0xc
264# define BCC_LE 0xd
265# define BCC_AL 0xe
266# define BCC_NV 0xf
267/* adapted and cut down to only tested and required by lightning,
268 * from data in binutils/aarch64-tbl.h */
269# define A64_ADCS 0x3a000000
270# define A64_SBCS 0x7a000000
271# define A64_ADDI 0x11000000
272# define A64_ADDSI 0xb1000000
273# define A64_SUBI 0x51000000
274# define A64_SUBSI 0x71000000
275# define A64_ADD 0x0b000000
276# define A64_ADDS 0x2b000000
277# define A64_SUB 0x4b000000
278# define A64_NEG 0x4b0003e0
279# define A64_SUBS 0x6b000000
280# define A64_CMP 0x6b00001f
281# define A64_SBFM 0x93400000
282# define A64_UBFM 0x53400000
283# define A64_UBFX 0x53000000
284# define A64_B 0x14000000
285# define A64_BL 0x94000000
286# define A64_BR 0xd61f0000
287# define A64_BLR 0xd63f0000
288# define A64_RET 0xd65f0000
289# define A64_CBZ 0x34000000
290# define A64_CBNZ 0x35000000
291# define A64_B_C 0x54000000
292# define A64_CSINC 0x1a800400
e0659411 293# define A64_CSSEL 0x1a800000
4a71579b
PC		 294# define A64_REV 0xdac00c00
295# define A64_UDIV 0x1ac00800
296# define A64_SDIV 0x1ac00c00
297# define A64_LSL 0x1ac02000
298# define A64_LSR 0x1ac02400
299# define A64_ASR 0x1ac02800
300# define A64_MUL 0x1b007c00
301# define A64_SMULL 0x9b207c00
302# define A64_SMULH 0x9b407c00
303# define A64_UMULL 0x9ba07c00
304# define A64_UMULH 0x9bc07c00
305# define A64_STRBI 0x39000000
306# define A64_LDRBI 0x39400000
307# define A64_LDRSBI 0x39800000
308# define A64_STRI 0xf9000000
309# define A64_LDRI 0xf9400000
310# define A64_STRHI 0x79000000
311# define A64_LDRHI 0x79400000
312# define A64_LDRSHI 0x79800000
313# define A64_STRWI 0xb9000000
314# define A64_LDRWI 0xb9400000
315# define A64_LDRSWI 0xb9800000
316# define A64_STRB 0x38206800
317# define A64_LDRB 0x38606800
318# define A64_LDRSB 0x38e06800
319# define A64_STR 0xf8206800
320# define A64_LDR 0xf8606800
ba3814c1
PC		 321# define A64_LDAXR 0xc85ffc00
322# define A64_STLXR 0xc800fc00
4a71579b
PC		 323# define A64_STRH 0x78206800
324# define A64_LDRH 0x78606800
325# define A64_LDRSH 0x78a06800
326# define A64_STRW 0xb8206800
327# define A64_LDRW 0xb8606800
328# define A64_LDRSW 0xb8a06800
329# define A64_STURB 0x38000000
330# define A64_LDURB 0x38400000
331# define A64_LDURSB 0x38800000
332# define A64_STUR 0xf8000000
333# define A64_LDUR 0xf8400000
334# define A64_STURH 0x78000000
335# define A64_LDURH 0x78400000
336# define A64_LDURSH 0x78800000
337# define A64_STURW 0xb8000000
338# define A64_LDURW 0xb8400000
339# define A64_LDURSW 0xb8800000
340# define A64_STP 0x29000000
341# define A64_LDP 0x29400000
342# define A64_STP_POS 0x29800000
343# define A64_LDP_PRE 0x28c00000
344# define A64_ANDI 0x12400000
345# define A64_ORRI 0x32400000
346# define A64_EORI 0x52400000
347# define A64_ANDSI 0x72000000
348# define A64_AND 0x0a000000
349# define A64_ORR 0x2a000000
350# define A64_MOV 0x2a0003e0 /* AKA orr Rd,xzr,Rm */
351# define A64_MVN 0x2a2003e0
79bfeef6
PC		 352# define A64_CLS 0x5ac01400
353# define A64_CLZ 0x5ac01000
354# define A64_RBIT 0x5ac00000
4a71579b
PC		 355# define A64_UXTW 0x2a0003e0 /* AKA MOV */
356# define A64_EOR 0x4a000000
357# define A64_ANDS 0x6a000000
358# define A64_MOVN 0x12800000
359# define A64_MOVZ 0x52800000
360# define A64_MOVK 0x72800000
361# define SBFM(Rd,Rn,ImmR,ImmS) oxxrs(A64_SBFM|XS,Rd,Rn,ImmR,ImmS)
362# define UBFM(Rd,Rn,ImmR,ImmS) oxxrs(A64_UBFM|XS,Rd,Rn,ImmR,ImmS)
363# define UBFX(Rd,Rn,ImmR,ImmS) oxxrs(A64_UBFX,Rd,Rn,ImmR,ImmS)
364# define CMP(Rn,Rm) oxx_(A64_CMP|XS,Rn,Rm)
365# define CMPI(Rn,Imm12) oxxi(A64_SUBSI|XS,XZR_REGNO,Rn,Imm12)
366# define CMPI_12(Rn,Imm12) oxxi(A64_SUBSI|XS|LSL_12,XZR_REGNO,Rn,Imm12)
367# define CMNI(Rn,Imm12) oxxi(A64_ADDSI|XS,XZR_REGNO,Rn,Imm12)
368# define CMNI_12(Rn,Imm12) oxxi(A64_ADDSI|XS|LSL_12,XZR_REGNO,Rn,Imm12)
369# define CSINC(Rd,Rn,Rm,Cc) oxxxc(A64_CSINC|XS,Rd,Rn,Rm,Cc)
370# define TST(Rn,Rm) oxxx(A64_ANDS|XS,XZR_REGNO,Rn,Rm)
371/* actually should use oxxrs but logical_immediate returns proper encoding */
372# define TSTI(Rn,Imm12) oxxi(A64_ANDSI,XZR_REGNO,Rn,Imm12)
373# define MOV(Rd,Rm) ox_x(A64_MOV|XS,Rd,Rm)
374# define MVN(Rd,Rm) ox_x(A64_MVN|XS,Rd,Rm)
375# define NEG(Rd,Rm) ox_x(A64_NEG|XS,Rd,Rm)
79bfeef6
PC		 376# define CLS(Rd,Rm) o_xx(A64_CLS|XS,Rd,Rm)
377# define CLZ(Rd,Rm) o_xx(A64_CLZ|XS,Rd,Rm)
378# define RBIT(Rd,Rm) o_xx(A64_RBIT|XS,Rd,Rm)
4a71579b
PC		 379# define MOVN(Rd,Imm16) ox_h(A64_MOVN|XS,Rd,Imm16)
380# define MOVN_16(Rd,Imm16) ox_h(A64_MOVN|XS|MOVI_LSL_16,Rd,Imm16)
381# define MOVN_32(Rd,Imm16) ox_h(A64_MOVN|XS|MOVI_LSL_32,Rd,Imm16)
382# define MOVN_48(Rd,Imm16) ox_h(A64_MOVN|XS|MOVI_LSL_48,Rd,Imm16)
383# define MOVZ(Rd,Imm16) ox_h(A64_MOVZ|XS,Rd,Imm16)
384# define MOVZ_16(Rd,Imm16) ox_h(A64_MOVZ|XS|MOVI_LSL_16,Rd,Imm16)
385# define MOVZ_32(Rd,Imm16) ox_h(A64_MOVZ|XS|MOVI_LSL_32,Rd,Imm16)
386# define MOVZ_48(Rd,Imm16) ox_h(A64_MOVZ|XS|MOVI_LSL_48,Rd,Imm16)
387# define MOVK(Rd,Imm16) ox_h(A64_MOVK|XS,Rd,Imm16)
388# define MOVK_16(Rd,Imm16) ox_h(A64_MOVK|XS|MOVI_LSL_16,Rd,Imm16)
389# define MOVK_32(Rd,Imm16) ox_h(A64_MOVK|XS|MOVI_LSL_32,Rd,Imm16)
390# define MOVK_48(Rd,Imm16) ox_h(A64_MOVK|XS|MOVI_LSL_48,Rd,Imm16)
391# define ADD(Rd,Rn,Rm) oxxx(A64_ADD|XS,Rd,Rn,Rm)
392# define ADDI(Rd,Rn,Imm12) oxxi(A64_ADDI|XS,Rd,Rn,Imm12)
393# define ADDI_12(Rd,Rn,Imm12) oxxi(A64_ADDI|XS|LSL_12,Rd,Rn,Imm12)
394# define MOV_XSP(Rd,Rn) ADDI(Rd,Rn,0)
395# define ADDS(Rd,Rn,Rm) oxxx(A64_ADDS|XS,Rd,Rn,Rm)
396# define ADDSI(Rd,Rn,Imm12) oxxi(A64_ADDSI|XS,Rd,Rn,Imm12)
397# define ADDSI_12(Rd,Rn,Imm12) oxxi(A64_ADDSI|XS|LSL_12,Rd,Rn,Imm12)
398# define ADCS(Rd,Rn,Rm) oxxx(A64_ADCS|XS,Rd,Rn,Rm)
399# define SUB(Rd,Rn,Rm) oxxx(A64_SUB|XS,Rd,Rn,Rm)
400# define SUBI(Rd,Rn,Imm12) oxxi(A64_SUBI|XS,Rd,Rn,Imm12)
401# define SUBI_12(Rd,Rn,Imm12) oxxi(A64_SUBI|XS|LSL_12,Rd,Rn,Imm12)
402# define SUBS(Rd,Rn,Rm) oxxx(A64_SUBS|XS,Rd,Rn,Rm)
403# define SUBSI(Rd,Rn,Imm12) oxxi(A64_SUBSI|XS,Rd,Rn,Imm12)
404# define SUBSI_12(Rd,Rn,Imm12) oxxi(A64_SUBSI|XS|LSL_12,Rd,Rn,Imm12)
405# define SBCS(Rd,Rn,Rm) oxxx(A64_SBCS|XS,Rd,Rn,Rm)
406# define MUL(Rd,Rn,Rm) oxxx(A64_MUL|XS,Rd,Rn,Rm)
407# define SMULL(Rd,Rn,Rm) oxxx(A64_SMULL,Rd,Rn,Rm)
408# define SMULH(Rd,Rn,Rm) oxxx(A64_SMULH,Rd,Rn,Rm)
409# define UMULL(Rd,Rn,Rm) oxxx(A64_UMULL,Rd,Rn,Rm)
410# define UMULH(Rd,Rn,Rm) oxxx(A64_UMULH,Rd,Rn,Rm)
411# define SDIV(Rd,Rn,Rm) oxxx(A64_SDIV|XS,Rd,Rn,Rm)
412# define UDIV(Rd,Rn,Rm) oxxx(A64_UDIV|XS,Rd,Rn,Rm)
413# define LSL(Rd,Rn,Rm) oxxx(A64_LSL|XS,Rd,Rn,Rm)
414# define LSLI(r0,r1,i0) UBFM(r0,r1,(64-i0)&63,63-i0)
415# define ASR(Rd,Rn,Rm) oxxx(A64_ASR|XS,Rd,Rn,Rm)
416# define ASRI(r0,r1,i0) SBFM(r0,r1,i0,63)
417# define LSR(Rd,Rn,Rm) oxxx(A64_LSR|XS,Rd,Rn,Rm)
418# define LSRI(r0,r1,i0) UBFM(r0,r1,i0,63)
419# define AND(Rd,Rn,Rm) oxxx(A64_AND|XS,Rd,Rn,Rm)
420/* actually should use oxxrs but logical_immediate returns proper encoding */
421# define ANDI(Rd,Rn,Imm12) oxxi(A64_ANDI|XS,Rd,Rn,Imm12)
422# define ORR(Rd,Rn,Rm) oxxx(A64_ORR|XS,Rd,Rn,Rm)
423/* actually should use oxxrs but logical_immediate returns proper encoding */
424# define ORRI(Rd,Rn,Imm12) oxxi(A64_ORRI|XS,Rd,Rn,Imm12)
425# define EOR(Rd,Rn,Rm) oxxx(A64_EOR|XS,Rd,Rn,Rm)
426/* actually should use oxxrs but logical_immediate returns proper encoding */
427# define EORI(Rd,Rn,Imm12) oxxi(A64_EORI|XS,Rd,Rn,Imm12)
428# define SXTB(Rd,Rn) SBFM(Rd,Rn,0,7)
429# define SXTH(Rd,Rn) SBFM(Rd,Rn,0,15)
430# define SXTW(Rd,Rn) SBFM(Rd,Rn,0,31)
431# define UXTB(Rd,Rn) UBFX(Rd,Rn,0,7)
432# define UXTH(Rd,Rn) UBFX(Rd,Rn,0,15)
433# define UXTW(Rd,Rm) ox_x(A64_UXTW,Rd,Rm)
434# define REV(Rd,Rn) o_xx(A64_REV,Rd,Rn)
435# define LDRSB(Rt,Rn,Rm) oxxx(A64_LDRSB,Rt,Rn,Rm)
436# define LDRSBI(Rt,Rn,Imm12) oxxi(A64_LDRSBI,Rt,Rn,Imm12)
437# define LDURSB(Rt,Rn,Imm9) oxx9(A64_LDURSB,Rt,Rn,Imm9)
438# define LDRB(Rt,Rn,Rm) oxxx(A64_LDRB,Rt,Rn,Rm)
439# define LDRBI(Rt,Rn,Imm12) oxxi(A64_LDRBI,Rt,Rn,Imm12)
440# define LDURB(Rt,Rn,Imm9) oxx9(A64_LDURB,Rt,Rn,Imm9)
441# define LDRSH(Rt,Rn,Rm) oxxx(A64_LDRSH,Rt,Rn,Rm)
442# define LDRSHI(Rt,Rn,Imm12) oxxi(A64_LDRSHI,Rt,Rn,Imm12)
443# define LDURSH(Rt,Rn,Imm9) oxx9(A64_LDURSH,Rt,Rn,Imm9)
444# define LDRH(Rt,Rn,Rm) oxxx(A64_LDRH,Rt,Rn,Rm)
445# define LDRHI(Rt,Rn,Imm12) oxxi(A64_LDRHI,Rt,Rn,Imm12)
446# define LDURH(Rt,Rn,Imm9) oxx9(A64_LDURH,Rt,Rn,Imm9)
447# define LDRSW(Rt,Rn,Rm) oxxx(A64_LDRSW,Rt,Rn,Rm)
448# define LDRSWI(Rt,Rn,Imm12) oxxi(A64_LDRSWI,Rt,Rn,Imm12)
449# define LDURSW(Rt,Rn,Imm9) oxx9(A64_LDURSW,Rt,Rn,Imm9)
450# define LDRW(Rt,Rn,Rm) oxxx(A64_LDRW,Rt,Rn,Rm)
451# define LDRWI(Rt,Rn,Imm12) oxxi(A64_LDRWI,Rt,Rn,Imm12)
452# define LDURW(Rt,Rn,Imm9) oxx9(A64_LDURW,Rt,Rn,Imm9)
453# define LDR(Rt,Rn,Rm) oxxx(A64_LDR,Rt,Rn,Rm)
454# define LDRI(Rt,Rn,Imm12) oxxi(A64_LDRI,Rt,Rn,Imm12)
455# define LDUR(Rt,Rn,Imm9) oxx9(A64_LDUR,Rt,Rn,Imm9)
ba3814c1
PC		 456# define LDAXR(Rt,Rn) o_xx(A64_LDAXR,Rt,Rn)
457# define STLXR(Rs,Rt,Rn) oxxx(A64_STLXR,Rs,Rn,Rt)
4a71579b
PC		 458# define STRB(Rt,Rn,Rm) oxxx(A64_STRB,Rt,Rn,Rm)
459# define STRBI(Rt,Rn,Imm12) oxxi(A64_STRBI,Rt,Rn,Imm12)
460# define STURB(Rt,Rn,Imm9) oxx9(A64_STURB,Rt,Rn,Imm9)
461# define STRH(Rt,Rn,Rm) oxxx(A64_STRH,Rt,Rn,Rm)
462# define STRHI(Rt,Rn,Imm12) oxxi(A64_STRHI,Rt,Rn,Imm12)
463# define STURH(Rt,Rn,Imm9) oxx9(A64_STURH,Rt,Rn,Imm9)
464# define STRW(Rt,Rn,Rm) oxxx(A64_STRW,Rt,Rn,Rm)
465# define STRWI(Rt,Rn,Imm12) oxxi(A64_STRWI,Rt,Rn,Imm12)
466# define STURW(Rt,Rn,Imm9) oxx9(A64_STURW,Rt,Rn,Imm9)
467# define STR(Rt,Rn,Rm) oxxx(A64_STR,Rt,Rn,Rm)
468# define STRI(Rt,Rn,Imm12) oxxi(A64_STRI,Rt,Rn,Imm12)
469# define STUR(Rt,Rn,Imm9) oxx9(A64_STUR,Rt,Rn,Imm9)
470# define LDPI(Rt,Rt2,Rn,Simm7) oxxx7(A64_LDP|XS,Rt,Rt2,Rn,Simm7)
471# define STPI(Rt,Rt2,Rn,Simm7) oxxx7(A64_STP|XS,Rt,Rt2,Rn,Simm7)
472# define LDPI_PRE(Rt,Rt2,Rn,Simm7) oxxx7(A64_LDP_PRE|XS,Rt,Rt2,Rn,Simm7)
473# define STPI_POS(Rt,Rt2,Rn,Simm7) oxxx7(A64_STP_POS|XS,Rt,Rt2,Rn,Simm7)
474# define CSET(Rd,Cc) CSINC(Rd,XZR_REGNO,XZR_REGNO,Cc)
e0659411 475# define CSEL(Rd,Rn,Rm,Cc) oxxxc(A64_CSSEL|XS,Rd,Rn,Rm,Cc)
4a71579b
PC		 476# define B(Simm26) o26(A64_B,Simm26)
477# define BL(Simm26) o26(A64_BL,Simm26)
478# define BR(Rn) o_x_(A64_BR,Rn)
479# define BLR(Rn) o_x_(A64_BLR,Rn)
480# define RET() o_x_(A64_RET,LR_REGNO)
481# define B_C(Cc,Simm19) oc19(A64_B_C,Cc,Simm19)
482# define CBZ(Rd,Simm19) ox19(A64_CBZ|XS,Rd,Simm19)
483# define CBNZ(Rd,Simm19) ox19(A64_CBNZ|XS,Rd,Simm19)
484# define NOP() ii(0xd503201f)
485static jit_int32_t logical_immediate(jit_word_t);
486# define oxxx(Op,Rd,Rn,Rm) _oxxx(_jit,Op,Rd,Rn,Rm)
487static void _oxxx(jit_state_t*,jit_int32_t,jit_int32_t,jit_int32_t,jit_int32_t);
488# define oxxi(Op,Rd,Rn,Imm12) _oxxi(_jit,Op,Rd,Rn,Imm12)
489static void _oxxi(jit_state_t*,jit_int32_t,jit_int32_t,jit_int32_t,jit_int32_t);
490# define oxx9(Op,Rd,Rn,Imm9) _oxx9(_jit,Op,Rd,Rn,Imm9)
491static void _oxx9(jit_state_t*,jit_int32_t,jit_int32_t,jit_int32_t,jit_int32_t);
492# define ox19(Op,Rd,Simm19) _ox19(_jit,Op,Rd,Simm19)
493static void _ox19(jit_state_t*,jit_int32_t,jit_int32_t,jit_int32_t);
494# define oc19(Op,Cc,Simm19) _oc19(_jit,Op,Cc,Simm19)
495static void _oc19(jit_state_t*,jit_int32_t,jit_int32_t,jit_int32_t);
496# define o26(Op,Simm26) _o26(_jit,Op,Simm26)
497static void _oc26(jit_state_t*,jit_int32_t,jit_int32_t);
498# define ox_x(Op,Rd,Rn) _ox_x(_jit,Op,Rd,Rn)
499static void _ox_x(jit_state_t*,jit_int32_t,jit_int32_t,jit_int32_t);
500# define o_xx(Op,Rd,Rn) _o_xx(_jit,Op,Rd,Rn)
501static void _o_xx(jit_state_t*,jit_int32_t,jit_int32_t,jit_int32_t);
502# define oxx_(Op,Rn,Rm) _oxx_(_jit,Op,Rn,Rm)
503static void _oxx_(jit_state_t*,jit_int32_t,jit_int32_t,jit_int32_t);
504# define o_x_(Op,Rn) _o_x_(_jit,Op,Rn)
505static void _o_x_(jit_state_t*,jit_int32_t,jit_int32_t);
506# define ox_h(Op,Rd,Imm16) _ox_h(_jit,Op,Rd,Imm16)
507static void _ox_h(jit_state_t*,jit_int32_t,jit_int32_t,jit_int32_t);
508# define oxxrs(Op,Rd,Rn,R,S) _oxxrs(_jit,Op,Rd,Rn,R,S)
509static void _oxxrs(jit_state_t*,jit_int32_t,jit_int32_t,
510 jit_int32_t,jit_int32_t,jit_int32_t);
511# define oxxxc(Op,Rd,Rn,Rm,Cc) _oxxxc(_jit,Op,Rd,Rn,Rm,Cc)
512static void _oxxxc(jit_state_t*,jit_int32_t,jit_int32_t,
513 jit_int32_t,jit_int32_t,jit_int32_t);
514# define oxxx7(Op,Rt,Rt2,Rn,Simm7) _oxxx7(_jit,Op,Rt,Rt2,Rn,Simm7)
515static void _oxxx7(jit_state_t*,jit_int32_t,
516 jit_int32_t,jit_int32_t,jit_int32_t,jit_int32_t);
517# define nop(i0) _nop(_jit,i0)
518static void _nop(jit_state_t*,jit_int32_t);
519# define addr(r0,r1,r2) ADD(r0,r1,r2)
520# define addi(r0,r1,i0) _addi(_jit,r0,r1,i0)
521static void _addi(jit_state_t*,jit_int32_t,jit_int32_t,jit_word_t);
522# define addcr(r0,r1,r2) ADDS(r0,r1,r2)
523# define addci(r0,r1,i0) _addci(_jit,r0,r1,i0)
524static void _addci(jit_state_t*,jit_int32_t,jit_int32_t,jit_word_t);
525# define addxr(r0,r1,r2) ADCS(r0,r1,r2)
526# define addxi(r0,r1,i0) _addxi(_jit,r0,r1,i0)
527static void _addxi(jit_state_t*,jit_int32_t,jit_int32_t,jit_word_t);
528# define subr(r0,r1,r2) SUB(r0,r1,r2)
529# define subi(r0,r1,i0) _subi(_jit,r0,r1,i0)
530static void _subi(jit_state_t*,jit_int32_t,jit_int32_t,jit_word_t);
531# define subcr(r0,r1,r2) SUBS(r0,r1,r2)
532# define subci(r0,r1,i0) _subci(_jit,r0,r1,i0)
533static void _subci(jit_state_t*,jit_int32_t,jit_int32_t,jit_word_t);
534# define subxr(r0,r1,r2) SBCS(r0,r1,r2)
535# define subxi(r0,r1,i0) _subxi(_jit,r0,r1,i0)
536static void _subxi(jit_state_t*,jit_int32_t,jit_int32_t,jit_word_t);
537# define rsbi(r0, r1, i0) _rsbi(_jit, r0, r1, i0)
538static void _rsbi(jit_state_t*,jit_int32_t,jit_int32_t,jit_word_t);
539# define mulr(r0,r1,r2) MUL(r0,r1,r2)
540# define muli(r0,r1,i0) _muli(_jit,r0,r1,i0)
541static void _muli(jit_state_t*,jit_int32_t,jit_int32_t,jit_word_t);
542# define qmulr(r0,r1,r2,r3) _qmulr(_jit,r0,r1,r2,r3)
543static void _qmulr(jit_state_t*,jit_int32_t,
544 jit_int32_t,jit_int32_t,jit_int32_t);
545# define qmuli(r0,r1,r2,i0) _qmuli(_jit,r0,r1,r2,i0)
546static void _qmuli(jit_state_t*,jit_int32_t,
547 jit_int32_t,jit_int32_t,jit_word_t);
548# define qmulr_u(r0,r1,r2,r3) _qmulr_u(_jit,r0,r1,r2,r3)
549static void _qmulr_u(jit_state_t*,jit_int32_t,
550 jit_int32_t,jit_int32_t,jit_int32_t);
551# define qmuli_u(r0,r1,r2,i0) _qmuli_u(_jit,r0,r1,r2,i0)
552static void _qmuli_u(jit_state_t*,jit_int32_t,
553 jit_int32_t,jit_int32_t,jit_word_t);
554# define divr(r0,r1,r2) SDIV(r0,r1,r2)
555# define divi(r0,r1,i0) _divi(_jit,r0,r1,i0)
556static void _divi(jit_state_t*,jit_int32_t,jit_int32_t,jit_word_t);
557# define divr_u(r0,r1,r2) UDIV(r0,r1,r2)
558# define divi_u(r0,r1,i0) _divi_u(_jit,r0,r1,i0)
559static void _divi_u(jit_state_t*,jit_int32_t,jit_int32_t,jit_word_t);
560# define qdivr(r0,r1,r2,r3) _iqdivr(_jit,1,r0,r1,r2,r3)
561# define qdivr_u(r0,r1,r2,r3) _iqdivr(_jit,0,r0,r1,r2,r3)
562static void _iqdivr(jit_state_t*,jit_bool_t,
563 jit_int32_t,jit_int32_t,jit_int32_t,jit_int32_t);
564# define qdivi(r0,r1,r2,i0) _qdivi(_jit,r0,r1,r2,i0)
565static void _qdivi(jit_state_t*,jit_int32_t,
566 jit_int32_t,jit_int32_t,jit_word_t);
567# define qdivi_u(r0,r1,r2,i0) _qdivi_u(_jit,r0,r1,r2,i0)
568static void _qdivi_u(jit_state_t*,jit_int32_t,
569 jit_int32_t,jit_int32_t,jit_word_t);
570# define remr(r0,r1,r2) _remr(_jit,r0,r1,r2)
571static void _remr(jit_state_t*,jit_int32_t,jit_int32_t,jit_int32_t);
572# define remi(r0,r1,i0) _remi(_jit,r0,r1,i0)
573static void _remi(jit_state_t*,jit_int32_t,jit_int32_t,jit_word_t);
574# define remr_u(r0,r1,r2) _remr_u(_jit,r0,r1,r2)
575static void _remr_u(jit_state_t*,jit_int32_t,jit_int32_t,jit_int32_t);
576# define remi_u(r0,r1,i0) _remi_u(_jit,r0,r1,i0)
577static void _remi_u(jit_state_t*,jit_int32_t,jit_int32_t,jit_word_t);
578# define lshr(r0,r1,r2) LSL(r0,r1,r2)
579# define lshi(r0,r1,i0) _lshi(_jit,r0,r1,i0)
580static void _lshi(jit_state_t*,jit_int32_t,jit_int32_t,jit_word_t);
581# define rshr(r0,r1,r2) ASR(r0,r1,r2)
582# define rshi(r0,r1,i0) _rshi(_jit,r0,r1,i0)
583static void _rshi(jit_state_t*,jit_int32_t,jit_int32_t,jit_word_t);
584# define rshr_u(r0,r1,r2) LSR(r0,r1,r2)
585# define rshi_u(r0,r1,i0) _rshi_u(_jit,r0,r1,i0)
586static void _rshi_u(jit_state_t*,jit_int32_t,jit_int32_t,jit_word_t);
e0659411
PC		 587# define movnr(r0,r1,r2) _movnr(_jit,r0,r1,r2)
588static void _movnr(jit_state_t*,jit_int32_t,jit_int32_t,jit_int32_t);
589# define movzr(r0,r1,r2) _movzr(_jit,r0,r1,r2)
590static void _movzr(jit_state_t*,jit_int32_t,jit_int32_t,jit_int32_t);
4a71579b
PC		 591# define negr(r0,r1) NEG(r0,r1)
592# define comr(r0,r1) MVN(r0,r1)
79bfeef6
PC		 593# define clor(r0, r1) _clor(_jit, r0, r1)
594static void _clor(jit_state_t*, jit_int32_t, jit_int32_t);
595# define clzr(r0, r1) CLZ(r0,r1)
596static void _clzr(jit_state_t*, jit_int32_t, jit_int32_t);
597# define ctor(r0, r1) _ctor(_jit, r0, r1)
598static void _ctor(jit_state_t*, jit_int32_t, jit_int32_t);
599# define ctzr(r0, r1) _ctzr(_jit, r0, r1)
600static void _ctzr(jit_state_t*, jit_int32_t, jit_int32_t);
4a71579b
PC		 601# define andr(r0,r1,r2) AND(r0,r1,r2)
602# define andi(r0,r1,i0) _andi(_jit,r0,r1,i0)
603static void _andi(jit_state_t*,jit_int32_t,jit_int32_t,jit_word_t);
604# define orr(r0,r1,r2) ORR(r0,r1,r2)
605# define ori(r0,r1,i0) _ori(_jit,r0,r1,i0)
606static void _ori(jit_state_t*,jit_int32_t,jit_int32_t,jit_word_t);
607# define xorr(r0,r1,r2) EOR(r0,r1,r2)
608# define xori(r0,r1,i0) _xori(_jit,r0,r1,i0)
609static void _xori(jit_state_t*,jit_int32_t,jit_int32_t,jit_word_t);
610# define ldr_c(r0,r1) LDRSBI(r0,r1,0)
611# define ldi_c(r0,i0) _ldi_c(_jit,r0,i0)
612static void _ldi_c(jit_state_t*,jit_int32_t,jit_word_t);
613# define ldr_uc(r0,r1) _ldr_uc(_jit,r0,r1)
614static void _ldr_uc(jit_state_t*,jit_int32_t,jit_int32_t);
615# define ldi_uc(r0,i0) _ldi_uc(_jit,r0,i0)
616static void _ldi_uc(jit_state_t*,jit_int32_t,jit_word_t);
617# define ldr_s(r0,r1) LDRSHI(r0,r1,0)
618# define ldi_s(r0,i0) _ldi_s(_jit,r0,i0)
619static void _ldi_s(jit_state_t*,jit_int32_t,jit_word_t);
620# define ldr_us(r0,r1) _ldr_us(_jit,r0,r1)
621static void _ldr_us(jit_state_t*,jit_int32_t,jit_int32_t);
622# define ldi_us(r0,i0) _ldi_us(_jit,r0,i0)
623static void _ldi_us(jit_state_t*,jit_int32_t,jit_word_t);
624# define ldr_i(r0,r1) LDRSWI(r0,r1,0)
625# define ldi_i(r0,i0) _ldi_i(_jit,r0,i0)
626static void _ldi_i(jit_state_t*,jit_int32_t,jit_word_t);
627# define ldr_ui(r0,r1) _ldr_ui(_jit,r0,r1)
628static void _ldr_ui(jit_state_t*,jit_int32_t,jit_int32_t);
629# define ldi_ui(r0,i0) _ldi_ui(_jit,r0,i0)
630static void _ldi_ui(jit_state_t*,jit_int32_t,jit_word_t);
631# define ldr_l(r0,r1) LDRI(r0,r1,0)
632static void _ldr_l(jit_state_t*,jit_int32_t,jit_int32_t);
633# define ldi_l(r0,i0) _ldi_l(_jit,r0,i0)
634static void _ldi_l(jit_state_t*,jit_int32_t,jit_word_t);
635# define ldxr_c(r0,r1,r2) _ldxr_c(_jit,r0,r1,r2)
636static void _ldxr_c(jit_state_t*,jit_int32_t,jit_int32_t,jit_int32_t);
637# define ldxi_c(r0,r1,i0) _ldxi_c(_jit,r0,r1,i0)
638static void _ldxi_c(jit_state_t*,jit_int32_t,jit_int32_t,jit_word_t);
639# define ldxr_uc(r0,r1,r2) _ldxr_uc(_jit,r0,r1,r2)
640static void _ldxr_uc(jit_state_t*,jit_int32_t,jit_int32_t,jit_int32_t);
641# define ldxi_uc(r0,r1,i0) _ldxi_uc(_jit,r0,r1,i0)
642static void _ldxi_uc(jit_state_t*,jit_int32_t,jit_int32_t,jit_word_t);
643# define ldxr_s(r0,r1,r2) LDRSH(r0,r1,r2)
644# define ldxi_s(r0,r1,i0) _ldxi_s(_jit,r0,r1,i0)
645static void _ldxi_s(jit_state_t*,jit_int32_t,jit_int32_t,jit_word_t);
646# define ldxr_us(r0,r1,r2) _ldxr_us(_jit,r0,r1,r2)
647static void _ldxr_us(jit_state_t*,jit_int32_t,jit_int32_t,jit_int32_t);
648# define ldxi_us(r0,r1,i0) _ldxi_us(_jit,r0,r1,i0)
649static void _ldxi_us(jit_state_t*,jit_int32_t,jit_int32_t,jit_word_t);
650# define ldxr_i(r0,r1,r2) LDRSW(r0,r1,r2)
651# define ldxi_i(r0,r1,i0) _ldxi_i(_jit,r0,r1,i0)
652static void _ldxi_i(jit_state_t*,jit_int32_t,jit_int32_t,jit_word_t);
653# define ldxr_ui(r0,r1,r2) _ldxr_ui(_jit,r0,r1,r2)
654static void _ldxr_ui(jit_state_t*,jit_int32_t,jit_int32_t,jit_int32_t);
655# define ldxi_ui(r0,r1,i0) _ldxi_ui(_jit,r0,r1,i0)
656static void _ldxi_ui(jit_state_t*,jit_int32_t,jit_int32_t,jit_word_t);
657# define ldxr_l(r0,r1,r2) LDR(r0,r1,r2)
658# define ldxi_l(r0,r1,i0) _ldxi_l(_jit,r0,r1,i0)
659static void _ldxi_l(jit_state_t*,jit_int32_t,jit_int32_t,jit_word_t);
660# define str_c(r0,r1) STRBI(r1,r0,0)
661# define sti_c(i0,r0) _sti_c(_jit,i0,r0)
662static void _sti_c(jit_state_t*,jit_word_t,jit_int32_t);
663# define str_s(r0,r1) STRHI(r1,r0,0)
664# define sti_s(i0,r0) _sti_s(_jit,i0,r0)
665static void _sti_s(jit_state_t*,jit_word_t,jit_int32_t);
666# define str_i(r0,r1) STRWI(r1,r0,0)
667# define sti_i(i0,r0) _sti_i(_jit,i0,r0)
668static void _sti_i(jit_state_t*,jit_word_t,jit_int32_t);
669# define str_l(r0,r1) STRI(r1,r0,0)
670# define sti_l(i0,r0) _sti_l(_jit,i0,r0)
671static void _sti_l(jit_state_t*,jit_word_t,jit_int32_t);
672# define stxr_c(r0,r1,r2) STRB(r2,r1,r0)
673# define stxi_c(i0,r0,r1) _stxi_c(_jit,i0,r0,r1)
674static void _stxi_c(jit_state_t*,jit_word_t,jit_int32_t,jit_int32_t);
675# define stxr_s(r0,r1,r2) STRH(r2,r1,r0)
676# define stxi_s(i0,r0,r1) _stxi_s(_jit,i0,r0,r1)
677static void _stxi_s(jit_state_t*,jit_word_t,jit_int32_t,jit_int32_t);
678# define stxr_i(r0,r1,r2) STRW(r2,r1,r0)
679# define stxi_i(i0,r0,r1) _stxi_i(_jit,i0,r0,r1)
680static void _stxi_i(jit_state_t*,jit_word_t,jit_int32_t,jit_int32_t);
681# define stxr_l(r0,r1,r2) STR(r2,r1,r0)
682# define stxi_l(i0,r0,r1) _stxi_l(_jit,i0,r0,r1)
683static void _stxi_l(jit_state_t*,jit_word_t,jit_int32_t,jit_int32_t);
40a44dcb
PC		 684# define bswapr_us(r0,r1) _bswapr_us(_jit,r0,r1)
685static void _bswapr_us(jit_state_t*,jit_int32_t,jit_int32_t);
686# define bswapr_ui(r0,r1) _bswapr_ui(_jit,r0,r1)
687static void _bswapr_ui(jit_state_t*,jit_int32_t,jit_int32_t);
688# define bswapr_ul(r0,r1) REV(r0,r1)
4a71579b
PC		 689# define extr_c(r0,r1) SXTB(r0,r1)
690# define extr_uc(r0,r1) UXTB(r0,r1)
691# define extr_s(r0,r1) SXTH(r0,r1)
692# define extr_us(r0,r1) UXTH(r0,r1)
693# define extr_i(r0,r1) SXTW(r0,r1)
694# define extr_ui(r0,r1) UXTW(r0,r1)
ba3814c1
PC		 695# define casx(r0, r1, r2, r3, i0) _casx(_jit, r0, r1, r2, r3, i0)
696static void _casx(jit_state_t *_jit,jit_int32_t,jit_int32_t,
697 jit_int32_t,jit_int32_t,jit_word_t);
698#define casr(r0, r1, r2, r3) casx(r0, r1, r2, r3, 0)
699#define casi(r0, i0, r1, r2) casx(r0, _NOREG, r1, r2, i0)
4a71579b
PC		 700# define movr(r0,r1) _movr(_jit,r0,r1)
701static void _movr(jit_state_t*,jit_int32_t,jit_int32_t);
702# define movi(r0,i0) _movi(_jit,r0,i0)
703static void _movi(jit_state_t*,jit_int32_t,jit_word_t);
704# define movi_p(r0,i0) _movi_p(_jit,r0,i0)
705static jit_word_t _movi_p(jit_state_t*,jit_int32_t,jit_word_t);
706# define ccr(cc,r0,r1,r2) _ccr(_jit,cc,r0,r1,r2)
707static void _ccr(jit_state_t*,jit_int32_t,jit_int32_t,jit_int32_t,jit_int32_t);
708# define cci(cc,r0,r1,i0) _cci(_jit,cc,r0,r1,i0)
709static void _cci(jit_state_t*,jit_int32_t,jit_int32_t,jit_int32_t,jit_word_t);
710# define ltr(r0,r1,r2) ccr(CC_LT,r0,r1,r2)
711# define lti(r0,r1,i0) cci(CC_LT,r0,r1,i0)
712# define ltr_u(r0,r1,r2) ccr(CC_CC,r0,r1,r2)
713# define lti_u(r0,r1,i0) cci(CC_CC,r0,r1,i0)
714# define ler(r0,r1,r2) ccr(CC_LE,r0,r1,r2)
715# define lei(r0,r1,i0) cci(CC_LE,r0,r1,i0)
716# define ler_u(r0,r1,r2) ccr(CC_LS,r0,r1,r2)
717# define lei_u(r0,r1,i0) cci(CC_LS,r0,r1,i0)
718# define eqr(r0,r1,r2) ccr(CC_EQ,r0,r1,r2)
719# define eqi(r0,r1,i0) cci(CC_EQ,r0,r1,i0)
720# define ger(r0,r1,r2) ccr(CC_GE,r0,r1,r2)
721# define gei(r0,r1,i0) cci(CC_GE,r0,r1,i0)
722# define ger_u(r0,r1,r2) ccr(CC_CS,r0,r1,r2)
723# define gei_u(r0,r1,i0) cci(CC_CS,r0,r1,i0)
724# define gtr(r0,r1,r2) ccr(CC_GT,r0,r1,r2)
725# define gti(r0,r1,i0) cci(CC_GT,r0,r1,i0)
726# define gtr_u(r0,r1,r2) ccr(CC_HI,r0,r1,r2)
727# define gti_u(r0,r1,i0) cci(CC_HI,r0,r1,i0)
728# define ner(r0,r1,r2) ccr(CC_NE,r0,r1,r2)
729# define nei(r0,r1,i0) cci(CC_NE,r0,r1,i0)
730# define bccr(cc,i0,r0,r1) _bccr(_jit,cc,i0,r0,r1)
731static jit_word_t
732_bccr(jit_state_t*,jit_int32_t,jit_word_t,jit_int32_t,jit_int32_t);
733# define bcci(cc,i0,r0,i1) _bcci(_jit,cc,i0,r0,i1)
734static jit_word_t
735_bcci(jit_state_t*,jit_int32_t,jit_word_t,jit_int32_t,jit_word_t);
736# define bltr(i0,r0,r1) bccr(BCC_LT,i0,r0,r1)
737# define blti(i0,r0,i1) bcci(BCC_LT,i0,r0,i1)
738# define bltr_u(i0,r0,r1) bccr(BCC_CC,i0,r0,r1)
739# define blti_u(i0,r0,i1) bcci(BCC_CC,i0,r0,i1)
740# define bler(i0,r0,r1) bccr(BCC_LE,i0,r0,r1)
741# define blei(i0,r0,i1) bcci(BCC_LE,i0,r0,i1)
742# define bler_u(i0,r0,r1) bccr(BCC_LS,i0,r0,r1)
743# define blei_u(i0,r0,i1) bcci(BCC_LS,i0,r0,i1)
744# define beqr(i0,r0,r1) bccr(BCC_EQ,i0,r0,r1)
745# define beqi(i0,r0,i1) _beqi(_jit,i0,r0,i1)
746static jit_word_t _beqi(jit_state_t*,jit_word_t,jit_int32_t,jit_word_t);
747# define bger(i0,r0,r1) bccr(BCC_GE,i0,r0,r1)
748# define bgei(i0,r0,i1) bcci(BCC_GE,i0,r0,i1)
749# define bger_u(i0,r0,r1) bccr(BCC_CS,i0,r0,r1)
750# define bgei_u(i0,r0,i1) bcci(BCC_CS,i0,r0,i1)
751# define bgtr(i0,r0,r1) bccr(BCC_GT,i0,r0,r1)
752# define bgti(i0,r0,i1) bcci(BCC_GT,i0,r0,i1)
753# define bgtr_u(i0,r0,r1) bccr(BCC_HI,i0,r0,r1)
754# define bgti_u(i0,r0,i1) bcci(BCC_HI,i0,r0,i1)
755# define bner(i0,r0,r1) bccr(BCC_NE,i0,r0,r1)
756# define bnei(i0,r0,i1) _bnei(_jit,i0,r0,i1)
757static jit_word_t _bnei(jit_state_t*,jit_word_t,jit_int32_t,jit_word_t);
758# define baddr(cc,i0,r0,r1) _baddr(_jit,cc,i0,r0,r1)
759static jit_word_t
760_baddr(jit_state_t*,jit_int32_t,jit_word_t,jit_int32_t,jit_int32_t);
761# define baddi(cc,i0,r0,i1) _baddi(_jit,cc,i0,r0,i1)
762static jit_word_t
763_baddi(jit_state_t*,jit_int32_t,jit_word_t,jit_int32_t,jit_word_t);
764# define boaddr(i0,r0,r1) baddr(BCC_VS,i0,r0,r1)
765# define boaddi(i0,r0,i1) baddi(BCC_VS,i0,r0,i1)
766# define boaddr_u(i0,r0,r1) baddr(BCC_HS,i0,r0,r1)
767# define boaddi_u(i0,r0,i1) baddi(BCC_HS,i0,r0,i1)
768# define bxaddr(i0,r0,r1) baddr(BCC_VC,i0,r0,r1)
769# define bxaddi(i0,r0,i1) baddi(BCC_VC,i0,r0,i1)
770# define bxaddr_u(i0,r0,r1) baddr(BCC_LO,i0,r0,r1)
771# define bxaddi_u(i0,r0,i1) baddi(BCC_LO,i0,r0,i1)
772# define bsubr(cc,i0,r0,r1) _bsubr(_jit,cc,i0,r0,r1)
773static jit_word_t
774_bsubr(jit_state_t*,jit_int32_t,jit_word_t,jit_int32_t,jit_int32_t);
775# define bsubi(cc,i0,r0,i1) _bsubi(_jit,cc,i0,r0,i1)
776static jit_word_t
777_bsubi(jit_state_t*,jit_int32_t,jit_word_t,jit_int32_t,jit_word_t);
778# define bosubr(i0,r0,r1) bsubr(BCC_VS,i0,r0,r1)
779# define bosubi(i0,r0,i1) bsubi(BCC_VS,i0,r0,i1)
780# define bosubr_u(i0,r0,r1) bsubr(BCC_LO,i0,r0,r1)
781# define bosubi_u(i0,r0,i1) bsubi(BCC_LO,i0,r0,i1)
782# define bxsubr(i0,r0,r1) bsubr(BCC_VC,i0,r0,r1)
783# define bxsubi(i0,r0,i1) bsubi(BCC_VC,i0,r0,i1)
784# define bxsubr_u(i0,r0,r1) bsubr(BCC_HS,i0,r0,r1)
785# define bxsubi_u(i0,r0,i1) bsubi(BCC_HS,i0,r0,i1)
786# define bmxr(cc,i0,r0,r1) _bmxr(_jit,cc,i0,r0,r1)
787static jit_word_t
788_bmxr(jit_state_t*,jit_int32_t,jit_word_t,jit_int32_t,jit_int32_t);
789# define bmxi(cc,i0,r0,r1) _bmxi(_jit,cc,i0,r0,r1)
790static jit_word_t
791_bmxi(jit_state_t*,jit_int32_t,jit_word_t,jit_int32_t,jit_word_t);
792# define bmsr(i0,r0,r1) bmxr(BCC_NE,i0,r0,r1)
793# define bmsi(i0,r0,i1) bmxi(BCC_NE,i0,r0,i1)
794# define bmcr(i0,r0,r1) bmxr(BCC_EQ,i0,r0,r1)
795# define bmci(i0,r0,i1) bmxi(BCC_EQ,i0,r0,i1)
796# define jmpr(r0) BR(r0)
797# define jmpi(i0) _jmpi(_jit,i0)
79bfeef6 798static jit_word_t _jmpi(jit_state_t*,jit_word_t);
4a71579b
PC		 799# define jmpi_p(i0) _jmpi_p(_jit,i0)
800static jit_word_t _jmpi_p(jit_state_t*,jit_word_t);
801# define callr(r0) BLR(r0)
802# define calli(i0) _calli(_jit,i0)
79bfeef6 803static jit_word_t _calli(jit_state_t*,jit_word_t);
4a71579b
PC		 804# define calli_p(i0) _calli_p(_jit,i0)
805static jit_word_t _calli_p(jit_state_t*,jit_word_t);
806# define prolog(i0) _prolog(_jit,i0)
807static void _prolog(jit_state_t*,jit_node_t*);
808# define epilog(i0) _epilog(_jit,i0)
809static void _epilog(jit_state_t*,jit_node_t*);
810# define vastart(r0) _vastart(_jit, r0)
811static void _vastart(jit_state_t*, jit_int32_t);
812# define vaarg(r0, r1) _vaarg(_jit, r0, r1)
813static void _vaarg(jit_state_t*, jit_int32_t, jit_int32_t);
814# define patch_at(jump,label) _patch_at(_jit,jump,label)
815static void _patch_at(jit_state_t*,jit_word_t,jit_word_t);
816#endif
817
818#if CODE
79bfeef6
PC		 819/* https://dougallj.wordpress.com/2021/10/30/bit-twiddling-optimising-aarch64-logical-immediate-encoding-and-decoding/ */
820#include "aarch64-logical-immediates.c"
4a71579b
PC		 821static jit_int32_t
822logical_immediate(jit_word_t imm)
823{
79bfeef6
PC		 824 jit_int32_t result = encodeLogicalImmediate64(imm);
825 if (result != ENCODE_FAILED) {
826 assert(isValidLogicalImmediate64(result));
827 return (result & 0xfff);
4a71579b 828 }
79bfeef6 829 return (-1);
4a71579b
PC		 830}
831
832static void
833_oxxx(jit_state_t *_jit, jit_int32_t Op,
834 jit_int32_t Rd, jit_int32_t Rn, jit_int32_t Rm)
835{
836 instr_t i;
837 assert(!(Rd & ~0x1f));
838 assert(!(Rn & ~0x1f));
839 assert(!(Rm & ~0x1f));
840 assert(!(Op & ~0xffe0fc00));
841 i.w = Op;
842 i.Rd.b = Rd;
843 i.Rn.b = Rn;
844 i.Rm.b = Rm;
845 ii(i.w);
846}
847
848static void
849_oxxi(jit_state_t *_jit, jit_int32_t Op,
850 jit_int32_t Rd, jit_int32_t Rn, jit_int32_t Imm12)
851{
852 instr_t i;
853 assert(!(Rd & ~0x1f));
854 assert(!(Rn & ~0x1f));
855 assert(!(Imm12 & ~0xfff));
856 assert(!(Op & ~0xffe00000));
857 i.w = Op;
858 i.Rd.b = Rd;
859 i.Rn.b = Rn;
860 i.imm12.b = Imm12;
861 ii(i.w);
862}
863
864static void
865_oxx9(jit_state_t *_jit, jit_int32_t Op,
866 jit_int32_t Rd, jit_int32_t Rn, jit_int32_t Imm9)
867{
868 instr_t i;
869 assert(!(Rd & ~0x1f));
870 assert(!(Rn & ~0x1f));
871 assert(!(Imm9 & ~0x1ff));
872 assert(!(Op & ~0xffe00000));
873 i.w = Op;
874 i.Rd.b = Rd;
875 i.Rn.b = Rn;
876 i.imm9.b = Imm9;
877 ii(i.w);
878}
879
880static void
881_ox19(jit_state_t *_jit, jit_int32_t Op, jit_int32_t Rd, jit_int32_t Simm19)
882{
883 instr_t i;
884 assert(!(Rd & ~0x1f));
885 assert(Simm19 >= -262148 && Simm19 <= 262143);
886 assert(!(Op & ~0xff000000));
887 i.w = Op;
888 i.Rd.b = Rd;
889 i.imm19.b = Simm19;
890 ii(i.w);
891}
892
893static void
894_oc19(jit_state_t *_jit, jit_int32_t Op, jit_int32_t Cc, jit_int32_t Simm19)
895{
896 instr_t i;
897 assert(!(Cc & ~0xf));
898 assert(Simm19 >= -262148 && Simm19 <= 262143);
899 assert(!(Op & ~0xff000000));
900 i.w = Op;
901 i.cond2.b = Cc;
902 i.imm19.b = Simm19;
903 ii(i.w);
904}
905
906static void
907_o26(jit_state_t *_jit, jit_int32_t Op, jit_int32_t Simm26)
908{
909 instr_t i;
79bfeef6 910 assert(s26_p(Simm26));
4a71579b
PC		 911 assert(!(Op & ~0xfc000000));
912 i.w = Op;
913 i.imm26.b = Simm26;
914 ii(i.w);
915}
916
917static void
918_ox_x(jit_state_t *_jit, jit_int32_t Op, jit_int32_t Rd, jit_int32_t Rm)
919{
920 instr_t i;
921 assert(!(Rd & ~0x1f));
922 assert(!(Rm & ~0x1f));
923 assert(!(Op & ~0xffe0ffe0));
924 i.w = Op;
925 i.Rd.b = Rd;
926 i.Rm.b = Rm;
927 ii(i.w);
928}
929
930static void
931_o_xx(jit_state_t *_jit, jit_int32_t Op, jit_int32_t Rd, jit_int32_t Rn)
932{
933 instr_t i;
934 assert(!(Rd & ~0x1f));
935 assert(!(Rn & ~0x1f));
936 assert(!(Op & ~0xfffffc00));
937 i.w = Op;
938 i.Rd.b = Rd;
939 i.Rn.b = Rn;
940 ii(i.w);
941}
942
943static void
944_oxx_(jit_state_t *_jit, jit_int32_t Op, jit_int32_t Rn, jit_int32_t Rm)
945{
946 instr_t i;
947 assert(!(Rn & ~0x1f));
948 assert(!(Rm & ~0x1f));
949 assert(!(Op & ~0xffc0fc1f));
950 i.w = Op;
951 i.Rn.b = Rn;
952 i.Rm.b = Rm;
953 ii(i.w);
954}
955
956static void
957_o_x_(jit_state_t *_jit, jit_int32_t Op, jit_int32_t Rn)
958{
959 instr_t i;
960 assert(!(Rn & ~0x1f));
961 assert(!(Op & 0x3e0));
962 i.w = Op;
963 i.Rn.b = Rn;
964 ii(i.w);
965}
966
967static void
968_ox_h(jit_state_t *_jit, jit_int32_t Op, jit_int32_t Rd, jit_int32_t Imm16)
969{
970 instr_t i;
971 assert(!(Rd & ~0x1f));
972 assert(!(Imm16 & ~0xffff));
973 assert(!(Op & ~0xffe00000));
974 i.w = Op;
975 i.Rd.b = Rd;
976 i.imm16.b = Imm16;
977 ii(i.w);
978}
979
980static void
981_oxxrs(jit_state_t *_jit, jit_int32_t Op,
982 jit_int32_t Rd, jit_int32_t Rn, jit_int32_t R, jit_int32_t S)
983{
984 instr_t i;
985 assert(!(Rd & ~0x1f));
986 assert(!(Rn & ~0x1f));
987 assert(!(R & ~0x3f));
988 assert(!(S & ~0x3f));
989 assert(!(Op & ~0xffc00000));
990 i.w = Op;
991 i.Rd.b = Rd;
992 i.Rn.b = Rn;
993 i.immr.b = R;
994 i.imms.b = S;
995 ii(i.w);
996}
997
998static void
999_oxxxc(jit_state_t *_jit, jit_int32_t Op,
1000 jit_int32_t Rd, jit_int32_t Rn, jit_int32_t Rm, jit_int32_t Cc)
1001{
1002 instr_t i;
1003 assert(!(Rd & ~0x1f));
1004 assert(!(Rn & ~0x1f));
1005 assert(!(Rm & ~0x1f));
1006 assert(!(Cc & ~0xf));
1007 assert(!(Op & ~0xffc00c00));
1008 i.w = Op;
1009 i.Rd.b = Rd;
1010 i.Rn.b = Rn;
1011 i.Rm.b = Rm;
1012 i.cond.b = Cc;
1013 ii(i.w);
1014}
1015
1016static void
1017_oxxx7(jit_state_t *_jit, jit_int32_t Op,
1018 jit_int32_t Rt, jit_int32_t Rt2, jit_int32_t Rn, jit_int32_t Simm7)
1019{
1020 instr_t i;
1021 assert(!(Rt & ~0x1f));
1022 assert(!(Rt2 & ~0x1f));
1023 assert(!(Rn & ~0x1f));
1024 assert(Simm7 >= -128 && Simm7 <= 127);
1025 assert(!(Op & ~0xffc003e0));
1026 i.w = Op;
1027 i.Rt.b = Rt;
1028 i.Rt2.b = Rt2;
1029 i.Rn.b = Rn;
1030 i.imm7.b = Simm7;
1031 ii(i.w);
1032}
1033
1034static void
1035_nop(jit_state_t *_jit, jit_int32_t i0)
1036{
1037 for (; i0 > 0; i0 -= 4)
1038 NOP();
1039 assert(i0 == 0);
1040}
1041
1042static void
1043_addi(jit_state_t *_jit, jit_int32_t r0, jit_int32_t r1, jit_word_t i0)
1044{
1045 jit_int32_t reg;
1046 jit_word_t is = i0 >> 12;
1047 jit_word_t in = -i0;
1048 jit_word_t iS = in >> 12;
1049 if ( i0 >= 0 && i0 <= 0xfff)
1050 ADDI (r0, r1, i0);
1051 else if ((is << 12) == i0 && is >= 0 && is <= 0xfff)
1052 ADDI_12(r0, r1, is);
1053 else if ( in >= 0 && in <= 0xfff)
1054 SUBI (r0, r1, in);
1055 else if ((iS << 12) == is && iS >= 0 && iS <= 0xfff)
1056 SUBI_12(r0, r1, iS);
1057 else {
1058 reg = jit_get_reg(jit_class_gpr);
1059 movi(rn(reg), i0);
1060 addr(r0, r1, rn(reg));
1061 jit_unget_reg(reg);
1062 }
1063}
1064
1065static void
1066_addci(jit_state_t *_jit, jit_int32_t r0, jit_int32_t r1, jit_word_t i0)
1067{
1068 jit_int32_t reg;
1069 jit_word_t is = i0 >> 12;
1070 jit_word_t in = -i0;
1071 jit_word_t iS = in >> 12;
1072 if ( i0 >= 0 && i0 <= 0xfff)
1073 ADDSI (r0, r1, i0);
1074 else if ((is << 12) == i0 && is >= 0 && is <= 0xfff)
1075 ADDSI_12(r0, r1, is);
1076 else if ( in >= 0 && in <= 0xfff)
1077 SUBSI (r0, r1, in);
1078 else if ((iS << 12) == is && iS >= 0 && iS <= 0xfff)
1079 SUBSI_12(r0, r1, iS);
1080 else {
1081 reg = jit_get_reg(jit_class_gpr);
1082 movi(rn(reg), i0);
1083 addcr(r0, r1, rn(reg));
1084 jit_unget_reg(reg);
1085 }
1086}
1087
1088static void
1089_addxi(jit_state_t *_jit, jit_int32_t r0, jit_int32_t r1, jit_word_t i0)
1090{
1091 jit_int32_t reg;
1092 reg = jit_get_reg(jit_class_gpr);
1093 movi(rn(reg), i0);
1094 addxr(r0, r1, rn(reg));
1095 jit_unget_reg(reg);
1096}
1097
1098static void
1099_subi(jit_state_t *_jit, jit_int32_t r0, jit_int32_t r1, jit_word_t i0)
1100{
1101 jit_int32_t reg;
1102 jit_word_t is = i0 >> 12;
1103 if ( i0 >= 0 && i0 <= 0xfff)
1104 SUBI (r0, r1, i0);
1105 else if ((is << 12) == i0 && is >= 0 && is <= 0xfff)
1106 SUBI_12(r0, r1, is);
1107 else {
1108 reg = jit_get_reg(jit_class_gpr);
1109 movi(rn(reg), i0);
1110 subr(r0, r1, rn(reg));
1111 jit_unget_reg(reg);
1112 }
1113}
1114
1115static void
1116_subci(jit_state_t *_jit, jit_int32_t r0, jit_int32_t r1, jit_word_t i0)
1117{
1118 jit_int32_t reg;
1119 jit_word_t is = i0 >> 12;
1120 if ( i0 >= 0 && i0 <= 0xfff)
1121 SUBSI (r0, r1, i0);
1122 else if ((is << 12) == i0 && is >= 0 && is <= 0xfff)
1123 SUBSI_12(r0, r1, is);
1124 else {
1125 reg = jit_get_reg(jit_class_gpr);
1126 movi(rn(reg), i0);
1127 subcr(r0, r1, rn(reg));
1128 jit_unget_reg(reg);
1129 }
1130}
1131
1132static void
1133_subxi(jit_state_t *_jit, jit_int32_t r0, jit_int32_t r1, jit_word_t i0)
1134{
1135 jit_int32_t reg;
1136 reg = jit_get_reg(jit_class_gpr);
1137 movi(rn(reg), i0);
1138 subxr(r0, r1, rn(reg));
1139 jit_unget_reg(reg);
1140}
1141
1142static void
1143_rsbi(jit_state_t *_jit, jit_int32_t r0, jit_int32_t r1, jit_word_t i0)
1144{
1145 subi(r0, r1, i0);
1146 negr(r0, r0);
1147}
1148
1149static void
1150_muli(jit_state_t *_jit, jit_int32_t r0, jit_int32_t r1, jit_word_t i0)
1151{
1152 jit_int32_t reg;
1153 reg = jit_get_reg(jit_class_gpr);
1154 movi(rn(reg), i0);
1155 mulr(r0, r1, rn(reg));
1156 jit_unget_reg(reg);
1157}
1158
1159static void
1160_qmulr(jit_state_t *_jit, jit_int32_t r0,
1161 jit_int32_t r1, jit_int32_t r2, jit_int32_t r3)
1162{
1163 jit_int32_t reg;
1164 if (r0 == r2 || r0 == r3) {
1165 reg = jit_get_reg(jit_class_gpr);
1166 mulr(rn(reg), r2, r3);
1167 }
1168 else
1169 mulr(r0, r2, r3);
1170 SMULH(r1, r2, r3);
1171 if (r0 == r2 || r0 == r3) {
1172 movr(r0, rn(reg));
1173 jit_unget_reg(reg);
1174 }
1175}
1176
1177static void
1178_qmuli(jit_state_t *_jit, jit_int32_t r0,
1179 jit_int32_t r1, jit_int32_t r2, jit_word_t i0)
1180{
1181 jit_int32_t reg;
1182 reg = jit_get_reg(jit_class_gpr);
1183 movi(rn(reg), i0);
1184 qmulr(r0, r1, r2, rn(reg));
1185 jit_unget_reg(reg);
1186}
1187
1188static void
1189_qmulr_u(jit_state_t *_jit, jit_int32_t r0,
1190 jit_int32_t r1, jit_int32_t r2, jit_int32_t r3)
1191{
1192 jit_int32_t reg;
1193 if (r0 == r2 || r0 == r3) {
1194 reg = jit_get_reg(jit_class_gpr);
1195 mulr(rn(reg), r2, r3);
1196 }
1197 else
1198 mulr(r0, r2, r3);
1199 UMULH(r1, r2, r3);
1200 if (r0 == r2 || r0 == r3) {
1201 movr(r0, rn(reg));
1202 jit_unget_reg(reg);
1203 }
1204}
1205
1206static void
1207_qmuli_u(jit_state_t *_jit, jit_int32_t r0,
1208 jit_int32_t r1, jit_int32_t r2, jit_word_t i0)
1209{
1210 jit_int32_t reg;
1211 reg = jit_get_reg(jit_class_gpr);
1212 movi(rn(reg), i0);
1213 qmulr_u(r0, r1, r2, rn(reg));
1214 jit_unget_reg(reg);
1215}
1216
1217static void
1218_divi(jit_state_t *_jit, jit_int32_t r0, jit_int32_t r1, jit_word_t i0)
1219{
1220 jit_int32_t reg;
1221 reg = jit_get_reg(jit_class_gpr);
1222 movi(rn(reg), i0);
1223 divr(r0, r1, rn(reg));
1224 jit_unget_reg(reg);
1225}
1226
1227static void
1228_divi_u(jit_state_t *_jit, jit_int32_t r0, jit_int32_t r1, jit_word_t i0)
1229{
1230 jit_int32_t reg;
1231 reg = jit_get_reg(jit_class_gpr);
1232 movi(rn(reg), i0);
1233 divr_u(r0, r1, rn(reg));
1234 jit_unget_reg(reg);
1235}
1236
1237static void
1238_iqdivr(jit_state_t *_jit, jit_bool_t sign,
1239 jit_int32_t r0, jit_int32_t r1, jit_int32_t r2, jit_int32_t r3)
1240{
1241 jit_int32_t sv0, rg0;
1242 jit_int32_t sv1, rg1;
1243 if (r0 == r2 || r0 == r3) {
1244 sv0 = jit_get_reg(jit_class_gpr);
1245 rg0 = rn(sv0);
1246 }
1247 else
1248 rg0 = r0;
1249 if (r1 == r2 || r1 == r3) {
1250 sv1 = jit_get_reg(jit_class_gpr);
1251 rg1 = rn(sv1);
1252 }
1253 else
1254 rg1 = r1;
1255 if (sign)
1256 divr(rg0, r2, r3);
1257 else
1258 divr_u(rg0, r2, r3);
1259 mulr(rg1, r3, rg0);
1260 subr(rg1, r2, rg1);
1261 if (rg0 != r0) {
1262 movr(r0, rg0);
1263 jit_unget_reg(sv0);
1264 }
1265 if (rg1 != r1) {
1266 movr(r1, rg1);
1267 jit_unget_reg(sv1);
1268 }
1269}
1270
1271static void
1272_qdivi(jit_state_t *_jit, jit_int32_t r0,
1273 jit_int32_t r1, jit_int32_t r2, jit_word_t i0)
1274{
1275 jit_int32_t reg;
1276 reg = jit_get_reg(jit_class_gpr);
1277 movi(rn(reg), i0);
1278 qdivr(r0, r1, r2, rn(reg));
1279 jit_unget_reg(reg);
1280}
1281
1282static void
1283_qdivi_u(jit_state_t *_jit, jit_int32_t r0,
1284 jit_int32_t r1, jit_int32_t r2, jit_word_t i0)
1285{
1286 jit_int32_t reg;
1287 reg = jit_get_reg(jit_class_gpr);
1288 movi(rn(reg), i0);
1289 qdivr_u(r0, r1, r2, rn(reg));
1290 jit_unget_reg(reg);
1291}
1292
1293static void
1294_remr(jit_state_t *_jit, jit_int32_t r0, jit_int32_t r1, jit_int32_t r2)
1295{
1296 jit_int32_t reg;
1297 if (r0 == r1 || r0 == r2) {
1298 reg = jit_get_reg(jit_class_gpr);
1299 divr(rn(reg), r1, r2);
1300 mulr(rn(reg), r2, rn(reg));
1301 subr(r0, r1, rn(reg));
1302 jit_unget_reg(reg);
1303 }
1304 else {
1305 divr(r0, r1, r2);
1306 mulr(r0, r2, r0);
1307 subr(r0, r1, r0);
1308 }
1309}
1310
1311static void
1312_remi(jit_state_t *_jit, jit_int32_t r0, jit_int32_t r1, jit_word_t i0)
1313{
1314 jit_int32_t reg;
1315 reg = jit_get_reg(jit_class_gpr);
1316 movi(rn(reg), i0);
1317 remr(r0, r1, rn(reg));
1318 jit_unget_reg(reg);
1319}
1320
1321static void
1322_remr_u(jit_state_t *_jit, jit_int32_t r0, jit_int32_t r1, jit_int32_t r2)
1323{
1324 jit_int32_t reg;
1325 if (r0 == r1 || r0 == r2) {
1326 reg = jit_get_reg(jit_class_gpr);
1327 divr_u(rn(reg), r1, r2);
1328 mulr(rn(reg), r2, rn(reg));
1329 subr(r0, r1, rn(reg));
1330 jit_unget_reg(reg);
1331 }
1332 else {
1333 divr_u(r0, r1, r2);
1334 mulr(r0, r2, r0);
1335 subr(r0, r1, r0);
1336 }
1337}
1338
1339static void
1340_remi_u(jit_state_t *_jit, jit_int32_t r0, jit_int32_t r1, jit_word_t i0)
1341{
1342 jit_int32_t reg;
1343 reg = jit_get_reg(jit_class_gpr);
1344 movi(rn(reg), i0);
1345 remr_u(r0, r1, rn(reg));
1346 jit_unget_reg(reg);
1347}
1348
1349static void
1350_lshi(jit_state_t *_jit, jit_int32_t r0, jit_int32_t r1, jit_word_t i0)
1351{
1352 if (i0 == 0)
1353 movr(r0, r1);
1354 else {
1355 assert(i0 > 0 && i0 < 64);
1356 LSLI(r0, r1, i0);
1357 }
1358}
1359
1360static void
1361_rshi(jit_state_t *_jit, jit_int32_t r0, jit_int32_t r1, jit_word_t i0)
1362{
1363 if (i0 == 0)
1364 movr(r0, r1);
1365 else {
1366 assert(i0 > 0 && i0 < 64);
1367 ASRI(r0, r1, i0);
1368 }
1369}
1370
1371static void
1372_rshi_u(jit_state_t *_jit, jit_int32_t r0, jit_int32_t r1, jit_word_t i0)
1373{
1374 if (i0 == 0)
1375 movr(r0, r1);
1376 else {
1377 assert(i0 > 0 && i0 < 64);
1378 LSRI(r0, r1, i0);
1379 }
1380}
1381
e0659411
PC		 1382static void
1383_movnr(jit_state_t *_jit, jit_int32_t r0, jit_int32_t r1, jit_int32_t r2)
1384{
1385 CMPI(r2, 0);
1386 CSEL(r0, r0, r1, CC_NE);
1387}
1388
1389static void
1390_movzr(jit_state_t *_jit, jit_int32_t r0, jit_int32_t r1, jit_int32_t r2)
1391{
1392 CMPI(r2, 0);
1393 CSEL(r0, r0, r1, CC_EQ);
1394}
1395
79bfeef6
PC		 1396static void
1397_clor(jit_state_t *_jit, jit_int32_t r0, jit_int32_t r1)
1398{
1399 comr(r0, r1);
1400 clzr(r0, r0);
1401}
1402
1403static void
1404_ctor(jit_state_t *_jit, jit_int32_t r0, jit_int32_t r1)
1405{
1406 RBIT(r0, r1);
1407 clor(r0, r0);
1408}
1409
1410static void
1411_ctzr(jit_state_t *_jit, jit_int32_t r0, jit_int32_t r1)
1412{
1413 RBIT(r0, r1);
1414 clzr(r0, r0);
1415}
1416
4a71579b
PC		 1417static void
1418_andi(jit_state_t *_jit, jit_int32_t r0, jit_int32_t r1, jit_word_t i0)
1419{
1420 jit_int32_t reg;
1421 jit_int32_t imm;
1422 if (i0 == 0)
1423 movi(r0, 0);
1424 else if (i0 == -1)
1425 movr(r0, r1);
1426 else {
1427 imm = logical_immediate(i0);
1428 if (imm != -1)
1429 ANDI(r0, r1, imm);
1430 else {
1431 reg = jit_get_reg(jit_class_gpr);
1432 movi(rn(reg), i0);
1433 andr(r0, r1, rn(reg));
1434 jit_unget_reg(reg);
1435 }
1436 }
1437}
1438
1439static void
1440_ori(jit_state_t *_jit, jit_int32_t r0, jit_int32_t r1, jit_word_t i0)
1441{
1442 jit_int32_t reg;
1443 jit_int32_t imm;
1444 if (i0 == 0)
1445 movr(r0, r1);
1446 else if (i0 == -1)
1447 movi(r0, -1);
1448 else {
1449 imm = logical_immediate(i0);
1450 if (imm != -1)
1451 ORRI(r0, r1, imm);
1452 else {
1453 reg = jit_get_reg(jit_class_gpr);
1454 movi(rn(reg), i0);
1455 orr(r0, r1, rn(reg));
1456 jit_unget_reg(reg);
1457 }
1458 }
1459}
1460
1461static void
1462_xori(jit_state_t *_jit, jit_int32_t r0, jit_int32_t r1, jit_word_t i0)
1463{
1464 jit_int32_t reg;
1465 jit_int32_t imm;
1466 if (i0 == 0)
1467 movr(r0, r1);
1468 else if (i0 == -1)
1469 comr(r0, r1);
1470 else {
1471 imm = logical_immediate(i0);
1472 if (imm != -1)
1473 EORI(r0, r1, imm);
1474 else {
1475 reg = jit_get_reg(jit_class_gpr);
1476 movi(rn(reg), i0);
1477 xorr(r0, r1, rn(reg));
1478 jit_unget_reg(reg);
1479 }
1480 }
1481}
1482
4a71579b 1483static void
40a44dcb 1484_bswapr_us(jit_state_t *_jit, jit_int32_t r0, jit_int32_t r1)
4a71579b 1485{
40a44dcb 1486 bswapr_ul(r0, r1);
4a71579b
PC		 1487 rshi_u(r0, r0, 48);
1488}
1489
1490static void
40a44dcb 1491_bswapr_ui(jit_state_t *_jit, jit_int32_t r0, jit_int32_t r1)
4a71579b 1492{
40a44dcb 1493 bswapr_ul(r0, r1);
4a71579b
PC		 1494 rshi_u(r0, r0, 32);
1495}
4a71579b
PC		 1496
1497static void
1498_ldi_c(jit_state_t *_jit, jit_int32_t r0, jit_word_t i0)
1499{
1500 jit_int32_t reg;
1501 reg = jit_get_reg(jit_class_gpr);
1502 movi(rn(reg), i0);
1503 ldr_c(r0, rn(reg));
1504 jit_unget_reg(reg);
1505}
1506
1507static void
1508_ldr_uc(jit_state_t *_jit, jit_int32_t r0, jit_int32_t r1)
1509{
1510 LDRBI(r0, r1, 0);
1511#if 0
1512 extr_uc(r0, r0);
1513#endif
1514}
1515
1516static void
1517_ldi_uc(jit_state_t *_jit, jit_int32_t r0, jit_word_t i0)
1518{
1519 jit_int32_t reg;
1520 reg = jit_get_reg(jit_class_gpr);
1521 movi(rn(reg), i0);
1522 ldr_uc(r0, rn(reg));
1523 jit_unget_reg(reg);
1524}
1525
1526static void
1527_ldi_s(jit_state_t *_jit, jit_int32_t r0, jit_word_t i0)
1528{
1529 jit_int32_t reg;
1530 reg = jit_get_reg(jit_class_gpr);
1531 movi(rn(reg), i0);
1532 ldr_s(r0, rn(reg));
1533 jit_unget_reg(reg);
1534}
1535
1536static void
1537_ldr_us(jit_state_t *_jit, jit_int32_t r0, jit_int32_t r1)
1538{
1539 LDRHI(r0, r1, 0);
1540#if 0
1541 extr_us(r0, r0);
1542#endif
1543}
1544
1545static void
1546_ldi_us(jit_state_t *_jit, jit_int32_t r0, jit_word_t i0)
1547{
1548 jit_int32_t reg;
1549 reg = jit_get_reg(jit_class_gpr);
1550 movi(rn(reg), i0);
1551 ldr_us(r0, rn(reg));
1552 jit_unget_reg(reg);
1553}
1554
1555static void
1556_ldi_i(jit_state_t *_jit, jit_int32_t r0, jit_word_t i0)
1557{
1558 jit_int32_t reg;
1559 reg = jit_get_reg(jit_class_gpr);
1560 movi(rn(reg), i0);
1561 ldr_i(r0, rn(reg));
1562 jit_unget_reg(reg);
1563}
1564
1565static void
1566_ldr_ui(jit_state_t *_jit, jit_int32_t r0, jit_int32_t r1)
1567{
1568 LDRWI(r0, r1, 0);
1569#if 0
1570 extr_ui(r0, r0);
1571#endif
1572}
1573
1574static void
1575_ldi_ui(jit_state_t *_jit, jit_int32_t r0, jit_word_t i0)
1576{
1577 jit_int32_t reg;
1578 reg = jit_get_reg(jit_class_gpr);
1579 movi(rn(reg), i0);
1580 ldr_ui(r0, rn(reg));
1581 jit_unget_reg(reg);
1582}
1583
1584static void
1585_ldi_l(jit_state_t *_jit, jit_int32_t r0, jit_word_t i0)
1586{
1587 jit_int32_t reg;
1588 reg = jit_get_reg(jit_class_gpr);
1589 movi(rn(reg), i0);
1590 ldr_l(r0, rn(reg));
1591 jit_unget_reg(reg);
1592}
1593
1594static void
1595_ldxr_c(jit_state_t *_jit, jit_int32_t r0, jit_int32_t r1, jit_int32_t r2)
1596{
1597 LDRSB(r0, r1, r2);
1598 extr_c(r0, r0);
1599}
1600
1601static void
1602_ldxi_c(jit_state_t *_jit, jit_int32_t r0, jit_int32_t r1, jit_word_t i0)
1603{
1604 jit_int32_t reg;
1605 if (i0 >= 0 && i0 <= 4095)
1606 LDRSBI(r0, r1, i0);
1607 else if (i0 > -256 && i0 < 0)
1608 LDURSB(r0, r1, i0 & 0x1ff);
1609 else {
1610 reg = jit_get_reg(jit_class_gpr);
1611 movi(rn(reg), i0);
1612 LDRSB(r0, r1, rn(reg));
1613 jit_unget_reg(reg);
1614 }
1615 extr_c(r0, r0);
1616}
1617
1618static void
1619_ldxr_uc(jit_state_t *_jit, jit_int32_t r0, jit_int32_t r1, jit_int32_t r2)
1620{
1621 LDRB(r0, r1, r2);
1622#if 0
1623 extr_uc(r0, r0);
1624#endif
1625}
1626
1627static void
1628_ldxi_uc(jit_state_t *_jit, jit_int32_t r0, jit_int32_t r1, jit_word_t i0)
1629{
1630 jit_int32_t reg;
1631 if (i0 >= 0 && i0 <= 4095)
1632 LDRBI(r0, r1, i0);
1633 else if (i0 > -256 && i0 < 0)
1634 LDURB(r0, r1, i0 & 0x1ff);
1635 else {
1636 reg = jit_get_reg(jit_class_gpr);
1637 addi(rn(reg), r1, i0);
1638 ldr_uc(r0, rn(reg));
1639 jit_unget_reg(reg);
1640 }
1641#if 0
1642 extr_uc(r0, r0);
1643#endif
1644}
1645
1646static void
1647_ldxi_s(jit_state_t *_jit, jit_int32_t r0, jit_int32_t r1, jit_word_t i0)
1648{
1649 jit_int32_t reg;
519a9ea1 1650 if (i0 >= 0 && i0 <= 8191 && !(i0 & 1))
4a71579b
PC		 1651 LDRSHI(r0, r1, i0 >> 1);
1652 else if (i0 > -256 && i0 < 0)
1653 LDURSH(r0, r1, i0 & 0x1ff);
1654 else {
1655 reg = jit_get_reg(jit_class_gpr);
1656 movi(rn(reg), i0);
1657 LDRSH(r0, r1, rn(reg));
1658 jit_unget_reg(reg);
1659 }
1660}
1661
1662static void
1663_ldxr_us(jit_state_t *_jit, jit_int32_t r0, jit_int32_t r1, jit_int32_t r2)
1664{
1665 LDRH(r0, r1, r2);
1666#if 0
1667 extr_us(r0, r0);
1668#endif
1669}
1670
1671static void
1672_ldxi_us(jit_state_t *_jit, jit_int32_t r0, jit_int32_t r1, jit_word_t i0)
1673{
1674 jit_int32_t reg;
519a9ea1 1675 if (i0 >= 0 && i0 <= 8191 && !(i0 & 1))
4a71579b
PC		 1676 LDRHI(r0, r1, i0 >> 1);
1677 else if (i0 > -256 && i0 < 0)
1678 LDURH(r0, r1, i0 & 0x1ff);
1679 else {
1680 reg = jit_get_reg(jit_class_gpr);
1681 movi(rn(reg), i0);
1682 LDRH(r0, r1, rn(reg));
1683 jit_unget_reg(reg);
1684 }
1685#if 0
1686 extr_us(r0, r0);
1687#endif
1688}
1689
1690static void
1691_ldxi_i(jit_state_t *_jit, jit_int32_t r0, jit_int32_t r1, jit_word_t i0)
1692{
1693 jit_int32_t reg;
519a9ea1 1694 if (i0 >= 0 && i0 <= 16383 && !(i0 & 3))
4a71579b
PC		 1695 LDRSWI(r0, r1, i0 >> 2);
1696 else if (i0 > -256 && i0 < 0)
1697 LDURSW(r0, r1, i0 & 0x1ff);
1698 else {
1699 reg = jit_get_reg(jit_class_gpr);
1700 addi(rn(reg), r1, i0);
1701 ldr_i(r0, rn(reg));
1702 jit_unget_reg(reg);
1703 }
1704}
1705
1706static void
1707_ldxr_ui(jit_state_t *_jit, jit_int32_t r0, jit_int32_t r1, jit_int32_t r2)
1708{
1709 LDRW(r0, r1, r2);
1710#if 0
1711 extr_ui(r0, r0);
1712#endif
1713}
1714
1715static void
1716_ldxi_ui(jit_state_t *_jit, jit_int32_t r0, jit_int32_t r1, jit_word_t i0)
1717{
1718 jit_int32_t reg;
519a9ea1 1719 if (i0 >= 0 && i0 <= 16383 && !(i0 & 3))
4a71579b
PC		 1720 LDRWI(r0, r1, i0 >> 2);
1721 else if (i0 > -256 && i0 < 0)
1722 LDURW(r0, r1, i0 & 0x1ff);
1723 else {
1724 reg = jit_get_reg(jit_class_gpr);
1725 movi(rn(reg), i0);
1726 LDRW(r0, r1, rn(reg));
1727 jit_unget_reg(reg);
1728 }
1729#if 0
1730 extr_ui(r0, r0);
1731#endif
1732}
1733
1734static void
1735_ldxi_l(jit_state_t *_jit, jit_int32_t r0, jit_int32_t r1, jit_word_t i0)
1736{
1737 jit_int32_t reg;
519a9ea1 1738 if (i0 >= 0 && i0 <= 32767 && !(i0 & 7))
4a71579b
PC		 1739 LDRI(r0, r1, i0 >> 3);
1740 else if (i0 > -256 && i0 < 0)
1741 LDUR(r0, r1, i0 & 0x1ff);
1742 else {
1743 reg = jit_get_reg(jit_class_gpr);
1744 addi(rn(reg), r1, i0);
1745 ldr_l(r0, rn(reg));
1746 jit_unget_reg(reg);
1747 }
1748}
1749
1750static void
1751_sti_c(jit_state_t *_jit, jit_word_t i0, jit_int32_t r0)
1752{
1753 jit_int32_t reg;
1754 reg = jit_get_reg(jit_class_gpr);
1755 movi(rn(reg), i0);
1756 str_c(rn(reg), r0);
1757 jit_unget_reg(reg);
1758}
1759
1760static void
1761_sti_s(jit_state_t *_jit, jit_word_t i0, jit_int32_t r0)
1762{
1763 jit_int32_t reg;
1764 reg = jit_get_reg(jit_class_gpr);
1765 movi(rn(reg), i0);
1766 str_s(rn(reg), r0);
1767 jit_unget_reg(reg);
1768}
1769
1770static void
1771_sti_i(jit_state_t *_jit, jit_word_t i0, jit_int32_t r0)
1772{
1773 jit_int32_t reg;
1774 reg = jit_get_reg(jit_class_gpr);
1775 movi(rn(reg), i0);
1776 str_i(rn(reg), r0);
1777 jit_unget_reg(reg);
1778}
1779
1780static void
1781_sti_l(jit_state_t *_jit, jit_word_t i0, jit_int32_t r0)
1782{
1783 jit_int32_t reg;
1784 reg = jit_get_reg(jit_class_gpr);
1785 movi(rn(reg), i0);
1786 str_l(rn(reg), r0);
1787 jit_unget_reg(reg);
1788}
1789
1790static void
1791_stxi_c(jit_state_t *_jit, jit_word_t i0, jit_int32_t r0, jit_int32_t r1)
1792{
1793 jit_int32_t reg;
1794 if (i0 >= 0 && i0 <= 4095)
1795 STRBI(r1, r0, i0);
1796 else if (i0 > -256 && i0 < 0)
1797 STURB(r1, r0, i0 & 0x1ff);
1798 else {
1799 reg = jit_get_reg(jit_class_gpr);
1800 addi(rn(reg), r0, i0);
1801 str_c(rn(reg), r1);
1802 jit_unget_reg(reg);
1803 }
1804}
1805
1806static void
1807_stxi_s(jit_state_t *_jit, jit_word_t i0, jit_int32_t r0, jit_int32_t r1)
1808{
1809 jit_int32_t reg;
519a9ea1 1810 if (i0 >= 0 && i0 <= 8191 && !(i0 & 1))
4a71579b
PC		 1811 STRHI(r1, r0, i0 >> 1);
1812 else if (i0 > -256 && i0 < 0)
1813 STURH(r1, r0, i0 & 0x1ff);
1814 else {
1815 reg = jit_get_reg(jit_class_gpr);
1816 addi(rn(reg), r0, i0);
1817 str_s(rn(reg), r1);
1818 jit_unget_reg(reg);
1819 }
1820}
1821
1822static void
1823_stxi_i(jit_state_t *_jit, jit_word_t i0, jit_int32_t r0, jit_int32_t r1)
1824{
1825 jit_int32_t reg;
519a9ea1 1826 if (i0 >= 0 && i0 <= 16383 && !(i0 & 3))
4a71579b
PC		 1827 STRWI(r1, r0, i0 >> 2);
1828 else if (i0 > -256 && i0 < 0)
1829 STURW(r1, r0, i0 & 0x1ff);
1830 else {
1831 reg = jit_get_reg(jit_class_gpr);
1832 addi(rn(reg), r0, i0);
1833 str_i(rn(reg), r1);
1834 jit_unget_reg(reg);
1835 }
1836}
1837
1838static void
1839_stxi_l(jit_state_t *_jit, jit_word_t i0, jit_int32_t r0, jit_int32_t r1)
1840{
1841 jit_int32_t reg;
519a9ea1 1842 if (i0 >= 0 && i0 <= 32767 && !(i0 & 7))
4a71579b
PC		 1843 STRI(r1, r0, i0 >> 3);
1844 else if (i0 > -256 && i0 < 0)
1845 STUR(r1, r0, i0 & 0x1ff);
1846 else {
1847 reg = jit_get_reg(jit_class_gpr);
1848 addi(rn(reg), r0, i0);
1849 str_l(rn(reg), r1);
1850 jit_unget_reg(reg);
1851 }
1852}
1853
ba3814c1
PC		 1854static void
1855_casx(jit_state_t *_jit, jit_int32_t r0, jit_int32_t r1,
1856 jit_int32_t r2, jit_int32_t r3, jit_word_t i0)
1857{
1858 jit_int32_t r1_reg, iscasi;
1859 jit_word_t retry, done, jump0, jump1;
1860 if ((iscasi = (r1 == _NOREG))) {
1861 r1_reg = jit_get_reg(jit_class_gpr);
1862 r1 = rn(r1_reg);
1863 movi(r1, i0);
1864 }
1865 /* retry: */
1866 retry = _jit->pc.w;
1867 LDAXR(r0, r1);
c0c16242 1868 eqr(r0, r0, r2);
79bfeef6 1869 jump0 = beqi(_jit->pc.w, r0, 0); /* beqi done r0 0 */
c0c16242 1870 STLXR(r3, r0, r1);
ba3814c1
PC		 1871 jump1 = bnei(_jit->pc.w, r0, 0); /* bnei retry r0 0 */
1872 /* done: */
1873 CSET(r0, CC_EQ);
1874 done = _jit->pc.w;
1875 patch_at(jump0, done);
1876 patch_at(jump1, retry);
1877 if (iscasi)
1878 jit_unget_reg(r1_reg);
1879}
1880
4a71579b
PC		 1881static void
1882_movr(jit_state_t *_jit, jit_int32_t r0, jit_int32_t r1)
1883{
1884 if (r0 != r1)
1885 MOV(r0, r1);
1886}
1887
1888static void
1889_movi(jit_state_t *_jit, jit_int32_t r0, jit_word_t i0)
1890{
1891 jit_word_t n0, ibit, nbit;
1892 n0 = ~i0;
1893 ibit = nbit = 0;
1894 if (i0 & 0x000000000000ffffL) ibit |= 1;
1895 if (i0 & 0x00000000ffff0000L) ibit |= 2;
1896 if (i0 & 0x0000ffff00000000L) ibit |= 4;
1897 if (i0 & 0xffff000000000000L) ibit |= 8;
1898 if (n0 & 0x000000000000ffffL) nbit |= 1;
1899 if (n0 & 0x00000000ffff0000L) nbit |= 2;
1900 if (n0 & 0x0000ffff00000000L) nbit |= 4;
1901 if (n0 & 0xffff000000000000L) nbit |= 8;
1902 switch (ibit) {
1903 case 0:
1904 MOVZ (r0, 0);
1905 break;
1906 case 1:
1907 MOVZ (r0, i0 & 0xffff);
1908 break;
1909 case 2:
1910 MOVZ_16(r0, (i0 >> 16) & 0xffff);
1911 break;
1912 case 3:
1913 MOVZ (r0, i0 & 0xffff);
1914 MOVK_16(r0, (i0 >> 16) & 0xffff);
1915 break;
1916 case 4:
1917 MOVZ_32(r0, (i0 >> 32) & 0xffff);
1918 break;
1919 case 5:
1920 MOVZ (r0, i0 & 0xffff);
1921 MOVK_32(r0, (i0 >> 32) & 0xffff);
1922 break;
1923 case 6:
1924 MOVZ_16(r0, (i0 >> 16) & 0xffff);
1925 MOVK_32(r0, (i0 >> 32) & 0xffff);
1926 break;
1927 case 7:
1928 if (nbit == 8)
1929 MOVN_48(r0, (n0 >> 48) & 0xffff);
1930 else {
1931 MOVZ (r0, i0 & 0xffff);
1932 MOVK_16(r0, (i0 >> 16) & 0xffff);
1933 MOVK_32(r0, (i0 >> 32) & 0xffff);
1934 }
1935 break;
1936 case 8:
1937 MOVZ_48(r0, (i0 >> 48) & 0xffff);
1938 break;
1939 case 9:
1940 MOVZ (r0, i0 & 0xffff);
1941 MOVK_48(r0, (i0 >> 48) & 0xffff);
1942 break;
1943 case 10:
1944 MOVZ_16(r0, (i0 >> 16) & 0xffff);
1945 MOVK_48(r0, (i0 >> 48) & 0xffff);
1946 break;
1947 case 11:
1948 if (nbit == 4)
1949 MOVN_32(r0, (n0 >> 32) & 0xffff);
1950 else {
1951 MOVZ (r0, i0 & 0xffff);
1952 MOVK_16(r0, (i0 >> 16) & 0xffff);
1953 MOVK_48(r0, (i0 >> 48) & 0xffff);
1954 }
1955 break;
1956 case 12:
1957 MOVZ_32(r0, (i0 >> 32) & 0xffff);
1958 MOVK_48(r0, (i0 >> 48) & 0xffff);
1959 break;
1960 case 13:
1961 if (nbit == 2)
1962 MOVN_16(r0, (n0 >> 16) & 0xffff);
1963 else {
1964 MOVZ (r0, i0 & 0xffff);
1965 MOVK_32(r0, (i0 >> 32) & 0xffff);
1966 MOVK_48(r0, (i0 >> 48) & 0xffff);
1967 }
1968 break;
1969 case 14:
1970 if (nbit == 1)
1971 MOVN (r0, (n0) & 0xffff);
1972 else {
1973 MOVZ_16(r0, (i0 >> 16) & 0xffff);
1974 MOVK_32(r0, (i0 >> 32) & 0xffff);
1975 MOVK_48(r0, (i0 >> 48) & 0xffff);
1976 }
1977 break;
1978 case 15:
1979 if (nbit == 0)
1980 MOVN (r0, 0);
1981 else if (nbit == 1)
1982 MOVN (r0, n0 & 0xffff);
1983 else if (nbit == 8)
1984 MOVN_48(r0, (n0 >> 48) & 0xffff);
1985 else {
1986 MOVZ (r0, i0 & 0xffff);
1987 MOVK_16(r0, (i0 >> 16) & 0xffff);
1988 MOVK_32(r0, (i0 >> 32) & 0xffff);
1989 MOVK_48(r0, (i0 >> 48) & 0xffff);
1990 }
1991 break;
1992 default:
1993 abort();
1994 }
1995}
1996
1997static jit_word_t
1998_movi_p(jit_state_t *_jit, jit_int32_t r0, jit_word_t i0)
1999{
2000 jit_word_t w;
2001 w = _jit->pc.w;
2002 MOVZ (r0, i0 & 0xffff);
2003 MOVK_16(r0, (i0 >> 16) & 0xffff);
2004 MOVK_32(r0, (i0 >> 32) & 0xffff);
2005 MOVK_48(r0, (i0 >> 48) & 0xffff);
2006 return (w);
2007}
2008
2009static void
2010_ccr(jit_state_t *_jit, jit_int32_t cc,
2011 jit_int32_t r0, jit_int32_t r1, jit_int32_t r2)
2012{
2013 CMP(r1, r2);
2014 CSET(r0, cc);
2015}
2016
2017static void
2018_cci(jit_state_t *_jit, jit_int32_t cc,
2019 jit_int32_t r0, jit_int32_t r1, jit_word_t i0)
2020{
2021 jit_int32_t reg;
2022 jit_word_t is = i0 >> 12;
2023 jit_word_t in = -i0;
2024 jit_word_t iS = in >> 12;
2025 if ( i0 >= 0 && i0 <= 0xfff)
2026 CMPI (r1, i0);
2027 else if ((is << 12) == i0 && is >= 0 && is <= 0xfff)
2028 CMPI_12(r1, is);
2029 else if ( in >= 0 && in <= 0xfff)
2030 CMNI (r1, in);
2031 else if ((iS << 12) == is && iS >= 0 && iS <= 0xfff)
2032 CMNI_12(r1, iS);
2033 else {
2034 reg = jit_get_reg(jit_class_gpr);
2035 movi(rn(reg), i0);
2036 CMP(r1, rn(reg));
2037 jit_unget_reg(reg);
2038 }
2039 CSET(r0, cc);
2040}
2041
2042static jit_word_t
2043_bccr(jit_state_t *_jit, jit_int32_t cc,
2044 jit_word_t i0, jit_int32_t r0, jit_int32_t r1)
2045{
2046 jit_word_t w, d;
2047 CMP(r0, r1);
2048 w = _jit->pc.w;
2049 d = (i0 - w) >> 2;
2050 B_C(cc, d);
2051 return (w);
2052}
2053
2054static jit_word_t
2055_bcci(jit_state_t *_jit, jit_int32_t cc,
2056 jit_word_t i0, jit_int32_t r0, jit_word_t i1)
2057{
2058 jit_int32_t reg;
2059 jit_word_t w, d;
2060 jit_word_t is = i1 >> 12;
2061 jit_word_t in = -i1;
2062 jit_word_t iS = in >> 12;
2063 if ( i1 >= 0 && i1 <= 0xfff)
2064 CMPI (r0, i1);
2065 else if ((is << 12) == i1 && is >= 0 && is <= 0xfff)
2066 CMPI_12(r0, is);
2067 else if ( in >= 0 && in <= 0xfff)
2068 CMNI (r0, in);
2069 else if ((iS << 12) == is && iS >= 0 && iS <= 0xfff)
2070 CMNI_12(r0, iS);
2071 else {
2072 reg = jit_get_reg(jit_class_gpr);
2073 movi(rn(reg), i1);
2074 CMP(r0, rn(reg));
2075 jit_unget_reg(reg);
2076 }
2077 w = _jit->pc.w;
2078 d = (i0 - w) >> 2;
2079 B_C(cc, d);
2080 return (w);
2081}
2082
2083static jit_word_t
2084_beqi(jit_state_t *_jit, jit_word_t i0, jit_int32_t r0, jit_word_t i1)
2085{
2086 jit_word_t w;
2087 if (i1 == 0) {
2088 w = _jit->pc.w;
2089 CBZ(r0, (i0 - w) >> 2);
2090 }
2091 else
2092 w = bcci(BCC_EQ, i0, r0, i1);
2093 return (w);
2094}
2095
2096static jit_word_t
2097_bnei(jit_state_t *_jit, jit_word_t i0, jit_int32_t r0, jit_word_t i1)
2098{
2099 jit_word_t w;
2100 if (i1 == 0) {
2101 w = _jit->pc.w;
2102 CBNZ(r0, (i0 - w) >> 2);
2103 }
2104 else
2105 w = bcci(BCC_NE, i0, r0, i1);
2106 return (w);
2107}
2108
2109static jit_word_t
2110_baddr(jit_state_t *_jit, jit_int32_t cc,
2111 jit_word_t i0, jit_int32_t r0, jit_int32_t r1)
2112{
2113 jit_word_t w;
2114 addcr(r0, r0, r1);
2115 w = _jit->pc.w;
2116 B_C(cc, (i0 - w) >> 2);
2117 return (w);
2118}
2119
2120static jit_word_t
2121_baddi(jit_state_t *_jit, jit_int32_t cc,
2122 jit_word_t i0, jit_int32_t r0, jit_word_t i1)
2123{
2124 jit_word_t w;
2125 addci(r0, r0, i1);
2126 w = _jit->pc.w;
2127 B_C(cc, (i0 - w) >> 2);
2128 return (w);
2129}
2130
2131static jit_word_t
2132_bsubr(jit_state_t *_jit, jit_int32_t cc,
2133 jit_word_t i0, jit_int32_t r0, jit_int32_t r1)
2134{
2135 jit_word_t w;
2136 subcr(r0, r0, r1);
2137 w = _jit->pc.w;
2138 B_C(cc, (i0 - w) >> 2);
2139 return (w);
2140}
2141
2142static jit_word_t
2143_bsubi(jit_state_t *_jit, jit_int32_t cc,
2144 jit_word_t i0, jit_int32_t r0, jit_word_t i1)
2145{
2146 jit_word_t w;
2147 subci(r0, r0, i1);
2148 w = _jit->pc.w;
2149 B_C(cc, (i0 - w) >> 2);
2150 return (w);
2151}
2152
2153static jit_word_t
2154_bmxr(jit_state_t *_jit, jit_int32_t cc,
2155 jit_word_t i0, jit_int32_t r0, jit_int32_t r1)
2156{
2157 jit_word_t w;
2158 TST(r0, r1);
2159 w = _jit->pc.w;
2160 B_C(cc, (i0 - w) >> 2);
2161 return (w);
2162}
2163
2164static jit_word_t
2165_bmxi(jit_state_t *_jit, jit_int32_t cc,
2166 jit_word_t i0, jit_int32_t r0, jit_word_t i1)
2167{
2168 jit_word_t w;
2169 jit_int32_t reg;
2170 jit_int32_t imm;
2171 imm = logical_immediate(i1);
2172 if (imm != -1)
2173 TSTI(r0, imm);
2174 else {
2175 reg = jit_get_reg(jit_class_gpr);
2176 movi(rn(reg), i1);
2177 TST(r0, rn(reg));
2178 jit_unget_reg(reg);
2179 }
2180 w = _jit->pc.w;
2181 B_C(cc, (i0 - w) >> 2);
2182 return (w);
2183}
2184
79bfeef6 2185static jit_word_t
4a71579b
PC		 2186_jmpi(jit_state_t *_jit, jit_word_t i0)
2187{
4a71579b 2188 jit_int32_t reg;
79bfeef6
PC		 2189 jit_word_t d, w;
2190 w = _jit->pc.w;
2191 d = (i0 - w) >> 2;
2192 if (s26_p(d))
2193 B(d);
4a71579b
PC		 2194 else {
2195 reg = jit_get_reg(jit_class_gpr|jit_class_nospill);
2196 movi(rn(reg), i0);
2197 jmpr(rn(reg));
2198 jit_unget_reg(reg);
2199 }
79bfeef6 2200 return (w);
4a71579b
PC		 2201}
2202
2203static jit_word_t
2204_jmpi_p(jit_state_t *_jit, jit_word_t i0)
2205{
2206 jit_word_t w;
2207 jit_int32_t reg;
2208 reg = jit_get_reg(jit_class_gpr|jit_class_nospill);
2209 w = movi_p(rn(reg), i0);
2210 jmpr(rn(reg));
2211 jit_unget_reg(reg);
2212 return (w);
2213}
2214
79bfeef6 2215static jit_word_t
4a71579b
PC		 2216_calli(jit_state_t *_jit, jit_word_t i0)
2217{
4a71579b 2218 jit_int32_t reg;
79bfeef6
PC		 2219 jit_word_t d, w;
2220 w = _jit->pc.w;
2221 d = (i0 - w) >> 2;
2222 if (s26_p(d))
2223 BL(d);
4a71579b
PC		 2224 else {
2225 reg = jit_get_reg(jit_class_gpr);
2226 movi(rn(reg), i0);
2227 callr(rn(reg));
2228 jit_unget_reg(reg);
2229 }
79bfeef6 2230 return (w);
4a71579b
PC		 2231}
2232
2233static jit_word_t
2234_calli_p(jit_state_t *_jit, jit_word_t i0)
2235{
2236 jit_word_t w;
2237 jit_int32_t reg;
2238 reg = jit_get_reg(jit_class_gpr);
2239 w = movi_p(rn(reg), i0);
2240 callr(rn(reg));
2241 jit_unget_reg(reg);
2242 return (w);
2243}
2244
4a71579b
PC		 2245static void
2246_prolog(jit_state_t *_jit, jit_node_t *node)
2247{
79bfeef6 2248 jit_int32_t reg, rreg, offs;
4a71579b
PC		 2249 if (_jitc->function->define_frame || _jitc->function->assume_frame) {
2250 jit_int32_t frame = -_jitc->function->frame;
79bfeef6 2251 jit_check_frame();
4a71579b
PC		 2252 assert(_jitc->function->self.aoff >= frame);
2253 if (_jitc->function->assume_frame)
2254 return;
2255 _jitc->function->self.aoff = frame;
2256 }
2257 if (_jitc->function->allocar)
2258 _jitc->function->self.aoff &= -16;
2259 _jitc->function->stack = ((_jitc->function->self.alen -
2260 /* align stack at 16 bytes */
2261 _jitc->function->self.aoff) + 15) & -16;
79bfeef6
PC		 2262
2263 if (!_jitc->function->need_frame) {
2264 /* check if any callee save register needs to be saved */
2265 for (reg = 0; reg < _jitc->reglen; ++reg)
2266 if (jit_regset_tstbit(&_jitc->function->regset, reg) &&
2267 (_rvs[reg].spec & jit_class_sav)) {
2268 jit_check_frame();
2269 break;
2270 }
2271 }
2272
2273 if (_jitc->function->need_frame) {
2274 STPI_POS(FP_REGNO, LR_REGNO, SP_REGNO, -(jit_framesize() >> 3));
2275 MOV_XSP(FP_REGNO, SP_REGNO);
2276 }
2277 /* callee save registers */
2278 for (reg = 0, offs = 2; reg < jit_size(iregs);) {
2279 if (jit_regset_tstbit(&_jitc->function->regset, iregs[reg])) {
2280 for (rreg = reg + 1; rreg < jit_size(iregs); rreg++) {
2281 if (jit_regset_tstbit(&_jitc->function->regset, iregs[rreg]))
2282 break;
2283 }
2284 if (rreg < jit_size(iregs)) {
2285 STPI(rn(iregs[reg]), rn(iregs[rreg]), SP_REGNO, offs);
2286 offs += 2;
2287 reg = rreg + 1;
2288 }
2289 else {
2290 STRI(rn(iregs[reg]), SP_REGNO, offs);
2291 ++offs;
2292 /* No pair found */
2293 break;
2294 }
2295 }
2296 else
2297 ++reg;
2298 }
2299 for (reg = 0, offs <<= 3; reg < jit_size(fregs); reg++) {
2300 if (jit_regset_tstbit(&_jitc->function->regset, fregs[reg])) {
2301 stxi_d(offs, SP_REGNO, rn(fregs[reg]));
2302 offs += sizeof(jit_float64_t);
2303 }
2304 }
2305
2306 if (_jitc->function->stack)
4a71579b
PC		 2307 subi(SP_REGNO, SP_REGNO, _jitc->function->stack);
2308 if (_jitc->function->allocar) {
2309 reg = jit_get_reg(jit_class_gpr);
2310 movi(rn(reg), _jitc->function->self.aoff);
2311 stxi_i(_jitc->function->aoffoff, FP_REGNO, rn(reg));
2312 jit_unget_reg(reg);
2313 }
2314
79bfeef6 2315#if !__APPLE__
4a71579b
PC		 2316 if (_jitc->function->self.call & jit_call_varargs) {
2317 /* Save gp registers in the save area, if any is a vararg */
2318 for (reg = 8 - _jitc->function->vagp / -8;
2319 jit_arg_reg_p(reg); ++reg)
2320 stxi(_jitc->function->vaoff + offsetof(jit_va_list_t, x0) +
2321 reg * 8, FP_REGNO, rn(JIT_RA0 - reg));
2322
2323 for (reg = 8 - _jitc->function->vafp / -16;
2324 jit_arg_f_reg_p(reg); ++reg)
2325 /* Save fp registers in the save area, if any is a vararg */
2326 /* Note that the full 16 byte register is not saved, because
2327 * lightning only handles float and double, and, while
2328 * attempting to provide a va_list compatible pointer as
2329 * jit_va_start return, does not guarantee it (on all ports). */
2330 stxi_d(_jitc->function->vaoff + offsetof(jit_va_list_t, q0) +
2331 reg * 16 + offsetof(jit_qreg_t, l), FP_REGNO, rn(_V0 - reg));
2332 }
79bfeef6 2333#endif
4a71579b
PC		 2334}
2335
2336static void
2337_epilog(jit_state_t *_jit, jit_node_t *node)
2338{
79bfeef6 2339 jit_int32_t reg, rreg, offs;
4a71579b
PC		 2340 if (_jitc->function->assume_frame)
2341 return;
2342 if (_jitc->function->stack)
2343 MOV_XSP(SP_REGNO, FP_REGNO);
79bfeef6
PC		 2344 /* callee save registers */
2345 for (reg = 0, offs = 2; reg < jit_size(iregs);) {
2346 if (jit_regset_tstbit(&_jitc->function->regset, iregs[reg])) {
2347 for (rreg = reg + 1; rreg < jit_size(iregs); rreg++) {
2348 if (jit_regset_tstbit(&_jitc->function->regset, iregs[rreg]))
2349 break;
2350 }
2351 if (rreg < jit_size(iregs)) {
2352 LDPI(rn(iregs[reg]), rn(iregs[rreg]), SP_REGNO, offs);
2353 offs += 2;
2354 reg = rreg + 1;
2355 }
2356 else {
2357 LDRI(rn(iregs[reg]), SP_REGNO, offs);
2358 ++offs;
2359 /* No pair found */
2360 break;
2361 }
2362 }
2363 else
2364 ++reg;
2365 }
2366 for (reg = 0, offs <<= 3; reg < jit_size(fregs); reg++) {
2367 if (jit_regset_tstbit(&_jitc->function->regset, fregs[reg])) {
2368 ldxi_d(rn(fregs[reg]), SP_REGNO, offs);
2369 offs += sizeof(jit_float64_t);
2370 }
2371 }
2372
2373 if (_jitc->function->need_frame)
2374 LDPI_PRE(FP_REGNO, LR_REGNO, SP_REGNO, jit_framesize() >> 3);
4a71579b
PC		 2375 RET();
2376}
2377
2378static void
2379_vastart(jit_state_t *_jit, jit_int32_t r0)
2380{
79bfeef6 2381#if !__APPLE__
4a71579b
PC		 2382 jit_int32_t reg;
2383
2384 assert(_jitc->function->self.call & jit_call_varargs);
2385
2386 /* Return jit_va_list_t in the register argument */
2387 addi(r0, FP_REGNO, _jitc->function->vaoff);
2388
2389 reg = jit_get_reg(jit_class_gpr);
2390
2391 /* Initialize stack pointer to the first stack argument. */
79bfeef6 2392 addi(rn(reg), FP_REGNO, jit_selfsize());
4a71579b
PC		 2393 stxi(offsetof(jit_va_list_t, stack), r0, rn(reg));
2394
2395 /* Initialize gp top pointer to the first stack argument. */
2396 addi(rn(reg), r0, va_gp_top_offset);
2397 stxi(offsetof(jit_va_list_t, gptop), r0, rn(reg));
2398
2399 /* Initialize fp top pointer to the first stack argument. */
2400 addi(rn(reg), r0, va_fp_top_offset);
2401 stxi(offsetof(jit_va_list_t, fptop), r0, rn(reg));
2402
2403 /* Initialize gp offset in the save area. */
2404 movi(rn(reg), _jitc->function->vagp);
2405 stxi_i(offsetof(jit_va_list_t, gpoff), r0, rn(reg));
2406
2407 /* Initialize fp offset in the save area. */
2408 movi(rn(reg), _jitc->function->vafp);
2409 stxi_i(offsetof(jit_va_list_t, fpoff), r0, rn(reg));
2410
2411 jit_unget_reg(reg);
79bfeef6
PC		 2412#else
2413 assert(_jitc->function->self.call & jit_call_varargs);
2414 addi(r0, FP_REGNO, jit_selfsize());
2415#endif
4a71579b
PC		 2416}
2417
2418static void
2419_vaarg(jit_state_t *_jit, jit_int32_t r0, jit_int32_t r1)
2420{
79bfeef6 2421#if !__APPLE__
4a71579b
PC		 2422 jit_word_t ge_code;
2423 jit_word_t lt_code;
2424 jit_int32_t rg0, rg1;
2425
2426 assert(_jitc->function->self.call & jit_call_varargs);
2427
2428 rg0 = jit_get_reg(jit_class_gpr);
2429 rg1 = jit_get_reg(jit_class_gpr);
2430
2431 /* Load the gp offset in save area in the first temporary. */
2432 ldxi_i(rn(rg0), r1, offsetof(jit_va_list_t, gpoff));
2433
2434 /* Jump over if there are no remaining arguments in the save area. */
2435 ge_code = bgei(_jit->pc.w, rn(rg0), 0);
2436
2437 /* Load the gp save pointer in the second temporary. */
2438 ldxi(rn(rg1), r1, offsetof(jit_va_list_t, gptop));
2439
2440 /* Load the vararg argument in the first argument. */
2441 ldxr(r0, rn(rg1), rn(rg0));
2442
2443 /* Update the gp offset. */
2444 addi(rn(rg0), rn(rg0), 8);
2445 stxi_i(offsetof(jit_va_list_t, gpoff), r1, rn(rg0));
2446
2447 /* Will only need one temporary register below. */
2448 jit_unget_reg(rg1);
2449
2450 /* Jump over overflow code. */
79bfeef6 2451 lt_code = jmpi(_jit->pc.w);
4a71579b
PC		 2452
2453 /* Where to land if argument is in overflow area. */
2454 patch_at(ge_code, _jit->pc.w);
2455
2456 /* Load stack pointer. */
2457 ldxi(rn(rg0), r1, offsetof(jit_va_list_t, stack));
2458
2459 /* Load argument. */
2460 ldr(r0, rn(rg0));
2461
2462 /* Update stack pointer. */
2463 addi(rn(rg0), rn(rg0), 8);
2464 stxi(offsetof(jit_va_list_t, stack), r1, rn(rg0));
2465
2466 /* Where to land if argument is in gp save area. */
2467 patch_at(lt_code, _jit->pc.w);
2468
2469 jit_unget_reg(rg0);
79bfeef6
PC		 2470#else
2471 assert(_jitc->function->self.call & jit_call_varargs);
2472 ldr(r0, r1);
2473 addi(r1, r1, sizeof(jit_word_t));
2474#endif
4a71579b
PC		 2475}
2476
2477static void
2478_patch_at(jit_state_t *_jit, jit_word_t instr, jit_word_t label)
2479{
2480 instr_t i;
2481 jit_word_t d;
2482 jit_int32_t fc, ff, ffc;
2483 union {
2484 jit_int32_t *i;
2485 jit_word_t w;
2486 } u;
2487 u.w = instr;
2488 i.w = u.i[0];
2489 fc = i.w & 0xfc000000;
2490 ff = i.w & 0xff000000;
2491 ffc = i.w & 0xffc00000;
2492 if (fc == A64_B || fc == A64_BL) {
2493 d = (label - instr) >> 2;
79bfeef6 2494 assert(s26_p(d));
4a71579b
PC		 2495 i.imm26.b = d;
2496 u.i[0] = i.w;
2497 }
2498 else if (ff == A64_B_C || ff == (A64_CBZ|XS) || ff == (A64_CBNZ|XS)) {
2499 d = (label - instr) >> 2;
2500 assert(d >= -262148 && d <= 262143);
2501 i.imm19.b = d;
2502 u.i[0] = i.w;
2503 }
2504 else if (ffc == (A64_MOVZ|XS)) {
2505 i.imm16.b = label;
2506 u.i[0] = i.w;
2507 i.w = u.i[1];
2508 assert((i.w & 0xffe00000) == (A64_MOVK|XS|MOVI_LSL_16));
2509 i.imm16.b = label >> 16;
2510 u.i[1] = i.w;
2511 i.w = u.i[2];
2512 assert((i.w & 0xffe00000) == (A64_MOVK|XS|MOVI_LSL_32));
2513 i.imm16.b = label >> 32;
2514 u.i[2] = i.w;
2515 i.w = u.i[3];
2516 assert((i.w & 0xffe00000) == (A64_MOVK|XS|MOVI_LSL_48));
2517 i.imm16.b = label >> 48;
2518 u.i[3] = i.w;
2519 }
2520 else
2521 abort();
2522}
2523#endif
ARM optimized PCSX fork