2 * Copyright (C) 2013-2022 Free Software Foundation, Inc.
4 * This file is part of GNU lightning.
6 * GNU lightning is free software; you can redistribute it and/or modify it
7 * under the terms of the GNU Lesser General Public License as published
8 * by the Free Software Foundation; either version 3, or (at your option)
11 * GNU lightning is distributed in the hope that it will be useful, but
12 * WITHOUT ANY WARRANTY; without even the implied warranty of MERCHANTABILITY
13 * or FITNESS FOR A PARTICULAR PURPOSE. See the GNU Lesser General Public
14 * License for more details.
17 * Paulo Cesar Pereira de Andrade
23 # define ui jit_uint32_t
24 # if __BYTE_ORDER == __LITTLE_ENDIAN
25 /* cond2: condition in truly conditional-executed inst. */
26 struct { ui b: 4; } cond2;
27 /* nzcv: flag bit specifier, encoded in the "nzcv" field. */
28 struct { ui b: 4; } nzcv;
29 /* defgh: d:e:f:g:h bits in AdvSIMD modified immediate. */
30 struct { ui _: 5; ui b: 5; } defgh;
31 /* abc: a:b:c bits in AdvSIMD modified immediate. */
32 struct { ui _: 16; ui b: 3; } abc;
33 /* imm19: e.g. in CBZ. */
34 struct { ui _: 5; ui b: 19; } imm19;
35 /* immhi: e.g. in ADRP. */
36 struct { ui _: 5; ui b: 19; } immhi;
37 /* immlo: e.g. in ADRP. */
38 struct { ui _: 29; ui b: 2; } immlo;
39 /* size: in most AdvSIMD and floating-point instructions. */
40 struct { ui _: 22; ui b: 2; } size;
41 /* vldst_size: size field in the AdvSIMD load/store inst. */
42 struct { ui _: 10; ui b: 2; } vldst_size;
43 /* op: in AdvSIMD modified immediate instructions. */
44 struct { ui _: 29; ui b: 1; } op;
45 /* Q: in most AdvSIMD instructions. */
46 struct { ui _: 30; ui b: 1; } Q;
47 /* Rt: in load/store instructions. */
48 struct { ui b: 5; } Rt;
49 /* Rd: in many integer instructions. */
50 struct { ui b: 5; } Rd;
51 /* Rn: in many integer instructions. */
52 struct { ui _: 5; ui b: 5; } Rn;
53 /* Rt2: in load/store pair instructions. */
54 struct { ui _: 10; ui b: 5; } Rt2;
55 /* Ra: in fp instructions. */
56 struct { ui _: 10; ui b: 5; } Ra;
57 /* op2: in the system instructions. */
58 struct { ui _: 5; ui b: 3; } op2;
59 /* CRm: in the system instructions. */
60 struct { ui _: 8; ui b: 4; } CRm;
61 /* CRn: in the system instructions. */
62 struct { ui _: 12; ui b: 4; } CRn;
63 /* op1: in the system instructions. */
64 struct { ui _: 16; ui b: 3; } op1;
65 /* op0: in the system instructions. */
66 struct { ui _: 19; ui b: 2; } op0;
67 /* imm3: in add/sub extended reg instructions. */
68 struct { ui _: 10; ui b: 3; } imm3;
69 /* cond: condition flags as a source operand. */
70 struct { ui _: 12; ui b: 4; } cond;
71 /* opcode: in advsimd load/store instructions. */
72 struct { ui _: 12; ui b: 4; } opcode;
73 /* cmode: in advsimd modified immediate instructions. */
74 struct { ui _: 12; ui b: 4; } cmode;
75 /* asisdlso_opcode: opcode in advsimd ld/st single element. */
76 struct { ui _: 13; ui b: 3; } asisdlso_opcode;
77 /* len: in advsimd tbl/tbx instructions. */
78 struct { ui _: 13; ui b: 2; } len;
79 /* Rm: in ld/st reg offset and some integer inst. */
80 struct { ui _: 16; ui b: 5; } Rm;
81 /* Rs: in load/store exclusive instructions. */
82 struct { ui _: 16; ui b: 5; } Rs;
83 /* option: in ld/st reg offset + add/sub extended reg inst. */
84 struct { ui _: 13; ui b: 3; } option;
85 /* S: in load/store reg offset instructions. */
86 struct { ui _: 12; ui b: 1; } S;
87 /* hw: in move wide constant instructions. */
88 struct { ui _: 21; ui b: 2; } hw;
89 /* opc: in load/store reg offset instructions. */
90 struct { ui _: 22; ui b: 2; } opc;
91 /* opc1: in load/store reg offset instructions. */
92 struct { ui _: 23; ui b: 1; } opc1;
93 /* shift: in add/sub reg/imm shifted instructions. */
94 struct { ui _: 22; ui b: 2; } shift;
95 /* type: floating point type field in fp data inst. */
96 struct { ui _: 22; ui b: 2; } type;
97 /* ldst_size: size field in ld/st reg offset inst. */
98 struct { ui _: 30; ui b: 2; } ldst_size;
99 /* imm6: in add/sub reg shifted instructions. */
100 struct { ui _: 10; ui b: 6; } imm6;
101 /* imm4: in advsimd ext and advsimd ins instructions. */
102 struct { ui _: 11; ui b: 4; } imm4;
103 /* imm5: in conditional compare (immediate) instructions. */
104 struct { ui _: 16; ui b: 5; } imm5;
105 /* imm7: in load/store pair pre/post index instructions. */
106 struct { ui _: 15; ui b: 7; } imm7;
107 /* imm8: in floating-point scalar move immediate inst. */
108 struct { ui _: 13; ui b: 8; } imm8;
109 /* imm9: in load/store pre/post index instructions. */
110 struct { ui _: 12; ui b: 9; } imm9;
111 /* imm12: in ld/st unsigned imm or add/sub shifted inst. */
112 struct { ui _: 10; ui b: 12; } imm12;
113 /* imm14: in test bit and branch instructions. */
114 struct { ui _: 5; ui b: 14; } imm14;
115 /* imm16: in exception instructions. */
116 struct { ui _: 5; ui b: 16; } imm16;
117 /* imm26: in unconditional branch instructions. */
118 struct { ui b: 26; } imm26;
119 /* imms: in bitfield and logical immediate instructions. */
120 struct { ui _: 10; ui b: 6; } imms;
121 /* immr: in bitfield and logical immediate instructions. */
122 struct { ui _: 16; ui b: 6; } immr;
123 /* immb: in advsimd shift by immediate instructions. */
124 struct { ui _: 16; ui b: 3; } immb;
125 /* immh: in advsimd shift by immediate instructions. */
126 struct { ui _: 19; ui b: 4; } immh;
127 /* N: in logical (immediate) instructions. */
128 struct { ui _: 22; ui b: 1; } N;
129 /* index: in ld/st inst deciding the pre/post-index. */
130 struct { ui _: 11; ui b: 1; } index;
131 /* index2: in ld/st pair inst deciding the pre/post-index. */
132 struct { ui _: 24; ui b: 1; } index2;
133 /* sf: in integer data processing instructions. */
134 struct { ui _: 31; ui b: 1; } sf;
135 /* H: in advsimd scalar x indexed element instructions. */
136 struct { ui _: 11; ui b: 1; } H;
137 /* L: in advsimd scalar x indexed element instructions. */
138 struct { ui _: 21; ui b: 1; } L;
139 /* M: in advsimd scalar x indexed element instructions. */
140 struct { ui _: 20; ui b: 1; } M;
141 /* b5: in the test bit and branch instructions. */
142 struct { ui _: 31; ui b: 1; } b5;
143 /* b40: in the test bit and branch instructions. */
144 struct { ui _: 19; ui b: 5; } b40;
145 /* scale: in the fixed-point scalar to fp converting inst. */
146 struct { ui _: 10; ui b: 6; } scale;
148 struct { ui _: 28; ui b: 4; } cond2;
149 struct { ui _: 28; ui b: 4; } nzcv;
150 struct { ui _: 22; ui b: 5; } defgh;
151 struct { ui _: 13; ui b: 3; } abc;
152 struct { ui _: 8; ui b: 19; } imm19;
153 struct { ui _: 8; ui b: 19; } immhi;
154 struct { ui _: 1; ui b: 29; } immlo;
155 struct { ui _: 8; ui b: 2; } size;
156 struct { ui _: 20; ui b: 2; } vldst_size;
157 struct { ui _: 2; ui b: 1; } op;
158 struct { ui _: 1; ui b: 1; } Q;
159 struct { ui _: 27; ui b: 1; } Rt;
160 struct { ui _: 27; ui b: 1; } Rd;
161 struct { ui _: 22; ui b: 5; } Rn;
162 struct { ui _: 17; ui b: 5; } Rt2;
163 struct { ui _: 17; ui b: 5; } Ra;
164 struct { ui _: 24; ui b: 3; } op2;
165 struct { ui _: 20; ui b: 4; } CRm;
166 struct { ui _: 16; ui b: 4; } CRn;
167 struct { ui _: 13; ui b: 3; } op1;
168 struct { ui _: 11; ui b: 2; } op0;
169 struct { ui _: 19; ui b: 3; } imm3;
170 struct { ui _: 16; ui b: 4; } cond;
171 struct { ui _: 16; ui b: 4; } opcode;
172 struct { ui _: 16; ui b: 4; } cmode;
173 struct { ui _: 16; ui b: 3; } asisdlso_opcode;
174 struct { ui _: 17; ui b: 2; } len;
175 struct { ui _: 11; ui b: 5; } Rm;
176 struct { ui _: 11; ui b: 5; } Rs;
177 struct { ui _: 16; ui b: 3; } option;
178 struct { ui _: 19; ui b: 1; } S;
179 struct { ui _: 9; ui b: 2; } hw;
180 struct { ui _: 8; ui b: 2; } opc;
181 struct { ui _: 8; ui b: 1; } opc1;
182 struct { ui _: 8; ui b: 2; } shift;
183 struct { ui _: 8; ui b: 2; } type;
184 struct { ui b: 2; } ldst_size;
185 struct { ui _: 16; ui b: 6; } imm6;
186 struct { ui _: 17; ui b: 4; } imm4;
187 struct { ui _: 11; ui b: 5; } imm5;
188 struct { ui _: 10; ui b: 7; } imm7;
189 struct { ui _: 11; ui b: 8; } imm8;
190 struct { ui _: 11; ui b: 9; } imm9;
191 struct { ui _: 10; ui b: 12; } imm12;
192 struct { ui _: 13; ui b: 14; } imm14;
193 struct { ui _: 11; ui b: 16; } imm16;
194 struct { ui _: 6; ui b: 26; } imm26;
195 struct { ui _: 16; ui b: 6; } imms;
196 struct { ui _: 10; ui b: 6; } immr;
197 struct { ui _: 13; ui b: 3; } immb;
198 struct { ui _: 9; ui b: 4; } immh;
199 struct { ui _: 9; ui b: 1; } N;
200 struct { ui _: 20; ui b: 1; } index;
201 struct { ui _: 7; ui b: 1; } index2;
202 struct { ui b: 1; } sf;
203 struct { ui _: 20; ui b: 1; } H;
204 struct { ui _: 10; ui b: 1; } L;
205 struct { ui _: 11; ui b: 1; } M;
206 struct { ui b: 1; } b5;
207 struct { ui _: 8; ui b: 5; } b40;
208 struct { ui _: 16; ui b: 6; } scale;
213 # define stack_framesize 160
214 # define ii(i) *_jit->pc.ui++ = i
215 # define ldr(r0,r1) ldr_l(r0,r1)
216 # define ldxr(r0,r1,r2) ldxr_l(r0,r1,r2)
217 # define ldxi(r0,r1,i0) ldxi_l(r0,r1,i0)
218 # define stxi(i0,r0,r1) stxi_l(i0,r0,r1)
219 # define FP_REGNO 0x1d
220 # define LR_REGNO 0x1e
221 # define SP_REGNO 0x1f
222 # define XZR_REGNO 0x1f
223 # define WZR_REGNO XZR_REGNO
224 # define LSL_12 0x00400000
225 # define MOVI_LSL_16 0x00200000
226 # define MOVI_LSL_32 0x00400000
227 # define MOVI_LSL_48 0x00600000
228 # define XS 0x80000000 /* Wn -> Xn */
229 # define DS 0x00400000 /* Sn -> Dn */
248 /* Branches need inverted condition */
252 # define BCC_HS BCC_CS
254 # define BCC_LO BCC_CC
267 /* adapted and cut down to only tested and required by lightning,
268 * from data in binutils/aarch64-tbl.h */
269 # define A64_ADCS 0x3a000000
270 # define A64_SBCS 0x7a000000
271 # define A64_ADDI 0x11000000
272 # define A64_ADDSI 0xb1000000
273 # define A64_SUBI 0x51000000
274 # define A64_SUBSI 0x71000000
275 # define A64_ADD 0x0b000000
276 # define A64_ADDS 0x2b000000
277 # define A64_SUB 0x4b000000
278 # define A64_NEG 0x4b0003e0
279 # define A64_SUBS 0x6b000000
280 # define A64_CMP 0x6b00001f
281 # define A64_SBFM 0x93400000
282 # define A64_UBFM 0x53400000
283 # define A64_UBFX 0x53000000
284 # define A64_B 0x14000000
285 # define A64_BL 0x94000000
286 # define A64_BR 0xd61f0000
287 # define A64_BLR 0xd63f0000
288 # define A64_RET 0xd65f0000
289 # define A64_CBZ 0x34000000
290 # define A64_CBNZ 0x35000000
291 # define A64_B_C 0x54000000
292 # define A64_CSINC 0x1a800400
293 # define A64_CSSEL 0x1a800000
294 # define A64_REV 0xdac00c00
295 # define A64_UDIV 0x1ac00800
296 # define A64_SDIV 0x1ac00c00
297 # define A64_LSL 0x1ac02000
298 # define A64_LSR 0x1ac02400
299 # define A64_ASR 0x1ac02800
300 # define A64_MUL 0x1b007c00
301 # define A64_SMULL 0x9b207c00
302 # define A64_SMULH 0x9b407c00
303 # define A64_UMULL 0x9ba07c00
304 # define A64_UMULH 0x9bc07c00
305 # define A64_STRBI 0x39000000
306 # define A64_LDRBI 0x39400000
307 # define A64_LDRSBI 0x39800000
308 # define A64_STRI 0xf9000000
309 # define A64_LDRI 0xf9400000
310 # define A64_STRHI 0x79000000
311 # define A64_LDRHI 0x79400000
312 # define A64_LDRSHI 0x79800000
313 # define A64_STRWI 0xb9000000
314 # define A64_LDRWI 0xb9400000
315 # define A64_LDRSWI 0xb9800000
316 # define A64_STRB 0x38206800
317 # define A64_LDRB 0x38606800
318 # define A64_LDRSB 0x38e06800
319 # define A64_STR 0xf8206800
320 # define A64_LDR 0xf8606800
321 # define A64_LDAXR 0xc85ffc00
322 # define A64_STLXR 0xc800fc00
323 # define A64_STRH 0x78206800
324 # define A64_LDRH 0x78606800
325 # define A64_LDRSH 0x78a06800
326 # define A64_STRW 0xb8206800
327 # define A64_LDRW 0xb8606800
328 # define A64_LDRSW 0xb8a06800
329 # define A64_STURB 0x38000000
330 # define A64_LDURB 0x38400000
331 # define A64_LDURSB 0x38800000
332 # define A64_STUR 0xf8000000
333 # define A64_LDUR 0xf8400000
334 # define A64_STURH 0x78000000
335 # define A64_LDURH 0x78400000
336 # define A64_LDURSH 0x78800000
337 # define A64_STURW 0xb8000000
338 # define A64_LDURW 0xb8400000
339 # define A64_LDURSW 0xb8800000
340 # define A64_STP 0x29000000
341 # define A64_LDP 0x29400000
342 # define A64_STP_POS 0x29800000
343 # define A64_LDP_PRE 0x28c00000
344 # define A64_ANDI 0x12400000
345 # define A64_ORRI 0x32400000
346 # define A64_EORI 0x52400000
347 # define A64_ANDSI 0x72000000
348 # define A64_AND 0x0a000000
349 # define A64_ORR 0x2a000000
350 # define A64_MOV 0x2a0003e0 /* AKA orr Rd,xzr,Rm */
351 # define A64_MVN 0x2a2003e0
352 # define A64_UXTW 0x2a0003e0 /* AKA MOV */
353 # define A64_EOR 0x4a000000
354 # define A64_ANDS 0x6a000000
355 # define A64_MOVN 0x12800000
356 # define A64_MOVZ 0x52800000
357 # define A64_MOVK 0x72800000
358 # define SBFM(Rd,Rn,ImmR,ImmS) oxxrs(A64_SBFM|XS,Rd,Rn,ImmR,ImmS)
359 # define UBFM(Rd,Rn,ImmR,ImmS) oxxrs(A64_UBFM|XS,Rd,Rn,ImmR,ImmS)
360 # define UBFX(Rd,Rn,ImmR,ImmS) oxxrs(A64_UBFX,Rd,Rn,ImmR,ImmS)
361 # define CMP(Rn,Rm) oxx_(A64_CMP|XS,Rn,Rm)
362 # define CMPI(Rn,Imm12) oxxi(A64_SUBSI|XS,XZR_REGNO,Rn,Imm12)
363 # define CMPI_12(Rn,Imm12) oxxi(A64_SUBSI|XS|LSL_12,XZR_REGNO,Rn,Imm12)
364 # define CMNI(Rn,Imm12) oxxi(A64_ADDSI|XS,XZR_REGNO,Rn,Imm12)
365 # define CMNI_12(Rn,Imm12) oxxi(A64_ADDSI|XS|LSL_12,XZR_REGNO,Rn,Imm12)
366 # define CSINC(Rd,Rn,Rm,Cc) oxxxc(A64_CSINC|XS,Rd,Rn,Rm,Cc)
367 # define TST(Rn,Rm) oxxx(A64_ANDS|XS,XZR_REGNO,Rn,Rm)
368 /* actually should use oxxrs but logical_immediate returns proper encoding */
369 # define TSTI(Rn,Imm12) oxxi(A64_ANDSI,XZR_REGNO,Rn,Imm12)
370 # define MOV(Rd,Rm) ox_x(A64_MOV|XS,Rd,Rm)
371 # define MVN(Rd,Rm) ox_x(A64_MVN|XS,Rd,Rm)
372 # define NEG(Rd,Rm) ox_x(A64_NEG|XS,Rd,Rm)
373 # define MOVN(Rd,Imm16) ox_h(A64_MOVN|XS,Rd,Imm16)
374 # define MOVN_16(Rd,Imm16) ox_h(A64_MOVN|XS|MOVI_LSL_16,Rd,Imm16)
375 # define MOVN_32(Rd,Imm16) ox_h(A64_MOVN|XS|MOVI_LSL_32,Rd,Imm16)
376 # define MOVN_48(Rd,Imm16) ox_h(A64_MOVN|XS|MOVI_LSL_48,Rd,Imm16)
377 # define MOVZ(Rd,Imm16) ox_h(A64_MOVZ|XS,Rd,Imm16)
378 # define MOVZ_16(Rd,Imm16) ox_h(A64_MOVZ|XS|MOVI_LSL_16,Rd,Imm16)
379 # define MOVZ_32(Rd,Imm16) ox_h(A64_MOVZ|XS|MOVI_LSL_32,Rd,Imm16)
380 # define MOVZ_48(Rd,Imm16) ox_h(A64_MOVZ|XS|MOVI_LSL_48,Rd,Imm16)
381 # define MOVK(Rd,Imm16) ox_h(A64_MOVK|XS,Rd,Imm16)
382 # define MOVK_16(Rd,Imm16) ox_h(A64_MOVK|XS|MOVI_LSL_16,Rd,Imm16)
383 # define MOVK_32(Rd,Imm16) ox_h(A64_MOVK|XS|MOVI_LSL_32,Rd,Imm16)
384 # define MOVK_48(Rd,Imm16) ox_h(A64_MOVK|XS|MOVI_LSL_48,Rd,Imm16)
385 # define ADD(Rd,Rn,Rm) oxxx(A64_ADD|XS,Rd,Rn,Rm)
386 # define ADDI(Rd,Rn,Imm12) oxxi(A64_ADDI|XS,Rd,Rn,Imm12)
387 # define ADDI_12(Rd,Rn,Imm12) oxxi(A64_ADDI|XS|LSL_12,Rd,Rn,Imm12)
388 # define MOV_XSP(Rd,Rn) ADDI(Rd,Rn,0)
389 # define ADDS(Rd,Rn,Rm) oxxx(A64_ADDS|XS,Rd,Rn,Rm)
390 # define ADDSI(Rd,Rn,Imm12) oxxi(A64_ADDSI|XS,Rd,Rn,Imm12)
391 # define ADDSI_12(Rd,Rn,Imm12) oxxi(A64_ADDSI|XS|LSL_12,Rd,Rn,Imm12)
392 # define ADCS(Rd,Rn,Rm) oxxx(A64_ADCS|XS,Rd,Rn,Rm)
393 # define SUB(Rd,Rn,Rm) oxxx(A64_SUB|XS,Rd,Rn,Rm)
394 # define SUBI(Rd,Rn,Imm12) oxxi(A64_SUBI|XS,Rd,Rn,Imm12)
395 # define SUBI_12(Rd,Rn,Imm12) oxxi(A64_SUBI|XS|LSL_12,Rd,Rn,Imm12)
396 # define SUBS(Rd,Rn,Rm) oxxx(A64_SUBS|XS,Rd,Rn,Rm)
397 # define SUBSI(Rd,Rn,Imm12) oxxi(A64_SUBSI|XS,Rd,Rn,Imm12)
398 # define SUBSI_12(Rd,Rn,Imm12) oxxi(A64_SUBSI|XS|LSL_12,Rd,Rn,Imm12)
399 # define SBCS(Rd,Rn,Rm) oxxx(A64_SBCS|XS,Rd,Rn,Rm)
400 # define MUL(Rd,Rn,Rm) oxxx(A64_MUL|XS,Rd,Rn,Rm)
401 # define SMULL(Rd,Rn,Rm) oxxx(A64_SMULL,Rd,Rn,Rm)
402 # define SMULH(Rd,Rn,Rm) oxxx(A64_SMULH,Rd,Rn,Rm)
403 # define UMULL(Rd,Rn,Rm) oxxx(A64_UMULL,Rd,Rn,Rm)
404 # define UMULH(Rd,Rn,Rm) oxxx(A64_UMULH,Rd,Rn,Rm)
405 # define SDIV(Rd,Rn,Rm) oxxx(A64_SDIV|XS,Rd,Rn,Rm)
406 # define UDIV(Rd,Rn,Rm) oxxx(A64_UDIV|XS,Rd,Rn,Rm)
407 # define LSL(Rd,Rn,Rm) oxxx(A64_LSL|XS,Rd,Rn,Rm)
408 # define LSLI(r0,r1,i0) UBFM(r0,r1,(64-i0)&63,63-i0)
409 # define ASR(Rd,Rn,Rm) oxxx(A64_ASR|XS,Rd,Rn,Rm)
410 # define ASRI(r0,r1,i0) SBFM(r0,r1,i0,63)
411 # define LSR(Rd,Rn,Rm) oxxx(A64_LSR|XS,Rd,Rn,Rm)
412 # define LSRI(r0,r1,i0) UBFM(r0,r1,i0,63)
413 # define AND(Rd,Rn,Rm) oxxx(A64_AND|XS,Rd,Rn,Rm)
414 /* actually should use oxxrs but logical_immediate returns proper encoding */
415 # define ANDI(Rd,Rn,Imm12) oxxi(A64_ANDI|XS,Rd,Rn,Imm12)
416 # define ORR(Rd,Rn,Rm) oxxx(A64_ORR|XS,Rd,Rn,Rm)
417 /* actually should use oxxrs but logical_immediate returns proper encoding */
418 # define ORRI(Rd,Rn,Imm12) oxxi(A64_ORRI|XS,Rd,Rn,Imm12)
419 # define EOR(Rd,Rn,Rm) oxxx(A64_EOR|XS,Rd,Rn,Rm)
420 /* actually should use oxxrs but logical_immediate returns proper encoding */
421 # define EORI(Rd,Rn,Imm12) oxxi(A64_EORI|XS,Rd,Rn,Imm12)
422 # define SXTB(Rd,Rn) SBFM(Rd,Rn,0,7)
423 # define SXTH(Rd,Rn) SBFM(Rd,Rn,0,15)
424 # define SXTW(Rd,Rn) SBFM(Rd,Rn,0,31)
425 # define UXTB(Rd,Rn) UBFX(Rd,Rn,0,7)
426 # define UXTH(Rd,Rn) UBFX(Rd,Rn,0,15)
427 # define UXTW(Rd,Rm) ox_x(A64_UXTW,Rd,Rm)
428 # define REV(Rd,Rn) o_xx(A64_REV,Rd,Rn)
429 # define LDRSB(Rt,Rn,Rm) oxxx(A64_LDRSB,Rt,Rn,Rm)
430 # define LDRSBI(Rt,Rn,Imm12) oxxi(A64_LDRSBI,Rt,Rn,Imm12)
431 # define LDURSB(Rt,Rn,Imm9) oxx9(A64_LDURSB,Rt,Rn,Imm9)
432 # define LDRB(Rt,Rn,Rm) oxxx(A64_LDRB,Rt,Rn,Rm)
433 # define LDRBI(Rt,Rn,Imm12) oxxi(A64_LDRBI,Rt,Rn,Imm12)
434 # define LDURB(Rt,Rn,Imm9) oxx9(A64_LDURB,Rt,Rn,Imm9)
435 # define LDRSH(Rt,Rn,Rm) oxxx(A64_LDRSH,Rt,Rn,Rm)
436 # define LDRSHI(Rt,Rn,Imm12) oxxi(A64_LDRSHI,Rt,Rn,Imm12)
437 # define LDURSH(Rt,Rn,Imm9) oxx9(A64_LDURSH,Rt,Rn,Imm9)
438 # define LDRH(Rt,Rn,Rm) oxxx(A64_LDRH,Rt,Rn,Rm)
439 # define LDRHI(Rt,Rn,Imm12) oxxi(A64_LDRHI,Rt,Rn,Imm12)
440 # define LDURH(Rt,Rn,Imm9) oxx9(A64_LDURH,Rt,Rn,Imm9)
441 # define LDRSW(Rt,Rn,Rm) oxxx(A64_LDRSW,Rt,Rn,Rm)
442 # define LDRSWI(Rt,Rn,Imm12) oxxi(A64_LDRSWI,Rt,Rn,Imm12)
443 # define LDURSW(Rt,Rn,Imm9) oxx9(A64_LDURSW,Rt,Rn,Imm9)
444 # define LDRW(Rt,Rn,Rm) oxxx(A64_LDRW,Rt,Rn,Rm)
445 # define LDRWI(Rt,Rn,Imm12) oxxi(A64_LDRWI,Rt,Rn,Imm12)
446 # define LDURW(Rt,Rn,Imm9) oxx9(A64_LDURW,Rt,Rn,Imm9)
447 # define LDR(Rt,Rn,Rm) oxxx(A64_LDR,Rt,Rn,Rm)
448 # define LDRI(Rt,Rn,Imm12) oxxi(A64_LDRI,Rt,Rn,Imm12)
449 # define LDUR(Rt,Rn,Imm9) oxx9(A64_LDUR,Rt,Rn,Imm9)
450 # define LDAXR(Rt,Rn) o_xx(A64_LDAXR,Rt,Rn)
451 # define STLXR(Rs,Rt,Rn) oxxx(A64_STLXR,Rs,Rn,Rt)
452 # define STRB(Rt,Rn,Rm) oxxx(A64_STRB,Rt,Rn,Rm)
453 # define STRBI(Rt,Rn,Imm12) oxxi(A64_STRBI,Rt,Rn,Imm12)
454 # define STURB(Rt,Rn,Imm9) oxx9(A64_STURB,Rt,Rn,Imm9)
455 # define STRH(Rt,Rn,Rm) oxxx(A64_STRH,Rt,Rn,Rm)
456 # define STRHI(Rt,Rn,Imm12) oxxi(A64_STRHI,Rt,Rn,Imm12)
457 # define STURH(Rt,Rn,Imm9) oxx9(A64_STURH,Rt,Rn,Imm9)
458 # define STRW(Rt,Rn,Rm) oxxx(A64_STRW,Rt,Rn,Rm)
459 # define STRWI(Rt,Rn,Imm12) oxxi(A64_STRWI,Rt,Rn,Imm12)
460 # define STURW(Rt,Rn,Imm9) oxx9(A64_STURW,Rt,Rn,Imm9)
461 # define STR(Rt,Rn,Rm) oxxx(A64_STR,Rt,Rn,Rm)
462 # define STRI(Rt,Rn,Imm12) oxxi(A64_STRI,Rt,Rn,Imm12)
463 # define STUR(Rt,Rn,Imm9) oxx9(A64_STUR,Rt,Rn,Imm9)
464 # define LDPI(Rt,Rt2,Rn,Simm7) oxxx7(A64_LDP|XS,Rt,Rt2,Rn,Simm7)
465 # define STPI(Rt,Rt2,Rn,Simm7) oxxx7(A64_STP|XS,Rt,Rt2,Rn,Simm7)
466 # define LDPI_PRE(Rt,Rt2,Rn,Simm7) oxxx7(A64_LDP_PRE|XS,Rt,Rt2,Rn,Simm7)
467 # define STPI_POS(Rt,Rt2,Rn,Simm7) oxxx7(A64_STP_POS|XS,Rt,Rt2,Rn,Simm7)
468 # define CSET(Rd,Cc) CSINC(Rd,XZR_REGNO,XZR_REGNO,Cc)
469 # define CSEL(Rd,Rn,Rm,Cc) oxxxc(A64_CSSEL|XS,Rd,Rn,Rm,Cc)
470 # define B(Simm26) o26(A64_B,Simm26)
471 # define BL(Simm26) o26(A64_BL,Simm26)
472 # define BR(Rn) o_x_(A64_BR,Rn)
473 # define BLR(Rn) o_x_(A64_BLR,Rn)
474 # define RET() o_x_(A64_RET,LR_REGNO)
475 # define B_C(Cc,Simm19) oc19(A64_B_C,Cc,Simm19)
476 # define CBZ(Rd,Simm19) ox19(A64_CBZ|XS,Rd,Simm19)
477 # define CBNZ(Rd,Simm19) ox19(A64_CBNZ|XS,Rd,Simm19)
478 # define NOP() ii(0xd503201f)
479 static jit_int32_t logical_immediate(jit_word_t);
480 # define oxxx(Op,Rd,Rn,Rm) _oxxx(_jit,Op,Rd,Rn,Rm)
481 static void _oxxx(jit_state_t*,jit_int32_t,jit_int32_t,jit_int32_t,jit_int32_t);
482 # define oxxi(Op,Rd,Rn,Imm12) _oxxi(_jit,Op,Rd,Rn,Imm12)
483 static void _oxxi(jit_state_t*,jit_int32_t,jit_int32_t,jit_int32_t,jit_int32_t);
484 # define oxx9(Op,Rd,Rn,Imm9) _oxx9(_jit,Op,Rd,Rn,Imm9)
485 static void _oxx9(jit_state_t*,jit_int32_t,jit_int32_t,jit_int32_t,jit_int32_t);
486 # define ox19(Op,Rd,Simm19) _ox19(_jit,Op,Rd,Simm19)
487 static void _ox19(jit_state_t*,jit_int32_t,jit_int32_t,jit_int32_t);
488 # define oc19(Op,Cc,Simm19) _oc19(_jit,Op,Cc,Simm19)
489 static void _oc19(jit_state_t*,jit_int32_t,jit_int32_t,jit_int32_t);
490 # define o26(Op,Simm26) _o26(_jit,Op,Simm26)
491 static void _oc26(jit_state_t*,jit_int32_t,jit_int32_t);
492 # define ox_x(Op,Rd,Rn) _ox_x(_jit,Op,Rd,Rn)
493 static void _ox_x(jit_state_t*,jit_int32_t,jit_int32_t,jit_int32_t);
494 # define o_xx(Op,Rd,Rn) _o_xx(_jit,Op,Rd,Rn)
495 static void _o_xx(jit_state_t*,jit_int32_t,jit_int32_t,jit_int32_t);
496 # define oxx_(Op,Rn,Rm) _oxx_(_jit,Op,Rn,Rm)
497 static void _oxx_(jit_state_t*,jit_int32_t,jit_int32_t,jit_int32_t);
498 # define o_x_(Op,Rn) _o_x_(_jit,Op,Rn)
499 static void _o_x_(jit_state_t*,jit_int32_t,jit_int32_t);
500 # define ox_h(Op,Rd,Imm16) _ox_h(_jit,Op,Rd,Imm16)
501 static void _ox_h(jit_state_t*,jit_int32_t,jit_int32_t,jit_int32_t);
502 # define oxxrs(Op,Rd,Rn,R,S) _oxxrs(_jit,Op,Rd,Rn,R,S)
503 static void _oxxrs(jit_state_t*,jit_int32_t,jit_int32_t,
504 jit_int32_t,jit_int32_t,jit_int32_t);
505 # define oxxxc(Op,Rd,Rn,Rm,Cc) _oxxxc(_jit,Op,Rd,Rn,Rm,Cc)
506 static void _oxxxc(jit_state_t*,jit_int32_t,jit_int32_t,
507 jit_int32_t,jit_int32_t,jit_int32_t);
508 # define oxxx7(Op,Rt,Rt2,Rn,Simm7) _oxxx7(_jit,Op,Rt,Rt2,Rn,Simm7)
509 static void _oxxx7(jit_state_t*,jit_int32_t,
510 jit_int32_t,jit_int32_t,jit_int32_t,jit_int32_t);
511 # define nop(i0) _nop(_jit,i0)
512 static void _nop(jit_state_t*,jit_int32_t);
513 # define addr(r0,r1,r2) ADD(r0,r1,r2)
514 # define addi(r0,r1,i0) _addi(_jit,r0,r1,i0)
515 static void _addi(jit_state_t*,jit_int32_t,jit_int32_t,jit_word_t);
516 # define addcr(r0,r1,r2) ADDS(r0,r1,r2)
517 # define addci(r0,r1,i0) _addci(_jit,r0,r1,i0)
518 static void _addci(jit_state_t*,jit_int32_t,jit_int32_t,jit_word_t);
519 # define addxr(r0,r1,r2) ADCS(r0,r1,r2)
520 # define addxi(r0,r1,i0) _addxi(_jit,r0,r1,i0)
521 static void _addxi(jit_state_t*,jit_int32_t,jit_int32_t,jit_word_t);
522 # define subr(r0,r1,r2) SUB(r0,r1,r2)
523 # define subi(r0,r1,i0) _subi(_jit,r0,r1,i0)
524 static void _subi(jit_state_t*,jit_int32_t,jit_int32_t,jit_word_t);
525 # define subcr(r0,r1,r2) SUBS(r0,r1,r2)
526 # define subci(r0,r1,i0) _subci(_jit,r0,r1,i0)
527 static void _subci(jit_state_t*,jit_int32_t,jit_int32_t,jit_word_t);
528 # define subxr(r0,r1,r2) SBCS(r0,r1,r2)
529 # define subxi(r0,r1,i0) _subxi(_jit,r0,r1,i0)
530 static void _subxi(jit_state_t*,jit_int32_t,jit_int32_t,jit_word_t);
531 # define rsbi(r0, r1, i0) _rsbi(_jit, r0, r1, i0)
532 static void _rsbi(jit_state_t*,jit_int32_t,jit_int32_t,jit_word_t);
533 # define mulr(r0,r1,r2) MUL(r0,r1,r2)
534 # define muli(r0,r1,i0) _muli(_jit,r0,r1,i0)
535 static void _muli(jit_state_t*,jit_int32_t,jit_int32_t,jit_word_t);
536 # define qmulr(r0,r1,r2,r3) _qmulr(_jit,r0,r1,r2,r3)
537 static void _qmulr(jit_state_t*,jit_int32_t,
538 jit_int32_t,jit_int32_t,jit_int32_t);
539 # define qmuli(r0,r1,r2,i0) _qmuli(_jit,r0,r1,r2,i0)
540 static void _qmuli(jit_state_t*,jit_int32_t,
541 jit_int32_t,jit_int32_t,jit_word_t);
542 # define qmulr_u(r0,r1,r2,r3) _qmulr_u(_jit,r0,r1,r2,r3)
543 static void _qmulr_u(jit_state_t*,jit_int32_t,
544 jit_int32_t,jit_int32_t,jit_int32_t);
545 # define qmuli_u(r0,r1,r2,i0) _qmuli_u(_jit,r0,r1,r2,i0)
546 static void _qmuli_u(jit_state_t*,jit_int32_t,
547 jit_int32_t,jit_int32_t,jit_word_t);
548 # define divr(r0,r1,r2) SDIV(r0,r1,r2)
549 # define divi(r0,r1,i0) _divi(_jit,r0,r1,i0)
550 static void _divi(jit_state_t*,jit_int32_t,jit_int32_t,jit_word_t);
551 # define divr_u(r0,r1,r2) UDIV(r0,r1,r2)
552 # define divi_u(r0,r1,i0) _divi_u(_jit,r0,r1,i0)
553 static void _divi_u(jit_state_t*,jit_int32_t,jit_int32_t,jit_word_t);
554 # define qdivr(r0,r1,r2,r3) _iqdivr(_jit,1,r0,r1,r2,r3)
555 # define qdivr_u(r0,r1,r2,r3) _iqdivr(_jit,0,r0,r1,r2,r3)
556 static void _iqdivr(jit_state_t*,jit_bool_t,
557 jit_int32_t,jit_int32_t,jit_int32_t,jit_int32_t);
558 # define qdivi(r0,r1,r2,i0) _qdivi(_jit,r0,r1,r2,i0)
559 static void _qdivi(jit_state_t*,jit_int32_t,
560 jit_int32_t,jit_int32_t,jit_word_t);
561 # define qdivi_u(r0,r1,r2,i0) _qdivi_u(_jit,r0,r1,r2,i0)
562 static void _qdivi_u(jit_state_t*,jit_int32_t,
563 jit_int32_t,jit_int32_t,jit_word_t);
564 # define remr(r0,r1,r2) _remr(_jit,r0,r1,r2)
565 static void _remr(jit_state_t*,jit_int32_t,jit_int32_t,jit_int32_t);
566 # define remi(r0,r1,i0) _remi(_jit,r0,r1,i0)
567 static void _remi(jit_state_t*,jit_int32_t,jit_int32_t,jit_word_t);
568 # define remr_u(r0,r1,r2) _remr_u(_jit,r0,r1,r2)
569 static void _remr_u(jit_state_t*,jit_int32_t,jit_int32_t,jit_int32_t);
570 # define remi_u(r0,r1,i0) _remi_u(_jit,r0,r1,i0)
571 static void _remi_u(jit_state_t*,jit_int32_t,jit_int32_t,jit_word_t);
572 # define lshr(r0,r1,r2) LSL(r0,r1,r2)
573 # define lshi(r0,r1,i0) _lshi(_jit,r0,r1,i0)
574 static void _lshi(jit_state_t*,jit_int32_t,jit_int32_t,jit_word_t);
575 # define rshr(r0,r1,r2) ASR(r0,r1,r2)
576 # define rshi(r0,r1,i0) _rshi(_jit,r0,r1,i0)
577 static void _rshi(jit_state_t*,jit_int32_t,jit_int32_t,jit_word_t);
578 # define rshr_u(r0,r1,r2) LSR(r0,r1,r2)
579 # define rshi_u(r0,r1,i0) _rshi_u(_jit,r0,r1,i0)
580 static void _rshi_u(jit_state_t*,jit_int32_t,jit_int32_t,jit_word_t);
581 # define movnr(r0,r1,r2) _movnr(_jit,r0,r1,r2)
582 static void _movnr(jit_state_t*,jit_int32_t,jit_int32_t,jit_int32_t);
583 # define movzr(r0,r1,r2) _movzr(_jit,r0,r1,r2)
584 static void _movzr(jit_state_t*,jit_int32_t,jit_int32_t,jit_int32_t);
585 # define negr(r0,r1) NEG(r0,r1)
586 # define comr(r0,r1) MVN(r0,r1)
587 # define andr(r0,r1,r2) AND(r0,r1,r2)
588 # define andi(r0,r1,i0) _andi(_jit,r0,r1,i0)
589 static void _andi(jit_state_t*,jit_int32_t,jit_int32_t,jit_word_t);
590 # define orr(r0,r1,r2) ORR(r0,r1,r2)
591 # define ori(r0,r1,i0) _ori(_jit,r0,r1,i0)
592 static void _ori(jit_state_t*,jit_int32_t,jit_int32_t,jit_word_t);
593 # define xorr(r0,r1,r2) EOR(r0,r1,r2)
594 # define xori(r0,r1,i0) _xori(_jit,r0,r1,i0)
595 static void _xori(jit_state_t*,jit_int32_t,jit_int32_t,jit_word_t);
596 # define ldr_c(r0,r1) LDRSBI(r0,r1,0)
597 # define ldi_c(r0,i0) _ldi_c(_jit,r0,i0)
598 static void _ldi_c(jit_state_t*,jit_int32_t,jit_word_t);
599 # define ldr_uc(r0,r1) _ldr_uc(_jit,r0,r1)
600 static void _ldr_uc(jit_state_t*,jit_int32_t,jit_int32_t);
601 # define ldi_uc(r0,i0) _ldi_uc(_jit,r0,i0)
602 static void _ldi_uc(jit_state_t*,jit_int32_t,jit_word_t);
603 # define ldr_s(r0,r1) LDRSHI(r0,r1,0)
604 # define ldi_s(r0,i0) _ldi_s(_jit,r0,i0)
605 static void _ldi_s(jit_state_t*,jit_int32_t,jit_word_t);
606 # define ldr_us(r0,r1) _ldr_us(_jit,r0,r1)
607 static void _ldr_us(jit_state_t*,jit_int32_t,jit_int32_t);
608 # define ldi_us(r0,i0) _ldi_us(_jit,r0,i0)
609 static void _ldi_us(jit_state_t*,jit_int32_t,jit_word_t);
610 # define ldr_i(r0,r1) LDRSWI(r0,r1,0)
611 # define ldi_i(r0,i0) _ldi_i(_jit,r0,i0)
612 static void _ldi_i(jit_state_t*,jit_int32_t,jit_word_t);
613 # define ldr_ui(r0,r1) _ldr_ui(_jit,r0,r1)
614 static void _ldr_ui(jit_state_t*,jit_int32_t,jit_int32_t);
615 # define ldi_ui(r0,i0) _ldi_ui(_jit,r0,i0)
616 static void _ldi_ui(jit_state_t*,jit_int32_t,jit_word_t);
617 # define ldr_l(r0,r1) LDRI(r0,r1,0)
618 static void _ldr_l(jit_state_t*,jit_int32_t,jit_int32_t);
619 # define ldi_l(r0,i0) _ldi_l(_jit,r0,i0)
620 static void _ldi_l(jit_state_t*,jit_int32_t,jit_word_t);
621 # define ldxr_c(r0,r1,r2) _ldxr_c(_jit,r0,r1,r2)
622 static void _ldxr_c(jit_state_t*,jit_int32_t,jit_int32_t,jit_int32_t);
623 # define ldxi_c(r0,r1,i0) _ldxi_c(_jit,r0,r1,i0)
624 static void _ldxi_c(jit_state_t*,jit_int32_t,jit_int32_t,jit_word_t);
625 # define ldxr_uc(r0,r1,r2) _ldxr_uc(_jit,r0,r1,r2)
626 static void _ldxr_uc(jit_state_t*,jit_int32_t,jit_int32_t,jit_int32_t);
627 # define ldxi_uc(r0,r1,i0) _ldxi_uc(_jit,r0,r1,i0)
628 static void _ldxi_uc(jit_state_t*,jit_int32_t,jit_int32_t,jit_word_t);
629 # define ldxr_s(r0,r1,r2) LDRSH(r0,r1,r2)
630 # define ldxi_s(r0,r1,i0) _ldxi_s(_jit,r0,r1,i0)
631 static void _ldxi_s(jit_state_t*,jit_int32_t,jit_int32_t,jit_word_t);
632 # define ldxr_us(r0,r1,r2) _ldxr_us(_jit,r0,r1,r2)
633 static void _ldxr_us(jit_state_t*,jit_int32_t,jit_int32_t,jit_int32_t);
634 # define ldxi_us(r0,r1,i0) _ldxi_us(_jit,r0,r1,i0)
635 static void _ldxi_us(jit_state_t*,jit_int32_t,jit_int32_t,jit_word_t);
636 # define ldxr_i(r0,r1,r2) LDRSW(r0,r1,r2)
637 # define ldxi_i(r0,r1,i0) _ldxi_i(_jit,r0,r1,i0)
638 static void _ldxi_i(jit_state_t*,jit_int32_t,jit_int32_t,jit_word_t);
639 # define ldxr_ui(r0,r1,r2) _ldxr_ui(_jit,r0,r1,r2)
640 static void _ldxr_ui(jit_state_t*,jit_int32_t,jit_int32_t,jit_int32_t);
641 # define ldxi_ui(r0,r1,i0) _ldxi_ui(_jit,r0,r1,i0)
642 static void _ldxi_ui(jit_state_t*,jit_int32_t,jit_int32_t,jit_word_t);
643 # define ldxr_l(r0,r1,r2) LDR(r0,r1,r2)
644 # define ldxi_l(r0,r1,i0) _ldxi_l(_jit,r0,r1,i0)
645 static void _ldxi_l(jit_state_t*,jit_int32_t,jit_int32_t,jit_word_t);
646 # define str_c(r0,r1) STRBI(r1,r0,0)
647 # define sti_c(i0,r0) _sti_c(_jit,i0,r0)
648 static void _sti_c(jit_state_t*,jit_word_t,jit_int32_t);
649 # define str_s(r0,r1) STRHI(r1,r0,0)
650 # define sti_s(i0,r0) _sti_s(_jit,i0,r0)
651 static void _sti_s(jit_state_t*,jit_word_t,jit_int32_t);
652 # define str_i(r0,r1) STRWI(r1,r0,0)
653 # define sti_i(i0,r0) _sti_i(_jit,i0,r0)
654 static void _sti_i(jit_state_t*,jit_word_t,jit_int32_t);
655 # define str_l(r0,r1) STRI(r1,r0,0)
656 # define sti_l(i0,r0) _sti_l(_jit,i0,r0)
657 static void _sti_l(jit_state_t*,jit_word_t,jit_int32_t);
658 # define stxr_c(r0,r1,r2) STRB(r2,r1,r0)
659 # define stxi_c(i0,r0,r1) _stxi_c(_jit,i0,r0,r1)
660 static void _stxi_c(jit_state_t*,jit_word_t,jit_int32_t,jit_int32_t);
661 # define stxr_s(r0,r1,r2) STRH(r2,r1,r0)
662 # define stxi_s(i0,r0,r1) _stxi_s(_jit,i0,r0,r1)
663 static void _stxi_s(jit_state_t*,jit_word_t,jit_int32_t,jit_int32_t);
664 # define stxr_i(r0,r1,r2) STRW(r2,r1,r0)
665 # define stxi_i(i0,r0,r1) _stxi_i(_jit,i0,r0,r1)
666 static void _stxi_i(jit_state_t*,jit_word_t,jit_int32_t,jit_int32_t);
667 # define stxr_l(r0,r1,r2) STR(r2,r1,r0)
668 # define stxi_l(i0,r0,r1) _stxi_l(_jit,i0,r0,r1)
669 static void _stxi_l(jit_state_t*,jit_word_t,jit_int32_t,jit_int32_t);
670 # define bswapr_us(r0,r1) _bswapr_us(_jit,r0,r1)
671 static void _bswapr_us(jit_state_t*,jit_int32_t,jit_int32_t);
672 # define bswapr_ui(r0,r1) _bswapr_ui(_jit,r0,r1)
673 static void _bswapr_ui(jit_state_t*,jit_int32_t,jit_int32_t);
674 # define bswapr_ul(r0,r1) REV(r0,r1)
675 # define extr_c(r0,r1) SXTB(r0,r1)
676 # define extr_uc(r0,r1) UXTB(r0,r1)
677 # define extr_s(r0,r1) SXTH(r0,r1)
678 # define extr_us(r0,r1) UXTH(r0,r1)
679 # define extr_i(r0,r1) SXTW(r0,r1)
680 # define extr_ui(r0,r1) UXTW(r0,r1)
681 # define casx(r0, r1, r2, r3, i0) _casx(_jit, r0, r1, r2, r3, i0)
682 static void _casx(jit_state_t *_jit,jit_int32_t,jit_int32_t,
683 jit_int32_t,jit_int32_t,jit_word_t);
684 #define casr(r0, r1, r2, r3) casx(r0, r1, r2, r3, 0)
685 #define casi(r0, i0, r1, r2) casx(r0, _NOREG, r1, r2, i0)
686 # define movr(r0,r1) _movr(_jit,r0,r1)
687 static void _movr(jit_state_t*,jit_int32_t,jit_int32_t);
688 # define movi(r0,i0) _movi(_jit,r0,i0)
689 static void _movi(jit_state_t*,jit_int32_t,jit_word_t);
690 # define movi_p(r0,i0) _movi_p(_jit,r0,i0)
691 static jit_word_t _movi_p(jit_state_t*,jit_int32_t,jit_word_t);
692 # define ccr(cc,r0,r1,r2) _ccr(_jit,cc,r0,r1,r2)
693 static void _ccr(jit_state_t*,jit_int32_t,jit_int32_t,jit_int32_t,jit_int32_t);
694 # define cci(cc,r0,r1,i0) _cci(_jit,cc,r0,r1,i0)
695 static void _cci(jit_state_t*,jit_int32_t,jit_int32_t,jit_int32_t,jit_word_t);
696 # define ltr(r0,r1,r2) ccr(CC_LT,r0,r1,r2)
697 # define lti(r0,r1,i0) cci(CC_LT,r0,r1,i0)
698 # define ltr_u(r0,r1,r2) ccr(CC_CC,r0,r1,r2)
699 # define lti_u(r0,r1,i0) cci(CC_CC,r0,r1,i0)
700 # define ler(r0,r1,r2) ccr(CC_LE,r0,r1,r2)
701 # define lei(r0,r1,i0) cci(CC_LE,r0,r1,i0)
702 # define ler_u(r0,r1,r2) ccr(CC_LS,r0,r1,r2)
703 # define lei_u(r0,r1,i0) cci(CC_LS,r0,r1,i0)
704 # define eqr(r0,r1,r2) ccr(CC_EQ,r0,r1,r2)
705 # define eqi(r0,r1,i0) cci(CC_EQ,r0,r1,i0)
706 # define ger(r0,r1,r2) ccr(CC_GE,r0,r1,r2)
707 # define gei(r0,r1,i0) cci(CC_GE,r0,r1,i0)
708 # define ger_u(r0,r1,r2) ccr(CC_CS,r0,r1,r2)
709 # define gei_u(r0,r1,i0) cci(CC_CS,r0,r1,i0)
710 # define gtr(r0,r1,r2) ccr(CC_GT,r0,r1,r2)
711 # define gti(r0,r1,i0) cci(CC_GT,r0,r1,i0)
712 # define gtr_u(r0,r1,r2) ccr(CC_HI,r0,r1,r2)
713 # define gti_u(r0,r1,i0) cci(CC_HI,r0,r1,i0)
714 # define ner(r0,r1,r2) ccr(CC_NE,r0,r1,r2)
715 # define nei(r0,r1,i0) cci(CC_NE,r0,r1,i0)
716 # define bccr(cc,i0,r0,r1) _bccr(_jit,cc,i0,r0,r1)
718 _bccr(jit_state_t*,jit_int32_t,jit_word_t,jit_int32_t,jit_int32_t);
719 # define bcci(cc,i0,r0,i1) _bcci(_jit,cc,i0,r0,i1)
721 _bcci(jit_state_t*,jit_int32_t,jit_word_t,jit_int32_t,jit_word_t);
722 # define bltr(i0,r0,r1) bccr(BCC_LT,i0,r0,r1)
723 # define blti(i0,r0,i1) bcci(BCC_LT,i0,r0,i1)
724 # define bltr_u(i0,r0,r1) bccr(BCC_CC,i0,r0,r1)
725 # define blti_u(i0,r0,i1) bcci(BCC_CC,i0,r0,i1)
726 # define bler(i0,r0,r1) bccr(BCC_LE,i0,r0,r1)
727 # define blei(i0,r0,i1) bcci(BCC_LE,i0,r0,i1)
728 # define bler_u(i0,r0,r1) bccr(BCC_LS,i0,r0,r1)
729 # define blei_u(i0,r0,i1) bcci(BCC_LS,i0,r0,i1)
730 # define beqr(i0,r0,r1) bccr(BCC_EQ,i0,r0,r1)
731 # define beqi(i0,r0,i1) _beqi(_jit,i0,r0,i1)
732 static jit_word_t _beqi(jit_state_t*,jit_word_t,jit_int32_t,jit_word_t);
733 # define bger(i0,r0,r1) bccr(BCC_GE,i0,r0,r1)
734 # define bgei(i0,r0,i1) bcci(BCC_GE,i0,r0,i1)
735 # define bger_u(i0,r0,r1) bccr(BCC_CS,i0,r0,r1)
736 # define bgei_u(i0,r0,i1) bcci(BCC_CS,i0,r0,i1)
737 # define bgtr(i0,r0,r1) bccr(BCC_GT,i0,r0,r1)
738 # define bgti(i0,r0,i1) bcci(BCC_GT,i0,r0,i1)
739 # define bgtr_u(i0,r0,r1) bccr(BCC_HI,i0,r0,r1)
740 # define bgti_u(i0,r0,i1) bcci(BCC_HI,i0,r0,i1)
741 # define bner(i0,r0,r1) bccr(BCC_NE,i0,r0,r1)
742 # define bnei(i0,r0,i1) _bnei(_jit,i0,r0,i1)
743 static jit_word_t _bnei(jit_state_t*,jit_word_t,jit_int32_t,jit_word_t);
744 # define baddr(cc,i0,r0,r1) _baddr(_jit,cc,i0,r0,r1)
746 _baddr(jit_state_t*,jit_int32_t,jit_word_t,jit_int32_t,jit_int32_t);
747 # define baddi(cc,i0,r0,i1) _baddi(_jit,cc,i0,r0,i1)
749 _baddi(jit_state_t*,jit_int32_t,jit_word_t,jit_int32_t,jit_word_t);
750 # define boaddr(i0,r0,r1) baddr(BCC_VS,i0,r0,r1)
751 # define boaddi(i0,r0,i1) baddi(BCC_VS,i0,r0,i1)
752 # define boaddr_u(i0,r0,r1) baddr(BCC_HS,i0,r0,r1)
753 # define boaddi_u(i0,r0,i1) baddi(BCC_HS,i0,r0,i1)
754 # define bxaddr(i0,r0,r1) baddr(BCC_VC,i0,r0,r1)
755 # define bxaddi(i0,r0,i1) baddi(BCC_VC,i0,r0,i1)
756 # define bxaddr_u(i0,r0,r1) baddr(BCC_LO,i0,r0,r1)
757 # define bxaddi_u(i0,r0,i1) baddi(BCC_LO,i0,r0,i1)
758 # define bsubr(cc,i0,r0,r1) _bsubr(_jit,cc,i0,r0,r1)
760 _bsubr(jit_state_t*,jit_int32_t,jit_word_t,jit_int32_t,jit_int32_t);
761 # define bsubi(cc,i0,r0,i1) _bsubi(_jit,cc,i0,r0,i1)
763 _bsubi(jit_state_t*,jit_int32_t,jit_word_t,jit_int32_t,jit_word_t);
764 # define bosubr(i0,r0,r1) bsubr(BCC_VS,i0,r0,r1)
765 # define bosubi(i0,r0,i1) bsubi(BCC_VS,i0,r0,i1)
766 # define bosubr_u(i0,r0,r1) bsubr(BCC_LO,i0,r0,r1)
767 # define bosubi_u(i0,r0,i1) bsubi(BCC_LO,i0,r0,i1)
768 # define bxsubr(i0,r0,r1) bsubr(BCC_VC,i0,r0,r1)
769 # define bxsubi(i0,r0,i1) bsubi(BCC_VC,i0,r0,i1)
770 # define bxsubr_u(i0,r0,r1) bsubr(BCC_HS,i0,r0,r1)
771 # define bxsubi_u(i0,r0,i1) bsubi(BCC_HS,i0,r0,i1)
772 # define bmxr(cc,i0,r0,r1) _bmxr(_jit,cc,i0,r0,r1)
774 _bmxr(jit_state_t*,jit_int32_t,jit_word_t,jit_int32_t,jit_int32_t);
775 # define bmxi(cc,i0,r0,r1) _bmxi(_jit,cc,i0,r0,r1)
777 _bmxi(jit_state_t*,jit_int32_t,jit_word_t,jit_int32_t,jit_word_t);
778 # define bmsr(i0,r0,r1) bmxr(BCC_NE,i0,r0,r1)
779 # define bmsi(i0,r0,i1) bmxi(BCC_NE,i0,r0,i1)
780 # define bmcr(i0,r0,r1) bmxr(BCC_EQ,i0,r0,r1)
781 # define bmci(i0,r0,i1) bmxi(BCC_EQ,i0,r0,i1)
782 # define jmpr(r0) BR(r0)
783 # define jmpi(i0) _jmpi(_jit,i0)
784 static void _jmpi(jit_state_t*,jit_word_t);
785 # define jmpi_p(i0) _jmpi_p(_jit,i0)
786 static jit_word_t _jmpi_p(jit_state_t*,jit_word_t);
787 # define callr(r0) BLR(r0)
788 # define calli(i0) _calli(_jit,i0)
789 static void _calli(jit_state_t*,jit_word_t);
790 # define calli_p(i0) _calli_p(_jit,i0)
791 static jit_word_t _calli_p(jit_state_t*,jit_word_t);
792 # define prolog(i0) _prolog(_jit,i0)
793 static void _prolog(jit_state_t*,jit_node_t*);
794 # define epilog(i0) _epilog(_jit,i0)
795 static void _epilog(jit_state_t*,jit_node_t*);
796 # define vastart(r0) _vastart(_jit, r0)
797 static void _vastart(jit_state_t*, jit_int32_t);
798 # define vaarg(r0, r1) _vaarg(_jit, r0, r1)
799 static void _vaarg(jit_state_t*, jit_int32_t, jit_int32_t);
800 # define patch_at(jump,label) _patch_at(_jit,jump,label)
801 static void _patch_at(jit_state_t*,jit_word_t,jit_word_t);
806 logical_immediate(jit_word_t imm)
808 /* There are 5334 possible immediate values, but to avoid the
809 * need of either too complex code or large lookup tables,
810 * only check for (simply) encodable common/small values */
812 case -16: return (0xf3b);
813 case -15: return (0xf3c);
814 case -13: return (0xf3d);
815 case -9: return (0xf3e);
816 case -8: return (0xf7c);
817 case -7: return (0xf7d);
818 case -5: return (0xf7e);
819 case -4: return (0xfbd);
820 case -3: return (0xfbe);
821 case -2: return (0xffe);
822 case 1: return (0x000);
823 case 2: return (0xfc0);
824 case 3: return (0x001);
825 case 4: return (0xf80);
826 case 6: return (0xfc1);
827 case 7: return (0x002);
828 case 8: return (0xf40);
829 case 12: return (0xf81);
830 case 14: return (0xfc2);
831 case 15: return (0x003);
832 case 16: return (0xf00);
833 default: return (-1);
838 _oxxx(jit_state_t *_jit, jit_int32_t Op,
839 jit_int32_t Rd, jit_int32_t Rn, jit_int32_t Rm)
842 assert(!(Rd & ~0x1f));
843 assert(!(Rn & ~0x1f));
844 assert(!(Rm & ~0x1f));
845 assert(!(Op & ~0xffe0fc00));
854 _oxxi(jit_state_t *_jit, jit_int32_t Op,
855 jit_int32_t Rd, jit_int32_t Rn, jit_int32_t Imm12)
858 assert(!(Rd & ~0x1f));
859 assert(!(Rn & ~0x1f));
860 assert(!(Imm12 & ~0xfff));
861 assert(!(Op & ~0xffe00000));
870 _oxx9(jit_state_t *_jit, jit_int32_t Op,
871 jit_int32_t Rd, jit_int32_t Rn, jit_int32_t Imm9)
874 assert(!(Rd & ~0x1f));
875 assert(!(Rn & ~0x1f));
876 assert(!(Imm9 & ~0x1ff));
877 assert(!(Op & ~0xffe00000));
886 _ox19(jit_state_t *_jit, jit_int32_t Op, jit_int32_t Rd, jit_int32_t Simm19)
889 assert(!(Rd & ~0x1f));
890 assert(Simm19 >= -262148 && Simm19 <= 262143);
891 assert(!(Op & ~0xff000000));
899 _oc19(jit_state_t *_jit, jit_int32_t Op, jit_int32_t Cc, jit_int32_t Simm19)
902 assert(!(Cc & ~0xf));
903 assert(Simm19 >= -262148 && Simm19 <= 262143);
904 assert(!(Op & ~0xff000000));
912 _o26(jit_state_t *_jit, jit_int32_t Op, jit_int32_t Simm26)
915 assert(Simm26 >= -33554432 && Simm26 <= 33554431);
916 assert(!(Op & ~0xfc000000));
923 _ox_x(jit_state_t *_jit, jit_int32_t Op, jit_int32_t Rd, jit_int32_t Rm)
926 assert(!(Rd & ~0x1f));
927 assert(!(Rm & ~0x1f));
928 assert(!(Op & ~0xffe0ffe0));
936 _o_xx(jit_state_t *_jit, jit_int32_t Op, jit_int32_t Rd, jit_int32_t Rn)
939 assert(!(Rd & ~0x1f));
940 assert(!(Rn & ~0x1f));
941 assert(!(Op & ~0xfffffc00));
949 _oxx_(jit_state_t *_jit, jit_int32_t Op, jit_int32_t Rn, jit_int32_t Rm)
952 assert(!(Rn & ~0x1f));
953 assert(!(Rm & ~0x1f));
954 assert(!(Op & ~0xffc0fc1f));
962 _o_x_(jit_state_t *_jit, jit_int32_t Op, jit_int32_t Rn)
965 assert(!(Rn & ~0x1f));
966 assert(!(Op & 0x3e0));
973 _ox_h(jit_state_t *_jit, jit_int32_t Op, jit_int32_t Rd, jit_int32_t Imm16)
976 assert(!(Rd & ~0x1f));
977 assert(!(Imm16 & ~0xffff));
978 assert(!(Op & ~0xffe00000));
986 _oxxrs(jit_state_t *_jit, jit_int32_t Op,
987 jit_int32_t Rd, jit_int32_t Rn, jit_int32_t R, jit_int32_t S)
990 assert(!(Rd & ~0x1f));
991 assert(!(Rn & ~0x1f));
992 assert(!(R & ~0x3f));
993 assert(!(S & ~0x3f));
994 assert(!(Op & ~0xffc00000));
1004 _oxxxc(jit_state_t *_jit, jit_int32_t Op,
1005 jit_int32_t Rd, jit_int32_t Rn, jit_int32_t Rm, jit_int32_t Cc)
1008 assert(!(Rd & ~0x1f));
1009 assert(!(Rn & ~0x1f));
1010 assert(!(Rm & ~0x1f));
1011 assert(!(Cc & ~0xf));
1012 assert(!(Op & ~0xffc00c00));
1022 _oxxx7(jit_state_t *_jit, jit_int32_t Op,
1023 jit_int32_t Rt, jit_int32_t Rt2, jit_int32_t Rn, jit_int32_t Simm7)
1026 assert(!(Rt & ~0x1f));
1027 assert(!(Rt2 & ~0x1f));
1028 assert(!(Rn & ~0x1f));
1029 assert(Simm7 >= -128 && Simm7 <= 127);
1030 assert(!(Op & ~0xffc003e0));
1040 _nop(jit_state_t *_jit, jit_int32_t i0)
1042 for (; i0 > 0; i0 -= 4)
1048 _addi(jit_state_t *_jit, jit_int32_t r0, jit_int32_t r1, jit_word_t i0)
1051 jit_word_t is = i0 >> 12;
1052 jit_word_t in = -i0;
1053 jit_word_t iS = in >> 12;
1054 if ( i0 >= 0 && i0 <= 0xfff)
1056 else if ((is << 12) == i0 && is >= 0 && is <= 0xfff)
1057 ADDI_12(r0, r1, is);
1058 else if ( in >= 0 && in <= 0xfff)
1060 else if ((iS << 12) == is && iS >= 0 && iS <= 0xfff)
1061 SUBI_12(r0, r1, iS);
1063 reg = jit_get_reg(jit_class_gpr);
1065 addr(r0, r1, rn(reg));
1071 _addci(jit_state_t *_jit, jit_int32_t r0, jit_int32_t r1, jit_word_t i0)
1074 jit_word_t is = i0 >> 12;
1075 jit_word_t in = -i0;
1076 jit_word_t iS = in >> 12;
1077 if ( i0 >= 0 && i0 <= 0xfff)
1079 else if ((is << 12) == i0 && is >= 0 && is <= 0xfff)
1080 ADDSI_12(r0, r1, is);
1081 else if ( in >= 0 && in <= 0xfff)
1083 else if ((iS << 12) == is && iS >= 0 && iS <= 0xfff)
1084 SUBSI_12(r0, r1, iS);
1086 reg = jit_get_reg(jit_class_gpr);
1088 addcr(r0, r1, rn(reg));
1094 _addxi(jit_state_t *_jit, jit_int32_t r0, jit_int32_t r1, jit_word_t i0)
1097 reg = jit_get_reg(jit_class_gpr);
1099 addxr(r0, r1, rn(reg));
1104 _subi(jit_state_t *_jit, jit_int32_t r0, jit_int32_t r1, jit_word_t i0)
1107 jit_word_t is = i0 >> 12;
1108 if ( i0 >= 0 && i0 <= 0xfff)
1110 else if ((is << 12) == i0 && is >= 0 && is <= 0xfff)
1111 SUBI_12(r0, r1, is);
1113 reg = jit_get_reg(jit_class_gpr);
1115 subr(r0, r1, rn(reg));
1121 _subci(jit_state_t *_jit, jit_int32_t r0, jit_int32_t r1, jit_word_t i0)
1124 jit_word_t is = i0 >> 12;
1125 if ( i0 >= 0 && i0 <= 0xfff)
1127 else if ((is << 12) == i0 && is >= 0 && is <= 0xfff)
1128 SUBSI_12(r0, r1, is);
1130 reg = jit_get_reg(jit_class_gpr);
1132 subcr(r0, r1, rn(reg));
1138 _subxi(jit_state_t *_jit, jit_int32_t r0, jit_int32_t r1, jit_word_t i0)
1141 reg = jit_get_reg(jit_class_gpr);
1143 subxr(r0, r1, rn(reg));
1148 _rsbi(jit_state_t *_jit, jit_int32_t r0, jit_int32_t r1, jit_word_t i0)
1155 _muli(jit_state_t *_jit, jit_int32_t r0, jit_int32_t r1, jit_word_t i0)
1158 reg = jit_get_reg(jit_class_gpr);
1160 mulr(r0, r1, rn(reg));
1165 _qmulr(jit_state_t *_jit, jit_int32_t r0,
1166 jit_int32_t r1, jit_int32_t r2, jit_int32_t r3)
1169 if (r0 == r2 || r0 == r3) {
1170 reg = jit_get_reg(jit_class_gpr);
1171 mulr(rn(reg), r2, r3);
1176 if (r0 == r2 || r0 == r3) {
1183 _qmuli(jit_state_t *_jit, jit_int32_t r0,
1184 jit_int32_t r1, jit_int32_t r2, jit_word_t i0)
1187 reg = jit_get_reg(jit_class_gpr);
1189 qmulr(r0, r1, r2, rn(reg));
1194 _qmulr_u(jit_state_t *_jit, jit_int32_t r0,
1195 jit_int32_t r1, jit_int32_t r2, jit_int32_t r3)
1198 if (r0 == r2 || r0 == r3) {
1199 reg = jit_get_reg(jit_class_gpr);
1200 mulr(rn(reg), r2, r3);
1205 if (r0 == r2 || r0 == r3) {
1212 _qmuli_u(jit_state_t *_jit, jit_int32_t r0,
1213 jit_int32_t r1, jit_int32_t r2, jit_word_t i0)
1216 reg = jit_get_reg(jit_class_gpr);
1218 qmulr_u(r0, r1, r2, rn(reg));
1223 _divi(jit_state_t *_jit, jit_int32_t r0, jit_int32_t r1, jit_word_t i0)
1226 reg = jit_get_reg(jit_class_gpr);
1228 divr(r0, r1, rn(reg));
1233 _divi_u(jit_state_t *_jit, jit_int32_t r0, jit_int32_t r1, jit_word_t i0)
1236 reg = jit_get_reg(jit_class_gpr);
1238 divr_u(r0, r1, rn(reg));
1243 _iqdivr(jit_state_t *_jit, jit_bool_t sign,
1244 jit_int32_t r0, jit_int32_t r1, jit_int32_t r2, jit_int32_t r3)
1246 jit_int32_t sv0, rg0;
1247 jit_int32_t sv1, rg1;
1248 if (r0 == r2 || r0 == r3) {
1249 sv0 = jit_get_reg(jit_class_gpr);
1254 if (r1 == r2 || r1 == r3) {
1255 sv1 = jit_get_reg(jit_class_gpr);
1263 divr_u(rg0, r2, r3);
1277 _qdivi(jit_state_t *_jit, jit_int32_t r0,
1278 jit_int32_t r1, jit_int32_t r2, jit_word_t i0)
1281 reg = jit_get_reg(jit_class_gpr);
1283 qdivr(r0, r1, r2, rn(reg));
1288 _qdivi_u(jit_state_t *_jit, jit_int32_t r0,
1289 jit_int32_t r1, jit_int32_t r2, jit_word_t i0)
1292 reg = jit_get_reg(jit_class_gpr);
1294 qdivr_u(r0, r1, r2, rn(reg));
1299 _remr(jit_state_t *_jit, jit_int32_t r0, jit_int32_t r1, jit_int32_t r2)
1302 if (r0 == r1 || r0 == r2) {
1303 reg = jit_get_reg(jit_class_gpr);
1304 divr(rn(reg), r1, r2);
1305 mulr(rn(reg), r2, rn(reg));
1306 subr(r0, r1, rn(reg));
1317 _remi(jit_state_t *_jit, jit_int32_t r0, jit_int32_t r1, jit_word_t i0)
1320 reg = jit_get_reg(jit_class_gpr);
1322 remr(r0, r1, rn(reg));
1327 _remr_u(jit_state_t *_jit, jit_int32_t r0, jit_int32_t r1, jit_int32_t r2)
1330 if (r0 == r1 || r0 == r2) {
1331 reg = jit_get_reg(jit_class_gpr);
1332 divr_u(rn(reg), r1, r2);
1333 mulr(rn(reg), r2, rn(reg));
1334 subr(r0, r1, rn(reg));
1345 _remi_u(jit_state_t *_jit, jit_int32_t r0, jit_int32_t r1, jit_word_t i0)
1348 reg = jit_get_reg(jit_class_gpr);
1350 remr_u(r0, r1, rn(reg));
1355 _lshi(jit_state_t *_jit, jit_int32_t r0, jit_int32_t r1, jit_word_t i0)
1360 assert(i0 > 0 && i0 < 64);
1366 _rshi(jit_state_t *_jit, jit_int32_t r0, jit_int32_t r1, jit_word_t i0)
1371 assert(i0 > 0 && i0 < 64);
1377 _rshi_u(jit_state_t *_jit, jit_int32_t r0, jit_int32_t r1, jit_word_t i0)
1382 assert(i0 > 0 && i0 < 64);
1388 _movnr(jit_state_t *_jit, jit_int32_t r0, jit_int32_t r1, jit_int32_t r2)
1391 CSEL(r0, r0, r1, CC_NE);
1395 _movzr(jit_state_t *_jit, jit_int32_t r0, jit_int32_t r1, jit_int32_t r2)
1398 CSEL(r0, r0, r1, CC_EQ);
1402 _andi(jit_state_t *_jit, jit_int32_t r0, jit_int32_t r1, jit_word_t i0)
1411 imm = logical_immediate(i0);
1415 reg = jit_get_reg(jit_class_gpr);
1417 andr(r0, r1, rn(reg));
1424 _ori(jit_state_t *_jit, jit_int32_t r0, jit_int32_t r1, jit_word_t i0)
1433 imm = logical_immediate(i0);
1437 reg = jit_get_reg(jit_class_gpr);
1439 orr(r0, r1, rn(reg));
1446 _xori(jit_state_t *_jit, jit_int32_t r0, jit_int32_t r1, jit_word_t i0)
1455 imm = logical_immediate(i0);
1459 reg = jit_get_reg(jit_class_gpr);
1461 xorr(r0, r1, rn(reg));
1468 _bswapr_us(jit_state_t *_jit, jit_int32_t r0, jit_int32_t r1)
1475 _bswapr_ui(jit_state_t *_jit, jit_int32_t r0, jit_int32_t r1)
1482 _ldi_c(jit_state_t *_jit, jit_int32_t r0, jit_word_t i0)
1485 reg = jit_get_reg(jit_class_gpr);
1492 _ldr_uc(jit_state_t *_jit, jit_int32_t r0, jit_int32_t r1)
1501 _ldi_uc(jit_state_t *_jit, jit_int32_t r0, jit_word_t i0)
1504 reg = jit_get_reg(jit_class_gpr);
1506 ldr_uc(r0, rn(reg));
1511 _ldi_s(jit_state_t *_jit, jit_int32_t r0, jit_word_t i0)
1514 reg = jit_get_reg(jit_class_gpr);
1521 _ldr_us(jit_state_t *_jit, jit_int32_t r0, jit_int32_t r1)
1530 _ldi_us(jit_state_t *_jit, jit_int32_t r0, jit_word_t i0)
1533 reg = jit_get_reg(jit_class_gpr);
1535 ldr_us(r0, rn(reg));
1540 _ldi_i(jit_state_t *_jit, jit_int32_t r0, jit_word_t i0)
1543 reg = jit_get_reg(jit_class_gpr);
1550 _ldr_ui(jit_state_t *_jit, jit_int32_t r0, jit_int32_t r1)
1559 _ldi_ui(jit_state_t *_jit, jit_int32_t r0, jit_word_t i0)
1562 reg = jit_get_reg(jit_class_gpr);
1564 ldr_ui(r0, rn(reg));
1569 _ldi_l(jit_state_t *_jit, jit_int32_t r0, jit_word_t i0)
1572 reg = jit_get_reg(jit_class_gpr);
1579 _ldxr_c(jit_state_t *_jit, jit_int32_t r0, jit_int32_t r1, jit_int32_t r2)
1586 _ldxi_c(jit_state_t *_jit, jit_int32_t r0, jit_int32_t r1, jit_word_t i0)
1589 if (i0 >= 0 && i0 <= 4095)
1591 else if (i0 > -256 && i0 < 0)
1592 LDURSB(r0, r1, i0 & 0x1ff);
1594 reg = jit_get_reg(jit_class_gpr);
1596 LDRSB(r0, r1, rn(reg));
1603 _ldxr_uc(jit_state_t *_jit, jit_int32_t r0, jit_int32_t r1, jit_int32_t r2)
1612 _ldxi_uc(jit_state_t *_jit, jit_int32_t r0, jit_int32_t r1, jit_word_t i0)
1615 if (i0 >= 0 && i0 <= 4095)
1617 else if (i0 > -256 && i0 < 0)
1618 LDURB(r0, r1, i0 & 0x1ff);
1620 reg = jit_get_reg(jit_class_gpr);
1621 addi(rn(reg), r1, i0);
1622 ldr_uc(r0, rn(reg));
1631 _ldxi_s(jit_state_t *_jit, jit_int32_t r0, jit_int32_t r1, jit_word_t i0)
1634 if (i0 >= 0 && i0 <= 8191 && !(i0 & 1))
1635 LDRSHI(r0, r1, i0 >> 1);
1636 else if (i0 > -256 && i0 < 0)
1637 LDURSH(r0, r1, i0 & 0x1ff);
1639 reg = jit_get_reg(jit_class_gpr);
1641 LDRSH(r0, r1, rn(reg));
1647 _ldxr_us(jit_state_t *_jit, jit_int32_t r0, jit_int32_t r1, jit_int32_t r2)
1656 _ldxi_us(jit_state_t *_jit, jit_int32_t r0, jit_int32_t r1, jit_word_t i0)
1659 if (i0 >= 0 && i0 <= 8191 && !(i0 & 1))
1660 LDRHI(r0, r1, i0 >> 1);
1661 else if (i0 > -256 && i0 < 0)
1662 LDURH(r0, r1, i0 & 0x1ff);
1664 reg = jit_get_reg(jit_class_gpr);
1666 LDRH(r0, r1, rn(reg));
1675 _ldxi_i(jit_state_t *_jit, jit_int32_t r0, jit_int32_t r1, jit_word_t i0)
1678 if (i0 >= 0 && i0 <= 16383 && !(i0 & 3))
1679 LDRSWI(r0, r1, i0 >> 2);
1680 else if (i0 > -256 && i0 < 0)
1681 LDURSW(r0, r1, i0 & 0x1ff);
1683 reg = jit_get_reg(jit_class_gpr);
1684 addi(rn(reg), r1, i0);
1691 _ldxr_ui(jit_state_t *_jit, jit_int32_t r0, jit_int32_t r1, jit_int32_t r2)
1700 _ldxi_ui(jit_state_t *_jit, jit_int32_t r0, jit_int32_t r1, jit_word_t i0)
1703 if (i0 >= 0 && i0 <= 16383 && !(i0 & 3))
1704 LDRWI(r0, r1, i0 >> 2);
1705 else if (i0 > -256 && i0 < 0)
1706 LDURW(r0, r1, i0 & 0x1ff);
1708 reg = jit_get_reg(jit_class_gpr);
1710 LDRW(r0, r1, rn(reg));
1719 _ldxi_l(jit_state_t *_jit, jit_int32_t r0, jit_int32_t r1, jit_word_t i0)
1722 if (i0 >= 0 && i0 <= 32767 && !(i0 & 7))
1723 LDRI(r0, r1, i0 >> 3);
1724 else if (i0 > -256 && i0 < 0)
1725 LDUR(r0, r1, i0 & 0x1ff);
1727 reg = jit_get_reg(jit_class_gpr);
1728 addi(rn(reg), r1, i0);
1735 _sti_c(jit_state_t *_jit, jit_word_t i0, jit_int32_t r0)
1738 reg = jit_get_reg(jit_class_gpr);
1745 _sti_s(jit_state_t *_jit, jit_word_t i0, jit_int32_t r0)
1748 reg = jit_get_reg(jit_class_gpr);
1755 _sti_i(jit_state_t *_jit, jit_word_t i0, jit_int32_t r0)
1758 reg = jit_get_reg(jit_class_gpr);
1765 _sti_l(jit_state_t *_jit, jit_word_t i0, jit_int32_t r0)
1768 reg = jit_get_reg(jit_class_gpr);
1775 _stxi_c(jit_state_t *_jit, jit_word_t i0, jit_int32_t r0, jit_int32_t r1)
1778 if (i0 >= 0 && i0 <= 4095)
1780 else if (i0 > -256 && i0 < 0)
1781 STURB(r1, r0, i0 & 0x1ff);
1783 reg = jit_get_reg(jit_class_gpr);
1784 addi(rn(reg), r0, i0);
1791 _stxi_s(jit_state_t *_jit, jit_word_t i0, jit_int32_t r0, jit_int32_t r1)
1794 if (i0 >= 0 && i0 <= 8191 && !(i0 & 1))
1795 STRHI(r1, r0, i0 >> 1);
1796 else if (i0 > -256 && i0 < 0)
1797 STURH(r1, r0, i0 & 0x1ff);
1799 reg = jit_get_reg(jit_class_gpr);
1800 addi(rn(reg), r0, i0);
1807 _stxi_i(jit_state_t *_jit, jit_word_t i0, jit_int32_t r0, jit_int32_t r1)
1810 if (i0 >= 0 && i0 <= 16383 && !(i0 & 3))
1811 STRWI(r1, r0, i0 >> 2);
1812 else if (i0 > -256 && i0 < 0)
1813 STURW(r1, r0, i0 & 0x1ff);
1815 reg = jit_get_reg(jit_class_gpr);
1816 addi(rn(reg), r0, i0);
1823 _stxi_l(jit_state_t *_jit, jit_word_t i0, jit_int32_t r0, jit_int32_t r1)
1826 if (i0 >= 0 && i0 <= 32767 && !(i0 & 7))
1827 STRI(r1, r0, i0 >> 3);
1828 else if (i0 > -256 && i0 < 0)
1829 STUR(r1, r0, i0 & 0x1ff);
1831 reg = jit_get_reg(jit_class_gpr);
1832 addi(rn(reg), r0, i0);
1839 _casx(jit_state_t *_jit, jit_int32_t r0, jit_int32_t r1,
1840 jit_int32_t r2, jit_int32_t r3, jit_word_t i0)
1842 jit_int32_t r1_reg, iscasi;
1843 jit_word_t retry, done, jump0, jump1;
1844 if ((iscasi = (r1 == _NOREG))) {
1845 r1_reg = jit_get_reg(jit_class_gpr);
1853 jump0 = beqi(_jit->pc.w r0, 0); /* beqi done r0 0 */
1855 jump1 = bnei(_jit->pc.w, r0, 0); /* bnei retry r0 0 */
1859 patch_at(jump0, done);
1860 patch_at(jump1, retry);
1862 jit_unget_reg(r1_reg);
1866 _movr(jit_state_t *_jit, jit_int32_t r0, jit_int32_t r1)
1873 _movi(jit_state_t *_jit, jit_int32_t r0, jit_word_t i0)
1875 jit_word_t n0, ibit, nbit;
1878 if (i0 & 0x000000000000ffffL) ibit |= 1;
1879 if (i0 & 0x00000000ffff0000L) ibit |= 2;
1880 if (i0 & 0x0000ffff00000000L) ibit |= 4;
1881 if (i0 & 0xffff000000000000L) ibit |= 8;
1882 if (n0 & 0x000000000000ffffL) nbit |= 1;
1883 if (n0 & 0x00000000ffff0000L) nbit |= 2;
1884 if (n0 & 0x0000ffff00000000L) nbit |= 4;
1885 if (n0 & 0xffff000000000000L) nbit |= 8;
1891 MOVZ (r0, i0 & 0xffff);
1894 MOVZ_16(r0, (i0 >> 16) & 0xffff);
1897 MOVZ (r0, i0 & 0xffff);
1898 MOVK_16(r0, (i0 >> 16) & 0xffff);
1901 MOVZ_32(r0, (i0 >> 32) & 0xffff);
1904 MOVZ (r0, i0 & 0xffff);
1905 MOVK_32(r0, (i0 >> 32) & 0xffff);
1908 MOVZ_16(r0, (i0 >> 16) & 0xffff);
1909 MOVK_32(r0, (i0 >> 32) & 0xffff);
1913 MOVN_48(r0, (n0 >> 48) & 0xffff);
1915 MOVZ (r0, i0 & 0xffff);
1916 MOVK_16(r0, (i0 >> 16) & 0xffff);
1917 MOVK_32(r0, (i0 >> 32) & 0xffff);
1921 MOVZ_48(r0, (i0 >> 48) & 0xffff);
1924 MOVZ (r0, i0 & 0xffff);
1925 MOVK_48(r0, (i0 >> 48) & 0xffff);
1928 MOVZ_16(r0, (i0 >> 16) & 0xffff);
1929 MOVK_48(r0, (i0 >> 48) & 0xffff);
1933 MOVN_32(r0, (n0 >> 32) & 0xffff);
1935 MOVZ (r0, i0 & 0xffff);
1936 MOVK_16(r0, (i0 >> 16) & 0xffff);
1937 MOVK_48(r0, (i0 >> 48) & 0xffff);
1941 MOVZ_32(r0, (i0 >> 32) & 0xffff);
1942 MOVK_48(r0, (i0 >> 48) & 0xffff);
1946 MOVN_16(r0, (n0 >> 16) & 0xffff);
1948 MOVZ (r0, i0 & 0xffff);
1949 MOVK_32(r0, (i0 >> 32) & 0xffff);
1950 MOVK_48(r0, (i0 >> 48) & 0xffff);
1955 MOVN (r0, (n0) & 0xffff);
1957 MOVZ_16(r0, (i0 >> 16) & 0xffff);
1958 MOVK_32(r0, (i0 >> 32) & 0xffff);
1959 MOVK_48(r0, (i0 >> 48) & 0xffff);
1966 MOVN (r0, n0 & 0xffff);
1968 MOVN_48(r0, (n0 >> 48) & 0xffff);
1970 MOVZ (r0, i0 & 0xffff);
1971 MOVK_16(r0, (i0 >> 16) & 0xffff);
1972 MOVK_32(r0, (i0 >> 32) & 0xffff);
1973 MOVK_48(r0, (i0 >> 48) & 0xffff);
1982 _movi_p(jit_state_t *_jit, jit_int32_t r0, jit_word_t i0)
1986 MOVZ (r0, i0 & 0xffff);
1987 MOVK_16(r0, (i0 >> 16) & 0xffff);
1988 MOVK_32(r0, (i0 >> 32) & 0xffff);
1989 MOVK_48(r0, (i0 >> 48) & 0xffff);
1994 _ccr(jit_state_t *_jit, jit_int32_t cc,
1995 jit_int32_t r0, jit_int32_t r1, jit_int32_t r2)
2002 _cci(jit_state_t *_jit, jit_int32_t cc,
2003 jit_int32_t r0, jit_int32_t r1, jit_word_t i0)
2006 jit_word_t is = i0 >> 12;
2007 jit_word_t in = -i0;
2008 jit_word_t iS = in >> 12;
2009 if ( i0 >= 0 && i0 <= 0xfff)
2011 else if ((is << 12) == i0 && is >= 0 && is <= 0xfff)
2013 else if ( in >= 0 && in <= 0xfff)
2015 else if ((iS << 12) == is && iS >= 0 && iS <= 0xfff)
2018 reg = jit_get_reg(jit_class_gpr);
2027 _bccr(jit_state_t *_jit, jit_int32_t cc,
2028 jit_word_t i0, jit_int32_t r0, jit_int32_t r1)
2039 _bcci(jit_state_t *_jit, jit_int32_t cc,
2040 jit_word_t i0, jit_int32_t r0, jit_word_t i1)
2044 jit_word_t is = i1 >> 12;
2045 jit_word_t in = -i1;
2046 jit_word_t iS = in >> 12;
2047 if ( i1 >= 0 && i1 <= 0xfff)
2049 else if ((is << 12) == i1 && is >= 0 && is <= 0xfff)
2051 else if ( in >= 0 && in <= 0xfff)
2053 else if ((iS << 12) == is && iS >= 0 && iS <= 0xfff)
2056 reg = jit_get_reg(jit_class_gpr);
2068 _beqi(jit_state_t *_jit, jit_word_t i0, jit_int32_t r0, jit_word_t i1)
2073 CBZ(r0, (i0 - w) >> 2);
2076 w = bcci(BCC_EQ, i0, r0, i1);
2081 _bnei(jit_state_t *_jit, jit_word_t i0, jit_int32_t r0, jit_word_t i1)
2086 CBNZ(r0, (i0 - w) >> 2);
2089 w = bcci(BCC_NE, i0, r0, i1);
2094 _baddr(jit_state_t *_jit, jit_int32_t cc,
2095 jit_word_t i0, jit_int32_t r0, jit_int32_t r1)
2100 B_C(cc, (i0 - w) >> 2);
2105 _baddi(jit_state_t *_jit, jit_int32_t cc,
2106 jit_word_t i0, jit_int32_t r0, jit_word_t i1)
2111 B_C(cc, (i0 - w) >> 2);
2116 _bsubr(jit_state_t *_jit, jit_int32_t cc,
2117 jit_word_t i0, jit_int32_t r0, jit_int32_t r1)
2122 B_C(cc, (i0 - w) >> 2);
2127 _bsubi(jit_state_t *_jit, jit_int32_t cc,
2128 jit_word_t i0, jit_int32_t r0, jit_word_t i1)
2133 B_C(cc, (i0 - w) >> 2);
2138 _bmxr(jit_state_t *_jit, jit_int32_t cc,
2139 jit_word_t i0, jit_int32_t r0, jit_int32_t r1)
2144 B_C(cc, (i0 - w) >> 2);
2149 _bmxi(jit_state_t *_jit, jit_int32_t cc,
2150 jit_word_t i0, jit_int32_t r0, jit_word_t i1)
2155 imm = logical_immediate(i1);
2159 reg = jit_get_reg(jit_class_gpr);
2165 B_C(cc, (i0 - w) >> 2);
2170 _jmpi(jit_state_t *_jit, jit_word_t i0)
2174 w = (i0 - _jit->pc.w) >> 2;
2175 if (w >= -33554432 && w <= 33554431)
2178 reg = jit_get_reg(jit_class_gpr|jit_class_nospill);
2186 _jmpi_p(jit_state_t *_jit, jit_word_t i0)
2190 reg = jit_get_reg(jit_class_gpr|jit_class_nospill);
2191 w = movi_p(rn(reg), i0);
2198 _calli(jit_state_t *_jit, jit_word_t i0)
2202 w = (i0 - _jit->pc.w) >> 2;
2203 if (w >= -33554432 && w <= 33554431)
2206 reg = jit_get_reg(jit_class_gpr);
2214 _calli_p(jit_state_t *_jit, jit_word_t i0)
2218 reg = jit_get_reg(jit_class_gpr);
2219 w = movi_p(rn(reg), i0);
2226 * prolog and epilog not as "optimized" as one would like, but the
2227 * problem of overallocating stack space to save callee save registers
2228 * exists on all ports, and is still a todo to use a variable
2230 * value, what would cause needing to patch some calls, most likely
2231 * the offset of jit_arg* of stack arguments.
2234 _prolog(jit_state_t *_jit, jit_node_t *node)
2237 if (_jitc->function->define_frame || _jitc->function->assume_frame) {
2238 jit_int32_t frame = -_jitc->function->frame;
2239 assert(_jitc->function->self.aoff >= frame);
2240 if (_jitc->function->assume_frame)
2242 _jitc->function->self.aoff = frame;
2244 if (_jitc->function->allocar)
2245 _jitc->function->self.aoff &= -16;
2246 _jitc->function->stack = ((_jitc->function->self.alen -
2247 /* align stack at 16 bytes */
2248 _jitc->function->self.aoff) + 15) & -16;
2249 STPI_POS(FP_REGNO, LR_REGNO, SP_REGNO, -(stack_framesize >> 3));
2250 MOV_XSP(FP_REGNO, SP_REGNO);
2251 #define SPILL(L, R, O) \
2253 if (jit_regset_tstbit(&_jitc->function->regset, _R##L)) { \
2254 if (jit_regset_tstbit(&_jitc->function->regset, _R##R)) \
2255 STPI(L, R, SP_REGNO, O); \
2257 STRI(L, SP_REGNO, O); \
2259 else if (jit_regset_tstbit(&_jitc->function->regset, _R##R)) \
2260 STRI(R, SP_REGNO, O + 1); \
2268 #define SPILL(R, O) \
2270 if (jit_regset_tstbit(&_jitc->function->regset, _V##R)) \
2271 stxi_d(O, SP_REGNO, R); \
2282 if (_jitc->function->stack)
2283 subi(SP_REGNO, SP_REGNO, _jitc->function->stack);
2284 if (_jitc->function->allocar) {
2285 reg = jit_get_reg(jit_class_gpr);
2286 movi(rn(reg), _jitc->function->self.aoff);
2287 stxi_i(_jitc->function->aoffoff, FP_REGNO, rn(reg));
2291 if (_jitc->function->self.call & jit_call_varargs) {
2292 /* Save gp registers in the save area, if any is a vararg */
2293 for (reg = 8 - _jitc->function->vagp / -8;
2294 jit_arg_reg_p(reg); ++reg)
2295 stxi(_jitc->function->vaoff + offsetof(jit_va_list_t, x0) +
2296 reg * 8, FP_REGNO, rn(JIT_RA0 - reg));
2298 for (reg = 8 - _jitc->function->vafp / -16;
2299 jit_arg_f_reg_p(reg); ++reg)
2300 /* Save fp registers in the save area, if any is a vararg */
2301 /* Note that the full 16 byte register is not saved, because
2302 * lightning only handles float and double, and, while
2303 * attempting to provide a va_list compatible pointer as
2304 * jit_va_start return, does not guarantee it (on all ports). */
2305 stxi_d(_jitc->function->vaoff + offsetof(jit_va_list_t, q0) +
2306 reg * 16 + offsetof(jit_qreg_t, l), FP_REGNO, rn(_V0 - reg));
2311 _epilog(jit_state_t *_jit, jit_node_t *node)
2313 if (_jitc->function->assume_frame)
2315 if (_jitc->function->stack)
2316 MOV_XSP(SP_REGNO, FP_REGNO);
2317 #define LOAD(L, R, O) \
2319 if (jit_regset_tstbit(&_jitc->function->regset, _R##L)) { \
2320 if (jit_regset_tstbit(&_jitc->function->regset, _R##R)) \
2321 LDPI(L, R, SP_REGNO, O); \
2323 LDRI(L, SP_REGNO, O); \
2325 else if (jit_regset_tstbit(&_jitc->function->regset, _R##R)) \
2326 LDRI(R, SP_REGNO, O + 1); \
2334 #define LOAD(R, O) \
2336 if (jit_regset_tstbit(&_jitc->function->regset, _V##R)) \
2337 ldxi_d(R, SP_REGNO, O); \
2348 LDPI_PRE(FP_REGNO, LR_REGNO, SP_REGNO, stack_framesize >> 3);
2353 _vastart(jit_state_t *_jit, jit_int32_t r0)
2357 assert(_jitc->function->self.call & jit_call_varargs);
2359 /* Return jit_va_list_t in the register argument */
2360 addi(r0, FP_REGNO, _jitc->function->vaoff);
2362 reg = jit_get_reg(jit_class_gpr);
2364 /* Initialize stack pointer to the first stack argument. */
2365 addi(rn(reg), FP_REGNO, _jitc->function->self.size);
2366 stxi(offsetof(jit_va_list_t, stack), r0, rn(reg));
2368 /* Initialize gp top pointer to the first stack argument. */
2369 addi(rn(reg), r0, va_gp_top_offset);
2370 stxi(offsetof(jit_va_list_t, gptop), r0, rn(reg));
2372 /* Initialize fp top pointer to the first stack argument. */
2373 addi(rn(reg), r0, va_fp_top_offset);
2374 stxi(offsetof(jit_va_list_t, fptop), r0, rn(reg));
2376 /* Initialize gp offset in the save area. */
2377 movi(rn(reg), _jitc->function->vagp);
2378 stxi_i(offsetof(jit_va_list_t, gpoff), r0, rn(reg));
2380 /* Initialize fp offset in the save area. */
2381 movi(rn(reg), _jitc->function->vafp);
2382 stxi_i(offsetof(jit_va_list_t, fpoff), r0, rn(reg));
2388 _vaarg(jit_state_t *_jit, jit_int32_t r0, jit_int32_t r1)
2392 jit_int32_t rg0, rg1;
2394 assert(_jitc->function->self.call & jit_call_varargs);
2396 rg0 = jit_get_reg(jit_class_gpr);
2397 rg1 = jit_get_reg(jit_class_gpr);
2399 /* Load the gp offset in save area in the first temporary. */
2400 ldxi_i(rn(rg0), r1, offsetof(jit_va_list_t, gpoff));
2402 /* Jump over if there are no remaining arguments in the save area. */
2403 ge_code = bgei(_jit->pc.w, rn(rg0), 0);
2405 /* Load the gp save pointer in the second temporary. */
2406 ldxi(rn(rg1), r1, offsetof(jit_va_list_t, gptop));
2408 /* Load the vararg argument in the first argument. */
2409 ldxr(r0, rn(rg1), rn(rg0));
2411 /* Update the gp offset. */
2412 addi(rn(rg0), rn(rg0), 8);
2413 stxi_i(offsetof(jit_va_list_t, gpoff), r1, rn(rg0));
2415 /* Will only need one temporary register below. */
2418 /* Jump over overflow code. */
2419 lt_code = jmpi_p(_jit->pc.w);
2421 /* Where to land if argument is in overflow area. */
2422 patch_at(ge_code, _jit->pc.w);
2424 /* Load stack pointer. */
2425 ldxi(rn(rg0), r1, offsetof(jit_va_list_t, stack));
2427 /* Load argument. */
2430 /* Update stack pointer. */
2431 addi(rn(rg0), rn(rg0), 8);
2432 stxi(offsetof(jit_va_list_t, stack), r1, rn(rg0));
2434 /* Where to land if argument is in gp save area. */
2435 patch_at(lt_code, _jit->pc.w);
2441 _patch_at(jit_state_t *_jit, jit_word_t instr, jit_word_t label)
2445 jit_int32_t fc, ff, ffc;
2452 fc = i.w & 0xfc000000;
2453 ff = i.w & 0xff000000;
2454 ffc = i.w & 0xffc00000;
2455 if (fc == A64_B || fc == A64_BL) {
2456 d = (label - instr) >> 2;
2457 assert(d >= -33554432 && d <= 33554431);
2461 else if (ff == A64_B_C || ff == (A64_CBZ|XS) || ff == (A64_CBNZ|XS)) {
2462 d = (label - instr) >> 2;
2463 assert(d >= -262148 && d <= 262143);
2467 else if (ffc == (A64_MOVZ|XS)) {
2471 assert((i.w & 0xffe00000) == (A64_MOVK|XS|MOVI_LSL_16));
2472 i.imm16.b = label >> 16;
2475 assert((i.w & 0xffe00000) == (A64_MOVK|XS|MOVI_LSL_32));
2476 i.imm16.b = label >> 32;
2479 assert((i.w & 0xffe00000) == (A64_MOVK|XS|MOVI_LSL_48));
2480 i.imm16.b = label >> 48;