2 * Copyright (C) 2012-2022 Free Software Foundation, Inc.
4 * This file is part of GNU lightning.
6 * GNU lightning is free software; you can redistribute it and/or modify it
7 * under the terms of the GNU Lesser General Public License as published
8 * by the Free Software Foundation; either version 3, or (at your option)
11 * GNU lightning is distributed in the hope that it will be useful, but
12 * WITHOUT ANY WARRANTY; without even the implied warranty of MERCHANTABILITY
13 * or FITNESS FOR A PARTICULAR PURPOSE. See the GNU Lesser General Public
14 * License for more details.
17 * Paulo Cesar Pereira de Andrade
20 /* avoid using it due to partial stalls */
24 # if __X32 || __X64_32
26 # define ldi(u, v) ldi_i(u, v)
27 # define ldr(u, v) ldr_i(u, v)
28 # define ldxr(u, v, w) ldxr_i(u, v, w)
29 # define ldxi(u, v, w) ldxi_i(u, v, w)
30 # define sti(u, v) sti_i(u, v)
31 # define stxi(u, v, w) stxi_i(u, v, w)
32 # define can_sign_extend_int_p(im) 1
33 # define can_zero_extend_int_p(im) 1
34 # define fits_uint32_p(im) 1
37 # define ldi(u, v) ldi_l(u, v)
38 # define ldr(u, v) ldr_l(u, v)
39 # define ldxr(u, v, w) ldxr_l(u, v, w)
40 # define ldxi(u, v, w) ldxi_l(u, v, w)
41 # define sti(u, v) sti_l(u, v)
42 # define stxi(u, v, w) stxi_l(u, v, w)
43 # define can_sign_extend_int_p(im) \
44 (((im) >= 0 && (long long)(im) <= 0x7fffffffLL) || \
45 ((im) < 0 && (long long)(im) > -0x80000000LL))
46 # define can_zero_extend_int_p(im) \
47 ((im) >= 0 && (im) < 0x80000000LL)
48 # define fits_uint32_p(im) (((im) & 0xffffffff00000000LL) == 0)
50 # if __X32 || __CYGWIN__ || __X64_32 || _WIN32
52 ((rn) >= _RAX_REGNO && (rn) <= _RBX_REGNO)
66 # define _R10_REGNO 10
67 # define _R11_REGNO 11
68 # define _R12_REGNO 12
69 # define _R13_REGNO 13
70 # define _R14_REGNO 14
71 # define _R15_REGNO 15
72 # define r7(reg) ((reg) & 7)
73 # define r8(reg) ((reg) & 15)
79 # define X86_OR 1 << 3
80 # define X86_ADC 2 << 3
81 # define X86_SBB 3 << 3
82 # define X86_AND 4 << 3
83 # define X86_SUB 5 << 3
84 # define X86_XOR 6 << 3
85 # define X86_CMP 7 << 3
100 # define X86_CC_NO 0x1
101 # define X86_CC_NAE 0x2
102 # define X86_CC_B 0x2
103 # define X86_CC_C 0x2
104 # define X86_CC_AE 0x3
105 # define X86_CC_NB 0x3
106 # define X86_CC_NC 0x3
107 # define X86_CC_E 0x4
108 # define X86_CC_Z 0x4
109 # define X86_CC_NE 0x5
110 # define X86_CC_NZ 0x5
111 # define X86_CC_BE 0x6
112 # define X86_CC_NA 0x6
113 # define X86_CC_A 0x7
114 # define X86_CC_NBE 0x7
115 # define X86_CC_S 0x8
116 # define X86_CC_NS 0x9
117 # define X86_CC_P 0xa
118 # define X86_CC_PE 0xa
119 # define X86_CC_NP 0xb
120 # define X86_CC_PO 0xb
121 # define X86_CC_L 0xc
122 # define X86_CC_NGE 0xc
123 # define X86_CC_GE 0xd
124 # define X86_CC_NL 0xd
125 # define X86_CC_LE 0xe
126 # define X86_CC_NG 0xe
127 # define X86_CC_G 0xf
128 # define X86_CC_NLE 0xf
129 # define mrm(md, r, m) *_jit->pc.uc++ = (md<<6) | (r<<3) | m
130 # define sib(sc, i, b) *_jit->pc.uc++ = (sc<<6) | (i<<3) | b
131 # define ic(c) *_jit->pc.uc++ = c
132 # define is(s) *_jit->pc.us++ = s
133 # define ii(i) *_jit->pc.ui++ = i
134 # if __X64 && !__X64_32
135 # define il(l) *_jit->pc.ul++ = l
139 # define patch_abs(instr, label) \
140 *(jit_word_t *)(instr - sizeof(jit_word_t)) = label
141 # define patch_rel(instr, label) \
142 *(jit_int32_t *)(instr - 4) = label - instr
143 # define patch_rel_char(instr, label) \
144 *(jit_int8_t *)(instr - 1) = label - instr
145 # define rex(l, w, r, x, b) _rex(_jit, l, w, r, x, b)
147 _rex(jit_state_t*,jit_int32_t,jit_int32_t,jit_int32_t,jit_int32_t,jit_int32_t);
148 # define rx(rd, md, rb, ri, ms) _rx(_jit, rd, md, rb, ri, ms)
150 _rx(jit_state_t*,jit_int32_t,jit_int32_t,jit_int32_t,jit_int32_t,jit_int32_t);
151 # define nop(n) _nop(_jit, n)
152 static void _nop(jit_state_t*, jit_int32_t);
153 # define emms() is(0x770f)
154 # define lea(md, rb, ri, ms, rd) _lea(_jit, md, rb, ri, ms, rd)
156 _lea(jit_state_t*,jit_int32_t,jit_int32_t,jit_int32_t,jit_int32_t,jit_int32_t);
157 # define pushr(r0) _pushr(_jit, r0)
158 static void _pushr(jit_state_t*, jit_int32_t) maybe_unused;
159 # define popr(r0) _popr(_jit, r0)
160 static void _popr(jit_state_t*, jit_int32_t) maybe_unused;
161 # define xchgr(r0, r1) _xchgr(_jit, r0, r1)
162 static void _xchgr(jit_state_t*, jit_int32_t, jit_int32_t);
163 # define testr(r0, r1) _testr(_jit, r0, r1)
164 static void _testr(jit_state_t*, jit_int32_t, jit_int32_t);
165 # define testi(r0, i0) _testi(_jit, r0, i0)
166 static void _testi(jit_state_t*, jit_int32_t, jit_word_t);
167 # define cc(code, r0) _cc(_jit, code, r0)
168 static void _cc(jit_state_t*, jit_int32_t, jit_int32_t);
169 # define icmpr(r0, r1) alur(X86_CMP, r0, r1)
170 # define alur(code, r0, r1) _alur(_jit, code, r0, r1)
171 static void _alur(jit_state_t*, jit_int32_t, jit_int32_t, jit_int32_t);
172 # define icmpi(r0, i0) alui(X86_CMP, r0, i0)
173 # define alui(code, r0, i0) _alui(_jit, code, r0, i0)
174 static void _alui(jit_state_t*, jit_int32_t, jit_int32_t, jit_word_t);
175 # define iaddr(r0, r1) alur(X86_ADD, r0, r1)
176 # define save(r0) _save(_jit, r0)
177 static void _save(jit_state_t*, jit_int32_t);
178 # define load(r0) _load(_jit, r0)
179 static void _load(jit_state_t*, jit_int32_t);
180 # define addr(r0, r1, r2) _addr(_jit, r0, r1, r2)
181 static void _addr(jit_state_t*, jit_int32_t, jit_int32_t, jit_int32_t);
182 # define iaddi(r0, i0) alui(X86_ADD, r0, i0)
183 # define addi(r0, r1, i0) _addi(_jit, r0, r1, i0)
184 static void _addi(jit_state_t*, jit_int32_t, jit_int32_t, jit_word_t);
185 #define addcr(r0, r1, r2) _addcr(_jit, r0, r1, r2)
186 static void _addcr(jit_state_t*, jit_int32_t, jit_int32_t, jit_int32_t);
187 #define addci(r0, r1, i0) _addci(_jit, r0, r1, i0)
188 static void _addci(jit_state_t*, jit_int32_t, jit_int32_t, jit_word_t);
189 # define iaddxr(r0, r1) alur(X86_ADC, r0, r1)
190 # define addxr(r0, r1, r2) _addxr(_jit, r0, r1, r2)
191 static void _addxr(jit_state_t*, jit_int32_t, jit_int32_t, jit_int32_t);
192 # define iaddxi(r0, i0) alui(X86_ADC, r0, i0)
193 # define addxi(r0, r1, i0) _addxi(_jit, r0, r1, i0)
194 static void _addxi(jit_state_t*, jit_int32_t, jit_int32_t, jit_word_t);
195 # define isubr(r0, r1) alur(X86_SUB, r0, r1)
196 # define subr(r0, r1, r2) _subr(_jit, r0, r1, r2)
197 static void _subr(jit_state_t*, jit_int32_t, jit_int32_t, jit_int32_t);
198 # define isubi(r0, i0) alui(X86_SUB, r0, i0)
199 # define subi(r0, r1, i0) _subi(_jit, r0, r1, i0)
200 static void _subi(jit_state_t*, jit_int32_t, jit_int32_t, jit_word_t);
201 # define subcr(r0, r1, r2) _subcr(_jit, r0, r1, r2)
202 static void _subcr(jit_state_t*,jit_int32_t,jit_int32_t,jit_int32_t);
203 # define subci(r0, r1, i0) _subci(_jit, r0, r1, i0)
204 static void _subci(jit_state_t*,jit_int32_t,jit_int32_t,jit_word_t);
205 # define isubxr(r0, r1) alur(X86_SBB, r0, r1)
206 # define subxr(r0, r1, r2) _subxr(_jit, r0, r1, r2)
207 static void _subxr(jit_state_t*,jit_int32_t,jit_int32_t,jit_int32_t);
208 # define isubxi(r0, i0) alui(X86_SBB, r0, i0)
209 # define subxi(r0, r1, i0) _subxi(_jit, r0, r1, i0)
210 static void _subxi(jit_state_t*,jit_int32_t,jit_int32_t,jit_word_t);
211 # define rsbi(r0, r1, i0) _rsbi(_jit, r0, r1, i0)
212 static void _rsbi(jit_state_t*,jit_int32_t,jit_int32_t,jit_word_t);
213 # define imulr(r0, r1) _imulr(_jit, r0, r1)
214 static void _imulr(jit_state_t*, jit_int32_t, jit_int32_t);
215 # define imuli(r0, r1, i0) _imuli(_jit, r0, r1, i0)
216 static void _imuli(jit_state_t*, jit_int32_t, jit_int32_t, jit_word_t);
217 # define mulr(r0, r1, r2) _mulr(_jit, r0, r1, r2)
218 static void _mulr(jit_state_t*, jit_int32_t, jit_int32_t, jit_int32_t);
219 # define muli(r0, r1, i0) _muli(_jit, r0, r1, i0)
220 static void _muli(jit_state_t*, jit_int32_t, jit_int32_t, jit_word_t);
221 # define umulr(r0) unr(X86_IMUL, r0)
222 # define umulr_u(r0) unr(X86_MUL, r0)
223 # define qmulr(r0, r1, r2, r3) _iqmulr(_jit, r0, r1, r2, r3, 1)
224 # define qmulr_u(r0, r1, r2, r3) _iqmulr(_jit, r0, r1, r2, r3, 0)
225 # define iqmulr(r0, r1, r2, r3, sign) _iqmulr(_jit, r0, r1, r2, r3, sign)
226 static void _iqmulr(jit_state_t*, jit_int32_t, jit_int32_t,
227 jit_int32_t,jit_int32_t, jit_bool_t);
228 # define qmuli(r0, r1, r2, i0) _iqmuli(_jit, r0, r1, r2, i0, 1)
229 # define qmuli_u(r0, r1, r2, i0) _iqmuli(_jit, r0, r1, r2, i0, 0)
230 # define iqmuli(r0, r1, r2, i0, sign) _iqmuli(_jit, r0, r1, r2, i0, sign)
231 static void _iqmuli(jit_state_t*, jit_int32_t, jit_int32_t,
232 jit_int32_t,jit_word_t, jit_bool_t);
233 # define sign_extend_rdx_rax() _sign_extend_rdx_rax(_jit)
234 static void _sign_extend_rdx_rax(jit_state_t*);
235 # define idivr(r0) unr(X86_IDIV, r0)
236 # define idivr_u(r0) unr(X86_DIV, r0)
237 # define divremr(r0, r1, r2, i0, i1) _divremr(_jit, r0, r1, r2, i0, i1)
239 _divremr(jit_state_t*,jit_int32_t,jit_int32_t,jit_int32_t,
240 jit_bool_t,jit_bool_t);
241 # define divremi(r0, r1, i0, i1, i2) _divremi(_jit, r0, r1, i0, i1, i2)
243 _divremi(jit_state_t*,jit_int32_t,jit_int32_t,jit_word_t,jit_bool_t,jit_bool_t);
244 # define divr(r0, r1, r2) divremr(r0, r1, r2, 1, 1)
245 # define divi(r0, r1, i0) divremi(r0, r1, i0, 1, 1)
246 # define divr_u(r0, r1, r2) divremr(r0, r1, r2, 0, 1)
247 # define divi_u(r0, r1, i0) divremi(r0, r1, i0, 0, 1)
248 # define qdivr(r0, r1, r2, r3) _iqdivr(_jit, r0, r1, r2, r3, 1)
249 # define qdivr_u(r0, r1, r2, r3) _iqdivr(_jit, r0, r1, r2, r3, 0)
250 # define iqdivr(r0, r1, r2, r3, sign) _iqdivr(_jit, r0, r1, r2, r3, sign)
251 static void _iqdivr(jit_state_t*, jit_int32_t, jit_int32_t,
252 jit_int32_t,jit_int32_t, jit_bool_t);
253 # define qdivi(r0, r1, r2, i0) _iqdivi(_jit, r0, r1, r2, i0, 1)
254 # define qdivi_u(r0, r1, r2, i0) _iqdivi(_jit, r0, r1, r2, i0, 0)
255 # define iqdivi(r0, r1, r2, i0, sign) _iqdivi(_jit, r0, r1, r2, i0, sign)
256 static void _iqdivi(jit_state_t*, jit_int32_t, jit_int32_t,
257 jit_int32_t,jit_word_t, jit_bool_t);
258 # define remr(r0, r1, r2) divremr(r0, r1, r2, 1, 0)
259 # define remi(r0, r1, i0) divremi(r0, r1, i0, 1, 0)
260 # define remr_u(r0, r1, r2) divremr(r0, r1, r2, 0, 0)
261 # define remi_u(r0, r1, i0) divremi(r0, r1, i0, 0, 0)
262 # define iandr(r0, r1) alur(X86_AND, r0, r1)
263 # define andr(r0, r1, r2) _andr(_jit, r0, r1, r2)
264 static void _andr(jit_state_t*,jit_int32_t,jit_int32_t,jit_int32_t);
265 # define iandi(r0, i0) alui(X86_AND, r0, i0)
266 # define andi(r0, r1, i0) _andi(_jit, r0, r1, i0)
267 static void _andi(jit_state_t*, jit_int32_t,jit_int32_t,jit_word_t);
268 # define iorr(r0, r1) alur(X86_OR, r0, r1)
269 # define orr(r0, r1, r2) _orr(_jit, r0, r1, r2)
270 static void _orr(jit_state_t*, jit_int32_t,jit_int32_t,jit_int32_t);
271 # define iori(r0, i0) alui(X86_OR, r0, i0)
272 # define ori(r0, r1, i0) _ori(_jit, r0, r1, i0)
273 static void _ori(jit_state_t*, jit_int32_t,jit_int32_t,jit_word_t);
274 # define ixorr(r0, r1) alur(X86_XOR, r0, r1)
275 # define xorr(r0, r1, r2) _xorr(_jit, r0, r1, r2)
276 static void _xorr(jit_state_t*, jit_int32_t,jit_int32_t,jit_int32_t);
277 # define ixori(r0, i0) alui(X86_XOR, r0, i0)
278 # define xori(r0, r1, i0) _xori(_jit, r0, r1, i0)
279 static void _xori(jit_state_t*, jit_int32_t,jit_int32_t,jit_word_t);
280 # define irotshr(code, r0) _irotshr(_jit, code, r0)
281 static void _irotshr(jit_state_t*, jit_int32_t, jit_int32_t);
282 # define rotshr(code, r0, r1, r2) _rotshr(_jit, code, r0, r1, r2)
284 _rotshr(jit_state_t*,jit_int32_t,jit_int32_t,jit_int32_t,jit_int32_t);
285 # define irotshi(code, r0, i0) _irotshi(_jit, code, r0, i0)
286 static void _irotshi(jit_state_t*, jit_int32_t, jit_int32_t, jit_word_t);
287 # define rotshi(code, r0, r1, i0) _rotshi(_jit, code, r0, r1, i0)
289 _rotshi(jit_state_t*,jit_int32_t,jit_int32_t,jit_int32_t,jit_word_t);
290 # define lshr(r0, r1, r2) rotshr(X86_SHL, r0, r1, r2)
291 # define lshi(r0, r1, i0) _lshi(_jit, r0, r1, i0)
292 static void _lshi(jit_state_t*, jit_int32_t, jit_int32_t, jit_word_t);
293 # define rshr(r0, r1, r2) rotshr(X86_SAR, r0, r1, r2)
294 # define rshi(r0, r1, i0) rotshi(X86_SAR, r0, r1, i0)
295 # define rshr_u(r0, r1, r2) rotshr(X86_SHR, r0, r1, r2)
296 # define rshi_u(r0, r1, i0) rotshi(X86_SHR, r0, r1, i0)
297 # define unr(code, r0) _unr(_jit, code, r0)
298 static void _unr(jit_state_t*, jit_int32_t, jit_int32_t);
299 # define inegr(r0) unr(X86_NEG, r0)
300 # define negr(r0, r1) _negr(_jit, r0, r1)
301 static void _negr(jit_state_t*, jit_int32_t, jit_int32_t);
302 # define icomr(r0) unr(X86_NOT, r0)
303 # define comr(r0, r1) _comr(_jit, r0, r1)
304 static void _comr(jit_state_t*, jit_int32_t, jit_int32_t);
306 # define incr(r0, r1) _incr(_jit, r0, r1)
307 static void _incr(jit_state_t*, jit_int32_t, jit_int32_t);
308 # define decr(r0, r1) _decr(_jit, r0, r1)
309 static void _decr(jit_state_t*, jit_int32_t, jit_int32_t);
311 # define cr(code, r0, r1, r2) _cr(_jit, code, r0, r1, r2)
313 _cr(jit_state_t*, jit_int32_t, jit_int32_t, jit_int32_t, jit_int32_t);
314 # define ci(code, r0, r1, i0) _ci(_jit, code, r0, r1, i0)
316 _ci(jit_state_t *_jit, jit_int32_t, jit_int32_t, jit_int32_t, jit_word_t);
317 # define ci0(code, r0, r1) _ci0(_jit, code, r0, r1)
318 static void _ci0(jit_state_t*, jit_int32_t, jit_int32_t, jit_int32_t);
319 # define ltr(r0, r1, r2) _ltr(_jit, r0, r1, r2)
320 static void _ltr(jit_state_t*, jit_int32_t, jit_int32_t, jit_int32_t);
321 # define lti(r0, r1, i0) _lti(_jit, r0, r1, i0)
322 static void _lti(jit_state_t*, jit_int32_t, jit_int32_t, jit_word_t);
323 # define ltr_u(r0, r1, r2) _ltr_u(_jit, r0, r1, r2)
324 static void _ltr_u(jit_state_t*, jit_int32_t, jit_int32_t, jit_int32_t);
325 # define lti_u(r0, r1, i0) ci(X86_CC_B, r0, r1, i0)
326 # define ler(r0, r1, r2) _ler(_jit, r0, r1, r2)
327 static void _ler(jit_state_t*, jit_int32_t, jit_int32_t, jit_int32_t);
328 # define lei(r0, r1, i0) ci(X86_CC_LE, r0, r1, i0)
329 # define ler_u(r0, r1, r2) _ler_u(_jit, r0, r1, r2)
330 static void _ler_u(jit_state_t*, jit_int32_t, jit_int32_t, jit_int32_t);
331 # define lei_u(r0, r1, i0) _lei_u(_jit, r0, r1, i0)
332 static void _lei_u(jit_state_t*, jit_int32_t, jit_int32_t, jit_word_t);
333 # define eqr(r0, r1, r2) _eqr(_jit, r0, r1, r2)
334 static void _eqr(jit_state_t*, jit_int32_t, jit_int32_t, jit_int32_t);
335 # define eqi(r0, r1, i0) _eqi(_jit, r0, r1, i0)
336 static void _eqi(jit_state_t*, jit_int32_t, jit_int32_t, jit_word_t);
337 # define ger(r0, r1, r2) _ger(_jit, r0, r1, r2)
338 static void _ger(jit_state_t*, jit_int32_t, jit_int32_t, jit_int32_t);
339 # define gei(r0, r1, i0) _gei(_jit, r0, r1, i0)
340 static void _gei(jit_state_t*, jit_int32_t, jit_int32_t, jit_word_t);
341 # define ger_u(r0, r1, r2) _ger_u(_jit, r0, r1, r2)
342 static void _ger_u(jit_state_t*, jit_int32_t, jit_int32_t, jit_int32_t);
343 # define gei_u(r0, r1, i0) _gei_u(_jit, r0, r1, i0)
344 static void _gei_u(jit_state_t*, jit_int32_t, jit_int32_t, jit_word_t);
345 # define gtr(r0, r1, r2) _gtr(_jit, r0, r1, r2)
346 static void _gtr(jit_state_t*, jit_int32_t, jit_int32_t, jit_int32_t);
347 # define gti(r0, r1, i0) _ci(_jit, X86_CC_G, r0, r1, i0)
348 # define gtr_u(r0, r1, r2) _gtr_u(_jit, r0, r1, r2)
349 static void _gtr_u(jit_state_t*, jit_int32_t, jit_int32_t, jit_int32_t);
350 # define gti_u(r0, r1, i0) _gti_u(_jit, r0, r1, i0)
351 static void _gti_u(jit_state_t*, jit_int32_t, jit_int32_t, jit_word_t);
352 # define ner(r0, r1, r2) _ner(_jit, r0, r1, r2)
353 static void _ner(jit_state_t*, jit_int32_t, jit_int32_t, jit_int32_t);
354 # define nei(r0, r1, i0) _nei(_jit, r0, r1, i0)
355 static void _nei(jit_state_t*, jit_int32_t, jit_int32_t, jit_word_t);
356 # define movr(r0, r1) _movr(_jit, r0, r1)
357 static void _movr(jit_state_t*, jit_int32_t, jit_int32_t);
358 # define imovi(r0, i0) _imovi(_jit, r0, i0)
359 static void _imovi(jit_state_t*, jit_int32_t, jit_word_t);
360 # define movi(r0, i0) _movi(_jit, r0, i0)
361 static void _movi(jit_state_t*, jit_int32_t, jit_word_t);
362 # define movi_p(r0, i0) _movi_p(_jit, r0, i0)
363 static jit_word_t _movi_p(jit_state_t*, jit_int32_t, jit_word_t);
364 # define movcr(r0, r1) _movcr(_jit, r0, r1)
365 static void _movcr(jit_state_t*,jit_int32_t,jit_int32_t);
366 # define movcr_u(r0, r1) _movcr_u(_jit, r0, r1)
367 static void _movcr_u(jit_state_t*,jit_int32_t,jit_int32_t);
368 # define movsr(r0, r1) _movsr(_jit, r0, r1)
369 static void _movsr(jit_state_t*,jit_int32_t,jit_int32_t);
370 # define movsr_u(r0, r1) _movsr_u(_jit, r0, r1)
371 static void _movsr_u(jit_state_t*,jit_int32_t,jit_int32_t);
372 # define casx(r0, r1, r2, r3, i0) _casx(_jit, r0, r1, r2, r3, i0)
373 static void _casx(jit_state_t *_jit,jit_int32_t,jit_int32_t,
374 jit_int32_t,jit_int32_t,jit_word_t);
375 #define casr(r0, r1, r2, r3) casx(r0, r1, r2, r3, 0)
376 #define casi(r0, i0, r1, r2) casx(r0, _NOREG, r1, r2, i0)
377 #define movnr(r0, r1, r2) _movnr(_jit, r0, r1, r2)
378 static void _movnr(jit_state_t*, jit_int32_t, jit_int32_t, jit_int32_t);
379 #define movzr(r0, r1, r2) _movzr(_jit, r0, r1, r2)
380 static void _movzr(jit_state_t*, jit_int32_t, jit_int32_t, jit_int32_t);
381 # if __X64 && !__X64_32
382 # define movir(r0, r1) _movir(_jit, r0, r1)
383 static void _movir(jit_state_t*,jit_int32_t,jit_int32_t);
384 # define movir_u(r0, r1) _movir_u(_jit, r0, r1)
385 static void _movir_u(jit_state_t*,jit_int32_t,jit_int32_t);
387 # define bswapr_us(r0, r1) _bswapr_us(_jit, r0, r1)
388 static void _bswapr_us(jit_state_t*,jit_int32_t,jit_int32_t);
389 # define bswapr_ui(r0, r1) _bswapr_ui(_jit, r0, r1)
390 static void _bswapr_ui(jit_state_t*,jit_int32_t,jit_int32_t);
391 # if __X64 && !__X64_32
392 #define bswapr_ul(r0, r1) _bswapr_ul(_jit, r0, r1)
393 static void _bswapr_ul(jit_state_t*,jit_int32_t,jit_int32_t);
395 # define extr_c(r0, r1) _extr_c(_jit, r0, r1)
396 static void _extr_c(jit_state_t*,jit_int32_t,jit_int32_t);
397 # define extr_uc(r0, r1) _extr_uc(_jit, r0, r1)
398 static void _extr_uc(jit_state_t*,jit_int32_t,jit_int32_t);
399 # define extr_s(r0, r1) movsr(r0, r1)
400 # define extr_us(r0, r1) movsr_u(r0, r1)
401 # if __X64 && !__X64_32
402 # define extr_i(r0, r1) movir(r0, r1)
403 # define extr_ui(r0, r1) movir_u(r0, r1)
405 # define ldr_c(r0, r1) _ldr_c(_jit, r0, r1)
406 static void _ldr_c(jit_state_t*, jit_int32_t, jit_int32_t);
407 # define ldi_c(r0, i0) _ldi_c(_jit, r0, i0)
408 static void _ldi_c(jit_state_t*, jit_int32_t, jit_word_t);
409 # define ldr_uc(r0, r1) _ldr_uc(_jit, r0, r1)
410 static void _ldr_uc(jit_state_t*, jit_int32_t, jit_int32_t);
411 # define ldi_uc(r0, i0) _ldi_uc(_jit, r0, i0)
412 static void _ldi_uc(jit_state_t*, jit_int32_t, jit_word_t);
413 # define ldr_s(r0, r1) _ldr_s(_jit, r0, r1)
414 static void _ldr_s(jit_state_t*, jit_int32_t, jit_int32_t);
415 # define ldi_s(r0, i0) _ldi_s(_jit, r0, i0)
416 static void _ldi_s(jit_state_t*, jit_int32_t, jit_word_t);
417 # define ldr_us(r0, r1) _ldr_us(_jit, r0, r1)
418 static void _ldr_us(jit_state_t*, jit_int32_t, jit_int32_t);
419 # define ldi_us(r0, i0) _ldi_us(_jit, r0, i0)
420 static void _ldi_us(jit_state_t*, jit_int32_t, jit_word_t);
421 # if __X32 || !__X64_32
422 # define ldr_i(r0, r1) _ldr_i(_jit, r0, r1)
423 static void _ldr_i(jit_state_t*, jit_int32_t, jit_int32_t);
424 # define ldi_i(r0, i0) _ldi_i(_jit, r0, i0)
425 static void _ldi_i(jit_state_t*, jit_int32_t, jit_word_t);
429 # define ldr_i(r0, r1) _ldr_ui(_jit, r0, r1)
430 # define ldi_i(r0, i0) _ldi_ui(_jit, r0, i0)
432 # define ldr_ui(r0, r1) _ldr_ui(_jit, r0, r1)
433 # define ldi_ui(r0, i0) _ldi_ui(_jit, r0, i0)
435 static void _ldr_ui(jit_state_t*, jit_int32_t, jit_int32_t);
436 static void _ldi_ui(jit_state_t*, jit_int32_t, jit_word_t);
438 # define ldr_l(r0, r1) _ldr_l(_jit, r0, r1)
439 static void _ldr_l(jit_state_t*, jit_int32_t, jit_int32_t);
440 # define ldi_l(r0, i0) _ldi_l(_jit, r0, i0)
441 static void _ldi_l(jit_state_t*, jit_int32_t, jit_word_t);
444 # define ldxr_c(r0, r1, r2) _ldxr_c(_jit, r0, r1, r2)
445 static void _ldxr_c(jit_state_t*, jit_int32_t, jit_int32_t, jit_int32_t);
446 # define ldxi_c(r0, r1, i0) _ldxi_c(_jit, r0, r1, i0)
447 static void _ldxi_c(jit_state_t*, jit_int32_t, jit_int32_t, jit_word_t);
448 # define ldxr_uc(r0, r1, r2) _ldxr_uc(_jit, r0, r1, r2)
449 static void _ldxr_uc(jit_state_t*, jit_int32_t, jit_int32_t, jit_int32_t);
450 # define ldxi_uc(r0, r1, i0) _ldxi_uc(_jit, r0, r1, i0)
451 static void _ldxi_uc(jit_state_t*, jit_int32_t, jit_int32_t, jit_word_t);
452 # define ldxr_s(r0, r1, r2) _ldxr_s(_jit, r0, r1, r2)
453 static void _ldxr_s(jit_state_t*, jit_int32_t, jit_int32_t, jit_int32_t);
454 # define ldxi_s(r0, r1, i0) _ldxi_s(_jit, r0, r1, i0)
455 static void _ldxi_s(jit_state_t*, jit_int32_t, jit_int32_t, jit_word_t);
456 # define ldxr_us(r0, r1, r2) _ldxr_us(_jit, r0, r1, r2)
457 static void _ldxr_us(jit_state_t*, jit_int32_t, jit_int32_t, jit_int32_t);
458 # define ldxi_us(r0, r1, i0) _ldxi_us(_jit, r0, r1, i0)
459 static void _ldxi_us(jit_state_t*, jit_int32_t, jit_int32_t, jit_word_t);
460 # if __X32 || !__X64_32
461 # define ldxr_i(r0, r1, r2) _ldxr_i(_jit, r0, r1, r2)
462 static void _ldxr_i(jit_state_t*, jit_int32_t, jit_int32_t, jit_int32_t);
463 # define ldxi_i(r0, r1, i0) _ldxi_i(_jit, r0, r1, i0)
464 static void _ldxi_i(jit_state_t*, jit_int32_t, jit_int32_t, jit_word_t);
468 # define ldxr_i(r0, r1, r2) _ldxr_ui(_jit, r0, r1, r2)
469 # define ldxi_i(r0, r1, i0) _ldxi_ui(_jit, r0, r1, i0)
471 # define ldxr_ui(r0, r1, r2) _ldxr_ui(_jit, r0, r1, r2)
472 # define ldxi_ui(r0, r1, i0) _ldxi_ui(_jit, r0, r1, i0)
474 static void _ldxr_ui(jit_state_t*, jit_int32_t, jit_int32_t, jit_int32_t);
475 static void _ldxi_ui(jit_state_t*, jit_int32_t, jit_int32_t, jit_word_t);
477 # define ldxr_l(r0, r1, r2) _ldxr_l(_jit, r0, r1, r2)
478 static void _ldxr_l(jit_state_t*, jit_int32_t, jit_int32_t, jit_int32_t);
479 # define ldxi_l(r0, r1, i0) _ldxi_l(_jit, r0, r1, i0)
480 static void _ldxi_l(jit_state_t*, jit_int32_t, jit_int32_t, jit_word_t);
483 # define str_c(r0, r1) _str_c(_jit, r0, r1)
484 static void _str_c(jit_state_t*, jit_int32_t, jit_int32_t);
485 # define sti_c(i0, r0) _sti_c(_jit, i0, r0)
486 static void _sti_c(jit_state_t*, jit_word_t, jit_int32_t);
487 # define str_s(r0, r1) _str_s(_jit, r0, r1)
488 static void _str_s(jit_state_t*, jit_int32_t, jit_int32_t);
489 # define sti_s(i0, r0) _sti_s(_jit, i0, r0)
490 static void _sti_s(jit_state_t*, jit_word_t, jit_int32_t);
491 # define str_i(r0, r1) _str_i(_jit, r0, r1)
492 static void _str_i(jit_state_t*, jit_int32_t, jit_int32_t);
493 # define sti_i(i0, r0) _sti_i(_jit, i0, r0)
494 static void _sti_i(jit_state_t*, jit_word_t, jit_int32_t);
495 # if __X64 && !__X64_32
496 # define str_l(r0, r1) _str_l(_jit, r0, r1)
497 static void _str_l(jit_state_t*, jit_int32_t, jit_int32_t);
498 # define sti_l(i0, r0) _sti_l(_jit, i0, r0)
499 static void _sti_l(jit_state_t*, jit_word_t, jit_int32_t);
501 # define stxr_c(r0, r1, r2) _stxr_c(_jit, r0, r1, r2)
502 static void _stxr_c(jit_state_t*, jit_int32_t, jit_int32_t, jit_int32_t);
503 # define stxi_c(i0, r0, r1) _stxi_c(_jit, i0, r0, r1)
504 static void _stxi_c(jit_state_t*, jit_word_t, jit_int32_t, jit_int32_t);
505 # define stxr_s(r0, r1, r2) _stxr_s(_jit, r0, r1, r2)
506 static void _stxr_s(jit_state_t*, jit_int32_t, jit_int32_t, jit_int32_t);
507 # define stxi_s(i0, r0, r1) _stxi_s(_jit, i0, r0, r1)
508 static void _stxi_s(jit_state_t*, jit_word_t, jit_int32_t, jit_int32_t);
509 # define stxr_i(r0, r1, r2) _stxr_i(_jit, r0, r1, r2)
510 static void _stxr_i(jit_state_t*, jit_int32_t, jit_int32_t, jit_int32_t);
511 # define stxi_i(i0, r0, r1) _stxi_i(_jit, i0, r0, r1)
512 static void _stxi_i(jit_state_t*, jit_word_t, jit_int32_t, jit_int32_t);
513 # if __X64 && !__X64_32
514 # define stxr_l(r0, r1, r2) _stxr_l(_jit, r0, r1, r2)
515 static void _stxr_l(jit_state_t*, jit_int32_t, jit_int32_t, jit_int32_t);
516 # define stxi_l(i0, r0, r1) _stxi_l(_jit, i0, r0, r1)
517 static void _stxi_l(jit_state_t*, jit_word_t, jit_int32_t, jit_int32_t);
519 # define jcc(code, i0) _jcc(_jit, code, i0)
520 # define jo(i0) jcc(X86_CC_O, i0)
521 # define jno(i0) jcc(X86_CC_NO, i0)
522 # define jnae(i0) jcc(X86_CC_NAE, i0)
523 # define jb(i0) jcc(X86_CC_B, i0)
524 # define jc(i0) jcc(X86_CC_C, i0)
525 # define jae(i0) jcc(X86_CC_AE, i0)
526 # define jnb(i0) jcc(X86_CC_NB, i0)
527 # define jnc(i0) jcc(X86_CC_NC, i0)
528 # define je(i0) jcc(X86_CC_E, i0)
529 # define jz(i0) jcc(X86_CC_Z, i0)
530 # define jne(i0) jcc(X86_CC_NE, i0)
531 # define jnz(i0) jcc(X86_CC_NZ, i0)
532 # define jbe(i0) jcc(X86_CC_BE, i0)
533 # define jna(i0) jcc(X86_CC_NA, i0)
534 # define ja(i0) jcc(X86_CC_A, i0)
535 # define jnbe(i0) jcc(X86_CC_NBE, i0)
536 # define js(i0) jcc(X86_CC_S, i0)
537 # define jns(i0) jcc(X86_CC_NS, i0)
538 # define jp(i0) jcc(X86_CC_P, i0)
539 # define jpe(i0) jcc(X86_CC_PE, i0)
540 # define jnp(i0) jcc(X86_CC_NP, i0)
541 # define jpo(i0) jcc(X86_CC_PO, i0)
542 # define jl(i0) jcc(X86_CC_L, i0)
543 # define jnge(i0) jcc(X86_CC_NGE, i0)
544 # define jge(i0) jcc(X86_CC_GE, i0)
545 # define jnl(i0) jcc(X86_CC_NL, i0)
546 # define jle(i0) jcc(X86_CC_LE, i0)
547 # define jng(i0) jcc(X86_CC_NG, i0)
548 # define jg(i0) jcc(X86_CC_G, i0)
549 # define jnle(i0) jcc(X86_CC_NLE, i0)
550 static void _jcc(jit_state_t*, jit_int32_t, jit_word_t);
551 # define jccs(code, i0) _jccs(_jit, code, i0)
552 # define jos(i0) jccs(X86_CC_O, i0)
553 # define jnos(i0) jccs(X86_CC_NO, i0)
554 # define jnaes(i0) jccs(X86_CC_NAE, i0)
555 # define jbs(i0) jccs(X86_CC_B, i0)
556 # define jcs(i0) jccs(X86_CC_C, i0)
557 # define jaes(i0) jccs(X86_CC_AE, i0)
558 # define jnbs(i0) jccs(X86_CC_NB, i0)
559 # define jncs(i0) jccs(X86_CC_NC, i0)
560 # define jes(i0) jccs(X86_CC_E, i0)
561 # define jzs(i0) jccs(X86_CC_Z, i0)
562 # define jnes(i0) jccs(X86_CC_NE, i0)
563 # define jnzs(i0) jccs(X86_CC_NZ, i0)
564 # define jbes(i0) jccs(X86_CC_BE, i0)
565 # define jnas(i0) jccs(X86_CC_NA, i0)
566 # define jas(i0) jccs(X86_CC_A, i0)
567 # define jnbes(i0) jccs(X86_CC_NBE, i0)
568 # define jss(i0) jccs(X86_CC_S, i0)
569 # define jnss(i0) jccs(X86_CC_NS, i0)
570 # define jps(i0) jccs(X86_CC_P, i0)
571 # define jpes(i0) jccs(X86_CC_PE, i0)
572 # define jnps(i0) jccs(X86_CC_NP, i0)
573 # define jpos(i0) jccs(X86_CC_PO, i0)
574 # define jls(i0) jccs(X86_CC_L, i0)
575 # define jnges(i0) jccs(X86_CC_NGE, i0)
576 # define jges(i0) jccs(X86_CC_GE, i0)
577 # define jnls(i0) jccs(X86_CC_NL, i0)
578 # define jles(i0) jccs(X86_CC_LE, i0)
579 # define jngs(i0) jccs(X86_CC_NG, i0)
580 # define jgs(i0) jccs(X86_CC_G, i0)
581 # define jnles(i0) jccs(X86_CC_NLE, i0)
582 static void _jccs(jit_state_t*, jit_int32_t, jit_word_t);
583 # define jcr(code, i0, r0, r1) _jcr(_jit, code, i0, r0, r1)
584 static void _jcr(jit_state_t*,jit_int32_t,jit_word_t,jit_int32_t,jit_int32_t);
585 # define jci(code, i0, r0, i1) _jci(_jit, code, i0, r0, i1)
586 static void _jci(jit_state_t*,jit_int32_t,jit_word_t,jit_int32_t,jit_word_t);
587 # define jci0(code, i0, r0) _jci0(_jit, code, i0, r0)
588 static void _jci0(jit_state_t*, jit_int32_t, jit_word_t, jit_int32_t);
589 # define bltr(i0, r0, r1) _bltr(_jit, i0, r0, r1)
590 static jit_word_t _bltr(jit_state_t*, jit_word_t, jit_int32_t, jit_int32_t);
591 # define blti(i0, r0, i1) _blti(_jit, i0, r0, i1)
592 static jit_word_t _blti(jit_state_t*, jit_word_t, jit_int32_t, jit_word_t);
593 # define bltr_u(i0, r0, r1) _bltr_u(_jit, i0, r0, r1)
594 static jit_word_t _bltr_u(jit_state_t*, jit_word_t, jit_int32_t, jit_int32_t);
595 # define blti_u(i0, r0, i1) _blti_u(_jit, i0, r0, i1)
596 static jit_word_t _blti_u(jit_state_t*, jit_word_t, jit_int32_t, jit_word_t);
597 # define bler(i0, r0, r1) _bler(_jit, i0, r0, r1)
598 static jit_word_t _bler(jit_state_t*, jit_word_t, jit_int32_t, jit_int32_t);
599 # define blei(i0, r0, i1) _blei(_jit, i0, r0, i1)
600 static jit_word_t _blei(jit_state_t*, jit_word_t, jit_int32_t, jit_word_t);
601 # define bler_u(i0, r0, r1) _bler_u(_jit, i0, r0, r1)
602 static jit_word_t _bler_u(jit_state_t*, jit_word_t, jit_int32_t, jit_int32_t);
603 # define blei_u(i0, r0, i1) _blei_u(_jit, i0, r0, i1)
604 static jit_word_t _blei_u(jit_state_t*, jit_word_t, jit_int32_t, jit_word_t);
605 # define beqr(i0, r0, r1) _beqr(_jit, i0, r0, r1)
606 static jit_word_t _beqr(jit_state_t*, jit_word_t, jit_int32_t, jit_int32_t);
607 # define beqi(i0, r0, i1) _beqi(_jit, i0, r0, i1)
608 static jit_word_t _beqi(jit_state_t*, jit_word_t, jit_int32_t, jit_word_t);
609 # define bger(i0, r0, r1) _bger(_jit, i0, r0, r1)
610 static jit_word_t _bger(jit_state_t*, jit_word_t, jit_int32_t, jit_int32_t);
611 # define bgei(i0, r0, i1) _bgei(_jit, i0, r0, i1)
612 static jit_word_t _bgei(jit_state_t*, jit_word_t, jit_int32_t, jit_word_t);
613 # define bger_u(i0, r0, r1) _bger_u(_jit, i0, r0, r1)
614 static jit_word_t _bger_u(jit_state_t*, jit_word_t, jit_int32_t, jit_int32_t);
615 # define bgei_u(i0, r0, i1) _bgei_u(_jit, i0, r0, i1)
616 static jit_word_t _bgei_u(jit_state_t*, jit_word_t, jit_int32_t, jit_word_t);
617 # define bgtr(i0, r0, r1) _bgtr(_jit, i0, r0, r1)
618 static jit_word_t _bgtr(jit_state_t*, jit_word_t, jit_int32_t, jit_int32_t);
619 # define bgti(i0, r0, i1) _bgti(_jit, i0, r0, i1)
620 static jit_word_t _bgti(jit_state_t*, jit_word_t, jit_int32_t, jit_word_t);
621 # define bgtr_u(i0, r0, r1) _bgtr_u(_jit, i0, r0, r1)
622 static jit_word_t _bgtr_u(jit_state_t*, jit_word_t, jit_int32_t, jit_int32_t);
623 # define bgti_u(i0, r0, i1) _bgti_u(_jit, i0, r0, i1)
624 static jit_word_t _bgti_u(jit_state_t*, jit_word_t, jit_int32_t, jit_word_t);
625 # define bner(i0, r0, r1) _bner(_jit, i0, r0, r1)
626 static jit_word_t _bner(jit_state_t*, jit_word_t, jit_int32_t, jit_int32_t);
627 # define bnei(i0, r0, i1) _bnei(_jit, i0, r0, i1)
628 static jit_word_t _bnei(jit_state_t*, jit_word_t, jit_int32_t, jit_word_t);
629 # define bmsr(i0, r0, r1) _bmsr(_jit, i0, r0, r1)
630 static jit_word_t _bmsr(jit_state_t*,jit_word_t,jit_int32_t,jit_int32_t);
631 # define bmsi(i0, r0, i1) _bmsi(_jit, i0, r0, i1)
632 static jit_word_t _bmsi(jit_state_t*,jit_word_t,jit_int32_t,jit_word_t);
633 # define bmcr(i0, r0, r1) _bmcr(_jit, i0, r0, r1)
634 static jit_word_t _bmcr(jit_state_t*,jit_word_t,jit_int32_t,jit_int32_t);
635 # define bmci(i0, r0, i1) _bmci(_jit, i0, r0, i1)
636 static jit_word_t _bmci(jit_state_t*,jit_word_t,jit_int32_t,jit_word_t);
637 # define boaddr(i0, r0, r1) _boaddr(_jit, i0, r0, r1)
638 static jit_word_t _boaddr(jit_state_t*,jit_word_t,jit_int32_t,jit_int32_t);
639 # define boaddi(i0, r0, i1) _boaddi(_jit, i0, r0, i1)
640 static jit_word_t _boaddi(jit_state_t*,jit_word_t,jit_int32_t,jit_word_t);
641 # define boaddr_u(i0, r0, r1) _boaddr_u(_jit, i0, r0, r1)
642 static jit_word_t _boaddr_u(jit_state_t*,jit_word_t,jit_int32_t,jit_int32_t);
643 # define boaddi_u(i0, r0, i1) _boaddi_u(_jit, i0, r0, i1)
644 static jit_word_t _boaddi_u(jit_state_t*,jit_word_t,jit_int32_t,jit_word_t);
645 # define bxaddr(i0, r0, r1) _bxaddr(_jit, i0, r0, r1)
646 static jit_word_t _bxaddr(jit_state_t*,jit_word_t,jit_int32_t,jit_int32_t);
647 # define bxaddi(i0, r0, i1) _bxaddi(_jit, i0, r0, i1)
648 static jit_word_t _bxaddi(jit_state_t*,jit_word_t,jit_int32_t,jit_word_t);
649 # define bxaddr_u(i0, r0, r1) _bxaddr_u(_jit, i0, r0, r1)
650 static jit_word_t _bxaddr_u(jit_state_t*,jit_word_t,jit_int32_t,jit_int32_t);
651 # define bxaddi_u(i0, r0, i1) _bxaddi_u(_jit, i0, r0, i1)
652 static jit_word_t _bxaddi_u(jit_state_t*,jit_word_t,jit_int32_t,jit_word_t);
653 # define bosubr(i0, r0, r1) _bosubr(_jit, i0, r0, r1)
654 static jit_word_t _bosubr(jit_state_t*,jit_word_t,jit_int32_t,jit_int32_t);
655 # define bosubi(i0, r0, i1) _bosubi(_jit, i0, r0, i1)
656 static jit_word_t _bosubi(jit_state_t*,jit_word_t,jit_int32_t,jit_word_t);
657 # define bosubr_u(i0, r0, r1) _bosubr_u(_jit, i0, r0, r1)
658 static jit_word_t _bosubr_u(jit_state_t*,jit_word_t,jit_int32_t,jit_int32_t);
659 # define bosubi_u(i0, r0, i1) _bosubi_u(_jit, i0, r0, i1)
660 static jit_word_t _bosubi_u(jit_state_t*,jit_word_t,jit_int32_t,jit_word_t);
661 # define bxsubr(i0, r0, r1) _bxsubr(_jit, i0, r0, r1)
662 static jit_word_t _bxsubr(jit_state_t*,jit_word_t,jit_int32_t,jit_int32_t);
663 # define bxsubi(i0, r0, i1) _bxsubi(_jit, i0, r0, i1)
664 static jit_word_t _bxsubi(jit_state_t*,jit_word_t,jit_int32_t,jit_word_t);
665 # define bxsubr_u(i0, r0, r1) _bxsubr_u(_jit, i0, r0, r1)
666 static jit_word_t _bxsubr_u(jit_state_t*,jit_word_t,jit_int32_t,jit_int32_t);
667 # define bxsubi_u(i0, r0, i1) _bxsubi_u(_jit, i0, r0, i1)
668 static jit_word_t _bxsubi_u(jit_state_t*,jit_word_t,jit_int32_t,jit_word_t);
669 # define callr(r0) _callr(_jit, r0)
670 static void _callr(jit_state_t*, jit_int32_t);
671 # define calli(i0) _calli(_jit, i0)
672 static jit_word_t _calli(jit_state_t*, jit_word_t);
674 # define calli_p(i0) _calli_p(_jit, i0)
675 static jit_word_t _calli_p(jit_state_t*, jit_word_t);
677 # define calli_p(i0) calli(i0)
679 # define jmpr(r0) _jmpr(_jit, r0)
680 static void _jmpr(jit_state_t*, jit_int32_t);
681 # define jmpi(i0) _jmpi(_jit, i0)
682 static jit_word_t _jmpi(jit_state_t*, jit_word_t);
684 # define jmpi_p(i0) _jmpi_p(_jit, i0)
685 static jit_word_t _jmpi_p(jit_state_t*, jit_word_t);
687 # define jmpi_p(i0) jmpi(i0)
689 # define jmpsi(i0) _jmpsi(_jit, i0)
690 static void _jmpsi(jit_state_t*, jit_uint8_t);
691 # define prolog(node) _prolog(_jit, node)
692 static void _prolog(jit_state_t*, jit_node_t*);
693 # define epilog(node) _epilog(_jit, node)
694 static void _epilog(jit_state_t*, jit_node_t*);
695 # define vastart(r0) _vastart(_jit, r0)
696 static void _vastart(jit_state_t*, jit_int32_t);
697 # define vaarg(r0, r1) _vaarg(_jit, r0, r1)
698 static void _vaarg(jit_state_t*, jit_int32_t, jit_int32_t);
699 # define vaarg_d(r0, r1, i0) _vaarg_d(_jit, r0, r1, i0)
700 static void _vaarg_d(jit_state_t*, jit_int32_t, jit_int32_t, jit_bool_t);
701 # define patch_at(node, instr, label) _patch_at(_jit, node, instr, label)
702 static void _patch_at(jit_state_t*, jit_node_t*, jit_word_t, jit_word_t);
703 # if !defined(HAVE_FFSL)
705 # define ffsl(i) __builtin_ffs(i)
707 # define ffsl(l) __builtin_ffsl(l)
710 # define jit_cmov_p() jit_cpu.cmov
715 _rex(jit_state_t *_jit, jit_int32_t l, jit_int32_t w,
716 jit_int32_t r, jit_int32_t x, jit_int32_t b)
719 jit_int32_t v = 0x40 | (w << 3);
733 _rx(jit_state_t *_jit, jit_int32_t rd, jit_int32_t md,
734 jit_int32_t rb, jit_int32_t ri, jit_int32_t ms)
739 mrm(0x00, r7(rd), 0x05);
741 mrm(0x00, r7(rd), 0x04);
742 sib(_SCL1, 0x04, 0x05);
746 else if (r7(rb) == _RSP_REGNO) {
748 mrm(0x00, r7(rd), 0x04);
751 else if ((jit_int8_t)md == md) {
752 mrm(0x01, r7(rd), 0x04);
757 mrm(0x02, r7(rd), 0x04);
763 if (md == 0 && r7(rb) != _RBP_REGNO)
764 mrm(0x00, r7(rd), r7(rb));
765 else if ((jit_int8_t)md == md) {
766 mrm(0x01, r7(rd), r7(rb));
770 mrm(0x02, r7(rd), r7(rb));
775 else if (rb == _NOREG) {
776 mrm(0x00, r7(rd), 0x04);
777 sib(ms, r7(ri), 0x05);
780 else if (r8(ri) != _RSP_REGNO) {
781 if (md == 0 && r7(rb) != _RBP_REGNO) {
782 mrm(0x00, r7(rd), 0x04);
783 sib(ms, r7(ri), r7(rb));
785 else if ((jit_int8_t)md == md) {
786 mrm(0x01, r7(rd), 0x04);
787 sib(ms, r7(ri), r7(rb));
791 mrm(0x02, r7(rd), 0x04);
792 sib(ms, r7(ri), r7(rb));
797 fprintf(stderr, "illegal index register");
803 _nop(jit_state_t *_jit, jit_int32_t count)
819 case 3: /* NOP DWORD ptr [EAX] */
820 ic(0x0f); ic(0x1f); ic(0x00);
822 case 4: /* NOP DWORD ptr [EAX + 00H] */
823 ic(0x0f); ic(0x1f); ic(0x40); ic(0x00);
825 case 5: /* NOP DWORD ptr [EAX + EAX*1 + 00H] */
826 ic(0x0f); ic(0x1f); ic(0x44); ic(0x00);
829 case 6: /* 66 NOP DWORD ptr [EAX + EAX*1 + 00H] */
830 ic(0x66); ic(0x0f); ic(0x1f); ic(0x44);
833 case 7: /* NOP DWORD ptr [EAX + 00000000H] */
834 ic(0x0f); ic(0x1f); ic(0x80); ii(0x0000);
836 case 8: /* NOP DWORD ptr [EAX + EAX*1 + 00000000H] */
837 ic(0x0f); ic(0x1f); ic(0x84); ic(0x00);
840 case 9: /* 66 NOP DWORD ptr [EAX + EAX*1 + 00000000H] */
841 ic(0x66); ic(0x0f); ic(0x1f); ic(0x84);
842 ic(0x00); ii(0x0000);
849 _lea(jit_state_t *_jit, jit_int32_t md, jit_int32_t rb,
850 jit_int32_t ri, jit_int32_t ms, jit_int32_t rd)
852 rex(0, WIDE, rd, ri, rb);
854 rx(rd, md, rb, ri, ms);
858 _pushr(jit_state_t *_jit, jit_int32_t r0)
860 rex(0, WIDE, 0, 0, r0);
865 _popr(jit_state_t *_jit, jit_int32_t r0)
867 rex(0, WIDE, 0, 0, r0);
872 _xchgr(jit_state_t *_jit, jit_int32_t r0, jit_int32_t r1)
874 rex(0, WIDE, r1, _NOREG, r0);
876 mrm(0x03, r7(r1), r7(r0));
880 _testr(jit_state_t *_jit, jit_int32_t r0, jit_int32_t r1)
882 rex(0, WIDE, r1, _NOREG, r0);
884 mrm(0x03, r7(r1), r7(r0));
888 _testi(jit_state_t *_jit, jit_int32_t r0, jit_word_t i0)
890 rex(0, WIDE, _NOREG, _NOREG, r0);
891 if (r0 == _RAX_REGNO)
895 mrm(0x03, 0x00, r7(r0));
901 _cc(jit_state_t *_jit, jit_int32_t code, jit_int32_t r0)
903 rex(0, 0, _NOREG, _NOREG, r0);
906 mrm(0x03, 0x00, r7(r0));
910 _alur(jit_state_t *_jit, jit_int32_t code, jit_int32_t r0, jit_int32_t r1)
912 rex(0, WIDE, r1, _NOREG, r0);
914 mrm(0x03, r7(r1), r7(r0));
918 _alui(jit_state_t *_jit, jit_int32_t code, jit_int32_t r0, jit_word_t i0)
921 if (can_sign_extend_int_p(i0)) {
922 rex(0, WIDE, _NOREG, _NOREG, r0);
923 if ((jit_int8_t)i0 == i0) {
925 ic(0xc0 | code | r7(r0));
929 if (r0 == _RAX_REGNO)
933 ic(0xc0 | code | r7(r0));
939 reg = jit_get_reg(jit_class_gpr);
941 alur(code, r0, rn(reg));
947 _save(jit_state_t *_jit, jit_int32_t r0)
949 if (!_jitc->function->regoff[r0]) {
950 _jitc->function->regoff[r0] = jit_allocai(sizeof(jit_word_t));
953 assert(!jit_regset_tstbit(&_jitc->regsav, r0));
954 jit_regset_setbit(&_jitc->regsav, r0);
955 stxi(_jitc->function->regoff[r0], _RBP_REGNO, r0);
959 _load(jit_state_t *_jit, jit_int32_t r0)
961 assert(_jitc->function->regoff[r0]);
962 assert(jit_regset_tstbit(&_jitc->regsav, r0));
963 jit_regset_clrbit(&_jitc->regsav, r0);
964 ldxi(r0, _RBP_REGNO, _jitc->function->regoff[r0]);
968 _addr(jit_state_t *_jit, jit_int32_t r0, jit_int32_t r1, jit_int32_t r2)
975 lea(0, r1, r2, _SCL1, r0);
979 _addi(jit_state_t *_jit, jit_int32_t r0, jit_int32_t r1, jit_word_t i0)
990 else if (can_sign_extend_int_p(i0)) {
994 lea(i0, r1, _NOREG, _SCL1, r0);
1001 reg = jit_get_reg(jit_class_gpr);
1009 _addcr(jit_state_t *_jit, jit_int32_t r0, jit_int32_t r1, jit_int32_t r2)
1020 _addci(jit_state_t *_jit, jit_int32_t r0, jit_int32_t r1, jit_word_t i0)
1023 if (can_sign_extend_int_p(i0)) {
1027 else if (r0 == r1) {
1028 reg = jit_get_reg(jit_class_gpr);
1040 _addxr(jit_state_t *_jit, jit_int32_t r0, jit_int32_t r1, jit_int32_t r2)
1051 _addxi(jit_state_t *_jit, jit_int32_t r0, jit_int32_t r1, jit_word_t i0)
1054 if (can_sign_extend_int_p(i0)) {
1058 else if (r0 == r1) {
1059 reg = jit_get_reg(jit_class_gpr);
1061 iaddxr(r0, rn(reg));
1071 _subr(jit_state_t *_jit, jit_int32_t r0, jit_int32_t r1, jit_int32_t r2)
1075 else if (r0 == r2) {
1086 _subi(jit_state_t *_jit, jit_int32_t r0, jit_int32_t r1, jit_word_t i0)
1097 else if (can_sign_extend_int_p(i0)) {
1101 lea(-i0, r1, _NOREG, _SCL1, r0);
1103 else if (r0 != r1) {
1108 reg = jit_get_reg(jit_class_gpr);
1116 _subcr(jit_state_t *_jit, jit_int32_t r0, jit_int32_t r1, jit_int32_t r2)
1119 if (r0 == r2 && r0 != r1) {
1120 reg = jit_get_reg(jit_class_gpr);
1133 _subci(jit_state_t *_jit, jit_int32_t r0, jit_int32_t r1, jit_word_t i0)
1137 if (can_sign_extend_int_p(i0))
1140 reg = jit_get_reg(jit_class_gpr);
1148 _subxr(jit_state_t *_jit, jit_int32_t r0, jit_int32_t r1, jit_int32_t r2)
1151 if (r0 == r2 && r0 != r1) {
1152 reg = jit_get_reg(jit_class_gpr);
1155 isubxr(r0, rn(reg));
1165 _subxi(jit_state_t *_jit, jit_int32_t r0, jit_int32_t r1, jit_word_t i0)
1169 if (can_sign_extend_int_p(i0))
1172 reg = jit_get_reg(jit_class_gpr);
1174 isubxr(r0, rn(reg));
1180 _rsbi(jit_state_t *_jit, jit_int32_t r0, jit_int32_t r1, jit_word_t i0)
1187 _imulr(jit_state_t *_jit, jit_int32_t r0, jit_int32_t r1)
1189 rex(0, WIDE, r0, _NOREG, r1);
1192 mrm(0x03, r7(r0), r7(r1));
1196 _imuli(jit_state_t *_jit, jit_int32_t r0, jit_int32_t r1, jit_word_t i0)
1199 if (can_sign_extend_int_p(i0)) {
1200 rex(0, WIDE, r0, _NOREG, r1);
1201 if ((jit_int8_t)i0 == i0) {
1203 mrm(0x03, r7(r0), r7(r1));
1208 mrm(0x03, r7(r0), r7(r1));
1213 reg = jit_get_reg(jit_class_gpr);
1221 _mulr(jit_state_t *_jit, jit_int32_t r0, jit_int32_t r1, jit_int32_t r2)
1234 _muli(jit_state_t *_jit, jit_int32_t r0, jit_int32_t r1, jit_word_t i0)
1247 lea(0, _NOREG, r1, _SCL2, r0);
1250 lea(0, _NOREG, r1, _SCL4, r0);
1253 lea(0, _NOREG, r1, _SCL8, r0);
1256 if (i0 > 0 && !(i0 & (i0 - 1)))
1257 lshi(r0, r1, ffsl(i0) - 1);
1258 else if (can_sign_extend_int_p(i0))
1260 else if (r0 != r1) {
1270 #define savset(rn) \
1273 if (r1 != rn && r2 != rn) \
1276 #define isavset(rn) \
1282 #define qsavset(rn) \
1283 if (r0 != rn && r1 != rn) { \
1285 if (r2 != rn && r3 != rn) \
1288 #define allocr(rn, rv) \
1289 if (set & (1 << rn)) \
1290 (void)jit_get_reg(rv|jit_class_gpr|jit_class_named); \
1291 if (sav & (1 << rn)) { \
1292 if ( jit_regset_tstbit(&_jitc->regsav, rv) || \
1293 !jit_regset_tstbit(&_jitc->reglive, rv)) \
1294 sav &= ~(1 << rn); \
1298 #define clear(rn, rv) \
1299 if (set & (1 << rn)) \
1300 jit_unget_reg(rv); \
1301 if (sav & (1 << rn)) \
1304 _iqmulr(jit_state_t *_jit, jit_int32_t r0, jit_int32_t r1,
1305 jit_int32_t r2, jit_int32_t r3, jit_bool_t sign)
1312 qsavset(_RDX_REGNO);
1313 qsavset(_RAX_REGNO);
1314 allocr(_RDX_REGNO, _RDX);
1315 allocr(_RAX_REGNO, _RAX);
1317 if (r3 == _RAX_REGNO)
1321 movr(_RAX_REGNO, r2);
1328 if (r0 == _RDX_REGNO && r1 == _RAX_REGNO)
1329 xchgr(_RAX_REGNO, _RDX_REGNO);
1331 if (r0 != _RDX_REGNO)
1332 movr(r0, _RAX_REGNO);
1333 movr(r1, _RDX_REGNO);
1334 if (r0 == _RDX_REGNO)
1335 movr(r0, _RAX_REGNO);
1338 clear(_RDX_REGNO, _RDX);
1339 clear(_RAX_REGNO, _RAX);
1343 _iqmuli(jit_state_t *_jit, jit_int32_t r0, jit_int32_t r1,
1344 jit_int32_t r2, jit_word_t i0, jit_bool_t sign)
1353 reg = jit_get_reg(jit_class_gpr);
1356 qmulr(r0, r1, r2, rn(reg));
1358 qmulr_u(r0, r1, r2, rn(reg));
1364 _sign_extend_rdx_rax(jit_state_t *_jit)
1366 rex(0, WIDE, 0, 0, 0);
1371 _divremr(jit_state_t *_jit, jit_int32_t r0, jit_int32_t r1, jit_int32_t r2,
1372 jit_bool_t sign, jit_bool_t divide)
1380 sav = set = use = 0;
1383 allocr(_RDX_REGNO, _RDX);
1384 allocr(_RAX_REGNO, _RAX);
1386 if (r2 == _RAX_REGNO) {
1387 if (r0 == _RAX_REGNO || r0 == _RDX_REGNO) {
1388 if ((reg = jit_get_reg(jit_class_gpr|jit_class_chk)) == JIT_NOREG)
1389 reg = jit_get_reg((r1 == _RCX_REGNO ? _RBX : _RCX) |
1390 jit_class_gpr|jit_class_named);
1393 movr(div, _RAX_REGNO);
1394 if (r1 != _RAX_REGNO)
1395 movr(_RAX_REGNO, r1);
1399 xchgr(r0, _RAX_REGNO);
1401 if (r0 != _RAX_REGNO)
1402 movr(r0, _RAX_REGNO);
1403 if (r1 != _RAX_REGNO)
1404 movr(_RAX_REGNO, r1);
1409 else if (r2 == _RDX_REGNO) {
1410 if (r0 == _RAX_REGNO || r0 == _RDX_REGNO) {
1411 if ((reg = jit_get_reg(jit_class_gpr|jit_class_chk)) == JIT_NOREG)
1412 reg = jit_get_reg((r1 == _RCX_REGNO ? _RBX : _RCX) |
1413 jit_class_gpr|jit_class_named);
1416 movr(div, _RDX_REGNO);
1417 if (r1 != _RAX_REGNO)
1418 movr(_RAX_REGNO, r1);
1421 if (r1 != _RAX_REGNO)
1422 movr(_RAX_REGNO, r1);
1423 movr(r0, _RDX_REGNO);
1428 if (r1 != _RAX_REGNO)
1429 movr(_RAX_REGNO, r1);
1434 sign_extend_rdx_rax();
1438 ixorr(_RDX_REGNO, _RDX_REGNO);
1446 movr(r0, _RAX_REGNO);
1448 movr(r0, _RDX_REGNO);
1450 clear(_RDX_REGNO, _RDX);
1451 clear(_RAX_REGNO, _RAX);
1455 _divremi(jit_state_t *_jit, jit_int32_t r0, jit_int32_t r1, jit_word_t i0,
1456 jit_bool_t sign, jit_bool_t divide)
1476 if (i0 > 0 && !(i0 & (i0 - 1))) {
1479 rshi(r0, r0, ffsl(i0) - 1);
1481 rshi_u(r0, r0, ffsl(i0) - 1);
1487 else if (i0 == 1 || (sign && i0 == -1)) {
1491 else if (!sign && i0 > 0 && !(i0 & (i0 - 1))) {
1492 if (can_sign_extend_int_p(i0)) {
1496 else if (r0 != r1) {
1501 reg = jit_get_reg(jit_class_gpr);
1502 movi(rn(reg), i0 - 1);
1509 sav = set = use = 0;
1510 isavset(_RDX_REGNO);
1511 isavset(_RAX_REGNO);
1512 allocr(_RDX_REGNO, _RDX);
1513 allocr(_RAX_REGNO, _RAX);
1515 if (r0 == _RAX_REGNO || r0 == _RDX_REGNO || r0 == r1) {
1516 if ((reg = jit_get_reg(jit_class_gpr|jit_class_chk)) == JIT_NOREG)
1517 reg = jit_get_reg((r1 == _RCX_REGNO ? _RBX : _RCX) |
1518 jit_class_gpr|jit_class_named);
1526 movr(_RAX_REGNO, r1);
1529 sign_extend_rdx_rax();
1533 ixorr(_RDX_REGNO, _RDX_REGNO);
1541 movr(r0, _RAX_REGNO);
1543 movr(r0, _RDX_REGNO);
1545 clear(_RDX_REGNO, _RDX);
1546 clear(_RAX_REGNO, _RAX);
1550 _iqdivr(jit_state_t *_jit, jit_int32_t r0, jit_int32_t r1,
1551 jit_int32_t r2, jit_int32_t r3, jit_bool_t sign)
1559 sav = set = use = 0;
1560 qsavset(_RDX_REGNO);
1561 qsavset(_RAX_REGNO);
1562 allocr(_RDX_REGNO, _RDX);
1563 allocr(_RAX_REGNO, _RAX);
1564 if (r3 == _RAX_REGNO) {
1565 if (r0 == _RAX_REGNO || r0 == _RDX_REGNO) {
1566 if ((reg = jit_get_reg(jit_class_gpr|jit_class_chk)) == JIT_NOREG)
1567 reg = jit_get_reg((r1 == _RCX_REGNO ? _RBX : _RCX) |
1568 jit_class_gpr|jit_class_named);
1571 movr(div, _RAX_REGNO);
1572 if (r2 != _RAX_REGNO)
1573 movr(_RAX_REGNO, r2);
1577 xchgr(r0, _RAX_REGNO);
1579 if (r0 != _RAX_REGNO)
1580 movr(r0, _RAX_REGNO);
1581 if (r2 != _RAX_REGNO)
1582 movr(_RAX_REGNO, r2);
1587 else if (r3 == _RDX_REGNO) {
1588 if (r0 == _RAX_REGNO || r0 == _RDX_REGNO) {
1589 if ((reg = jit_get_reg(jit_class_gpr|jit_class_chk)) == JIT_NOREG)
1590 reg = jit_get_reg((r1 == _RCX_REGNO ? _RBX : _RCX) |
1591 jit_class_gpr|jit_class_named);
1594 movr(div, _RDX_REGNO);
1595 if (r2 != _RAX_REGNO)
1596 movr(_RAX_REGNO, r2);
1599 if (r2 != _RAX_REGNO)
1600 movr(_RAX_REGNO, r2);
1601 movr(r0, _RDX_REGNO);
1606 if (r2 != _RAX_REGNO)
1607 movr(_RAX_REGNO, r2);
1611 sign_extend_rdx_rax();
1615 ixorr(_RDX_REGNO, _RDX_REGNO);
1621 if (r0 == _RDX_REGNO && r1 == _RAX_REGNO)
1622 xchgr(_RAX_REGNO, _RDX_REGNO);
1624 if (r0 != _RDX_REGNO)
1625 movr(r0, _RAX_REGNO);
1626 movr(r1, _RDX_REGNO);
1627 if (r0 == _RDX_REGNO)
1628 movr(r0, _RAX_REGNO);
1631 clear(_RDX_REGNO, _RDX);
1632 clear(_RAX_REGNO, _RAX);
1636 _iqdivi(jit_state_t *_jit, jit_int32_t r0, jit_int32_t r1,
1637 jit_int32_t r2, jit_word_t i0, jit_bool_t sign)
1641 reg = jit_get_reg(jit_class_gpr);
1644 qdivr(r0, r1, r2, rn(reg));
1646 qdivr_u(r0, r1, r2, rn(reg));
1654 _andr(jit_state_t *_jit, jit_int32_t r0, jit_int32_t r1, jit_int32_t r2)
1669 _andi(jit_state_t *_jit, jit_int32_t r0, jit_int32_t r1, jit_word_t i0)
1677 else if (r0 == r1) {
1678 if (can_sign_extend_int_p(i0))
1681 reg = jit_get_reg(jit_class_gpr);
1694 _orr(jit_state_t *_jit, jit_int32_t r0, jit_int32_t r1, jit_int32_t r2)
1709 _ori(jit_state_t *_jit, jit_int32_t r0, jit_int32_t r1, jit_word_t i0)
1716 else if (can_sign_extend_int_p(i0)) {
1720 else if (r0 != r1) {
1725 reg = jit_get_reg(jit_class_gpr);
1733 _xorr(jit_state_t *_jit, jit_int32_t r0, jit_int32_t r1, jit_int32_t r2)
1748 _xori(jit_state_t *_jit, jit_int32_t r0, jit_int32_t r1, jit_word_t i0)
1755 else if (can_sign_extend_int_p(i0)) {
1759 else if (r0 != r1) {
1764 reg = jit_get_reg(jit_class_gpr);
1772 _irotshr(jit_state_t *_jit, jit_int32_t code, jit_int32_t r0)
1774 rex(0, WIDE, _RCX_REGNO, _NOREG, r0);
1776 mrm(0x03, code, r7(r0));
1780 _rotshr(jit_state_t *_jit, jit_int32_t code,
1781 jit_int32_t r0, jit_int32_t r1, jit_int32_t r2)
1786 if (r0 == _RCX_REGNO) {
1787 reg = jit_get_reg(jit_class_gpr);
1789 if (r2 != _RCX_REGNO)
1790 movr(_RCX_REGNO, r2);
1791 irotshr(code, rn(reg));
1792 movr(_RCX_REGNO, rn(reg));
1795 else if (r2 != _RCX_REGNO) {
1796 use = !jit_reg_free_p(_RCX);
1798 reg = jit_get_reg(jit_class_gpr);
1799 movr(rn(reg), _RCX_REGNO);
1803 if (r1 == _RCX_REGNO) {
1805 xchgr(r0, _RCX_REGNO);
1808 movr(_RCX_REGNO, r2);
1812 movr(_RCX_REGNO, r2);
1817 movr(_RCX_REGNO, rn(reg));
1828 _irotshi(jit_state_t *_jit, jit_int32_t code, jit_int32_t r0, jit_word_t i0)
1830 rex(0, WIDE, _NOREG, _NOREG, r0);
1833 mrm(0x03, code, r7(r0));
1837 mrm(0x03, code, r7(r0));
1843 _rotshi(jit_state_t *_jit, jit_int32_t code,
1844 jit_int32_t r0, jit_int32_t r1, jit_word_t i0)
1848 irotshi(code, r0, i0);
1852 _lshi(jit_state_t *_jit, jit_int32_t r0, jit_int32_t r1, jit_word_t i0)
1857 lea(0, _NOREG, r1, i0 == 1 ? _SCL2 : i0 == 2 ? _SCL4 : _SCL8, r0);
1859 rotshi(X86_SHL, r0, r1, i0);
1863 _unr(jit_state_t *_jit, jit_int32_t code, jit_int32_t r0)
1865 rex(0, WIDE, _NOREG, _NOREG, r0);
1867 mrm(0x03, code, r7(r0));
1871 _negr(jit_state_t *_jit, jit_int32_t r0, jit_int32_t r1)
1882 _comr(jit_state_t *_jit, jit_int32_t r0, jit_int32_t r1)
1890 _incr(jit_state_t *_jit, jit_int32_t r0, jit_int32_t r1)
1894 rex(0, WIDE, _NOREG, _NOREG, r0);
1903 _decr(jit_state_t *_jit, jit_int32_t r0, jit_int32_t r1)
1907 rex(0, WIDE, _NOREG, _NOREG, r0);
1917 _cr(jit_state_t *_jit,
1918 jit_int32_t code, jit_int32_t r0, jit_int32_t r1, jit_int32_t r2)
1923 same = r0 == r1 || r0 == r2;
1932 reg = jit_get_reg(jit_class_gpr|jit_class_rg8);
1933 ixorr(rn(reg), rn(reg));
1942 _ci(jit_state_t *_jit,
1943 jit_int32_t code, jit_int32_t r0, jit_int32_t r1, jit_word_t i0)
1957 reg = jit_get_reg(jit_class_gpr|jit_class_rg8);
1958 ixorr(rn(reg), rn(reg));
1967 _ci0(jit_state_t *_jit, jit_int32_t code, jit_int32_t r0, jit_int32_t r1)
1981 reg = jit_get_reg(jit_class_gpr|jit_class_rg8);
1982 ixorr(rn(reg), rn(reg));
1991 _ltr(jit_state_t *_jit, jit_int32_t r0, jit_int32_t r1, jit_int32_t r2)
1996 cr(X86_CC_L, r0, r1, r2);
2000 _lti(jit_state_t *_jit, jit_int32_t r0, jit_int32_t r1, jit_word_t i0)
2003 ci(X86_CC_L, r0, r1, i0);
2005 ci0(X86_CC_S, r0, r1);
2009 _ltr_u(jit_state_t *_jit, jit_int32_t r0, jit_int32_t r1, jit_int32_t r2)
2014 cr(X86_CC_B, r0, r1, r2);
2018 _ler(jit_state_t *_jit, jit_int32_t r0, jit_int32_t r1, jit_int32_t r2)
2023 cr(X86_CC_LE, r0, r1, r2);
2027 _ler_u(jit_state_t *_jit, jit_int32_t r0, jit_int32_t r1, jit_int32_t r2)
2032 cr(X86_CC_BE, r0, r1, r2);
2036 _lei_u(jit_state_t *_jit, jit_int32_t r0, jit_int32_t r1, jit_word_t i0)
2039 ci(X86_CC_BE, r0, r1, i0);
2041 ci0(X86_CC_E, r0, r1);
2045 _eqr(jit_state_t *_jit, jit_int32_t r0, jit_int32_t r1, jit_int32_t r2)
2050 cr(X86_CC_E, r0, r1, r2);
2054 _eqi(jit_state_t *_jit, jit_int32_t r0, jit_int32_t r1, jit_word_t i0)
2057 ci(X86_CC_E, r0, r1, i0);
2059 ci0(X86_CC_E, r0, r1);
2063 _ger(jit_state_t *_jit, jit_int32_t r0, jit_int32_t r1, jit_int32_t r2)
2068 cr(X86_CC_GE, r0, r1, r2);
2072 _gei(jit_state_t *_jit, jit_int32_t r0, jit_int32_t r1, jit_word_t i0)
2075 ci(X86_CC_GE, r0, r1, i0);
2077 ci0(X86_CC_NS, r0, r1);
2081 _ger_u(jit_state_t *_jit, jit_int32_t r0, jit_int32_t r1, jit_int32_t r2)
2086 cr(X86_CC_AE, r0, r1, r2);
2090 _gei_u(jit_state_t *_jit, jit_int32_t r0, jit_int32_t r1, jit_word_t i0)
2093 ci(X86_CC_AE, r0, r1, i0);
2095 ci0(X86_CC_NB, r0, r1);
2099 _gtr(jit_state_t *_jit, jit_int32_t r0, jit_int32_t r1, jit_int32_t r2)
2104 cr(X86_CC_G, r0, r1, r2);
2108 _gtr_u(jit_state_t *_jit, jit_int32_t r0, jit_int32_t r1, jit_int32_t r2)
2113 cr(X86_CC_A, r0, r1, r2);
2117 _gti_u(jit_state_t *_jit, jit_int32_t r0, jit_int32_t r1, jit_word_t i0)
2120 ci(X86_CC_A, r0, r1, i0);
2122 ci0(X86_CC_NE, r0, r1);
2126 _ner(jit_state_t *_jit, jit_int32_t r0, jit_int32_t r1, jit_int32_t r2)
2131 cr(X86_CC_NE, r0, r1, r2);
2135 _nei(jit_state_t *_jit, jit_int32_t r0, jit_int32_t r1, jit_word_t i0)
2138 ci(X86_CC_NE, r0, r1, i0);
2140 ci0(X86_CC_NE, r0, r1);
2144 _movr(jit_state_t *_jit, jit_int32_t r0, jit_int32_t r1)
2147 rex(0, 1, r1, _NOREG, r0);
2149 ic(0xc0 | (r1 << 3) | r7(r0));
2154 _imovi(jit_state_t *_jit, jit_int32_t r0, jit_word_t i0)
2158 if (fits_uint32_p(i0)) {
2160 rex(0, 0, _NOREG, _NOREG, r0);
2166 rex(0, 1, _NOREG, _NOREG, r0);
2178 _movi(jit_state_t *_jit, jit_int32_t r0, jit_word_t i0)
2187 _movi_p(jit_state_t *_jit, jit_int32_t r0, jit_word_t i0)
2189 rex(0, WIDE, _NOREG, _NOREG, r0);
2192 return (_jit->pc.w);
2196 _movcr(jit_state_t *_jit, jit_int32_t r0, jit_int32_t r1)
2198 rex(0, WIDE, r0, _NOREG, r1);
2201 mrm(0x03, r7(r0), r7(r1));
2205 _movcr_u(jit_state_t *_jit, jit_int32_t r0, jit_int32_t r1)
2207 rex(0, WIDE, r0, _NOREG, r1);
2210 mrm(0x03, r7(r0), r7(r1));
2214 _movsr(jit_state_t *_jit, jit_int32_t r0, jit_int32_t r1)
2216 rex(0, WIDE, r0, _NOREG, r1);
2219 mrm(0x03, r7(r0), r7(r1));
2223 _movsr_u(jit_state_t *_jit, jit_int32_t r0, jit_int32_t r1)
2225 rex(0, WIDE, r0, _NOREG, r1);
2228 mrm(0x03, r7(r0), r7(r1));
2232 _casx(jit_state_t *_jit, jit_int32_t r0, jit_int32_t r1,
2233 jit_int32_t r2, jit_int32_t r3, jit_word_t i0)
2235 jit_int32_t save_rax, restore_rax;
2236 jit_int32_t ascasr_reg, ascasr_use;
2237 if (r0 != _RAX_REGNO) { /* result not in %rax */
2238 if (r2 != _RAX_REGNO) { /* old value not in %rax */
2239 save_rax = jit_get_reg(jit_class_gpr);
2240 movr(rn(save_rax), _RAX_REGNO);
2248 if (r2 != _RAX_REGNO)
2249 movr(_RAX_REGNO, r2);
2250 if (r1 == _NOREG) { /* using immediate address */
2251 if (!can_sign_extend_int_p(i0)) {
2252 ascasr_reg = jit_get_reg(jit_class_gpr);
2253 if (ascasr_reg == _RAX) {
2254 ascasr_reg = jit_get_reg(jit_class_gpr);
2255 jit_unget_reg(_RAX);
2258 movi(rn(ascasr_reg), i0);
2265 ic(0xf0); /* lock */
2267 rex(0, WIDE, r3, _NOREG, rn(ascasr_reg));
2269 rex(0, WIDE, r3, _NOREG, r1);
2272 if (r1 != _NOREG) /* casr */
2273 rx(r3, 0, r1, _NOREG, _SCL1);
2276 rx(r3, 0, rn(ascasr_reg), _NOREG, _SCL1); /* address in reg */
2278 rx(r3, i0, _NOREG, _NOREG, _SCL1); /* address in offset */
2281 if (r0 != _RAX_REGNO)
2282 movr(r0, _RAX_REGNO);
2284 movr(_RAX_REGNO, rn(save_rax));
2285 jit_unget_reg(save_rax);
2288 jit_unget_reg(ascasr_reg);
2292 _movnr(jit_state_t *_jit, jit_int32_t r0, jit_int32_t r1, jit_int32_t r2)
2294 assert(jit_cmov_p());
2298 rex(0, WIDE, r0, _NOREG, r1);
2301 mrm(0x03, r7(r0), r7(r1));
2305 _movzr(jit_state_t *_jit, jit_int32_t r0, jit_int32_t r1, jit_int32_t r2)
2307 assert(jit_cmov_p());
2311 rex(0, WIDE, r0, _NOREG, r1);
2314 mrm(0x03, r7(r0), r7(r1));
2319 _movir(jit_state_t *_jit, jit_int32_t r0, jit_int32_t r1)
2321 rex(0, 1, r0, _NOREG, r1);
2323 mrm(0x03, r7(r0), r7(r1));
2327 _movir_u(jit_state_t *_jit, jit_int32_t r0, jit_int32_t r1)
2329 rex(0, 0, r1, _NOREG, r0);
2331 ic(0xc0 | (r1 << 3) | r7(r0));
2336 _bswapr_us(jit_state_t *_jit, jit_int32_t r0, jit_int32_t r1)
2340 rex(0, 0, _NOREG, _NOREG, r0);
2342 mrm(0x03, X86_ROR, r7(r0));
2347 _bswapr_ui(jit_state_t *_jit, jit_int32_t r0, jit_int32_t r1)
2350 rex(0, 0, _NOREG, _NOREG, r0);
2355 #if __X64 && !__X64_32
2357 _bswapr_ul(jit_state_t *_jit, jit_int32_t r0, jit_int32_t r1)
2360 rex(0, 1, _NOREG, _NOREG, r0);
2367 _extr_c(jit_state_t *_jit, jit_int32_t r0, jit_int32_t r1)
2373 reg = jit_get_reg(jit_class_gpr|jit_class_rg8);
2381 _extr_uc(jit_state_t *_jit, jit_int32_t r0, jit_int32_t r1)
2387 reg = jit_get_reg(jit_class_gpr|jit_class_rg8);
2389 movcr_u(r0, rn(reg));
2395 _ldr_c(jit_state_t *_jit, jit_int32_t r0, jit_int32_t r1)
2397 rex(0, WIDE, r0, _NOREG, r1);
2400 rx(r0, 0, r1, _NOREG, _SCL1);
2404 _ldi_c(jit_state_t *_jit, jit_int32_t r0, jit_word_t i0)
2407 if (can_sign_extend_int_p(i0)) {
2408 rex(0, WIDE, r0, _NOREG, _NOREG);
2411 rx(r0, i0, _NOREG, _NOREG, _SCL1);
2414 reg = jit_get_reg(jit_class_gpr);
2422 _ldr_uc(jit_state_t *_jit, jit_int32_t r0, jit_int32_t r1)
2424 rex(0, WIDE, r0, _NOREG, r1);
2427 rx(r0, 0, r1, _NOREG, _SCL1);
2431 _ldi_uc(jit_state_t *_jit, jit_int32_t r0, jit_word_t i0)
2434 if (can_sign_extend_int_p(i0)) {
2435 rex(0, WIDE, r0, _NOREG, _NOREG);
2438 rx(r0, i0, _NOREG, _NOREG, _SCL1);
2441 reg = jit_get_reg(jit_class_gpr);
2443 ldr_uc(r0, rn(reg));
2449 _ldr_s(jit_state_t *_jit, jit_int32_t r0, jit_int32_t r1)
2451 rex(0, WIDE, r0, _NOREG, r1);
2454 rx(r0, 0, r1, _NOREG, _SCL1);
2458 _ldi_s(jit_state_t *_jit, jit_int32_t r0, jit_word_t i0)
2461 if (can_sign_extend_int_p(i0)) {
2462 rex(0, WIDE, r0, _NOREG, _NOREG);
2465 rx(r0, i0, _NOREG, _NOREG, _SCL1);
2468 reg = jit_get_reg(jit_class_gpr);
2476 _ldr_us(jit_state_t *_jit, jit_int32_t r0, jit_int32_t r1)
2478 rex(0, WIDE, r0, _NOREG, r1);
2481 rx(r0, 0, r1, _NOREG, _SCL1);
2485 _ldi_us(jit_state_t *_jit, jit_int32_t r0, jit_word_t i0)
2488 if (can_sign_extend_int_p(i0)) {
2489 rex(0, WIDE, r0, _NOREG, _NOREG);
2492 rx(r0, i0, _NOREG, _NOREG, _SCL1);
2495 reg = jit_get_reg(jit_class_gpr);
2497 ldr_us(r0, rn(reg));
2502 #if __X32 || !__X64_32
2504 _ldr_i(jit_state_t *_jit, jit_int32_t r0, jit_int32_t r1)
2507 rex(0, WIDE, r0, _NOREG, r1);
2512 rx(r0, 0, r1, _NOREG, _SCL1);
2516 _ldi_i(jit_state_t *_jit, jit_int32_t r0, jit_word_t i0)
2519 if (can_sign_extend_int_p(i0)) {
2521 rex(0, WIDE, r0, _NOREG, _NOREG);
2526 rx(r0, i0, _NOREG, _NOREG, _SCL1);
2529 reg = jit_get_reg(jit_class_gpr);
2539 _ldr_ui(jit_state_t *_jit, jit_int32_t r0, jit_int32_t r1)
2541 rex(0, 0, r0, _NOREG, r1);
2543 rx(r0, 0, r1, _NOREG, _SCL1);
2547 _ldi_ui(jit_state_t *_jit, jit_int32_t r0, jit_word_t i0)
2550 if (can_sign_extend_int_p(i0)) {
2551 rex(0, 0, r0, _NOREG, _NOREG);
2553 rx(r0, i0, _NOREG, _NOREG, _SCL1);
2556 reg = jit_get_reg(jit_class_gpr);
2558 ldr_ui(r0, rn(reg));
2565 _ldr_l(jit_state_t *_jit, jit_int32_t r0, jit_int32_t r1)
2567 rex(0, 1, r0, _NOREG, r1);
2569 rx(r0, 0, r1, _NOREG, _SCL1);
2573 _ldi_l(jit_state_t *_jit, jit_int32_t r0, jit_word_t i0)
2576 if (can_sign_extend_int_p(i0)) {
2577 rex(0, 1, r0, _NOREG, _NOREG);
2579 rx(r0, i0, _NOREG, _NOREG, _SCL1);
2582 reg = jit_get_reg(jit_class_gpr);
2592 _ldxr_c(jit_state_t *_jit, jit_int32_t r0, jit_int32_t r1, jit_int32_t r2)
2598 rex(0, WIDE, r0, r1, r2);
2601 rx(r0, 0, r2, r1, _SCL1);
2606 _ldxi_c(jit_state_t *_jit, jit_int32_t r0, jit_int32_t r1, jit_word_t i0)
2609 if (can_sign_extend_int_p(i0)) {
2610 rex(0, WIDE, r0, _NOREG, r1);
2613 rx(r0, i0, r1, _NOREG, _SCL1);
2616 reg = jit_get_reg(jit_class_gpr);
2618 ldxr_c(r0, r1, rn(reg));
2624 _ldxr_uc(jit_state_t *_jit, jit_int32_t r0, jit_int32_t r1, jit_int32_t r2)
2630 rex(0, WIDE, r0, r1, r2);
2633 rx(r0, 0, r2, r1, _SCL1);
2638 _ldxi_uc(jit_state_t *_jit, jit_int32_t r0, jit_int32_t r1, jit_word_t i0)
2641 if (can_sign_extend_int_p(i0)) {
2642 rex(0, WIDE, r0, _NOREG, r1);
2645 rx(r0, i0, r1, _NOREG, _SCL1);
2648 reg = jit_get_reg(jit_class_gpr);
2650 ldxr_uc(r0, r1, rn(reg));
2656 _ldxr_s(jit_state_t *_jit, jit_int32_t r0, jit_int32_t r1, jit_int32_t r2)
2662 rex(0, WIDE, r0, r1, r2);
2665 rx(r0, 0, r2, r1, _SCL1);
2670 _ldxi_s(jit_state_t *_jit, jit_int32_t r0, jit_int32_t r1, jit_word_t i0)
2673 if (can_sign_extend_int_p(i0)) {
2674 rex(0, WIDE, r0, _NOREG, r1);
2677 rx(r0, i0, r1, _NOREG, _SCL1);
2680 reg = jit_get_reg(jit_class_gpr);
2682 ldxr_s(r0, r1, rn(reg));
2688 _ldxr_us(jit_state_t *_jit, jit_int32_t r0, jit_int32_t r1, jit_int32_t r2)
2694 rex(0, WIDE, r0, r1, r2);
2697 rx(r0, 0, r2, r1, _SCL1);
2702 _ldxi_us(jit_state_t *_jit, jit_int32_t r0, jit_int32_t r1, jit_word_t i0)
2705 if (can_sign_extend_int_p(i0)) {
2706 rex(0, WIDE, r0, _NOREG, r1);
2709 rx(r0, i0, r1, _NOREG, _SCL1);
2712 reg = jit_get_reg(jit_class_gpr);
2714 ldxr_us(r0, r1, rn(reg));
2719 #if __X64 || !__X64_32
2721 _ldxr_i(jit_state_t *_jit, jit_int32_t r0, jit_int32_t r1, jit_int32_t r2)
2724 rex(0, WIDE, r0, r1, r2);
2729 rx(r0, 0, r2, r1, _SCL1);
2733 _ldxi_i(jit_state_t *_jit, jit_int32_t r0, jit_int32_t r1, jit_word_t i0)
2736 if (can_sign_extend_int_p(i0)) {
2738 rex(0, WIDE, r0, _NOREG, r1);
2743 rx(r0, i0, r1, _NOREG, _SCL1);
2746 reg = jit_get_reg(jit_class_gpr);
2748 ldxr_i(r0, r1, rn(reg));
2756 _ldxr_ui(jit_state_t *_jit, jit_int32_t r0, jit_int32_t r1, jit_int32_t r2)
2760 /* to avoid confusion with macro renames */
2761 _ldr_ui(_jit, r0, r0);
2763 rex(0, 0, r0, r1, r2);
2765 rx(r0, 0, r2, r1, _SCL1);
2770 _ldxi_ui(jit_state_t *_jit, jit_int32_t r0, jit_int32_t r1, jit_word_t i0)
2773 if (can_sign_extend_int_p(i0)) {
2774 rex(0, 0, r0, _NOREG, r1);
2776 rx(r0, i0, r1, _NOREG, _SCL1);
2779 reg = jit_get_reg(jit_class_gpr);
2781 ldxr_ui(r0, r1, rn(reg));
2788 _ldxr_l(jit_state_t *_jit, jit_int32_t r0, jit_int32_t r1, jit_int32_t r2)
2790 rex(0, 1, r0, r1, r2);
2792 rx(r0, 0, r2, r1, _SCL1);
2796 _ldxi_l(jit_state_t *_jit, jit_int32_t r0, jit_int32_t r1, jit_word_t i0)
2799 if (can_sign_extend_int_p(i0)) {
2800 rex(0, 1, r0, _NOREG, r1);
2802 rx(r0, i0, r1, _NOREG, _SCL1);
2805 reg = jit_get_reg(jit_class_gpr);
2807 ldxr_l(r0, r1, rn(reg));
2815 _str_c(jit_state_t *_jit, jit_int32_t r0, jit_int32_t r1)
2819 rex(0, 0, r1, _NOREG, r0);
2821 rx(r1, 0, r0, _NOREG, _SCL1);
2824 reg = jit_get_reg(jit_class_gpr|jit_class_rg8);
2826 rex(0, 0, rn(reg), _NOREG, r0);
2828 rx(rn(reg), 0, r0, _NOREG, _SCL1);
2834 _sti_c(jit_state_t *_jit, jit_word_t i0, jit_int32_t r0)
2837 if (can_sign_extend_int_p(i0)) {
2839 rex(0, 0, r0, _NOREG, _NOREG);
2841 rx(r0, i0, _NOREG, _NOREG, _SCL1);
2844 reg = jit_get_reg(jit_class_gpr|jit_class_rg8);
2846 rex(0, 0, rn(reg), _NOREG, _NOREG);
2848 rx(rn(reg), i0, _NOREG, _NOREG, _SCL1);
2853 reg = jit_get_reg(jit_class_gpr);
2861 _str_s(jit_state_t *_jit, jit_int32_t r0, jit_int32_t r1)
2864 rex(0, 0, r1, _NOREG, r0);
2866 rx(r1, 0, r0, _NOREG, _SCL1);
2870 _sti_s(jit_state_t *_jit, jit_word_t i0, jit_int32_t r0)
2873 if (can_sign_extend_int_p(i0)) {
2875 rex(0, 0, r0, _NOREG, _NOREG);
2877 rx(r0, i0, _NOREG, _NOREG, _SCL1);
2880 reg = jit_get_reg(jit_class_gpr);
2888 _str_i(jit_state_t *_jit, jit_int32_t r0, jit_int32_t r1)
2890 rex(0, 0, r1, _NOREG, r0);
2892 rx(r1, 0, r0, _NOREG, _SCL1);
2896 _sti_i(jit_state_t *_jit, jit_word_t i0, jit_int32_t r0)
2899 if (can_sign_extend_int_p(i0)) {
2900 rex(0, 0, r0, _NOREG, _NOREG);
2902 rx(r0, i0, _NOREG, _NOREG, _SCL1);
2905 reg = jit_get_reg(jit_class_gpr);
2912 #if __X64 && !__X64_32
2914 _str_l(jit_state_t *_jit, jit_int32_t r0, jit_int32_t r1)
2916 rex(0, 1, r1, _NOREG, r0);
2918 rx(r1, 0, r0, _NOREG, _SCL1);
2922 _sti_l(jit_state_t *_jit, jit_word_t i0, jit_int32_t r0)
2925 if (can_sign_extend_int_p(i0)) {
2926 rex(0, 1, r0, _NOREG, _NOREG);
2928 rx(r0, i0, _NOREG, _NOREG, _SCL1);
2931 reg = jit_get_reg(jit_class_gpr);
2940 _stxr_c(jit_state_t *_jit, jit_int32_t r0, jit_int32_t r1, jit_int32_t r2)
2944 reg = jit_get_reg(jit_class_gpr);
2945 addr(rn(reg), r0, r1);
2950 rex(0, 0, r2, r1, r0);
2952 rx(r2, 0, r0, r1, _SCL1);
2955 reg = jit_get_reg(jit_class_gpr|jit_class_rg8);
2957 rex(0, 0, rn(reg), r1, r0);
2959 rx(rn(reg), 0, r0, r1, _SCL1);
2966 _stxi_c(jit_state_t *_jit, jit_word_t i0, jit_int32_t r0, jit_int32_t r1)
2969 if (can_sign_extend_int_p(i0)) {
2971 rex(0, 0, r1, _NOREG, r0);
2973 rx(r1, i0, r0, _NOREG, _SCL1);
2976 reg = jit_get_reg(jit_class_gpr|jit_class_rg8);
2978 rex(0, 0, rn(reg), _NOREG, r0);
2980 rx(rn(reg), i0, r0, _NOREG, _SCL1);
2985 reg = jit_get_reg(jit_class_gpr);
2987 stxr_c(rn(reg), r0, r1);
2993 _stxr_s(jit_state_t *_jit, jit_int32_t r0, jit_int32_t r1, jit_int32_t r2)
2997 reg = jit_get_reg(jit_class_gpr);
2998 addr(rn(reg), r0, r1);
3003 rex(0, 0, r2, r1, r0);
3005 rx(r2, 0, r0, r1, _SCL1);
3010 _stxi_s(jit_state_t *_jit, jit_word_t i0, jit_int32_t r0, jit_int32_t r1)
3013 if (can_sign_extend_int_p(i0)) {
3015 rex(0, 0, r1, _NOREG, r0);
3017 rx(r1, i0, r0, _NOREG, _SCL1);
3020 reg = jit_get_reg(jit_class_gpr);
3022 stxr_s(rn(reg), r0, r1);
3028 _stxr_i(jit_state_t *_jit, jit_int32_t r0, jit_int32_t r1, jit_int32_t r2)
3032 reg = jit_get_reg(jit_class_gpr);
3033 addr(rn(reg), r0, r1);
3037 rex(0, 0, r2, r1, r0);
3039 rx(r2, 0, r0, r1, _SCL1);
3044 _stxi_i(jit_state_t *_jit, jit_word_t i0, jit_int32_t r0, jit_int32_t r1)
3047 if (can_sign_extend_int_p(i0)) {
3048 rex(0, 0, r1, _NOREG, r0);
3050 rx(r1, i0, r0, _NOREG, _SCL1);
3053 reg = jit_get_reg(jit_class_gpr);
3055 stxr_i(rn(reg), r0, r1);
3060 #if __X64 && !__X64_32
3062 _stxr_l(jit_state_t *_jit, jit_int32_t r0, jit_int32_t r1, jit_int32_t r2)
3064 rex(0, 1, r2, r1, r0);
3066 rx(r2, 0, r0, r1, _SCL1);
3070 _stxi_l(jit_state_t *_jit, jit_word_t i0, jit_int32_t r0, jit_int32_t r1)
3073 if (can_sign_extend_int_p(i0)) {
3074 rex(0, 1, r1, _NOREG, r0);
3076 rx(r1, i0, r0, _NOREG, _SCL1);
3079 reg = jit_get_reg(jit_class_gpr);
3081 stxr_l(rn(reg), r0, r1);
3088 _jccs(jit_state_t *_jit, jit_int32_t code, jit_word_t i0)
3092 w = i0 - (_jit->pc.w + 1);
3097 _jcc(jit_state_t *_jit, jit_int32_t code, jit_word_t i0)
3102 w = i0 - (_jit->pc.w + 4);
3107 _jcr(jit_state_t *_jit,
3108 jit_int32_t code, jit_word_t i0, jit_int32_t r0, jit_int32_t r1)
3110 alur(X86_CMP, r0, r1);
3115 _jci(jit_state_t *_jit,
3116 jit_int32_t code, jit_word_t i0, jit_int32_t r0, jit_word_t i1)
3118 alui(X86_CMP, r0, i1);
3123 _jci0(jit_state_t *_jit, jit_int32_t code, jit_word_t i0, jit_int32_t r0)
3130 _bltr(jit_state_t *_jit, jit_word_t i0, jit_int32_t r0, jit_int32_t r1)
3132 jcr(X86_CC_L, i0, r0, r1);
3133 return (_jit->pc.w);
3137 _blti(jit_state_t *_jit, jit_word_t i0, jit_int32_t r0, jit_word_t i1)
3139 if (i1) jci (X86_CC_L, i0, r0, i1);
3140 else jci0(X86_CC_S, i0, r0);
3141 return (_jit->pc.w);
3145 _bltr_u(jit_state_t *_jit, jit_word_t i0, jit_int32_t r0, jit_int32_t r1)
3147 jcr(X86_CC_B, i0, r0, r1);
3148 return (_jit->pc.w);
3152 _blti_u(jit_state_t *_jit, jit_word_t i0, jit_int32_t r0, jit_word_t i1)
3154 if (i1) jci (X86_CC_B, i0, r0, i1);
3155 else jci0(X86_CC_B, i0, r0);
3156 return (_jit->pc.w);
3160 _bler(jit_state_t *_jit, jit_word_t i0, jit_int32_t r0, jit_int32_t r1)
3162 if (r0 == r1) jmpi(i0);
3163 else jcr (X86_CC_LE, i0, r0, r1);
3164 return (_jit->pc.w);
3168 _blei(jit_state_t *_jit, jit_word_t i0, jit_int32_t r0, jit_word_t i1)
3170 if (i1) jci (X86_CC_LE, i0, r0, i1);
3171 else jci0(X86_CC_LE, i0, r0);
3172 return (_jit->pc.w);
3176 _bler_u(jit_state_t *_jit, jit_word_t i0, jit_int32_t r0, jit_int32_t r1)
3178 if (r0 == r1) jmpi(i0);
3179 else jcr (X86_CC_BE, i0, r0, r1);
3180 return (_jit->pc.w);
3184 _blei_u(jit_state_t *_jit, jit_word_t i0, jit_int32_t r0, jit_word_t i1)
3186 if (i1) jci (X86_CC_BE, i0, r0, i1);
3187 else jci0(X86_CC_BE, i0, r0);
3188 return (_jit->pc.w);
3192 _beqr(jit_state_t *_jit, jit_word_t i0, jit_int32_t r0, jit_int32_t r1)
3194 if (r0 == r1) jmpi(i0);
3195 else jcr (X86_CC_E, i0, r0, r1);
3196 return (_jit->pc.w);
3200 _beqi(jit_state_t *_jit, jit_word_t i0, jit_int32_t r0, jit_word_t i1)
3202 if (i1) jci (X86_CC_E, i0, r0, i1);
3203 else jci0(X86_CC_E, i0, r0);
3204 return (_jit->pc.w);
3208 _bger(jit_state_t *_jit, jit_word_t i0, jit_int32_t r0, jit_int32_t r1)
3210 if (r0 == r1) jmpi(i0);
3211 else jcr (X86_CC_GE, i0, r0, r1);
3212 return (_jit->pc.w);
3216 _bgei(jit_state_t *_jit, jit_word_t i0, jit_int32_t r0, jit_word_t i1)
3218 if (i1) jci (X86_CC_GE, i0, r0, i1);
3219 else jci0(X86_CC_NS, i0, r0);
3220 return (_jit->pc.w);
3224 _bger_u(jit_state_t *_jit, jit_word_t i0, jit_int32_t r0, jit_int32_t r1)
3226 if (r0 == r1) jmpi(i0);
3227 else jcr (X86_CC_AE, i0, r0, r1);
3228 return (_jit->pc.w);
3232 _bgei_u(jit_state_t *_jit, jit_word_t i0, jit_int32_t r0, jit_word_t i1)
3234 if (i1) jci (X86_CC_AE, i0, r0, i1);
3236 return (_jit->pc.w);
3240 _bgtr(jit_state_t *_jit, jit_word_t i0, jit_int32_t r0, jit_int32_t r1)
3242 jcr(X86_CC_G, i0, r0, r1);
3243 return (_jit->pc.w);
3247 _bgti(jit_state_t *_jit, jit_word_t i0, jit_int32_t r0, jit_word_t i1)
3249 jci(X86_CC_G, i0, r0, i1);
3250 return (_jit->pc.w);
3254 _bgtr_u(jit_state_t *_jit, jit_word_t i0, jit_int32_t r0, jit_int32_t r1)
3256 jcr(X86_CC_A, i0, r0, r1);
3257 return (_jit->pc.w);
3261 _bgti_u(jit_state_t *_jit, jit_word_t i0, jit_int32_t r0, jit_word_t i1)
3263 if (i1) jci (X86_CC_A, i0, r0, i1);
3264 else jci0(X86_CC_NE, i0, r0);
3265 return (_jit->pc.w);
3269 _bner(jit_state_t *_jit, jit_word_t i0, jit_int32_t r0, jit_int32_t r1)
3271 jcr(X86_CC_NE, i0, r0, r1);
3272 return (_jit->pc.w);
3276 _bnei(jit_state_t *_jit, jit_word_t i0, jit_int32_t r0, jit_word_t i1)
3278 if (i1) jci (X86_CC_NE, i0, r0, i1);
3279 else jci0(X86_CC_NE, i0, r0);
3280 return (_jit->pc.w);
3284 _bmsr(jit_state_t *_jit, jit_word_t i0, jit_int32_t r0, jit_int32_t r1)
3288 return (_jit->pc.w);
3292 _bmsi(jit_state_t *_jit, jit_word_t i0, jit_int32_t r0, jit_word_t i1)
3295 if (can_zero_extend_int_p(i1))
3298 reg = jit_get_reg(jit_class_gpr);
3304 return (_jit->pc.w);
3308 _bmcr(jit_state_t *_jit, jit_word_t i0, jit_int32_t r0, jit_int32_t r1)
3312 return (_jit->pc.w);
3316 _bmci(jit_state_t *_jit, jit_word_t i0, jit_int32_t r0, jit_word_t i1)
3319 if (can_zero_extend_int_p(i1))
3322 reg = jit_get_reg(jit_class_gpr);
3328 return (_jit->pc.w);
3332 _boaddr(jit_state_t *_jit, jit_word_t i0, jit_int32_t r0, jit_int32_t r1)
3336 return (_jit->pc.w);
3340 _boaddi(jit_state_t *_jit, jit_word_t i0, jit_int32_t r0, jit_word_t i1)
3343 if (can_sign_extend_int_p(i1)) {
3346 return (_jit->pc.w);
3348 reg = jit_get_reg(jit_class_gpr|jit_class_nospill);
3351 return (boaddr(i0, r0, rn(reg)));
3355 _boaddr_u(jit_state_t *_jit, jit_word_t i0, jit_int32_t r0, jit_int32_t r1)
3359 return (_jit->pc.w);
3363 _boaddi_u(jit_state_t *_jit, jit_word_t i0, jit_int32_t r0, jit_word_t i1)
3366 if (can_sign_extend_int_p(i1)) {
3369 return (_jit->pc.w);
3371 reg = jit_get_reg(jit_class_gpr|jit_class_nospill);
3374 return (boaddr_u(i0, r0, rn(reg)));
3378 _bxaddr(jit_state_t *_jit, jit_word_t i0, jit_int32_t r0, jit_int32_t r1)
3382 return (_jit->pc.w);
3386 _bxaddi(jit_state_t *_jit, jit_word_t i0, jit_int32_t r0, jit_word_t i1)
3389 if (can_sign_extend_int_p(i1)) {
3392 return (_jit->pc.w);
3394 reg = jit_get_reg(jit_class_gpr|jit_class_nospill);
3397 return (bxaddr(i0, r0, rn(reg)));
3401 _bxaddr_u(jit_state_t *_jit, jit_word_t i0, jit_int32_t r0, jit_int32_t r1)
3405 return (_jit->pc.w);
3409 _bxaddi_u(jit_state_t *_jit, jit_word_t i0, jit_int32_t r0, jit_word_t i1)
3412 if (can_sign_extend_int_p(i1)) {
3415 return (_jit->pc.w);
3417 reg = jit_get_reg(jit_class_gpr|jit_class_nospill);
3420 return (bxaddr_u(i0, r0, rn(reg)));
3424 _bosubr(jit_state_t *_jit, jit_word_t i0, jit_int32_t r0, jit_int32_t r1)
3428 return (_jit->pc.w);
3432 _bosubi(jit_state_t *_jit, jit_word_t i0, jit_int32_t r0, jit_word_t i1)
3435 if (can_sign_extend_int_p(i1)) {
3438 return (_jit->pc.w);
3440 reg = jit_get_reg(jit_class_gpr|jit_class_nospill);
3443 return (bosubr(i0, r0, rn(reg)));
3447 _bosubr_u(jit_state_t *_jit, jit_word_t i0, jit_int32_t r0, jit_int32_t r1)
3451 return (_jit->pc.w);
3455 _bosubi_u(jit_state_t *_jit, jit_word_t i0, jit_int32_t r0, jit_word_t i1)
3458 if (can_sign_extend_int_p(i1)) {
3461 return (_jit->pc.w);
3463 reg = jit_get_reg(jit_class_gpr|jit_class_nospill);
3466 return (bosubr_u(i0, r0, rn(reg)));
3470 _bxsubr(jit_state_t *_jit, jit_word_t i0, jit_int32_t r0, jit_int32_t r1)
3474 return (_jit->pc.w);
3478 _bxsubi(jit_state_t *_jit, jit_word_t i0, jit_int32_t r0, jit_word_t i1)
3481 if (can_sign_extend_int_p(i1)) {
3484 return (_jit->pc.w);
3486 reg = jit_get_reg(jit_class_gpr|jit_class_nospill);
3489 return (bxsubr(i0, r0, rn(reg)));
3493 _bxsubr_u(jit_state_t *_jit, jit_word_t i0, jit_int32_t r0, jit_int32_t r1)
3497 return (_jit->pc.w);
3501 _bxsubi_u(jit_state_t *_jit, jit_word_t i0, jit_int32_t r0, jit_word_t i1)
3504 if (can_sign_extend_int_p(i1)) {
3507 return (_jit->pc.w);
3509 reg = jit_get_reg(jit_class_gpr|jit_class_nospill);
3512 return (bxsubr_u(i0, r0, rn(reg)));
3516 _callr(jit_state_t *_jit, jit_int32_t r0)
3518 rex(0, 0, _NOREG, _NOREG, r0);
3520 mrm(0x03, 0x02, r7(r0));
3524 _calli(jit_state_t *_jit, jit_word_t i0)
3529 w = i0 - (_jit->pc.w + 5);
3530 if ((jit_int32_t)w == w) {
3533 w = i0 - (_jit->pc.w + 4);
3546 _calli_p(jit_state_t *_jit, jit_word_t i0)
3550 reg = jit_get_reg(jit_class_gpr);
3551 word = movi_p(rn(reg), i0);
3559 _jmpr(jit_state_t *_jit, jit_int32_t r0)
3561 rex(0, 0, _NOREG, _NOREG, r0);
3563 mrm(0x03, 0x04, r7(r0));
3567 _jmpi(jit_state_t *_jit, jit_word_t i0)
3572 w = i0 - (_jit->pc.w + 5);
3573 if ((jit_int32_t)w == w) {
3576 w = i0 - (_jit->pc.w + 4);
3589 _jmpi_p(jit_state_t *_jit, jit_word_t i0)
3593 reg = jit_get_reg(jit_class_gpr|jit_class_nospill);
3594 word = movi_p(rn(reg), i0);
3602 _jmpsi(jit_state_t *_jit, jit_uint8_t i0)
3609 _prolog(jit_state_t *_jit, jit_node_t *node)
3612 if (_jitc->function->define_frame || _jitc->function->assume_frame) {
3613 jit_int32_t frame = -_jitc->function->frame;
3614 assert(_jitc->function->self.aoff >= frame);
3615 if (_jitc->function->assume_frame)
3617 _jitc->function->self.aoff = frame;
3619 if (_jitc->function->allocar)
3620 _jitc->function->self.aoff &= -16;
3621 #if __X64 && (__CYGWIN__ || _WIN32)
3622 _jitc->function->stack = (((/* first 32 bytes must be allocated */
3623 (_jitc->function->self.alen > 32 ?
3624 _jitc->function->self.alen : 32) -
3625 /* align stack at 16 bytes */
3626 _jitc->function->self.aoff) + 15) & -16) +
3629 _jitc->function->stack = (((_jitc->function->self.alen -
3630 _jitc->function->self.aoff) + 15) & -16) +
3633 subi(_RSP_REGNO, _RSP_REGNO, stack_framesize - REAL_WORDSIZE);
3634 /* callee save registers */
3636 if (jit_regset_tstbit(&_jitc->function->regset, _RDI))
3637 stxi(12, _RSP_REGNO, _RDI_REGNO);
3638 if (jit_regset_tstbit(&_jitc->function->regset, _RSI))
3639 stxi( 8, _RSP_REGNO, _RSI_REGNO);
3640 if (jit_regset_tstbit(&_jitc->function->regset, _RBX))
3641 stxi( 4, _RSP_REGNO, _RBX_REGNO);
3643 # if __CYGWIN__ || _WIN32
3644 if (jit_regset_tstbit(&_jitc->function->regset, _XMM15))
3645 sse_stxi_d(136, _RSP_REGNO, _XMM15_REGNO);
3646 if (jit_regset_tstbit(&_jitc->function->regset, _XMM14))
3647 sse_stxi_d(128, _RSP_REGNO, _XMM14_REGNO);
3648 if (jit_regset_tstbit(&_jitc->function->regset, _XMM13))
3649 sse_stxi_d(120, _RSP_REGNO, _XMM13_REGNO);
3650 if (jit_regset_tstbit(&_jitc->function->regset, _XMM12))
3651 sse_stxi_d(112, _RSP_REGNO, _XMM12_REGNO);
3652 if (jit_regset_tstbit(&_jitc->function->regset, _XMM11))
3653 sse_stxi_d(104, _RSP_REGNO, _XMM11_REGNO);
3654 if (jit_regset_tstbit(&_jitc->function->regset, _XMM10))
3655 sse_stxi_d(96, _RSP_REGNO, _XMM10_REGNO);
3656 if (jit_regset_tstbit(&_jitc->function->regset, _XMM9))
3657 sse_stxi_d(88, _RSP_REGNO, _XMM9_REGNO);
3658 if (jit_regset_tstbit(&_jitc->function->regset, _XMM8))
3659 sse_stxi_d(80, _RSP_REGNO, _XMM8_REGNO);
3660 if (jit_regset_tstbit(&_jitc->function->regset, _XMM7))
3661 sse_stxi_d(72, _RSP_REGNO, _XMM7_REGNO);
3662 if (jit_regset_tstbit(&_jitc->function->regset, _XMM6))
3663 sse_stxi_d(64, _RSP_REGNO, _XMM6_REGNO);
3664 if (jit_regset_tstbit(&_jitc->function->regset, _R15))
3665 stxi(56, _RSP_REGNO, _R15_REGNO);
3666 if (jit_regset_tstbit(&_jitc->function->regset, _R14))
3667 stxi(48, _RSP_REGNO, _R14_REGNO);
3668 if (jit_regset_tstbit(&_jitc->function->regset, _R13))
3669 stxi(40, _RSP_REGNO, _R13_REGNO);
3670 if (jit_regset_tstbit(&_jitc->function->regset, _R12))
3671 stxi(32, _RSP_REGNO, _R12_REGNO);
3672 if (jit_regset_tstbit(&_jitc->function->regset, _RSI))
3673 stxi(24, _RSP_REGNO, _RSI_REGNO);
3674 if (jit_regset_tstbit(&_jitc->function->regset, _RDI))
3675 stxi(16, _RSP_REGNO, _RDI_REGNO);
3676 if (jit_regset_tstbit(&_jitc->function->regset, _RBX))
3677 stxi( 8, _RSP_REGNO, _RBX_REGNO);
3679 if (jit_regset_tstbit(&_jitc->function->regset, _RBX))
3680 stxi(40, _RSP_REGNO, _RBX_REGNO);
3681 if (jit_regset_tstbit(&_jitc->function->regset, _R12))
3682 stxi(32, _RSP_REGNO, _R12_REGNO);
3683 if (jit_regset_tstbit(&_jitc->function->regset, _R13))
3684 stxi(24, _RSP_REGNO, _R13_REGNO);
3685 if (jit_regset_tstbit(&_jitc->function->regset, _R14))
3686 stxi(16, _RSP_REGNO, _R14_REGNO);
3687 if (jit_regset_tstbit(&_jitc->function->regset, _R15))
3688 stxi( 8, _RSP_REGNO, _R15_REGNO);
3691 stxi(0, _RSP_REGNO, _RBP_REGNO);
3692 movr(_RBP_REGNO, _RSP_REGNO);
3695 subi(_RSP_REGNO, _RSP_REGNO, _jitc->function->stack);
3696 if (_jitc->function->allocar) {
3697 reg = jit_get_reg(jit_class_gpr);
3698 movi(rn(reg), _jitc->function->self.aoff);
3699 stxi_i(_jitc->function->aoffoff, _RBP_REGNO, rn(reg));
3703 #if __X64 && !(__CYGWIN__ || _WIN32)
3704 if (_jitc->function->self.call & jit_call_varargs) {
3705 jit_word_t nofp_code;
3707 /* Save gp registers in the save area, if any is a vararg */
3708 for (reg = first_gp_from_offset(_jitc->function->vagp);
3709 jit_arg_reg_p(reg); ++reg)
3710 stxi(_jitc->function->vaoff + first_gp_offset +
3711 reg * 8, _RBP_REGNO, rn(JIT_RA0 - reg));
3713 reg = first_fp_from_offset(_jitc->function->vafp);
3714 if (jit_arg_f_reg_p(reg)) {
3715 /* Skip over if no float registers were passed as argument */
3720 nofp_code = _jit->pc.w;
3722 /* Save fp registers in the save area, if any is a vararg */
3723 /* Note that the full 16 byte xmm is not saved, because
3724 * lightning only handles float and double, and, while
3725 * attempting to provide a va_list compatible pointer as
3726 * jit_va_start return, does not guarantee it (on all ports). */
3727 for (; jit_arg_f_reg_p(reg); ++reg)
3728 sse_stxi_d(_jitc->function->vaoff + first_fp_offset +
3729 reg * va_fp_increment, _RBP_REGNO, rn(_XMM0 - reg));
3731 patch_rel_char(nofp_code, _jit->pc.w);
3738 _epilog(jit_state_t *_jit, jit_node_t *node)
3740 if (_jitc->function->assume_frame)
3742 /* callee save registers */
3743 movr(_RSP_REGNO, _RBP_REGNO);
3745 if (jit_regset_tstbit(&_jitc->function->regset, _RDI))
3746 ldxi(_RDI_REGNO, _RSP_REGNO, 12);
3747 if (jit_regset_tstbit(&_jitc->function->regset, _RSI))
3748 ldxi(_RSI_REGNO, _RSP_REGNO, 8);
3749 if (jit_regset_tstbit(&_jitc->function->regset, _RBX))
3750 ldxi(_RBX_REGNO, _RSP_REGNO, 4);
3752 # if __CYGWIN__ || _WIN32
3753 if (jit_regset_tstbit(&_jitc->function->regset, _XMM15))
3754 sse_ldxi_d(_XMM15_REGNO, _RSP_REGNO, 136);
3755 if (jit_regset_tstbit(&_jitc->function->regset, _XMM14))
3756 sse_ldxi_d(_XMM14_REGNO, _RSP_REGNO, 128);
3757 if (jit_regset_tstbit(&_jitc->function->regset, _XMM13))
3758 sse_ldxi_d(_XMM13_REGNO, _RSP_REGNO, 120);
3759 if (jit_regset_tstbit(&_jitc->function->regset, _XMM12))
3760 sse_ldxi_d(_XMM12_REGNO, _RSP_REGNO, 112);
3761 if (jit_regset_tstbit(&_jitc->function->regset, _XMM11))
3762 sse_ldxi_d(_XMM11_REGNO, _RSP_REGNO, 104);
3763 if (jit_regset_tstbit(&_jitc->function->regset, _XMM10))
3764 sse_ldxi_d(_XMM10_REGNO, _RSP_REGNO, 96);
3765 if (jit_regset_tstbit(&_jitc->function->regset, _XMM9))
3766 sse_ldxi_d(_XMM9_REGNO, _RSP_REGNO, 88);
3767 if (jit_regset_tstbit(&_jitc->function->regset, _XMM8))
3768 sse_ldxi_d(_XMM8_REGNO, _RSP_REGNO, 80);
3769 if (jit_regset_tstbit(&_jitc->function->regset, _XMM7))
3770 sse_ldxi_d(_XMM7_REGNO, _RSP_REGNO, 72);
3771 if (jit_regset_tstbit(&_jitc->function->regset, _XMM6))
3772 sse_ldxi_d(_XMM6_REGNO, _RSP_REGNO, 64);
3773 if (jit_regset_tstbit(&_jitc->function->regset, _R15))
3774 ldxi(_R15_REGNO, _RSP_REGNO, 56);
3775 if (jit_regset_tstbit(&_jitc->function->regset, _R14))
3776 ldxi(_R14_REGNO, _RSP_REGNO, 48);
3777 if (jit_regset_tstbit(&_jitc->function->regset, _R13))
3778 ldxi(_R13_REGNO, _RSP_REGNO, 40);
3779 if (jit_regset_tstbit(&_jitc->function->regset, _R12))
3780 ldxi(_R12_REGNO, _RSP_REGNO, 32);
3781 if (jit_regset_tstbit(&_jitc->function->regset, _RSI))
3782 ldxi(_RSI_REGNO, _RSP_REGNO, 24);
3783 if (jit_regset_tstbit(&_jitc->function->regset, _RDI))
3784 ldxi(_RDI_REGNO, _RSP_REGNO, 16);
3785 if (jit_regset_tstbit(&_jitc->function->regset, _RBX))
3786 ldxi(_RBX_REGNO, _RSP_REGNO, 8);
3788 if (jit_regset_tstbit(&_jitc->function->regset, _RBX))
3789 ldxi(_RBX_REGNO, _RSP_REGNO, 40);
3790 if (jit_regset_tstbit(&_jitc->function->regset, _R12))
3791 ldxi(_R12_REGNO, _RSP_REGNO, 32);
3792 if (jit_regset_tstbit(&_jitc->function->regset, _R13))
3793 ldxi(_R13_REGNO, _RSP_REGNO, 24);
3794 if (jit_regset_tstbit(&_jitc->function->regset, _R14))
3795 ldxi(_R14_REGNO, _RSP_REGNO, 16);
3796 if (jit_regset_tstbit(&_jitc->function->regset, _R15))
3797 ldxi(_R15_REGNO, _RSP_REGNO, 8);
3800 ldxi(_RBP_REGNO, _RSP_REGNO, 0);
3801 addi(_RSP_REGNO, _RSP_REGNO, stack_framesize - REAL_WORDSIZE);
3807 _vastart(jit_state_t *_jit, jit_int32_t r0)
3809 #if __X32 || __CYGWIN__ || _WIN32
3810 assert(_jitc->function->self.call & jit_call_varargs);
3811 addi(r0, _RBP_REGNO, _jitc->function->self.size);
3815 assert(_jitc->function->self.call & jit_call_varargs);
3817 /* Return jit_va_list_t in the register argument */
3818 addi(r0, _RBP_REGNO, _jitc->function->vaoff);
3819 reg = jit_get_reg(jit_class_gpr);
3821 /* Initialize gp offset in the save area. */
3822 movi(rn(reg), _jitc->function->vagp);
3823 stxi_i(offsetof(jit_va_list_t, gpoff), r0, rn(reg));
3825 /* Initialize fp offset in the save area. */
3826 movi(rn(reg), _jitc->function->vafp);
3827 stxi_i(offsetof(jit_va_list_t, fpoff), r0, rn(reg));
3829 /* Initialize overflow pointer to the first stack argument. */
3830 addi(rn(reg), _RBP_REGNO, _jitc->function->self.size);
3831 stxi(offsetof(jit_va_list_t, over), r0, rn(reg));
3833 /* Initialize register save area pointer. */
3834 addi(rn(reg), r0, first_gp_offset);
3835 stxi(offsetof(jit_va_list_t, save), r0, rn(reg));
3842 _vaarg(jit_state_t *_jit, jit_int32_t r0, jit_int32_t r1)
3844 #if __X32 || __CYGWIN__ || _WIN32
3845 assert(_jitc->function->self.call & jit_call_varargs);
3847 addi(r1, r1, va_gp_increment);
3854 assert(_jitc->function->self.call & jit_call_varargs);
3856 rg0 = jit_get_reg(jit_class_gpr);
3857 rg1 = jit_get_reg(jit_class_gpr);
3859 /* Load the gp offset in save area in the first temporary. */
3860 ldxi_i(rn(rg0), r1, offsetof(jit_va_list_t, gpoff));
3862 /* Jump over if there are no remaining arguments in the save area. */
3863 icmpi(rn(rg0), va_gp_max_offset);
3865 ge_code = _jit->pc.w;
3867 /* Load the save area pointer in the second temporary. */
3868 ldxi(rn(rg1), r1, offsetof(jit_va_list_t, save));
3870 /* Load the vararg argument in the first argument. */
3871 ldxr(r0, rn(rg1), rn(rg0));
3873 /* Update the gp offset. */
3874 addi(rn(rg0), rn(rg0), 8);
3875 stxi_i(offsetof(jit_va_list_t, gpoff), r1, rn(rg0));
3877 /* Will only need one temporary register below. */
3880 /* Jump over overflow code. */
3882 lt_code = _jit->pc.w;
3884 /* Where to land if argument is in overflow area. */
3885 patch_rel_char(ge_code, _jit->pc.w);
3887 /* Load overflow pointer. */
3888 ldxi(rn(rg0), r1, offsetof(jit_va_list_t, over));
3890 /* Load argument. */
3893 /* Update overflow pointer. */
3894 addi(rn(rg0), rn(rg0), va_gp_increment);
3895 stxi(offsetof(jit_va_list_t, over), r1, rn(rg0));
3897 /* Where to land if argument is in save area. */
3898 patch_rel_char(lt_code, _jit->pc.w);
3904 /* The x87 boolean argument tells if will put the result in a x87
3905 * register if non false, in a sse register otherwise. */
3907 _vaarg_d(jit_state_t *_jit, jit_int32_t r0, jit_int32_t r1, jit_bool_t x87)
3909 #if __X32 || __CYGWIN__ || _WIN32
3910 assert(_jitc->function->self.call & jit_call_varargs);
3922 assert(_jitc->function->self.call & jit_call_varargs);
3924 rg0 = jit_get_reg(jit_class_gpr);
3925 rg1 = jit_get_reg(jit_class_gpr);
3927 /* Load the fp offset in save area in the first temporary. */
3928 ldxi_i(rn(rg0), r1, offsetof(jit_va_list_t, fpoff));
3930 /* Jump over if there are no remaining arguments in the save area. */
3931 icmpi(rn(rg0), va_fp_max_offset);
3933 ge_code = _jit->pc.w;
3935 /* Load the save area pointer in the second temporary. */
3936 ldxi(rn(rg1), r1, offsetof(jit_va_list_t, save));
3938 /* Load the vararg argument in the first argument. */
3940 x87_ldxr_d(r0, rn(rg1), rn(rg0));
3942 sse_ldxr_d(r0, rn(rg1), rn(rg0));
3944 /* Update the fp offset. */
3945 addi(rn(rg0), rn(rg0), va_fp_increment);
3946 stxi_i(offsetof(jit_va_list_t, fpoff), r1, rn(rg0));
3948 /* Will only need one temporary register below. */
3951 /* Jump over overflow code. */
3953 lt_code = _jit->pc.w;
3955 /* Where to land if argument is in overflow area. */
3956 patch_rel_char(ge_code, _jit->pc.w);
3958 /* Load overflow pointer. */
3959 ldxi(rn(rg0), r1, offsetof(jit_va_list_t, over));
3961 /* Load argument. */
3963 x87_ldr_d(r0, rn(rg0));
3965 sse_ldr_d(r0, rn(rg0));
3967 /* Update overflow pointer. */
3968 addi(rn(rg0), rn(rg0), 8);
3969 stxi(offsetof(jit_va_list_t, over), r1, rn(rg0));
3971 /* Where to land if argument is in save area. */
3972 patch_rel_char(lt_code, _jit->pc.w);
3979 _patch_at(jit_state_t *_jit, jit_node_t *node,
3980 jit_word_t instr, jit_word_t label)
3982 switch (node->code) {
3984 case jit_code_calli:
3988 patch_abs(instr, label);
3991 patch_rel(instr, label);