2 * Copyright (C) 2012-2019 Free Software Foundation, Inc.
4 * This file is part of GNU lightning.
6 * GNU lightning is free software; you can redistribute it and/or modify it
7 * under the terms of the GNU Lesser General Public License as published
8 * by the Free Software Foundation; either version 3, or (at your option)
11 * GNU lightning is distributed in the hope that it will be useful, but
12 * WITHOUT ANY WARRANTY; without even the implied warranty of MERCHANTABILITY
13 * or FITNESS FOR A PARTICULAR PURPOSE. See the GNU Lesser General Public
14 * License for more details.
17 * Paulo Cesar Pereira de Andrade
20 /* avoid using it due to partial stalls */
24 # if __X32 || __X64_32
26 # define ldi(u, v) ldi_i(u, v)
27 # define ldr(u, v) ldr_i(u, v)
28 # define ldxr(u, v, w) ldxr_i(u, v, w)
29 # define ldxi(u, v, w) ldxi_i(u, v, w)
30 # define sti(u, v) sti_i(u, v)
31 # define stxi(u, v, w) stxi_i(u, v, w)
32 # define can_sign_extend_int_p(im) 1
33 # define can_zero_extend_int_p(im) 1
34 # define fits_uint32_p(im) 1
37 # define ldi(u, v) ldi_l(u, v)
38 # define ldr(u, v) ldr_l(u, v)
39 # define ldxr(u, v, w) ldxr_l(u, v, w)
40 # define ldxi(u, v, w) ldxi_l(u, v, w)
41 # define sti(u, v) sti_l(u, v)
42 # define stxi(u, v, w) stxi_l(u, v, w)
43 # define can_sign_extend_int_p(im) \
44 (((im) >= 0 && (long long)(im) <= 0x7fffffffLL) || \
45 ((im) < 0 && (long long)(im) > -0x80000000LL))
46 # define can_zero_extend_int_p(im) \
47 ((im) >= 0 && (im) < 0x80000000LL)
48 # define fits_uint32_p(im) (((im) & 0xffffffff00000000LL) == 0)
50 # if __X32 || __CYGWIN__ || __X64_32 || _WIN32
52 ((rn) >= _RAX_REGNO && (rn) <= _RBX_REGNO)
66 # define _R10_REGNO 10
67 # define _R11_REGNO 11
68 # define _R12_REGNO 12
69 # define _R13_REGNO 13
70 # define _R14_REGNO 14
71 # define _R15_REGNO 15
72 # define r7(reg) ((reg) & 7)
73 # define r8(reg) ((reg) & 15)
79 # define X86_OR 1 << 3
80 # define X86_ADC 2 << 3
81 # define X86_SBB 3 << 3
82 # define X86_AND 4 << 3
83 # define X86_SUB 5 << 3
84 # define X86_XOR 6 << 3
85 # define X86_CMP 7 << 3
100 # define X86_CC_NO 0x1
101 # define X86_CC_NAE 0x2
102 # define X86_CC_B 0x2
103 # define X86_CC_C 0x2
104 # define X86_CC_AE 0x3
105 # define X86_CC_NB 0x3
106 # define X86_CC_NC 0x3
107 # define X86_CC_E 0x4
108 # define X86_CC_Z 0x4
109 # define X86_CC_NE 0x5
110 # define X86_CC_NZ 0x5
111 # define X86_CC_BE 0x6
112 # define X86_CC_NA 0x6
113 # define X86_CC_A 0x7
114 # define X86_CC_NBE 0x7
115 # define X86_CC_S 0x8
116 # define X86_CC_NS 0x9
117 # define X86_CC_P 0xa
118 # define X86_CC_PE 0xa
119 # define X86_CC_NP 0xb
120 # define X86_CC_PO 0xb
121 # define X86_CC_L 0xc
122 # define X86_CC_NGE 0xc
123 # define X86_CC_GE 0xd
124 # define X86_CC_NL 0xd
125 # define X86_CC_LE 0xe
126 # define X86_CC_NG 0xe
127 # define X86_CC_G 0xf
128 # define X86_CC_NLE 0xf
129 # define mrm(md, r, m) *_jit->pc.uc++ = (md<<6) | (r<<3) | m
130 # define sib(sc, i, b) *_jit->pc.uc++ = (sc<<6) | (i<<3) | b
131 # define ic(c) *_jit->pc.uc++ = c
132 # define is(s) *_jit->pc.us++ = s
133 # define ii(i) *_jit->pc.ui++ = i
134 # if __X64 && !__X64_32
135 # define il(l) *_jit->pc.ul++ = l
139 # define patch_abs(instr, label) \
140 *(jit_word_t *)(instr - sizeof(jit_word_t)) = label
141 # define patch_rel(instr, label) \
142 *(jit_int32_t *)(instr - 4) = label - instr
143 # define patch_rel_char(instr, label) \
144 *(jit_int8_t *)(instr - 1) = label - instr
145 # define rex(l, w, r, x, b) _rex(_jit, l, w, r, x, b)
147 _rex(jit_state_t*,jit_int32_t,jit_int32_t,jit_int32_t,jit_int32_t,jit_int32_t);
148 # define rx(rd, md, rb, ri, ms) _rx(_jit, rd, md, rb, ri, ms)
150 _rx(jit_state_t*,jit_int32_t,jit_int32_t,jit_int32_t,jit_int32_t,jit_int32_t);
151 # define nop(n) _nop(_jit, n)
152 static void _nop(jit_state_t*, jit_int32_t);
153 # define emms() is(0x770f)
154 # define lea(md, rb, ri, ms, rd) _lea(_jit, md, rb, ri, ms, rd)
156 _lea(jit_state_t*,jit_int32_t,jit_int32_t,jit_int32_t,jit_int32_t,jit_int32_t);
157 # define pushr(r0) _pushr(_jit, r0)
158 static void _pushr(jit_state_t*, jit_int32_t) maybe_unused;
159 # define popr(r0) _popr(_jit, r0)
160 static void _popr(jit_state_t*, jit_int32_t) maybe_unused;
161 # define xchgr(r0, r1) _xchgr(_jit, r0, r1)
162 static void _xchgr(jit_state_t*, jit_int32_t, jit_int32_t);
163 # define testr(r0, r1) _testr(_jit, r0, r1)
164 static void _testr(jit_state_t*, jit_int32_t, jit_int32_t);
165 # define testi(r0, i0) _testi(_jit, r0, i0)
166 static void _testi(jit_state_t*, jit_int32_t, jit_word_t);
167 # define cc(code, r0) _cc(_jit, code, r0)
168 static void _cc(jit_state_t*, jit_int32_t, jit_int32_t);
169 # define icmpr(r0, r1) alur(X86_CMP, r0, r1)
170 # define alur(code, r0, r1) _alur(_jit, code, r0, r1)
171 static void _alur(jit_state_t*, jit_int32_t, jit_int32_t, jit_int32_t);
172 # define icmpi(r0, i0) alui(X86_CMP, r0, i0)
173 # define alui(code, r0, i0) _alui(_jit, code, r0, i0)
174 static void _alui(jit_state_t*, jit_int32_t, jit_int32_t, jit_word_t);
175 # define iaddr(r0, r1) alur(X86_ADD, r0, r1)
176 # define save(r0) _save(_jit, r0)
177 static void _save(jit_state_t*, jit_int32_t);
178 # define load(r0) _load(_jit, r0)
179 static void _load(jit_state_t*, jit_int32_t);
180 # define addr(r0, r1, r2) _addr(_jit, r0, r1, r2)
181 static void _addr(jit_state_t*, jit_int32_t, jit_int32_t, jit_int32_t);
182 # define iaddi(r0, i0) alui(X86_ADD, r0, i0)
183 # define addi(r0, r1, i0) _addi(_jit, r0, r1, i0)
184 static void _addi(jit_state_t*, jit_int32_t, jit_int32_t, jit_word_t);
185 #define addcr(r0, r1, r2) _addcr(_jit, r0, r1, r2)
186 static void _addcr(jit_state_t*, jit_int32_t, jit_int32_t, jit_int32_t);
187 #define addci(r0, r1, i0) _addci(_jit, r0, r1, i0)
188 static void _addci(jit_state_t*, jit_int32_t, jit_int32_t, jit_word_t);
189 # define iaddxr(r0, r1) alur(X86_ADC, r0, r1)
190 # define addxr(r0, r1, r2) _addxr(_jit, r0, r1, r2)
191 static void _addxr(jit_state_t*, jit_int32_t, jit_int32_t, jit_int32_t);
192 # define iaddxi(r0, i0) alui(X86_ADC, r0, i0)
193 # define addxi(r0, r1, i0) _addxi(_jit, r0, r1, i0)
194 static void _addxi(jit_state_t*, jit_int32_t, jit_int32_t, jit_word_t);
195 # define isubr(r0, r1) alur(X86_SUB, r0, r1)
196 # define subr(r0, r1, r2) _subr(_jit, r0, r1, r2)
197 static void _subr(jit_state_t*, jit_int32_t, jit_int32_t, jit_int32_t);
198 # define isubi(r0, i0) alui(X86_SUB, r0, i0)
199 # define subi(r0, r1, i0) _subi(_jit, r0, r1, i0)
200 static void _subi(jit_state_t*, jit_int32_t, jit_int32_t, jit_word_t);
201 # define subcr(r0, r1, r2) _subcr(_jit, r0, r1, r2)
202 static void _subcr(jit_state_t*,jit_int32_t,jit_int32_t,jit_int32_t);
203 # define subci(r0, r1, i0) _subci(_jit, r0, r1, i0)
204 static void _subci(jit_state_t*,jit_int32_t,jit_int32_t,jit_word_t);
205 # define isubxr(r0, r1) alur(X86_SBB, r0, r1)
206 # define subxr(r0, r1, r2) _subxr(_jit, r0, r1, r2)
207 static void _subxr(jit_state_t*,jit_int32_t,jit_int32_t,jit_int32_t);
208 # define isubxi(r0, i0) alui(X86_SBB, r0, i0)
209 # define subxi(r0, r1, i0) _subxi(_jit, r0, r1, i0)
210 static void _subxi(jit_state_t*,jit_int32_t,jit_int32_t,jit_word_t);
211 # define rsbi(r0, r1, i0) _rsbi(_jit, r0, r1, i0)
212 static void _rsbi(jit_state_t*,jit_int32_t,jit_int32_t,jit_word_t);
213 # define imulr(r0, r1) _imulr(_jit, r0, r1)
214 static void _imulr(jit_state_t*, jit_int32_t, jit_int32_t);
215 # define imuli(r0, r1, i0) _imuli(_jit, r0, r1, i0)
216 static void _imuli(jit_state_t*, jit_int32_t, jit_int32_t, jit_word_t);
217 # define mulr(r0, r1, r2) _mulr(_jit, r0, r1, r2)
218 static void _mulr(jit_state_t*, jit_int32_t, jit_int32_t, jit_int32_t);
219 # define muli(r0, r1, i0) _muli(_jit, r0, r1, i0)
220 static void _muli(jit_state_t*, jit_int32_t, jit_int32_t, jit_word_t);
221 # define umulr(r0) unr(X86_IMUL, r0)
222 # define umulr_u(r0) unr(X86_MUL, r0)
223 # define qmulr(r0, r1, r2, r3) _iqmulr(_jit, r0, r1, r2, r3, 1)
224 # define qmulr_u(r0, r1, r2, r3) _iqmulr(_jit, r0, r1, r2, r3, 0)
225 # define iqmulr(r0, r1, r2, r3, sign) _iqmulr(_jit, r0, r1, r2, r3, sign)
226 static void _iqmulr(jit_state_t*, jit_int32_t, jit_int32_t,
227 jit_int32_t,jit_int32_t, jit_bool_t);
228 # define qmuli(r0, r1, r2, i0) _iqmuli(_jit, r0, r1, r2, i0, 1)
229 # define qmuli_u(r0, r1, r2, i0) _iqmuli(_jit, r0, r1, r2, i0, 0)
230 # define iqmuli(r0, r1, r2, i0, sign) _iqmuli(_jit, r0, r1, r2, i0, sign)
231 static void _iqmuli(jit_state_t*, jit_int32_t, jit_int32_t,
232 jit_int32_t,jit_word_t, jit_bool_t);
233 # define sign_extend_rdx_rax() _sign_extend_rdx_rax(_jit)
234 static void _sign_extend_rdx_rax(jit_state_t*);
235 # define idivr(r0) unr(X86_IDIV, r0)
236 # define idivr_u(r0) unr(X86_DIV, r0)
237 # define divremr(r0, r1, r2, i0, i1) _divremr(_jit, r0, r1, r2, i0, i1)
239 _divremr(jit_state_t*,jit_int32_t,jit_int32_t,jit_int32_t,
240 jit_bool_t,jit_bool_t);
241 # define divremi(r0, r1, i0, i1, i2) _divremi(_jit, r0, r1, i0, i1, i2)
243 _divremi(jit_state_t*,jit_int32_t,jit_int32_t,jit_word_t,jit_bool_t,jit_bool_t);
244 # define divr(r0, r1, r2) divremr(r0, r1, r2, 1, 1)
245 # define divi(r0, r1, i0) divremi(r0, r1, i0, 1, 1)
246 # define divr_u(r0, r1, r2) divremr(r0, r1, r2, 0, 1)
247 # define divi_u(r0, r1, i0) divremi(r0, r1, i0, 0, 1)
248 # define qdivr(r0, r1, r2, r3) _iqdivr(_jit, r0, r1, r2, r3, 1)
249 # define qdivr_u(r0, r1, r2, r3) _iqdivr(_jit, r0, r1, r2, r3, 0)
250 # define iqdivr(r0, r1, r2, r3, sign) _iqdivr(_jit, r0, r1, r2, r3, sign)
251 static void _iqdivr(jit_state_t*, jit_int32_t, jit_int32_t,
252 jit_int32_t,jit_int32_t, jit_bool_t);
253 # define qdivi(r0, r1, r2, i0) _iqdivi(_jit, r0, r1, r2, i0, 1)
254 # define qdivi_u(r0, r1, r2, i0) _iqdivi(_jit, r0, r1, r2, i0, 0)
255 # define iqdivi(r0, r1, r2, i0, sign) _iqdivi(_jit, r0, r1, r2, i0, sign)
256 static void _iqdivi(jit_state_t*, jit_int32_t, jit_int32_t,
257 jit_int32_t,jit_word_t, jit_bool_t);
258 # define remr(r0, r1, r2) divremr(r0, r1, r2, 1, 0)
259 # define remi(r0, r1, i0) divremi(r0, r1, i0, 1, 0)
260 # define remr_u(r0, r1, r2) divremr(r0, r1, r2, 0, 0)
261 # define remi_u(r0, r1, i0) divremi(r0, r1, i0, 0, 0)
262 # define iandr(r0, r1) alur(X86_AND, r0, r1)
263 # define andr(r0, r1, r2) _andr(_jit, r0, r1, r2)
264 static void _andr(jit_state_t*,jit_int32_t,jit_int32_t,jit_int32_t);
265 # define iandi(r0, i0) alui(X86_AND, r0, i0)
266 # define andi(r0, r1, i0) _andi(_jit, r0, r1, i0)
267 static void _andi(jit_state_t*, jit_int32_t,jit_int32_t,jit_word_t);
268 # define iorr(r0, r1) alur(X86_OR, r0, r1)
269 # define orr(r0, r1, r2) _orr(_jit, r0, r1, r2)
270 static void _orr(jit_state_t*, jit_int32_t,jit_int32_t,jit_int32_t);
271 # define iori(r0, i0) alui(X86_OR, r0, i0)
272 # define ori(r0, r1, i0) _ori(_jit, r0, r1, i0)
273 static void _ori(jit_state_t*, jit_int32_t,jit_int32_t,jit_word_t);
274 # define ixorr(r0, r1) alur(X86_XOR, r0, r1)
275 # define xorr(r0, r1, r2) _xorr(_jit, r0, r1, r2)
276 static void _xorr(jit_state_t*, jit_int32_t,jit_int32_t,jit_int32_t);
277 # define ixori(r0, i0) alui(X86_XOR, r0, i0)
278 # define xori(r0, r1, i0) _xori(_jit, r0, r1, i0)
279 static void _xori(jit_state_t*, jit_int32_t,jit_int32_t,jit_word_t);
280 # define irotshr(code, r0) _irotshr(_jit, code, r0)
281 static void _irotshr(jit_state_t*, jit_int32_t, jit_int32_t);
282 # define rotshr(code, r0, r1, r2) _rotshr(_jit, code, r0, r1, r2)
284 _rotshr(jit_state_t*,jit_int32_t,jit_int32_t,jit_int32_t,jit_int32_t);
285 # define irotshi(code, r0, i0) _irotshi(_jit, code, r0, i0)
286 static void _irotshi(jit_state_t*, jit_int32_t, jit_int32_t, jit_word_t);
287 # define rotshi(code, r0, r1, i0) _rotshi(_jit, code, r0, r1, i0)
289 _rotshi(jit_state_t*,jit_int32_t,jit_int32_t,jit_int32_t,jit_word_t);
290 # define lshr(r0, r1, r2) rotshr(X86_SHL, r0, r1, r2)
291 # define lshi(r0, r1, i0) _lshi(_jit, r0, r1, i0)
292 static void _lshi(jit_state_t*, jit_int32_t, jit_int32_t, jit_word_t);
293 # define rshr(r0, r1, r2) rotshr(X86_SAR, r0, r1, r2)
294 # define rshi(r0, r1, i0) rotshi(X86_SAR, r0, r1, i0)
295 # define rshr_u(r0, r1, r2) rotshr(X86_SHR, r0, r1, r2)
296 # define rshi_u(r0, r1, i0) rotshi(X86_SHR, r0, r1, i0)
297 # define unr(code, r0) _unr(_jit, code, r0)
298 static void _unr(jit_state_t*, jit_int32_t, jit_int32_t);
299 # define inegr(r0) unr(X86_NEG, r0)
300 # define negr(r0, r1) _negr(_jit, r0, r1)
301 static void _negr(jit_state_t*, jit_int32_t, jit_int32_t);
302 # define icomr(r0) unr(X86_NOT, r0)
303 # define comr(r0, r1) _comr(_jit, r0, r1)
304 static void _comr(jit_state_t*, jit_int32_t, jit_int32_t);
306 # define incr(r0, r1) _incr(_jit, r0, r1)
307 static void _incr(jit_state_t*, jit_int32_t, jit_int32_t);
308 # define decr(r0, r1) _decr(_jit, r0, r1)
309 static void _decr(jit_state_t*, jit_int32_t, jit_int32_t);
311 # define cr(code, r0, r1, r2) _cr(_jit, code, r0, r1, r2)
313 _cr(jit_state_t*, jit_int32_t, jit_int32_t, jit_int32_t, jit_int32_t);
314 # define ci(code, r0, r1, i0) _ci(_jit, code, r0, r1, i0)
316 _ci(jit_state_t *_jit, jit_int32_t, jit_int32_t, jit_int32_t, jit_word_t);
317 # define ci0(code, r0, r1) _ci0(_jit, code, r0, r1)
318 static void _ci0(jit_state_t*, jit_int32_t, jit_int32_t, jit_int32_t);
319 # define ltr(r0, r1, r2) _ltr(_jit, r0, r1, r2)
320 static void _ltr(jit_state_t*, jit_int32_t, jit_int32_t, jit_int32_t);
321 # define lti(r0, r1, i0) _lti(_jit, r0, r1, i0)
322 static void _lti(jit_state_t*, jit_int32_t, jit_int32_t, jit_word_t);
323 # define ltr_u(r0, r1, r2) _ltr_u(_jit, r0, r1, r2)
324 static void _ltr_u(jit_state_t*, jit_int32_t, jit_int32_t, jit_int32_t);
325 # define lti_u(r0, r1, i0) ci(X86_CC_B, r0, r1, i0)
326 # define ler(r0, r1, r2) _ler(_jit, r0, r1, r2)
327 static void _ler(jit_state_t*, jit_int32_t, jit_int32_t, jit_int32_t);
328 # define lei(r0, r1, i0) ci(X86_CC_LE, r0, r1, i0)
329 # define ler_u(r0, r1, r2) _ler_u(_jit, r0, r1, r2)
330 static void _ler_u(jit_state_t*, jit_int32_t, jit_int32_t, jit_int32_t);
331 # define lei_u(r0, r1, i0) _lei_u(_jit, r0, r1, i0)
332 static void _lei_u(jit_state_t*, jit_int32_t, jit_int32_t, jit_word_t);
333 # define eqr(r0, r1, r2) _eqr(_jit, r0, r1, r2)
334 static void _eqr(jit_state_t*, jit_int32_t, jit_int32_t, jit_int32_t);
335 # define eqi(r0, r1, i0) _eqi(_jit, r0, r1, i0)
336 static void _eqi(jit_state_t*, jit_int32_t, jit_int32_t, jit_word_t);
337 # define ger(r0, r1, r2) _ger(_jit, r0, r1, r2)
338 static void _ger(jit_state_t*, jit_int32_t, jit_int32_t, jit_int32_t);
339 # define gei(r0, r1, i0) _gei(_jit, r0, r1, i0)
340 static void _gei(jit_state_t*, jit_int32_t, jit_int32_t, jit_word_t);
341 # define ger_u(r0, r1, r2) _ger_u(_jit, r0, r1, r2)
342 static void _ger_u(jit_state_t*, jit_int32_t, jit_int32_t, jit_int32_t);
343 # define gei_u(r0, r1, i0) _gei_u(_jit, r0, r1, i0)
344 static void _gei_u(jit_state_t*, jit_int32_t, jit_int32_t, jit_word_t);
345 # define gtr(r0, r1, r2) _gtr(_jit, r0, r1, r2)
346 static void _gtr(jit_state_t*, jit_int32_t, jit_int32_t, jit_int32_t);
347 # define gti(r0, r1, i0) _ci(_jit, X86_CC_G, r0, r1, i0)
348 # define gtr_u(r0, r1, r2) _gtr_u(_jit, r0, r1, r2)
349 static void _gtr_u(jit_state_t*, jit_int32_t, jit_int32_t, jit_int32_t);
350 # define gti_u(r0, r1, i0) _gti_u(_jit, r0, r1, i0)
351 static void _gti_u(jit_state_t*, jit_int32_t, jit_int32_t, jit_word_t);
352 # define ner(r0, r1, r2) _ner(_jit, r0, r1, r2)
353 static void _ner(jit_state_t*, jit_int32_t, jit_int32_t, jit_int32_t);
354 # define nei(r0, r1, i0) _nei(_jit, r0, r1, i0)
355 static void _nei(jit_state_t*, jit_int32_t, jit_int32_t, jit_word_t);
356 # define movr(r0, r1) _movr(_jit, r0, r1)
357 static void _movr(jit_state_t*, jit_int32_t, jit_int32_t);
358 # define imovi(r0, i0) _imovi(_jit, r0, i0)
359 static void _imovi(jit_state_t*, jit_int32_t, jit_word_t);
360 # define movi(r0, i0) _movi(_jit, r0, i0)
361 static void _movi(jit_state_t*, jit_int32_t, jit_word_t);
362 # define movi_p(r0, i0) _movi_p(_jit, r0, i0)
363 static jit_word_t _movi_p(jit_state_t*, jit_int32_t, jit_word_t);
364 # define movcr(r0, r1) _movcr(_jit, r0, r1)
365 static void _movcr(jit_state_t*,jit_int32_t,jit_int32_t);
366 # define movcr_u(r0, r1) _movcr_u(_jit, r0, r1)
367 static void _movcr_u(jit_state_t*,jit_int32_t,jit_int32_t);
368 # define movsr(r0, r1) _movsr(_jit, r0, r1)
369 static void _movsr(jit_state_t*,jit_int32_t,jit_int32_t);
370 # define movsr_u(r0, r1) _movsr_u(_jit, r0, r1)
371 static void _movsr_u(jit_state_t*,jit_int32_t,jit_int32_t);
372 #define movnr(r0, r1, r2) _movnr(_jit, r0, r1, r2)
373 static void _movnr(jit_state_t*, jit_int32_t, jit_int32_t, jit_int32_t);
374 #define movzr(r0, r1, r2) _movzr(_jit, r0, r1, r2)
375 static void _movzr(jit_state_t*, jit_int32_t, jit_int32_t, jit_int32_t);
376 # if __X64 && !__X64_32
377 # define movir(r0, r1) _movir(_jit, r0, r1)
378 static void _movir(jit_state_t*,jit_int32_t,jit_int32_t);
379 # define movir_u(r0, r1) _movir_u(_jit, r0, r1)
380 static void _movir_u(jit_state_t*,jit_int32_t,jit_int32_t);
382 # define bswapr_us(r0, r1) _bswapr_us(_jit, r0, r1)
383 static void _bswapr_us(jit_state_t*,jit_int32_t,jit_int32_t);
384 # define bswapr_ui(r0, r1) _bswapr_ui(_jit, r0, r1)
385 static void _bswapr_ui(jit_state_t*,jit_int32_t,jit_int32_t);
386 # if __X64 && !__X64_32
387 #define bswapr_ul(r0, r1) _bswapr_ul(_jit, r0, r1)
388 static void _bswapr_ul(jit_state_t*,jit_int32_t,jit_int32_t);
390 # define extr_c(r0, r1) _extr_c(_jit, r0, r1)
391 static void _extr_c(jit_state_t*,jit_int32_t,jit_int32_t);
392 # define extr_uc(r0, r1) _extr_uc(_jit, r0, r1)
393 static void _extr_uc(jit_state_t*,jit_int32_t,jit_int32_t);
394 # define extr_s(r0, r1) movsr(r0, r1)
395 # define extr_us(r0, r1) movsr_u(r0, r1)
396 # if __X64 && !__X64_32
397 # define extr_i(r0, r1) movir(r0, r1)
398 # define extr_ui(r0, r1) movir_u(r0, r1)
400 # define ldr_c(r0, r1) _ldr_c(_jit, r0, r1)
401 static void _ldr_c(jit_state_t*, jit_int32_t, jit_int32_t);
402 # define ldi_c(r0, i0) _ldi_c(_jit, r0, i0)
403 static void _ldi_c(jit_state_t*, jit_int32_t, jit_word_t);
404 # define ldr_uc(r0, r1) _ldr_uc(_jit, r0, r1)
405 static void _ldr_uc(jit_state_t*, jit_int32_t, jit_int32_t);
406 # define ldi_uc(r0, i0) _ldi_uc(_jit, r0, i0)
407 static void _ldi_uc(jit_state_t*, jit_int32_t, jit_word_t);
408 # define ldr_s(r0, r1) _ldr_s(_jit, r0, r1)
409 static void _ldr_s(jit_state_t*, jit_int32_t, jit_int32_t);
410 # define ldi_s(r0, i0) _ldi_s(_jit, r0, i0)
411 static void _ldi_s(jit_state_t*, jit_int32_t, jit_word_t);
412 # define ldr_us(r0, r1) _ldr_us(_jit, r0, r1)
413 static void _ldr_us(jit_state_t*, jit_int32_t, jit_int32_t);
414 # define ldi_us(r0, i0) _ldi_us(_jit, r0, i0)
415 static void _ldi_us(jit_state_t*, jit_int32_t, jit_word_t);
416 # if __X32 || !__X64_32
417 # define ldr_i(r0, r1) _ldr_i(_jit, r0, r1)
418 static void _ldr_i(jit_state_t*, jit_int32_t, jit_int32_t);
419 # define ldi_i(r0, i0) _ldi_i(_jit, r0, i0)
420 static void _ldi_i(jit_state_t*, jit_int32_t, jit_word_t);
424 # define ldr_i(r0, r1) _ldr_ui(_jit, r0, r1)
425 # define ldi_i(r0, i0) _ldi_ui(_jit, r0, i0)
427 # define ldr_ui(r0, r1) _ldr_ui(_jit, r0, r1)
428 # define ldi_ui(r0, i0) _ldi_ui(_jit, r0, i0)
430 static void _ldr_ui(jit_state_t*, jit_int32_t, jit_int32_t);
431 static void _ldi_ui(jit_state_t*, jit_int32_t, jit_word_t);
433 # define ldr_l(r0, r1) _ldr_l(_jit, r0, r1)
434 static void _ldr_l(jit_state_t*, jit_int32_t, jit_int32_t);
435 # define ldi_l(r0, i0) _ldi_l(_jit, r0, i0)
436 static void _ldi_l(jit_state_t*, jit_int32_t, jit_word_t);
439 # define ldxr_c(r0, r1, r2) _ldxr_c(_jit, r0, r1, r2)
440 static void _ldxr_c(jit_state_t*, jit_int32_t, jit_int32_t, jit_int32_t);
441 # define ldxi_c(r0, r1, i0) _ldxi_c(_jit, r0, r1, i0)
442 static void _ldxi_c(jit_state_t*, jit_int32_t, jit_int32_t, jit_word_t);
443 # define ldxr_uc(r0, r1, r2) _ldxr_uc(_jit, r0, r1, r2)
444 static void _ldxr_uc(jit_state_t*, jit_int32_t, jit_int32_t, jit_int32_t);
445 # define ldxi_uc(r0, r1, i0) _ldxi_uc(_jit, r0, r1, i0)
446 static void _ldxi_uc(jit_state_t*, jit_int32_t, jit_int32_t, jit_word_t);
447 # define ldxr_s(r0, r1, r2) _ldxr_s(_jit, r0, r1, r2)
448 static void _ldxr_s(jit_state_t*, jit_int32_t, jit_int32_t, jit_int32_t);
449 # define ldxi_s(r0, r1, i0) _ldxi_s(_jit, r0, r1, i0)
450 static void _ldxi_s(jit_state_t*, jit_int32_t, jit_int32_t, jit_word_t);
451 # define ldxr_us(r0, r1, r2) _ldxr_us(_jit, r0, r1, r2)
452 static void _ldxr_us(jit_state_t*, jit_int32_t, jit_int32_t, jit_int32_t);
453 # define ldxi_us(r0, r1, i0) _ldxi_us(_jit, r0, r1, i0)
454 static void _ldxi_us(jit_state_t*, jit_int32_t, jit_int32_t, jit_word_t);
455 # if __X32 || !__X64_32
456 # define ldxr_i(r0, r1, r2) _ldxr_i(_jit, r0, r1, r2)
457 static void _ldxr_i(jit_state_t*, jit_int32_t, jit_int32_t, jit_int32_t);
458 # define ldxi_i(r0, r1, i0) _ldxi_i(_jit, r0, r1, i0)
459 static void _ldxi_i(jit_state_t*, jit_int32_t, jit_int32_t, jit_word_t);
463 # define ldxr_i(r0, r1, r2) _ldxr_ui(_jit, r0, r1, r2)
464 # define ldxi_i(r0, r1, i0) _ldxi_ui(_jit, r0, r1, i0)
466 # define ldxr_ui(r0, r1, r2) _ldxr_ui(_jit, r0, r1, r2)
467 # define ldxi_ui(r0, r1, i0) _ldxi_ui(_jit, r0, r1, i0)
469 static void _ldxr_ui(jit_state_t*, jit_int32_t, jit_int32_t, jit_int32_t);
470 static void _ldxi_ui(jit_state_t*, jit_int32_t, jit_int32_t, jit_word_t);
472 # define ldxr_l(r0, r1, r2) _ldxr_l(_jit, r0, r1, r2)
473 static void _ldxr_l(jit_state_t*, jit_int32_t, jit_int32_t, jit_int32_t);
474 # define ldxi_l(r0, r1, i0) _ldxi_l(_jit, r0, r1, i0)
475 static void _ldxi_l(jit_state_t*, jit_int32_t, jit_int32_t, jit_word_t);
478 # define str_c(r0, r1) _str_c(_jit, r0, r1)
479 static void _str_c(jit_state_t*, jit_int32_t, jit_int32_t);
480 # define sti_c(i0, r0) _sti_c(_jit, i0, r0)
481 static void _sti_c(jit_state_t*, jit_word_t, jit_int32_t);
482 # define str_s(r0, r1) _str_s(_jit, r0, r1)
483 static void _str_s(jit_state_t*, jit_int32_t, jit_int32_t);
484 # define sti_s(i0, r0) _sti_s(_jit, i0, r0)
485 static void _sti_s(jit_state_t*, jit_word_t, jit_int32_t);
486 # define str_i(r0, r1) _str_i(_jit, r0, r1)
487 static void _str_i(jit_state_t*, jit_int32_t, jit_int32_t);
488 # define sti_i(i0, r0) _sti_i(_jit, i0, r0)
489 static void _sti_i(jit_state_t*, jit_word_t, jit_int32_t);
490 # if __X64 && !__X64_32
491 # define str_l(r0, r1) _str_l(_jit, r0, r1)
492 static void _str_l(jit_state_t*, jit_int32_t, jit_int32_t);
493 # define sti_l(i0, r0) _sti_l(_jit, i0, r0)
494 static void _sti_l(jit_state_t*, jit_word_t, jit_int32_t);
496 # define stxr_c(r0, r1, r2) _stxr_c(_jit, r0, r1, r2)
497 static void _stxr_c(jit_state_t*, jit_int32_t, jit_int32_t, jit_int32_t);
498 # define stxi_c(i0, r0, r1) _stxi_c(_jit, i0, r0, r1)
499 static void _stxi_c(jit_state_t*, jit_word_t, jit_int32_t, jit_int32_t);
500 # define stxr_s(r0, r1, r2) _stxr_s(_jit, r0, r1, r2)
501 static void _stxr_s(jit_state_t*, jit_int32_t, jit_int32_t, jit_int32_t);
502 # define stxi_s(i0, r0, r1) _stxi_s(_jit, i0, r0, r1)
503 static void _stxi_s(jit_state_t*, jit_word_t, jit_int32_t, jit_int32_t);
504 # define stxr_i(r0, r1, r2) _stxr_i(_jit, r0, r1, r2)
505 static void _stxr_i(jit_state_t*, jit_int32_t, jit_int32_t, jit_int32_t);
506 # define stxi_i(i0, r0, r1) _stxi_i(_jit, i0, r0, r1)
507 static void _stxi_i(jit_state_t*, jit_word_t, jit_int32_t, jit_int32_t);
508 # if __X64 && !__X64_32
509 # define stxr_l(r0, r1, r2) _stxr_l(_jit, r0, r1, r2)
510 static void _stxr_l(jit_state_t*, jit_int32_t, jit_int32_t, jit_int32_t);
511 # define stxi_l(i0, r0, r1) _stxi_l(_jit, i0, r0, r1)
512 static void _stxi_l(jit_state_t*, jit_word_t, jit_int32_t, jit_int32_t);
514 # define jcc(code, i0) _jcc(_jit, code, i0)
515 # define jo(i0) jcc(X86_CC_O, i0)
516 # define jno(i0) jcc(X86_CC_NO, i0)
517 # define jnae(i0) jcc(X86_CC_NAE, i0)
518 # define jb(i0) jcc(X86_CC_B, i0)
519 # define jc(i0) jcc(X86_CC_C, i0)
520 # define jae(i0) jcc(X86_CC_AE, i0)
521 # define jnb(i0) jcc(X86_CC_NB, i0)
522 # define jnc(i0) jcc(X86_CC_NC, i0)
523 # define je(i0) jcc(X86_CC_E, i0)
524 # define jz(i0) jcc(X86_CC_Z, i0)
525 # define jne(i0) jcc(X86_CC_NE, i0)
526 # define jnz(i0) jcc(X86_CC_NZ, i0)
527 # define jbe(i0) jcc(X86_CC_BE, i0)
528 # define jna(i0) jcc(X86_CC_NA, i0)
529 # define ja(i0) jcc(X86_CC_A, i0)
530 # define jnbe(i0) jcc(X86_CC_NBE, i0)
531 # define js(i0) jcc(X86_CC_S, i0)
532 # define jns(i0) jcc(X86_CC_NS, i0)
533 # define jp(i0) jcc(X86_CC_P, i0)
534 # define jpe(i0) jcc(X86_CC_PE, i0)
535 # define jnp(i0) jcc(X86_CC_NP, i0)
536 # define jpo(i0) jcc(X86_CC_PO, i0)
537 # define jl(i0) jcc(X86_CC_L, i0)
538 # define jnge(i0) jcc(X86_CC_NGE, i0)
539 # define jge(i0) jcc(X86_CC_GE, i0)
540 # define jnl(i0) jcc(X86_CC_NL, i0)
541 # define jle(i0) jcc(X86_CC_LE, i0)
542 # define jng(i0) jcc(X86_CC_NG, i0)
543 # define jg(i0) jcc(X86_CC_G, i0)
544 # define jnle(i0) jcc(X86_CC_NLE, i0)
545 static void _jcc(jit_state_t*, jit_int32_t, jit_word_t);
546 # define jccs(code, i0) _jccs(_jit, code, i0)
547 # define jos(i0) jccs(X86_CC_O, i0)
548 # define jnos(i0) jccs(X86_CC_NO, i0)
549 # define jnaes(i0) jccs(X86_CC_NAE, i0)
550 # define jbs(i0) jccs(X86_CC_B, i0)
551 # define jcs(i0) jccs(X86_CC_C, i0)
552 # define jaes(i0) jccs(X86_CC_AE, i0)
553 # define jnbs(i0) jccs(X86_CC_NB, i0)
554 # define jncs(i0) jccs(X86_CC_NC, i0)
555 # define jes(i0) jccs(X86_CC_E, i0)
556 # define jzs(i0) jccs(X86_CC_Z, i0)
557 # define jnes(i0) jccs(X86_CC_NE, i0)
558 # define jnzs(i0) jccs(X86_CC_NZ, i0)
559 # define jbes(i0) jccs(X86_CC_BE, i0)
560 # define jnas(i0) jccs(X86_CC_NA, i0)
561 # define jas(i0) jccs(X86_CC_A, i0)
562 # define jnbes(i0) jccs(X86_CC_NBE, i0)
563 # define jss(i0) jccs(X86_CC_S, i0)
564 # define jnss(i0) jccs(X86_CC_NS, i0)
565 # define jps(i0) jccs(X86_CC_P, i0)
566 # define jpes(i0) jccs(X86_CC_PE, i0)
567 # define jnps(i0) jccs(X86_CC_NP, i0)
568 # define jpos(i0) jccs(X86_CC_PO, i0)
569 # define jls(i0) jccs(X86_CC_L, i0)
570 # define jnges(i0) jccs(X86_CC_NGE, i0)
571 # define jges(i0) jccs(X86_CC_GE, i0)
572 # define jnls(i0) jccs(X86_CC_NL, i0)
573 # define jles(i0) jccs(X86_CC_LE, i0)
574 # define jngs(i0) jccs(X86_CC_NG, i0)
575 # define jgs(i0) jccs(X86_CC_G, i0)
576 # define jnles(i0) jccs(X86_CC_NLE, i0)
577 static void _jccs(jit_state_t*, jit_int32_t, jit_word_t);
578 # define jcr(code, i0, r0, r1) _jcr(_jit, code, i0, r0, r1)
579 static void _jcr(jit_state_t*,jit_int32_t,jit_word_t,jit_int32_t,jit_int32_t);
580 # define jci(code, i0, r0, i1) _jci(_jit, code, i0, r0, i1)
581 static void _jci(jit_state_t*,jit_int32_t,jit_word_t,jit_int32_t,jit_word_t);
582 # define jci0(code, i0, r0) _jci0(_jit, code, i0, r0)
583 static void _jci0(jit_state_t*, jit_int32_t, jit_word_t, jit_int32_t);
584 # define bltr(i0, r0, r1) _bltr(_jit, i0, r0, r1)
585 static jit_word_t _bltr(jit_state_t*, jit_word_t, jit_int32_t, jit_int32_t);
586 # define blti(i0, r0, i1) _blti(_jit, i0, r0, i1)
587 static jit_word_t _blti(jit_state_t*, jit_word_t, jit_int32_t, jit_word_t);
588 # define bltr_u(i0, r0, r1) _bltr_u(_jit, i0, r0, r1)
589 static jit_word_t _bltr_u(jit_state_t*, jit_word_t, jit_int32_t, jit_int32_t);
590 # define blti_u(i0, r0, i1) _blti_u(_jit, i0, r0, i1)
591 static jit_word_t _blti_u(jit_state_t*, jit_word_t, jit_int32_t, jit_word_t);
592 # define bler(i0, r0, r1) _bler(_jit, i0, r0, r1)
593 static jit_word_t _bler(jit_state_t*, jit_word_t, jit_int32_t, jit_int32_t);
594 # define blei(i0, r0, i1) _blei(_jit, i0, r0, i1)
595 static jit_word_t _blei(jit_state_t*, jit_word_t, jit_int32_t, jit_word_t);
596 # define bler_u(i0, r0, r1) _bler_u(_jit, i0, r0, r1)
597 static jit_word_t _bler_u(jit_state_t*, jit_word_t, jit_int32_t, jit_int32_t);
598 # define blei_u(i0, r0, i1) _blei_u(_jit, i0, r0, i1)
599 static jit_word_t _blei_u(jit_state_t*, jit_word_t, jit_int32_t, jit_word_t);
600 # define beqr(i0, r0, r1) _beqr(_jit, i0, r0, r1)
601 static jit_word_t _beqr(jit_state_t*, jit_word_t, jit_int32_t, jit_int32_t);
602 # define beqi(i0, r0, i1) _beqi(_jit, i0, r0, i1)
603 static jit_word_t _beqi(jit_state_t*, jit_word_t, jit_int32_t, jit_word_t);
604 # define bger(i0, r0, r1) _bger(_jit, i0, r0, r1)
605 static jit_word_t _bger(jit_state_t*, jit_word_t, jit_int32_t, jit_int32_t);
606 # define bgei(i0, r0, i1) _bgei(_jit, i0, r0, i1)
607 static jit_word_t _bgei(jit_state_t*, jit_word_t, jit_int32_t, jit_word_t);
608 # define bger_u(i0, r0, r1) _bger_u(_jit, i0, r0, r1)
609 static jit_word_t _bger_u(jit_state_t*, jit_word_t, jit_int32_t, jit_int32_t);
610 # define bgei_u(i0, r0, i1) _bgei_u(_jit, i0, r0, i1)
611 static jit_word_t _bgei_u(jit_state_t*, jit_word_t, jit_int32_t, jit_word_t);
612 # define bgtr(i0, r0, r1) _bgtr(_jit, i0, r0, r1)
613 static jit_word_t _bgtr(jit_state_t*, jit_word_t, jit_int32_t, jit_int32_t);
614 # define bgti(i0, r0, i1) _bgti(_jit, i0, r0, i1)
615 static jit_word_t _bgti(jit_state_t*, jit_word_t, jit_int32_t, jit_word_t);
616 # define bgtr_u(i0, r0, r1) _bgtr_u(_jit, i0, r0, r1)
617 static jit_word_t _bgtr_u(jit_state_t*, jit_word_t, jit_int32_t, jit_int32_t);
618 # define bgti_u(i0, r0, i1) _bgti_u(_jit, i0, r0, i1)
619 static jit_word_t _bgti_u(jit_state_t*, jit_word_t, jit_int32_t, jit_word_t);
620 # define bner(i0, r0, r1) _bner(_jit, i0, r0, r1)
621 static jit_word_t _bner(jit_state_t*, jit_word_t, jit_int32_t, jit_int32_t);
622 # define bnei(i0, r0, i1) _bnei(_jit, i0, r0, i1)
623 static jit_word_t _bnei(jit_state_t*, jit_word_t, jit_int32_t, jit_word_t);
624 # define bmsr(i0, r0, r1) _bmsr(_jit, i0, r0, r1)
625 static jit_word_t _bmsr(jit_state_t*,jit_word_t,jit_int32_t,jit_int32_t);
626 # define bmsi(i0, r0, i1) _bmsi(_jit, i0, r0, i1)
627 static jit_word_t _bmsi(jit_state_t*,jit_word_t,jit_int32_t,jit_word_t);
628 # define bmcr(i0, r0, r1) _bmcr(_jit, i0, r0, r1)
629 static jit_word_t _bmcr(jit_state_t*,jit_word_t,jit_int32_t,jit_int32_t);
630 # define bmci(i0, r0, i1) _bmci(_jit, i0, r0, i1)
631 static jit_word_t _bmci(jit_state_t*,jit_word_t,jit_int32_t,jit_word_t);
632 # define boaddr(i0, r0, r1) _boaddr(_jit, i0, r0, r1)
633 static jit_word_t _boaddr(jit_state_t*,jit_word_t,jit_int32_t,jit_int32_t);
634 # define boaddi(i0, r0, i1) _boaddi(_jit, i0, r0, i1)
635 static jit_word_t _boaddi(jit_state_t*,jit_word_t,jit_int32_t,jit_word_t);
636 # define boaddr_u(i0, r0, r1) _boaddr_u(_jit, i0, r0, r1)
637 static jit_word_t _boaddr_u(jit_state_t*,jit_word_t,jit_int32_t,jit_int32_t);
638 # define boaddi_u(i0, r0, i1) _boaddi_u(_jit, i0, r0, i1)
639 static jit_word_t _boaddi_u(jit_state_t*,jit_word_t,jit_int32_t,jit_word_t);
640 # define bxaddr(i0, r0, r1) _bxaddr(_jit, i0, r0, r1)
641 static jit_word_t _bxaddr(jit_state_t*,jit_word_t,jit_int32_t,jit_int32_t);
642 # define bxaddi(i0, r0, i1) _bxaddi(_jit, i0, r0, i1)
643 static jit_word_t _bxaddi(jit_state_t*,jit_word_t,jit_int32_t,jit_word_t);
644 # define bxaddr_u(i0, r0, r1) _bxaddr_u(_jit, i0, r0, r1)
645 static jit_word_t _bxaddr_u(jit_state_t*,jit_word_t,jit_int32_t,jit_int32_t);
646 # define bxaddi_u(i0, r0, i1) _bxaddi_u(_jit, i0, r0, i1)
647 static jit_word_t _bxaddi_u(jit_state_t*,jit_word_t,jit_int32_t,jit_word_t);
648 # define bosubr(i0, r0, r1) _bosubr(_jit, i0, r0, r1)
649 static jit_word_t _bosubr(jit_state_t*,jit_word_t,jit_int32_t,jit_int32_t);
650 # define bosubi(i0, r0, i1) _bosubi(_jit, i0, r0, i1)
651 static jit_word_t _bosubi(jit_state_t*,jit_word_t,jit_int32_t,jit_word_t);
652 # define bosubr_u(i0, r0, r1) _bosubr_u(_jit, i0, r0, r1)
653 static jit_word_t _bosubr_u(jit_state_t*,jit_word_t,jit_int32_t,jit_int32_t);
654 # define bosubi_u(i0, r0, i1) _bosubi_u(_jit, i0, r0, i1)
655 static jit_word_t _bosubi_u(jit_state_t*,jit_word_t,jit_int32_t,jit_word_t);
656 # define bxsubr(i0, r0, r1) _bxsubr(_jit, i0, r0, r1)
657 static jit_word_t _bxsubr(jit_state_t*,jit_word_t,jit_int32_t,jit_int32_t);
658 # define bxsubi(i0, r0, i1) _bxsubi(_jit, i0, r0, i1)
659 static jit_word_t _bxsubi(jit_state_t*,jit_word_t,jit_int32_t,jit_word_t);
660 # define bxsubr_u(i0, r0, r1) _bxsubr_u(_jit, i0, r0, r1)
661 static jit_word_t _bxsubr_u(jit_state_t*,jit_word_t,jit_int32_t,jit_int32_t);
662 # define bxsubi_u(i0, r0, i1) _bxsubi_u(_jit, i0, r0, i1)
663 static jit_word_t _bxsubi_u(jit_state_t*,jit_word_t,jit_int32_t,jit_word_t);
664 # define callr(r0) _callr(_jit, r0)
665 static void _callr(jit_state_t*, jit_int32_t);
666 # define calli(i0) _calli(_jit, i0)
667 static jit_word_t _calli(jit_state_t*, jit_word_t);
669 # define calli_p(i0) _calli_p(_jit, i0)
670 static jit_word_t _calli_p(jit_state_t*, jit_word_t);
672 # define calli_p(i0) calli(i0)
674 # define jmpr(r0) _jmpr(_jit, r0)
675 static void _jmpr(jit_state_t*, jit_int32_t);
676 # define jmpi(i0) _jmpi(_jit, i0)
677 static jit_word_t _jmpi(jit_state_t*, jit_word_t);
679 # define jmpi_p(i0) _jmpi_p(_jit, i0)
680 static jit_word_t _jmpi_p(jit_state_t*, jit_word_t);
682 # define jmpi_p(i0) jmpi(i0)
684 # define jmpsi(i0) _jmpsi(_jit, i0)
685 static void _jmpsi(jit_state_t*, jit_uint8_t);
686 # define prolog(node) _prolog(_jit, node)
687 static void _prolog(jit_state_t*, jit_node_t*);
688 # define epilog(node) _epilog(_jit, node)
689 static void _epilog(jit_state_t*, jit_node_t*);
690 # define vastart(r0) _vastart(_jit, r0)
691 static void _vastart(jit_state_t*, jit_int32_t);
692 # define vaarg(r0, r1) _vaarg(_jit, r0, r1)
693 static void _vaarg(jit_state_t*, jit_int32_t, jit_int32_t);
694 # define vaarg_d(r0, r1, i0) _vaarg_d(_jit, r0, r1, i0)
695 static void _vaarg_d(jit_state_t*, jit_int32_t, jit_int32_t, jit_bool_t);
696 # define patch_at(node, instr, label) _patch_at(_jit, node, instr, label)
697 static void _patch_at(jit_state_t*, jit_node_t*, jit_word_t, jit_word_t);
698 # if !defined(HAVE_FFSL)
700 # define ffsl(i) __builtin_ffs(i)
702 # define ffsl(l) __builtin_ffsl(l)
705 # define jit_cmov_p() jit_cpu.cmov
710 _rex(jit_state_t *_jit, jit_int32_t l, jit_int32_t w,
711 jit_int32_t r, jit_int32_t x, jit_int32_t b)
714 jit_int32_t v = 0x40 | (w << 3);
728 _rx(jit_state_t *_jit, jit_int32_t rd, jit_int32_t md,
729 jit_int32_t rb, jit_int32_t ri, jit_int32_t ms)
734 mrm(0x00, r7(rd), 0x05);
736 mrm(0x00, r7(rd), 0x04);
737 sib(_SCL1, 0x04, 0x05);
741 else if (r7(rb) == _RSP_REGNO) {
743 mrm(0x00, r7(rd), 0x04);
746 else if ((jit_int8_t)md == md) {
747 mrm(0x01, r7(rd), 0x04);
752 mrm(0x02, r7(rd), 0x04);
758 if (md == 0 && r7(rb) != _RBP_REGNO)
759 mrm(0x00, r7(rd), r7(rb));
760 else if ((jit_int8_t)md == md) {
761 mrm(0x01, r7(rd), r7(rb));
765 mrm(0x02, r7(rd), r7(rb));
770 else if (rb == _NOREG) {
771 mrm(0x00, r7(rd), 0x04);
772 sib(ms, r7(ri), 0x05);
775 else if (r8(ri) != _RSP_REGNO) {
776 if (md == 0 && r7(rb) != _RBP_REGNO) {
777 mrm(0x00, r7(rd), 0x04);
778 sib(ms, r7(ri), r7(rb));
780 else if ((jit_int8_t)md == md) {
781 mrm(0x01, r7(rd), 0x04);
782 sib(ms, r7(ri), r7(rb));
786 mrm(0x02, r7(rd), 0x04);
787 sib(ms, r7(ri), r7(rb));
792 fprintf(stderr, "illegal index register");
798 _nop(jit_state_t *_jit, jit_int32_t count)
808 case 3: /* NOP DWORD ptr [EAX] */
809 ic(0x0f); ic(0x1f); ic(0x00);
811 case 4: /* NOP DWORD ptr [EAX + 00H] */
812 ic(0x0f); ic(0x1f); ic(0x40); ic(0x00);
814 case 5: /* NOP DWORD ptr [EAX + EAX*1 + 00H] */
815 ic(0x0f); ic(0x1f); ic(0x44); ic(0x00);
818 case 6: /* 66 NOP DWORD ptr [EAX + EAX*1 + 00H] */
819 ic(0x66); ic(0x0f); ic(0x1f); ic(0x44);
822 case 7: /* NOP DWORD ptr [EAX + 00000000H] */
823 ic(0x0f); ic(0x1f); ic(0x80); ii(0x0000);
825 case 8: /* NOP DWORD ptr [EAX + EAX*1 + 00000000H] */
826 ic(0x0f); ic(0x1f); ic(0x84); ic(0x00);
829 case 9: /* 66 NOP DWORD ptr [EAX + EAX*1 + 00000000H] */
830 ic(0x66); ic(0x0f); ic(0x1f); ic(0x84);
831 ic(0x00); ii(0x0000);
839 _lea(jit_state_t *_jit, jit_int32_t md, jit_int32_t rb,
840 jit_int32_t ri, jit_int32_t ms, jit_int32_t rd)
842 rex(0, WIDE, rd, ri, rb);
844 rx(rd, md, rb, ri, ms);
848 _pushr(jit_state_t *_jit, jit_int32_t r0)
850 rex(0, WIDE, 0, 0, r0);
855 _popr(jit_state_t *_jit, jit_int32_t r0)
857 rex(0, WIDE, 0, 0, r0);
862 _xchgr(jit_state_t *_jit, jit_int32_t r0, jit_int32_t r1)
864 rex(0, WIDE, r1, _NOREG, r0);
866 mrm(0x03, r7(r1), r7(r0));
870 _testr(jit_state_t *_jit, jit_int32_t r0, jit_int32_t r1)
872 rex(0, WIDE, r1, _NOREG, r0);
874 mrm(0x03, r7(r1), r7(r0));
878 _testi(jit_state_t *_jit, jit_int32_t r0, jit_word_t i0)
880 rex(0, WIDE, _NOREG, _NOREG, r0);
881 if (r0 == _RAX_REGNO)
885 mrm(0x03, 0x00, r7(r0));
891 _cc(jit_state_t *_jit, jit_int32_t code, jit_int32_t r0)
893 rex(0, 0, _NOREG, _NOREG, r0);
896 mrm(0x03, 0x00, r7(r0));
900 _alur(jit_state_t *_jit, jit_int32_t code, jit_int32_t r0, jit_int32_t r1)
902 rex(0, WIDE, r1, _NOREG, r0);
904 mrm(0x03, r7(r1), r7(r0));
908 _alui(jit_state_t *_jit, jit_int32_t code, jit_int32_t r0, jit_word_t i0)
911 if (can_sign_extend_int_p(i0)) {
912 rex(0, WIDE, _NOREG, _NOREG, r0);
913 if ((jit_int8_t)i0 == i0) {
915 ic(0xc0 | code | r7(r0));
919 if (r0 == _RAX_REGNO)
923 ic(0xc0 | code | r7(r0));
929 reg = jit_get_reg(jit_class_gpr);
931 alur(code, r0, rn(reg));
937 _save(jit_state_t *_jit, jit_int32_t r0)
939 if (!_jitc->function->regoff[r0]) {
940 _jitc->function->regoff[r0] = jit_allocai(sizeof(jit_word_t));
943 assert(!jit_regset_tstbit(&_jitc->regsav, r0));
944 jit_regset_setbit(&_jitc->regsav, r0);
945 stxi(_jitc->function->regoff[r0], _RBP_REGNO, r0);
949 _load(jit_state_t *_jit, jit_int32_t r0)
951 assert(_jitc->function->regoff[r0]);
952 assert(jit_regset_tstbit(&_jitc->regsav, r0));
953 jit_regset_clrbit(&_jitc->regsav, r0);
954 ldxi(r0, _RBP_REGNO, _jitc->function->regoff[r0]);
958 _addr(jit_state_t *_jit, jit_int32_t r0, jit_int32_t r1, jit_int32_t r2)
965 lea(0, r1, r2, _SCL1, r0);
969 _addi(jit_state_t *_jit, jit_int32_t r0, jit_int32_t r1, jit_word_t i0)
980 else if (can_sign_extend_int_p(i0)) {
984 lea(i0, r1, _NOREG, _SCL1, r0);
991 reg = jit_get_reg(jit_class_gpr);
999 _addcr(jit_state_t *_jit, jit_int32_t r0, jit_int32_t r1, jit_int32_t r2)
1010 _addci(jit_state_t *_jit, jit_int32_t r0, jit_int32_t r1, jit_word_t i0)
1013 if (can_sign_extend_int_p(i0)) {
1017 else if (r0 == r1) {
1018 reg = jit_get_reg(jit_class_gpr);
1030 _addxr(jit_state_t *_jit, jit_int32_t r0, jit_int32_t r1, jit_int32_t r2)
1041 _addxi(jit_state_t *_jit, jit_int32_t r0, jit_int32_t r1, jit_word_t i0)
1044 if (can_sign_extend_int_p(i0)) {
1048 else if (r0 == r1) {
1049 reg = jit_get_reg(jit_class_gpr);
1051 iaddxr(r0, rn(reg));
1061 _subr(jit_state_t *_jit, jit_int32_t r0, jit_int32_t r1, jit_int32_t r2)
1065 else if (r0 == r2) {
1076 _subi(jit_state_t *_jit, jit_int32_t r0, jit_int32_t r1, jit_word_t i0)
1087 else if (can_sign_extend_int_p(i0)) {
1091 lea(-i0, r1, _NOREG, _SCL1, r0);
1093 else if (r0 != r1) {
1098 reg = jit_get_reg(jit_class_gpr);
1106 _subcr(jit_state_t *_jit, jit_int32_t r0, jit_int32_t r1, jit_int32_t r2)
1109 if (r0 == r2 && r0 != r1) {
1110 reg = jit_get_reg(jit_class_gpr);
1123 _subci(jit_state_t *_jit, jit_int32_t r0, jit_int32_t r1, jit_word_t i0)
1127 if (can_sign_extend_int_p(i0))
1130 reg = jit_get_reg(jit_class_gpr);
1138 _subxr(jit_state_t *_jit, jit_int32_t r0, jit_int32_t r1, jit_int32_t r2)
1141 if (r0 == r2 && r0 != r1) {
1142 reg = jit_get_reg(jit_class_gpr);
1145 isubxr(r0, rn(reg));
1155 _subxi(jit_state_t *_jit, jit_int32_t r0, jit_int32_t r1, jit_word_t i0)
1159 if (can_sign_extend_int_p(i0))
1162 reg = jit_get_reg(jit_class_gpr);
1164 isubxr(r0, rn(reg));
1170 _rsbi(jit_state_t *_jit, jit_int32_t r0, jit_int32_t r1, jit_word_t i0)
1177 _imulr(jit_state_t *_jit, jit_int32_t r0, jit_int32_t r1)
1179 rex(0, WIDE, r0, _NOREG, r1);
1182 mrm(0x03, r7(r0), r7(r1));
1186 _imuli(jit_state_t *_jit, jit_int32_t r0, jit_int32_t r1, jit_word_t i0)
1189 if (can_sign_extend_int_p(i0)) {
1190 rex(0, WIDE, r0, _NOREG, r1);
1191 if ((jit_int8_t)i0 == i0) {
1193 mrm(0x03, r7(r0), r7(r1));
1198 mrm(0x03, r7(r0), r7(r1));
1203 reg = jit_get_reg(jit_class_gpr);
1211 _mulr(jit_state_t *_jit, jit_int32_t r0, jit_int32_t r1, jit_int32_t r2)
1224 _muli(jit_state_t *_jit, jit_int32_t r0, jit_int32_t r1, jit_word_t i0)
1237 lea(0, _NOREG, r1, _SCL2, r0);
1240 lea(0, _NOREG, r1, _SCL4, r0);
1243 lea(0, _NOREG, r1, _SCL8, r0);
1246 if (i0 > 0 && !(i0 & (i0 - 1)))
1247 lshi(r0, r1, ffsl(i0) - 1);
1248 else if (can_sign_extend_int_p(i0))
1250 else if (r0 != r1) {
1260 #define savset(rn) \
1263 if (r1 != rn && r2 != rn) \
1266 #define isavset(rn) \
1272 #define qsavset(rn) \
1273 if (r0 != rn && r1 != rn) { \
1275 if (r2 != rn && r3 != rn) \
1278 #define allocr(rn, rv) \
1279 if (set & (1 << rn)) \
1280 (void)jit_get_reg(rv|jit_class_gpr|jit_class_named); \
1281 if (sav & (1 << rn)) { \
1282 if ( jit_regset_tstbit(&_jitc->regsav, rv) || \
1283 !jit_regset_tstbit(&_jitc->reglive, rv)) \
1284 sav &= ~(1 << rn); \
1288 #define clear(rn, rv) \
1289 if (set & (1 << rn)) \
1290 jit_unget_reg(rv); \
1291 if (sav & (1 << rn)) \
1294 _iqmulr(jit_state_t *_jit, jit_int32_t r0, jit_int32_t r1,
1295 jit_int32_t r2, jit_int32_t r3, jit_bool_t sign)
1302 qsavset(_RDX_REGNO);
1303 qsavset(_RAX_REGNO);
1304 allocr(_RDX_REGNO, _RDX);
1305 allocr(_RAX_REGNO, _RAX);
1307 if (r3 == _RAX_REGNO)
1311 movr(_RAX_REGNO, r2);
1318 if (r0 == _RDX_REGNO && r1 == _RAX_REGNO)
1319 xchgr(_RAX_REGNO, _RDX_REGNO);
1321 if (r0 != _RDX_REGNO)
1322 movr(r0, _RAX_REGNO);
1323 movr(r1, _RDX_REGNO);
1324 if (r0 == _RDX_REGNO)
1325 movr(r0, _RAX_REGNO);
1328 clear(_RDX_REGNO, _RDX);
1329 clear(_RAX_REGNO, _RAX);
1333 _iqmuli(jit_state_t *_jit, jit_int32_t r0, jit_int32_t r1,
1334 jit_int32_t r2, jit_word_t i0, jit_bool_t sign)
1343 reg = jit_get_reg(jit_class_gpr);
1346 qmulr(r0, r1, r2, rn(reg));
1348 qmulr_u(r0, r1, r2, rn(reg));
1354 _sign_extend_rdx_rax(jit_state_t *_jit)
1356 rex(0, WIDE, 0, 0, 0);
1361 _divremr(jit_state_t *_jit, jit_int32_t r0, jit_int32_t r1, jit_int32_t r2,
1362 jit_bool_t sign, jit_bool_t divide)
1370 sav = set = use = 0;
1373 allocr(_RDX_REGNO, _RDX);
1374 allocr(_RAX_REGNO, _RAX);
1376 if (r2 == _RAX_REGNO) {
1377 if (r0 == _RAX_REGNO || r0 == _RDX_REGNO) {
1378 if ((reg = jit_get_reg(jit_class_gpr|jit_class_chk)) == JIT_NOREG)
1379 reg = jit_get_reg((r1 == _RCX_REGNO ? _RBX : _RCX) |
1380 jit_class_gpr|jit_class_named);
1383 movr(div, _RAX_REGNO);
1384 if (r1 != _RAX_REGNO)
1385 movr(_RAX_REGNO, r1);
1389 xchgr(r0, _RAX_REGNO);
1391 if (r0 != _RAX_REGNO)
1392 movr(r0, _RAX_REGNO);
1393 if (r1 != _RAX_REGNO)
1394 movr(_RAX_REGNO, r1);
1399 else if (r2 == _RDX_REGNO) {
1400 if (r0 == _RAX_REGNO || r0 == _RDX_REGNO) {
1401 if ((reg = jit_get_reg(jit_class_gpr|jit_class_chk)) == JIT_NOREG)
1402 reg = jit_get_reg((r1 == _RCX_REGNO ? _RBX : _RCX) |
1403 jit_class_gpr|jit_class_named);
1406 movr(div, _RDX_REGNO);
1407 if (r1 != _RAX_REGNO)
1408 movr(_RAX_REGNO, r1);
1411 if (r1 != _RAX_REGNO)
1412 movr(_RAX_REGNO, r1);
1413 movr(r0, _RDX_REGNO);
1418 if (r1 != _RAX_REGNO)
1419 movr(_RAX_REGNO, r1);
1424 sign_extend_rdx_rax();
1428 ixorr(_RDX_REGNO, _RDX_REGNO);
1436 movr(r0, _RAX_REGNO);
1438 movr(r0, _RDX_REGNO);
1440 clear(_RDX_REGNO, _RDX);
1441 clear(_RAX_REGNO, _RAX);
1445 _divremi(jit_state_t *_jit, jit_int32_t r0, jit_int32_t r1, jit_word_t i0,
1446 jit_bool_t sign, jit_bool_t divide)
1466 if (i0 > 0 && !(i0 & (i0 - 1))) {
1469 rshi(r0, r0, ffsl(i0) - 1);
1471 rshi_u(r0, r0, ffsl(i0) - 1);
1477 else if (i0 == 1 || (sign && i0 == -1)) {
1481 else if (!sign && i0 > 0 && !(i0 & (i0 - 1))) {
1482 if (can_sign_extend_int_p(i0)) {
1486 else if (r0 != r1) {
1491 reg = jit_get_reg(jit_class_gpr);
1492 movi(rn(reg), i0 - 1);
1499 sav = set = use = 0;
1500 isavset(_RDX_REGNO);
1501 isavset(_RAX_REGNO);
1502 allocr(_RDX_REGNO, _RDX);
1503 allocr(_RAX_REGNO, _RAX);
1505 if (r0 == _RAX_REGNO || r0 == _RDX_REGNO || r0 == r1) {
1506 if ((reg = jit_get_reg(jit_class_gpr|jit_class_chk)) == JIT_NOREG)
1507 reg = jit_get_reg((r1 == _RCX_REGNO ? _RBX : _RCX) |
1508 jit_class_gpr|jit_class_named);
1516 movr(_RAX_REGNO, r1);
1519 sign_extend_rdx_rax();
1523 ixorr(_RDX_REGNO, _RDX_REGNO);
1531 movr(r0, _RAX_REGNO);
1533 movr(r0, _RDX_REGNO);
1535 clear(_RDX_REGNO, _RDX);
1536 clear(_RAX_REGNO, _RAX);
1540 _iqdivr(jit_state_t *_jit, jit_int32_t r0, jit_int32_t r1,
1541 jit_int32_t r2, jit_int32_t r3, jit_bool_t sign)
1549 sav = set = use = 0;
1550 qsavset(_RDX_REGNO);
1551 qsavset(_RAX_REGNO);
1552 allocr(_RDX_REGNO, _RDX);
1553 allocr(_RAX_REGNO, _RAX);
1554 if (r3 == _RAX_REGNO) {
1555 if (r0 == _RAX_REGNO || r0 == _RDX_REGNO) {
1556 if ((reg = jit_get_reg(jit_class_gpr|jit_class_chk)) == JIT_NOREG)
1557 reg = jit_get_reg((r1 == _RCX_REGNO ? _RBX : _RCX) |
1558 jit_class_gpr|jit_class_named);
1561 movr(div, _RAX_REGNO);
1562 if (r2 != _RAX_REGNO)
1563 movr(_RAX_REGNO, r2);
1567 xchgr(r0, _RAX_REGNO);
1569 if (r0 != _RAX_REGNO)
1570 movr(r0, _RAX_REGNO);
1571 if (r2 != _RAX_REGNO)
1572 movr(_RAX_REGNO, r2);
1577 else if (r3 == _RDX_REGNO) {
1578 if (r0 == _RAX_REGNO || r0 == _RDX_REGNO) {
1579 if ((reg = jit_get_reg(jit_class_gpr|jit_class_chk)) == JIT_NOREG)
1580 reg = jit_get_reg((r1 == _RCX_REGNO ? _RBX : _RCX) |
1581 jit_class_gpr|jit_class_named);
1584 movr(div, _RDX_REGNO);
1585 if (r2 != _RAX_REGNO)
1586 movr(_RAX_REGNO, r2);
1589 if (r2 != _RAX_REGNO)
1590 movr(_RAX_REGNO, r2);
1591 movr(r0, _RDX_REGNO);
1596 if (r2 != _RAX_REGNO)
1597 movr(_RAX_REGNO, r2);
1601 sign_extend_rdx_rax();
1605 ixorr(_RDX_REGNO, _RDX_REGNO);
1611 if (r0 == _RDX_REGNO && r1 == _RAX_REGNO)
1612 xchgr(_RAX_REGNO, _RDX_REGNO);
1614 if (r0 != _RDX_REGNO)
1615 movr(r0, _RAX_REGNO);
1616 movr(r1, _RDX_REGNO);
1617 if (r0 == _RDX_REGNO)
1618 movr(r0, _RAX_REGNO);
1621 clear(_RDX_REGNO, _RDX);
1622 clear(_RAX_REGNO, _RAX);
1626 _iqdivi(jit_state_t *_jit, jit_int32_t r0, jit_int32_t r1,
1627 jit_int32_t r2, jit_word_t i0, jit_bool_t sign)
1631 reg = jit_get_reg(jit_class_gpr);
1634 qdivr(r0, r1, r2, rn(reg));
1636 qdivr_u(r0, r1, r2, rn(reg));
1644 _andr(jit_state_t *_jit, jit_int32_t r0, jit_int32_t r1, jit_int32_t r2)
1659 _andi(jit_state_t *_jit, jit_int32_t r0, jit_int32_t r1, jit_word_t i0)
1667 else if (r0 == r1) {
1668 if (can_sign_extend_int_p(i0))
1671 reg = jit_get_reg(jit_class_gpr);
1684 _orr(jit_state_t *_jit, jit_int32_t r0, jit_int32_t r1, jit_int32_t r2)
1699 _ori(jit_state_t *_jit, jit_int32_t r0, jit_int32_t r1, jit_word_t i0)
1706 else if (can_sign_extend_int_p(i0)) {
1710 else if (r0 != r1) {
1715 reg = jit_get_reg(jit_class_gpr);
1723 _xorr(jit_state_t *_jit, jit_int32_t r0, jit_int32_t r1, jit_int32_t r2)
1738 _xori(jit_state_t *_jit, jit_int32_t r0, jit_int32_t r1, jit_word_t i0)
1745 else if (can_sign_extend_int_p(i0)) {
1749 else if (r0 != r1) {
1754 reg = jit_get_reg(jit_class_gpr);
1762 _irotshr(jit_state_t *_jit, jit_int32_t code, jit_int32_t r0)
1764 rex(0, WIDE, _RCX_REGNO, _NOREG, r0);
1766 mrm(0x03, code, r7(r0));
1770 _rotshr(jit_state_t *_jit, jit_int32_t code,
1771 jit_int32_t r0, jit_int32_t r1, jit_int32_t r2)
1776 if (r0 == _RCX_REGNO) {
1777 reg = jit_get_reg(jit_class_gpr);
1779 if (r2 != _RCX_REGNO)
1780 movr(_RCX_REGNO, r2);
1781 irotshr(code, rn(reg));
1782 movr(_RCX_REGNO, rn(reg));
1785 else if (r2 != _RCX_REGNO) {
1786 use = !jit_reg_free_p(_RCX);
1788 reg = jit_get_reg(jit_class_gpr);
1789 movr(rn(reg), _RCX_REGNO);
1793 if (r1 == _RCX_REGNO) {
1795 xchgr(r0, _RCX_REGNO);
1798 movr(_RCX_REGNO, r2);
1802 movr(_RCX_REGNO, r2);
1807 movr(_RCX_REGNO, rn(reg));
1818 _irotshi(jit_state_t *_jit, jit_int32_t code, jit_int32_t r0, jit_word_t i0)
1820 rex(0, WIDE, _NOREG, _NOREG, r0);
1823 mrm(0x03, code, r7(r0));
1827 mrm(0x03, code, r7(r0));
1833 _rotshi(jit_state_t *_jit, jit_int32_t code,
1834 jit_int32_t r0, jit_int32_t r1, jit_word_t i0)
1838 irotshi(code, r0, i0);
1842 _lshi(jit_state_t *_jit, jit_int32_t r0, jit_int32_t r1, jit_word_t i0)
1847 lea(0, _NOREG, r1, i0 == 1 ? _SCL2 : i0 == 2 ? _SCL4 : _SCL8, r0);
1849 rotshi(X86_SHL, r0, r1, i0);
1853 _unr(jit_state_t *_jit, jit_int32_t code, jit_int32_t r0)
1855 rex(0, WIDE, _NOREG, _NOREG, r0);
1857 mrm(0x03, code, r7(r0));
1861 _negr(jit_state_t *_jit, jit_int32_t r0, jit_int32_t r1)
1872 _comr(jit_state_t *_jit, jit_int32_t r0, jit_int32_t r1)
1880 _incr(jit_state_t *_jit, jit_int32_t r0, jit_int32_t r1)
1884 rex(0, WIDE, _NOREG, _NOREG, r0);
1893 _decr(jit_state_t *_jit, jit_int32_t r0, jit_int32_t r1)
1897 rex(0, WIDE, _NOREG, _NOREG, r0);
1907 _cr(jit_state_t *_jit,
1908 jit_int32_t code, jit_int32_t r0, jit_int32_t r1, jit_int32_t r2)
1913 same = r0 == r1 || r0 == r2;
1922 reg = jit_get_reg(jit_class_gpr|jit_class_rg8);
1923 ixorr(rn(reg), rn(reg));
1932 _ci(jit_state_t *_jit,
1933 jit_int32_t code, jit_int32_t r0, jit_int32_t r1, jit_word_t i0)
1947 reg = jit_get_reg(jit_class_gpr|jit_class_rg8);
1948 ixorr(rn(reg), rn(reg));
1957 _ci0(jit_state_t *_jit, jit_int32_t code, jit_int32_t r0, jit_int32_t r1)
1971 reg = jit_get_reg(jit_class_gpr|jit_class_rg8);
1972 ixorr(rn(reg), rn(reg));
1981 _ltr(jit_state_t *_jit, jit_int32_t r0, jit_int32_t r1, jit_int32_t r2)
1986 cr(X86_CC_L, r0, r1, r2);
1990 _lti(jit_state_t *_jit, jit_int32_t r0, jit_int32_t r1, jit_word_t i0)
1993 ci(X86_CC_L, r0, r1, i0);
1995 ci0(X86_CC_S, r0, r1);
1999 _ltr_u(jit_state_t *_jit, jit_int32_t r0, jit_int32_t r1, jit_int32_t r2)
2004 cr(X86_CC_B, r0, r1, r2);
2008 _ler(jit_state_t *_jit, jit_int32_t r0, jit_int32_t r1, jit_int32_t r2)
2013 cr(X86_CC_LE, r0, r1, r2);
2017 _ler_u(jit_state_t *_jit, jit_int32_t r0, jit_int32_t r1, jit_int32_t r2)
2022 cr(X86_CC_BE, r0, r1, r2);
2026 _lei_u(jit_state_t *_jit, jit_int32_t r0, jit_int32_t r1, jit_word_t i0)
2029 ci(X86_CC_BE, r0, r1, i0);
2031 ci0(X86_CC_E, r0, r1);
2035 _eqr(jit_state_t *_jit, jit_int32_t r0, jit_int32_t r1, jit_int32_t r2)
2040 cr(X86_CC_E, r0, r1, r2);
2044 _eqi(jit_state_t *_jit, jit_int32_t r0, jit_int32_t r1, jit_word_t i0)
2047 ci(X86_CC_E, r0, r1, i0);
2049 ci0(X86_CC_E, r0, r1);
2053 _ger(jit_state_t *_jit, jit_int32_t r0, jit_int32_t r1, jit_int32_t r2)
2058 cr(X86_CC_GE, r0, r1, r2);
2062 _gei(jit_state_t *_jit, jit_int32_t r0, jit_int32_t r1, jit_word_t i0)
2065 ci(X86_CC_GE, r0, r1, i0);
2067 ci0(X86_CC_NS, r0, r1);
2071 _ger_u(jit_state_t *_jit, jit_int32_t r0, jit_int32_t r1, jit_int32_t r2)
2076 cr(X86_CC_AE, r0, r1, r2);
2080 _gei_u(jit_state_t *_jit, jit_int32_t r0, jit_int32_t r1, jit_word_t i0)
2083 ci(X86_CC_AE, r0, r1, i0);
2085 ci0(X86_CC_NB, r0, r1);
2089 _gtr(jit_state_t *_jit, jit_int32_t r0, jit_int32_t r1, jit_int32_t r2)
2094 cr(X86_CC_G, r0, r1, r2);
2098 _gtr_u(jit_state_t *_jit, jit_int32_t r0, jit_int32_t r1, jit_int32_t r2)
2103 cr(X86_CC_A, r0, r1, r2);
2107 _gti_u(jit_state_t *_jit, jit_int32_t r0, jit_int32_t r1, jit_word_t i0)
2110 ci(X86_CC_A, r0, r1, i0);
2112 ci0(X86_CC_NE, r0, r1);
2116 _ner(jit_state_t *_jit, jit_int32_t r0, jit_int32_t r1, jit_int32_t r2)
2121 cr(X86_CC_NE, r0, r1, r2);
2125 _nei(jit_state_t *_jit, jit_int32_t r0, jit_int32_t r1, jit_word_t i0)
2128 ci(X86_CC_NE, r0, r1, i0);
2130 ci0(X86_CC_NE, r0, r1);
2134 _movr(jit_state_t *_jit, jit_int32_t r0, jit_int32_t r1)
2137 rex(0, 1, r1, _NOREG, r0);
2139 ic(0xc0 | (r1 << 3) | r7(r0));
2144 _imovi(jit_state_t *_jit, jit_int32_t r0, jit_word_t i0)
2148 if (fits_uint32_p(i0)) {
2150 rex(0, 0, _NOREG, _NOREG, r0);
2156 rex(0, 1, _NOREG, _NOREG, r0);
2168 _movi(jit_state_t *_jit, jit_int32_t r0, jit_word_t i0)
2177 _movi_p(jit_state_t *_jit, jit_int32_t r0, jit_word_t i0)
2179 rex(0, WIDE, _NOREG, _NOREG, r0);
2182 return (_jit->pc.w);
2186 _movcr(jit_state_t *_jit, jit_int32_t r0, jit_int32_t r1)
2188 rex(0, WIDE, r0, _NOREG, r1);
2191 mrm(0x03, r7(r0), r7(r1));
2195 _movcr_u(jit_state_t *_jit, jit_int32_t r0, jit_int32_t r1)
2197 rex(0, WIDE, r0, _NOREG, r1);
2200 mrm(0x03, r7(r0), r7(r1));
2204 _movsr(jit_state_t *_jit, jit_int32_t r0, jit_int32_t r1)
2206 rex(0, WIDE, r0, _NOREG, r1);
2209 mrm(0x03, r7(r0), r7(r1));
2213 _movsr_u(jit_state_t *_jit, jit_int32_t r0, jit_int32_t r1)
2215 rex(0, WIDE, r0, _NOREG, r1);
2218 mrm(0x03, r7(r0), r7(r1));
2222 _movnr(jit_state_t *_jit, jit_int32_t r0, jit_int32_t r1, jit_int32_t r2)
2224 assert(jit_cmov_p());
2228 rex(0, WIDE, r0, _NOREG, r1);
2231 mrm(0x03, r7(r0), r7(r1));
2235 _movzr(jit_state_t *_jit, jit_int32_t r0, jit_int32_t r1, jit_int32_t r2)
2237 assert(jit_cmov_p());
2241 rex(0, WIDE, r0, _NOREG, r1);
2244 mrm(0x03, r7(r0), r7(r1));
2249 _movir(jit_state_t *_jit, jit_int32_t r0, jit_int32_t r1)
2251 rex(0, 1, r0, _NOREG, r1);
2253 mrm(0x03, r7(r0), r7(r1));
2257 _movir_u(jit_state_t *_jit, jit_int32_t r0, jit_int32_t r1)
2259 rex(0, 0, r1, _NOREG, r0);
2261 ic(0xc0 | (r1 << 3) | r7(r0));
2266 _bswapr_us(jit_state_t *_jit, jit_int32_t r0, jit_int32_t r1)
2270 rex(0, 0, _NOREG, _NOREG, r0);
2272 mrm(0x03, X86_ROR, r7(r0));
2277 _bswapr_ui(jit_state_t *_jit, jit_int32_t r0, jit_int32_t r1)
2280 rex(0, 0, _NOREG, _NOREG, r0);
2285 #if __X64 && !__X64_32
2287 _bswapr_ul(jit_state_t *_jit, jit_int32_t r0, jit_int32_t r1)
2290 rex(0, 1, _NOREG, _NOREG, r0);
2297 _extr_c(jit_state_t *_jit, jit_int32_t r0, jit_int32_t r1)
2303 reg = jit_get_reg(jit_class_gpr|jit_class_rg8);
2311 _extr_uc(jit_state_t *_jit, jit_int32_t r0, jit_int32_t r1)
2317 reg = jit_get_reg(jit_class_gpr|jit_class_rg8);
2319 movcr_u(r0, rn(reg));
2325 _ldr_c(jit_state_t *_jit, jit_int32_t r0, jit_int32_t r1)
2327 rex(0, WIDE, r0, _NOREG, r1);
2330 rx(r0, 0, r1, _NOREG, _SCL1);
2334 _ldi_c(jit_state_t *_jit, jit_int32_t r0, jit_word_t i0)
2337 if (can_sign_extend_int_p(i0)) {
2338 rex(0, WIDE, r0, _NOREG, _NOREG);
2341 rx(r0, i0, _NOREG, _NOREG, _SCL1);
2344 reg = jit_get_reg(jit_class_gpr);
2352 _ldr_uc(jit_state_t *_jit, jit_int32_t r0, jit_int32_t r1)
2354 rex(0, WIDE, r0, _NOREG, r1);
2357 rx(r0, 0, r1, _NOREG, _SCL1);
2361 _ldi_uc(jit_state_t *_jit, jit_int32_t r0, jit_word_t i0)
2364 if (can_sign_extend_int_p(i0)) {
2365 rex(0, WIDE, r0, _NOREG, _NOREG);
2368 rx(r0, i0, _NOREG, _NOREG, _SCL1);
2371 reg = jit_get_reg(jit_class_gpr);
2373 ldr_uc(r0, rn(reg));
2379 _ldr_s(jit_state_t *_jit, jit_int32_t r0, jit_int32_t r1)
2381 rex(0, WIDE, r0, _NOREG, r1);
2384 rx(r0, 0, r1, _NOREG, _SCL1);
2388 _ldi_s(jit_state_t *_jit, jit_int32_t r0, jit_word_t i0)
2391 if (can_sign_extend_int_p(i0)) {
2392 rex(0, WIDE, r0, _NOREG, _NOREG);
2395 rx(r0, i0, _NOREG, _NOREG, _SCL1);
2398 reg = jit_get_reg(jit_class_gpr);
2406 _ldr_us(jit_state_t *_jit, jit_int32_t r0, jit_int32_t r1)
2408 rex(0, WIDE, r0, _NOREG, r1);
2411 rx(r0, 0, r1, _NOREG, _SCL1);
2415 _ldi_us(jit_state_t *_jit, jit_int32_t r0, jit_word_t i0)
2418 if (can_sign_extend_int_p(i0)) {
2419 rex(0, WIDE, r0, _NOREG, _NOREG);
2422 rx(r0, i0, _NOREG, _NOREG, _SCL1);
2425 reg = jit_get_reg(jit_class_gpr);
2427 ldr_us(r0, rn(reg));
2432 #if __X32 || !__X64_32
2434 _ldr_i(jit_state_t *_jit, jit_int32_t r0, jit_int32_t r1)
2437 rex(0, WIDE, r0, _NOREG, r1);
2442 rx(r0, 0, r1, _NOREG, _SCL1);
2446 _ldi_i(jit_state_t *_jit, jit_int32_t r0, jit_word_t i0)
2449 if (can_sign_extend_int_p(i0)) {
2451 rex(0, WIDE, r0, _NOREG, _NOREG);
2456 rx(r0, i0, _NOREG, _NOREG, _SCL1);
2459 reg = jit_get_reg(jit_class_gpr);
2469 _ldr_ui(jit_state_t *_jit, jit_int32_t r0, jit_int32_t r1)
2471 rex(0, 0, r0, _NOREG, r1);
2473 rx(r0, 0, r1, _NOREG, _SCL1);
2477 _ldi_ui(jit_state_t *_jit, jit_int32_t r0, jit_word_t i0)
2480 if (can_sign_extend_int_p(i0)) {
2481 rex(0, 0, r0, _NOREG, _NOREG);
2483 rx(r0, i0, _NOREG, _NOREG, _SCL1);
2486 reg = jit_get_reg(jit_class_gpr);
2488 ldr_ui(r0, rn(reg));
2495 _ldr_l(jit_state_t *_jit, jit_int32_t r0, jit_int32_t r1)
2497 rex(0, 1, r0, _NOREG, r1);
2499 rx(r0, 0, r1, _NOREG, _SCL1);
2503 _ldi_l(jit_state_t *_jit, jit_int32_t r0, jit_word_t i0)
2506 if (can_sign_extend_int_p(i0)) {
2507 rex(0, 1, r0, _NOREG, _NOREG);
2509 rx(r0, i0, _NOREG, _NOREG, _SCL1);
2512 reg = jit_get_reg(jit_class_gpr);
2522 _ldxr_c(jit_state_t *_jit, jit_int32_t r0, jit_int32_t r1, jit_int32_t r2)
2528 rex(0, WIDE, r0, r1, r2);
2531 rx(r0, 0, r2, r1, _SCL1);
2536 _ldxi_c(jit_state_t *_jit, jit_int32_t r0, jit_int32_t r1, jit_word_t i0)
2539 if (can_sign_extend_int_p(i0)) {
2540 rex(0, WIDE, r0, _NOREG, r1);
2543 rx(r0, i0, r1, _NOREG, _SCL1);
2546 reg = jit_get_reg(jit_class_gpr);
2548 ldxr_c(r0, r1, rn(reg));
2554 _ldxr_uc(jit_state_t *_jit, jit_int32_t r0, jit_int32_t r1, jit_int32_t r2)
2560 rex(0, WIDE, r0, r1, r2);
2563 rx(r0, 0, r2, r1, _SCL1);
2568 _ldxi_uc(jit_state_t *_jit, jit_int32_t r0, jit_int32_t r1, jit_word_t i0)
2571 if (can_sign_extend_int_p(i0)) {
2572 rex(0, WIDE, r0, _NOREG, r1);
2575 rx(r0, i0, r1, _NOREG, _SCL1);
2578 reg = jit_get_reg(jit_class_gpr);
2580 ldxr_uc(r0, r1, rn(reg));
2586 _ldxr_s(jit_state_t *_jit, jit_int32_t r0, jit_int32_t r1, jit_int32_t r2)
2592 rex(0, WIDE, r0, r1, r2);
2595 rx(r0, 0, r2, r1, _SCL1);
2600 _ldxi_s(jit_state_t *_jit, jit_int32_t r0, jit_int32_t r1, jit_word_t i0)
2603 if (can_sign_extend_int_p(i0)) {
2604 rex(0, WIDE, r0, _NOREG, r1);
2607 rx(r0, i0, r1, _NOREG, _SCL1);
2610 reg = jit_get_reg(jit_class_gpr);
2612 ldxr_s(r0, r1, rn(reg));
2618 _ldxr_us(jit_state_t *_jit, jit_int32_t r0, jit_int32_t r1, jit_int32_t r2)
2624 rex(0, WIDE, r0, r1, r2);
2627 rx(r0, 0, r2, r1, _SCL1);
2632 _ldxi_us(jit_state_t *_jit, jit_int32_t r0, jit_int32_t r1, jit_word_t i0)
2635 if (can_sign_extend_int_p(i0)) {
2636 rex(0, WIDE, r0, _NOREG, r1);
2639 rx(r0, i0, r1, _NOREG, _SCL1);
2642 reg = jit_get_reg(jit_class_gpr);
2644 ldxr_us(r0, r1, rn(reg));
2649 #if __X64 || !__X64_32
2651 _ldxr_i(jit_state_t *_jit, jit_int32_t r0, jit_int32_t r1, jit_int32_t r2)
2654 rex(0, WIDE, r0, r1, r2);
2659 rx(r0, 0, r2, r1, _SCL1);
2663 _ldxi_i(jit_state_t *_jit, jit_int32_t r0, jit_int32_t r1, jit_word_t i0)
2666 if (can_sign_extend_int_p(i0)) {
2668 rex(0, WIDE, r0, _NOREG, r1);
2673 rx(r0, i0, r1, _NOREG, _SCL1);
2676 reg = jit_get_reg(jit_class_gpr);
2678 ldxr_i(r0, r1, rn(reg));
2686 _ldxr_ui(jit_state_t *_jit, jit_int32_t r0, jit_int32_t r1, jit_int32_t r2)
2690 /* to avoid confusion with macro renames */
2691 _ldr_ui(_jit, r0, r0);
2693 rex(0, 0, r0, r1, r2);
2695 rx(r0, 0, r2, r1, _SCL1);
2700 _ldxi_ui(jit_state_t *_jit, jit_int32_t r0, jit_int32_t r1, jit_word_t i0)
2703 if (can_sign_extend_int_p(i0)) {
2704 rex(0, 0, r0, _NOREG, r1);
2706 rx(r0, i0, r1, _NOREG, _SCL1);
2709 reg = jit_get_reg(jit_class_gpr);
2711 ldxr_ui(r0, r1, rn(reg));
2718 _ldxr_l(jit_state_t *_jit, jit_int32_t r0, jit_int32_t r1, jit_int32_t r2)
2720 rex(0, 1, r0, r1, r2);
2722 rx(r0, 0, r2, r1, _SCL1);
2726 _ldxi_l(jit_state_t *_jit, jit_int32_t r0, jit_int32_t r1, jit_word_t i0)
2729 if (can_sign_extend_int_p(i0)) {
2730 rex(0, 1, r0, _NOREG, r1);
2732 rx(r0, i0, r1, _NOREG, _SCL1);
2735 reg = jit_get_reg(jit_class_gpr);
2737 ldxr_l(r0, r1, rn(reg));
2745 _str_c(jit_state_t *_jit, jit_int32_t r0, jit_int32_t r1)
2749 rex(0, 0, r1, _NOREG, r0);
2751 rx(r1, 0, r0, _NOREG, _SCL1);
2754 reg = jit_get_reg(jit_class_gpr|jit_class_rg8);
2756 rex(0, 0, rn(reg), _NOREG, r0);
2758 rx(rn(reg), 0, r0, _NOREG, _SCL1);
2764 _sti_c(jit_state_t *_jit, jit_word_t i0, jit_int32_t r0)
2767 if (can_sign_extend_int_p(i0)) {
2769 rex(0, 0, r0, _NOREG, _NOREG);
2771 rx(r0, i0, _NOREG, _NOREG, _SCL1);
2774 reg = jit_get_reg(jit_class_gpr|jit_class_rg8);
2776 rex(0, 0, rn(reg), _NOREG, _NOREG);
2778 rx(rn(reg), i0, _NOREG, _NOREG, _SCL1);
2783 reg = jit_get_reg(jit_class_gpr);
2791 _str_s(jit_state_t *_jit, jit_int32_t r0, jit_int32_t r1)
2794 rex(0, 0, r1, _NOREG, r0);
2796 rx(r1, 0, r0, _NOREG, _SCL1);
2800 _sti_s(jit_state_t *_jit, jit_word_t i0, jit_int32_t r0)
2803 if (can_sign_extend_int_p(i0)) {
2805 rex(0, 0, r0, _NOREG, _NOREG);
2807 rx(r0, i0, _NOREG, _NOREG, _SCL1);
2810 reg = jit_get_reg(jit_class_gpr);
2818 _str_i(jit_state_t *_jit, jit_int32_t r0, jit_int32_t r1)
2820 rex(0, 0, r1, _NOREG, r0);
2822 rx(r1, 0, r0, _NOREG, _SCL1);
2826 _sti_i(jit_state_t *_jit, jit_word_t i0, jit_int32_t r0)
2829 if (can_sign_extend_int_p(i0)) {
2830 rex(0, 0, r0, _NOREG, _NOREG);
2832 rx(r0, i0, _NOREG, _NOREG, _SCL1);
2835 reg = jit_get_reg(jit_class_gpr);
2842 #if __X64 && !__X64_32
2844 _str_l(jit_state_t *_jit, jit_int32_t r0, jit_int32_t r1)
2846 rex(0, 1, r1, _NOREG, r0);
2848 rx(r1, 0, r0, _NOREG, _SCL1);
2852 _sti_l(jit_state_t *_jit, jit_word_t i0, jit_int32_t r0)
2855 if (can_sign_extend_int_p(i0)) {
2856 rex(0, 1, r0, _NOREG, _NOREG);
2858 rx(r0, i0, _NOREG, _NOREG, _SCL1);
2861 reg = jit_get_reg(jit_class_gpr);
2870 _stxr_c(jit_state_t *_jit, jit_int32_t r0, jit_int32_t r1, jit_int32_t r2)
2874 reg = jit_get_reg(jit_class_gpr);
2875 addr(rn(reg), r0, r1);
2880 rex(0, 0, r2, r1, r0);
2882 rx(r2, 0, r0, r1, _SCL1);
2885 reg = jit_get_reg(jit_class_gpr|jit_class_rg8);
2887 rex(0, 0, rn(reg), r1, r0);
2889 rx(rn(reg), 0, r0, r1, _SCL1);
2896 _stxi_c(jit_state_t *_jit, jit_word_t i0, jit_int32_t r0, jit_int32_t r1)
2899 if (can_sign_extend_int_p(i0)) {
2901 rex(0, 0, r1, _NOREG, r0);
2903 rx(r1, i0, r0, _NOREG, _SCL1);
2906 reg = jit_get_reg(jit_class_gpr|jit_class_rg8);
2908 rex(0, 0, rn(reg), _NOREG, r0);
2910 rx(rn(reg), i0, r0, _NOREG, _SCL1);
2915 reg = jit_get_reg(jit_class_gpr);
2917 stxr_c(rn(reg), r0, r1);
2923 _stxr_s(jit_state_t *_jit, jit_int32_t r0, jit_int32_t r1, jit_int32_t r2)
2927 reg = jit_get_reg(jit_class_gpr);
2928 addr(rn(reg), r0, r1);
2933 rex(0, 0, r2, r1, r0);
2935 rx(r2, 0, r0, r1, _SCL1);
2940 _stxi_s(jit_state_t *_jit, jit_word_t i0, jit_int32_t r0, jit_int32_t r1)
2943 if (can_sign_extend_int_p(i0)) {
2945 rex(0, 0, r1, _NOREG, r0);
2947 rx(r1, i0, r0, _NOREG, _SCL1);
2950 reg = jit_get_reg(jit_class_gpr);
2952 stxr_s(rn(reg), r0, r1);
2958 _stxr_i(jit_state_t *_jit, jit_int32_t r0, jit_int32_t r1, jit_int32_t r2)
2962 reg = jit_get_reg(jit_class_gpr);
2963 addr(rn(reg), r0, r1);
2967 rex(0, 0, r2, r1, r0);
2969 rx(r2, 0, r0, r1, _SCL1);
2974 _stxi_i(jit_state_t *_jit, jit_word_t i0, jit_int32_t r0, jit_int32_t r1)
2977 if (can_sign_extend_int_p(i0)) {
2978 rex(0, 0, r1, _NOREG, r0);
2980 rx(r1, i0, r0, _NOREG, _SCL1);
2983 reg = jit_get_reg(jit_class_gpr);
2985 stxr_i(rn(reg), r0, r1);
2990 #if __X64 && !__X64_32
2992 _stxr_l(jit_state_t *_jit, jit_int32_t r0, jit_int32_t r1, jit_int32_t r2)
2994 rex(0, 1, r2, r1, r0);
2996 rx(r2, 0, r0, r1, _SCL1);
3000 _stxi_l(jit_state_t *_jit, jit_word_t i0, jit_int32_t r0, jit_int32_t r1)
3003 if (can_sign_extend_int_p(i0)) {
3004 rex(0, 1, r1, _NOREG, r0);
3006 rx(r1, i0, r0, _NOREG, _SCL1);
3009 reg = jit_get_reg(jit_class_gpr);
3011 stxr_l(rn(reg), r0, r1);
3018 _jccs(jit_state_t *_jit, jit_int32_t code, jit_word_t i0)
3022 w = i0 - (_jit->pc.w + 1);
3027 _jcc(jit_state_t *_jit, jit_int32_t code, jit_word_t i0)
3032 w = i0 - (_jit->pc.w + 4);
3037 _jcr(jit_state_t *_jit,
3038 jit_int32_t code, jit_word_t i0, jit_int32_t r0, jit_int32_t r1)
3040 alur(X86_CMP, r0, r1);
3045 _jci(jit_state_t *_jit,
3046 jit_int32_t code, jit_word_t i0, jit_int32_t r0, jit_word_t i1)
3048 alui(X86_CMP, r0, i1);
3053 _jci0(jit_state_t *_jit, jit_int32_t code, jit_word_t i0, jit_int32_t r0)
3060 _bltr(jit_state_t *_jit, jit_word_t i0, jit_int32_t r0, jit_int32_t r1)
3062 jcr(X86_CC_L, i0, r0, r1);
3063 return (_jit->pc.w);
3067 _blti(jit_state_t *_jit, jit_word_t i0, jit_int32_t r0, jit_word_t i1)
3069 if (i1) jci (X86_CC_L, i0, r0, i1);
3070 else jci0(X86_CC_S, i0, r0);
3071 return (_jit->pc.w);
3075 _bltr_u(jit_state_t *_jit, jit_word_t i0, jit_int32_t r0, jit_int32_t r1)
3077 jcr(X86_CC_B, i0, r0, r1);
3078 return (_jit->pc.w);
3082 _blti_u(jit_state_t *_jit, jit_word_t i0, jit_int32_t r0, jit_word_t i1)
3084 if (i1) jci (X86_CC_B, i0, r0, i1);
3085 else jci0(X86_CC_B, i0, r0);
3086 return (_jit->pc.w);
3090 _bler(jit_state_t *_jit, jit_word_t i0, jit_int32_t r0, jit_int32_t r1)
3092 if (r0 == r1) jmpi(i0);
3093 else jcr (X86_CC_LE, i0, r0, r1);
3094 return (_jit->pc.w);
3098 _blei(jit_state_t *_jit, jit_word_t i0, jit_int32_t r0, jit_word_t i1)
3100 if (i1) jci (X86_CC_LE, i0, r0, i1);
3101 else jci0(X86_CC_LE, i0, r0);
3102 return (_jit->pc.w);
3106 _bler_u(jit_state_t *_jit, jit_word_t i0, jit_int32_t r0, jit_int32_t r1)
3108 if (r0 == r1) jmpi(i0);
3109 else jcr (X86_CC_BE, i0, r0, r1);
3110 return (_jit->pc.w);
3114 _blei_u(jit_state_t *_jit, jit_word_t i0, jit_int32_t r0, jit_word_t i1)
3116 if (i1) jci (X86_CC_BE, i0, r0, i1);
3117 else jci0(X86_CC_BE, i0, r0);
3118 return (_jit->pc.w);
3122 _beqr(jit_state_t *_jit, jit_word_t i0, jit_int32_t r0, jit_int32_t r1)
3124 if (r0 == r1) jmpi(i0);
3125 else jcr (X86_CC_E, i0, r0, r1);
3126 return (_jit->pc.w);
3130 _beqi(jit_state_t *_jit, jit_word_t i0, jit_int32_t r0, jit_word_t i1)
3132 if (i1) jci (X86_CC_E, i0, r0, i1);
3133 else jci0(X86_CC_E, i0, r0);
3134 return (_jit->pc.w);
3138 _bger(jit_state_t *_jit, jit_word_t i0, jit_int32_t r0, jit_int32_t r1)
3140 if (r0 == r1) jmpi(i0);
3141 else jcr (X86_CC_GE, i0, r0, r1);
3142 return (_jit->pc.w);
3146 _bgei(jit_state_t *_jit, jit_word_t i0, jit_int32_t r0, jit_word_t i1)
3148 if (i1) jci (X86_CC_GE, i0, r0, i1);
3149 else jci0(X86_CC_NS, i0, r0);
3150 return (_jit->pc.w);
3154 _bger_u(jit_state_t *_jit, jit_word_t i0, jit_int32_t r0, jit_int32_t r1)
3156 if (r0 == r1) jmpi(i0);
3157 else jcr (X86_CC_AE, i0, r0, r1);
3158 return (_jit->pc.w);
3162 _bgei_u(jit_state_t *_jit, jit_word_t i0, jit_int32_t r0, jit_word_t i1)
3164 if (i1) jci (X86_CC_AE, i0, r0, i1);
3166 return (_jit->pc.w);
3170 _bgtr(jit_state_t *_jit, jit_word_t i0, jit_int32_t r0, jit_int32_t r1)
3172 jcr(X86_CC_G, i0, r0, r1);
3173 return (_jit->pc.w);
3177 _bgti(jit_state_t *_jit, jit_word_t i0, jit_int32_t r0, jit_word_t i1)
3179 jci(X86_CC_G, i0, r0, i1);
3180 return (_jit->pc.w);
3184 _bgtr_u(jit_state_t *_jit, jit_word_t i0, jit_int32_t r0, jit_int32_t r1)
3186 jcr(X86_CC_A, i0, r0, r1);
3187 return (_jit->pc.w);
3191 _bgti_u(jit_state_t *_jit, jit_word_t i0, jit_int32_t r0, jit_word_t i1)
3193 if (i1) jci (X86_CC_A, i0, r0, i1);
3194 else jci0(X86_CC_NE, i0, r0);
3195 return (_jit->pc.w);
3199 _bner(jit_state_t *_jit, jit_word_t i0, jit_int32_t r0, jit_int32_t r1)
3201 jcr(X86_CC_NE, i0, r0, r1);
3202 return (_jit->pc.w);
3206 _bnei(jit_state_t *_jit, jit_word_t i0, jit_int32_t r0, jit_word_t i1)
3208 if (i1) jci (X86_CC_NE, i0, r0, i1);
3209 else jci0(X86_CC_NE, i0, r0);
3210 return (_jit->pc.w);
3214 _bmsr(jit_state_t *_jit, jit_word_t i0, jit_int32_t r0, jit_int32_t r1)
3218 return (_jit->pc.w);
3222 _bmsi(jit_state_t *_jit, jit_word_t i0, jit_int32_t r0, jit_word_t i1)
3225 if (can_zero_extend_int_p(i1))
3228 reg = jit_get_reg(jit_class_gpr);
3234 return (_jit->pc.w);
3238 _bmcr(jit_state_t *_jit, jit_word_t i0, jit_int32_t r0, jit_int32_t r1)
3242 return (_jit->pc.w);
3246 _bmci(jit_state_t *_jit, jit_word_t i0, jit_int32_t r0, jit_word_t i1)
3249 if (can_zero_extend_int_p(i1))
3252 reg = jit_get_reg(jit_class_gpr);
3258 return (_jit->pc.w);
3262 _boaddr(jit_state_t *_jit, jit_word_t i0, jit_int32_t r0, jit_int32_t r1)
3266 return (_jit->pc.w);
3270 _boaddi(jit_state_t *_jit, jit_word_t i0, jit_int32_t r0, jit_word_t i1)
3273 if (can_sign_extend_int_p(i1)) {
3276 return (_jit->pc.w);
3278 reg = jit_get_reg(jit_class_gpr|jit_class_nospill);
3281 return (boaddr(i0, r0, rn(reg)));
3285 _boaddr_u(jit_state_t *_jit, jit_word_t i0, jit_int32_t r0, jit_int32_t r1)
3289 return (_jit->pc.w);
3293 _boaddi_u(jit_state_t *_jit, jit_word_t i0, jit_int32_t r0, jit_word_t i1)
3296 if (can_sign_extend_int_p(i1)) {
3299 return (_jit->pc.w);
3301 reg = jit_get_reg(jit_class_gpr|jit_class_nospill);
3304 return (boaddr_u(i0, r0, rn(reg)));
3308 _bxaddr(jit_state_t *_jit, jit_word_t i0, jit_int32_t r0, jit_int32_t r1)
3312 return (_jit->pc.w);
3316 _bxaddi(jit_state_t *_jit, jit_word_t i0, jit_int32_t r0, jit_word_t i1)
3319 if (can_sign_extend_int_p(i1)) {
3322 return (_jit->pc.w);
3324 reg = jit_get_reg(jit_class_gpr|jit_class_nospill);
3327 return (bxaddr(i0, r0, rn(reg)));
3331 _bxaddr_u(jit_state_t *_jit, jit_word_t i0, jit_int32_t r0, jit_int32_t r1)
3335 return (_jit->pc.w);
3339 _bxaddi_u(jit_state_t *_jit, jit_word_t i0, jit_int32_t r0, jit_word_t i1)
3342 if (can_sign_extend_int_p(i1)) {
3345 return (_jit->pc.w);
3347 reg = jit_get_reg(jit_class_gpr|jit_class_nospill);
3350 return (bxaddr_u(i0, r0, rn(reg)));
3354 _bosubr(jit_state_t *_jit, jit_word_t i0, jit_int32_t r0, jit_int32_t r1)
3358 return (_jit->pc.w);
3362 _bosubi(jit_state_t *_jit, jit_word_t i0, jit_int32_t r0, jit_word_t i1)
3365 if (can_sign_extend_int_p(i1)) {
3368 return (_jit->pc.w);
3370 reg = jit_get_reg(jit_class_gpr|jit_class_nospill);
3373 return (bosubr(i0, r0, rn(reg)));
3377 _bosubr_u(jit_state_t *_jit, jit_word_t i0, jit_int32_t r0, jit_int32_t r1)
3381 return (_jit->pc.w);
3385 _bosubi_u(jit_state_t *_jit, jit_word_t i0, jit_int32_t r0, jit_word_t i1)
3388 if (can_sign_extend_int_p(i1)) {
3391 return (_jit->pc.w);
3393 reg = jit_get_reg(jit_class_gpr|jit_class_nospill);
3396 return (bosubr_u(i0, r0, rn(reg)));
3400 _bxsubr(jit_state_t *_jit, jit_word_t i0, jit_int32_t r0, jit_int32_t r1)
3404 return (_jit->pc.w);
3408 _bxsubi(jit_state_t *_jit, jit_word_t i0, jit_int32_t r0, jit_word_t i1)
3411 if (can_sign_extend_int_p(i1)) {
3414 return (_jit->pc.w);
3416 reg = jit_get_reg(jit_class_gpr|jit_class_nospill);
3419 return (bxsubr(i0, r0, rn(reg)));
3423 _bxsubr_u(jit_state_t *_jit, jit_word_t i0, jit_int32_t r0, jit_int32_t r1)
3427 return (_jit->pc.w);
3431 _bxsubi_u(jit_state_t *_jit, jit_word_t i0, jit_int32_t r0, jit_word_t i1)
3434 if (can_sign_extend_int_p(i1)) {
3437 return (_jit->pc.w);
3439 reg = jit_get_reg(jit_class_gpr|jit_class_nospill);
3442 return (bxsubr_u(i0, r0, rn(reg)));
3446 _callr(jit_state_t *_jit, jit_int32_t r0)
3448 rex(0, 0, _NOREG, _NOREG, r0);
3450 mrm(0x03, 0x02, r7(r0));
3454 _calli(jit_state_t *_jit, jit_word_t i0)
3459 w = i0 - (_jit->pc.w + 5);
3460 if ((jit_int32_t)w == w) {
3463 w = i0 - (_jit->pc.w + 4);
3476 _calli_p(jit_state_t *_jit, jit_word_t i0)
3480 reg = jit_get_reg(jit_class_gpr);
3481 word = movi_p(rn(reg), i0);
3489 _jmpr(jit_state_t *_jit, jit_int32_t r0)
3491 rex(0, 0, _NOREG, _NOREG, r0);
3493 mrm(0x03, 0x04, r7(r0));
3497 _jmpi(jit_state_t *_jit, jit_word_t i0)
3502 w = i0 - (_jit->pc.w + 5);
3503 if ((jit_int32_t)w == w) {
3506 w = i0 - (_jit->pc.w + 4);
3519 _jmpi_p(jit_state_t *_jit, jit_word_t i0)
3523 reg = jit_get_reg(jit_class_gpr|jit_class_nospill);
3524 word = movi_p(rn(reg), i0);
3532 _jmpsi(jit_state_t *_jit, jit_uint8_t i0)
3539 _prolog(jit_state_t *_jit, jit_node_t *node)
3542 if (_jitc->function->define_frame || _jitc->function->assume_frame) {
3543 jit_int32_t frame = -_jitc->function->frame;
3544 assert(_jitc->function->self.aoff >= frame);
3545 if (_jitc->function->assume_frame)
3547 _jitc->function->self.aoff = frame;
3549 if (_jitc->function->allocar)
3550 _jitc->function->self.aoff &= -16;
3551 #if __X64 && (__CYGWIN__ || _WIN32)
3552 _jitc->function->stack = (((/* first 32 bytes must be allocated */
3553 (_jitc->function->self.alen > 32 ?
3554 _jitc->function->self.alen : 32) -
3555 /* align stack at 16 bytes */
3556 _jitc->function->self.aoff) + 15) & -16) +
3559 _jitc->function->stack = (((_jitc->function->self.alen -
3560 _jitc->function->self.aoff) + 15) & -16) +
3563 subi(_RSP_REGNO, _RSP_REGNO, stack_framesize - REAL_WORDSIZE);
3564 /* callee save registers */
3566 if (jit_regset_tstbit(&_jitc->function->regset, _RDI))
3567 stxi(12, _RSP_REGNO, _RDI_REGNO);
3568 if (jit_regset_tstbit(&_jitc->function->regset, _RSI))
3569 stxi( 8, _RSP_REGNO, _RSI_REGNO);
3570 if (jit_regset_tstbit(&_jitc->function->regset, _RBX))
3571 stxi( 4, _RSP_REGNO, _RBX_REGNO);
3573 # if __CYGWIN__ || _WIN32
3574 if (jit_regset_tstbit(&_jitc->function->regset, _XMM15))
3575 sse_stxi_d(136, _RSP_REGNO, _XMM15_REGNO);
3576 if (jit_regset_tstbit(&_jitc->function->regset, _XMM14))
3577 sse_stxi_d(128, _RSP_REGNO, _XMM14_REGNO);
3578 if (jit_regset_tstbit(&_jitc->function->regset, _XMM13))
3579 sse_stxi_d(120, _RSP_REGNO, _XMM13_REGNO);
3580 if (jit_regset_tstbit(&_jitc->function->regset, _XMM12))
3581 sse_stxi_d(112, _RSP_REGNO, _XMM12_REGNO);
3582 if (jit_regset_tstbit(&_jitc->function->regset, _XMM11))
3583 sse_stxi_d(104, _RSP_REGNO, _XMM11_REGNO);
3584 if (jit_regset_tstbit(&_jitc->function->regset, _XMM10))
3585 sse_stxi_d(96, _RSP_REGNO, _XMM10_REGNO);
3586 if (jit_regset_tstbit(&_jitc->function->regset, _XMM9))
3587 sse_stxi_d(88, _RSP_REGNO, _XMM9_REGNO);
3588 if (jit_regset_tstbit(&_jitc->function->regset, _XMM8))
3589 sse_stxi_d(80, _RSP_REGNO, _XMM8_REGNO);
3590 if (jit_regset_tstbit(&_jitc->function->regset, _XMM7))
3591 sse_stxi_d(72, _RSP_REGNO, _XMM7_REGNO);
3592 if (jit_regset_tstbit(&_jitc->function->regset, _XMM6))
3593 sse_stxi_d(64, _RSP_REGNO, _XMM6_REGNO);
3594 if (jit_regset_tstbit(&_jitc->function->regset, _R15))
3595 stxi(56, _RSP_REGNO, _R15_REGNO);
3596 if (jit_regset_tstbit(&_jitc->function->regset, _R14))
3597 stxi(48, _RSP_REGNO, _R14_REGNO);
3598 if (jit_regset_tstbit(&_jitc->function->regset, _R13))
3599 stxi(40, _RSP_REGNO, _R13_REGNO);
3600 if (jit_regset_tstbit(&_jitc->function->regset, _R12))
3601 stxi(32, _RSP_REGNO, _R12_REGNO);
3602 if (jit_regset_tstbit(&_jitc->function->regset, _RSI))
3603 stxi(24, _RSP_REGNO, _RSI_REGNO);
3604 if (jit_regset_tstbit(&_jitc->function->regset, _RDI))
3605 stxi(16, _RSP_REGNO, _RDI_REGNO);
3606 if (jit_regset_tstbit(&_jitc->function->regset, _RBX))
3607 stxi( 8, _RSP_REGNO, _RBX_REGNO);
3609 if (jit_regset_tstbit(&_jitc->function->regset, _RBX))
3610 stxi(40, _RSP_REGNO, _RBX_REGNO);
3611 if (jit_regset_tstbit(&_jitc->function->regset, _R12))
3612 stxi(32, _RSP_REGNO, _R12_REGNO);
3613 if (jit_regset_tstbit(&_jitc->function->regset, _R13))
3614 stxi(24, _RSP_REGNO, _R13_REGNO);
3615 if (jit_regset_tstbit(&_jitc->function->regset, _R14))
3616 stxi(16, _RSP_REGNO, _R14_REGNO);
3617 if (jit_regset_tstbit(&_jitc->function->regset, _R15))
3618 stxi( 8, _RSP_REGNO, _R15_REGNO);
3621 stxi(0, _RSP_REGNO, _RBP_REGNO);
3622 movr(_RBP_REGNO, _RSP_REGNO);
3625 subi(_RSP_REGNO, _RSP_REGNO, _jitc->function->stack);
3626 if (_jitc->function->allocar) {
3627 reg = jit_get_reg(jit_class_gpr);
3628 movi(rn(reg), _jitc->function->self.aoff);
3629 stxi_i(_jitc->function->aoffoff, _RBP_REGNO, rn(reg));
3633 #if __X64 && !(__CYGWIN__ || _WIN32)
3634 if (_jitc->function->self.call & jit_call_varargs) {
3635 jit_word_t nofp_code;
3637 /* Save gp registers in the save area, if any is a vararg */
3638 for (reg = first_gp_from_offset(_jitc->function->vagp);
3639 jit_arg_reg_p(reg); ++reg)
3640 stxi(_jitc->function->vaoff + first_gp_offset +
3641 reg * 8, _RBP_REGNO, rn(JIT_RA0 - reg));
3643 reg = first_fp_from_offset(_jitc->function->vafp);
3644 if (jit_arg_f_reg_p(reg)) {
3645 /* Skip over if no float registers were passed as argument */
3650 nofp_code = _jit->pc.w;
3652 /* Save fp registers in the save area, if any is a vararg */
3653 /* Note that the full 16 byte xmm is not saved, because
3654 * lightning only handles float and double, and, while
3655 * attempting to provide a va_list compatible pointer as
3656 * jit_va_start return, does not guarantee it (on all ports). */
3657 for (; jit_arg_f_reg_p(reg); ++reg)
3658 sse_stxi_d(_jitc->function->vaoff + first_fp_offset +
3659 reg * va_fp_increment, _RBP_REGNO, rn(_XMM0 - reg));
3661 patch_rel_char(nofp_code, _jit->pc.w);
3668 _epilog(jit_state_t *_jit, jit_node_t *node)
3670 if (_jitc->function->assume_frame)
3672 /* callee save registers */
3673 movr(_RSP_REGNO, _RBP_REGNO);
3675 if (jit_regset_tstbit(&_jitc->function->regset, _RDI))
3676 ldxi(_RDI_REGNO, _RSP_REGNO, 12);
3677 if (jit_regset_tstbit(&_jitc->function->regset, _RSI))
3678 ldxi(_RSI_REGNO, _RSP_REGNO, 8);
3679 if (jit_regset_tstbit(&_jitc->function->regset, _RBX))
3680 ldxi(_RBX_REGNO, _RSP_REGNO, 4);
3682 # if __CYGWIN__ || _WIN32
3683 if (jit_regset_tstbit(&_jitc->function->regset, _XMM15))
3684 sse_ldxi_d(_XMM15_REGNO, _RSP_REGNO, 136);
3685 if (jit_regset_tstbit(&_jitc->function->regset, _XMM14))
3686 sse_ldxi_d(_XMM14_REGNO, _RSP_REGNO, 128);
3687 if (jit_regset_tstbit(&_jitc->function->regset, _XMM13))
3688 sse_ldxi_d(_XMM13_REGNO, _RSP_REGNO, 120);
3689 if (jit_regset_tstbit(&_jitc->function->regset, _XMM12))
3690 sse_ldxi_d(_XMM12_REGNO, _RSP_REGNO, 112);
3691 if (jit_regset_tstbit(&_jitc->function->regset, _XMM11))
3692 sse_ldxi_d(_XMM11_REGNO, _RSP_REGNO, 104);
3693 if (jit_regset_tstbit(&_jitc->function->regset, _XMM10))
3694 sse_ldxi_d(_XMM10_REGNO, _RSP_REGNO, 96);
3695 if (jit_regset_tstbit(&_jitc->function->regset, _XMM9))
3696 sse_ldxi_d(_XMM9_REGNO, _RSP_REGNO, 88);
3697 if (jit_regset_tstbit(&_jitc->function->regset, _XMM8))
3698 sse_ldxi_d(_XMM8_REGNO, _RSP_REGNO, 80);
3699 if (jit_regset_tstbit(&_jitc->function->regset, _XMM7))
3700 sse_ldxi_d(_XMM7_REGNO, _RSP_REGNO, 72);
3701 if (jit_regset_tstbit(&_jitc->function->regset, _XMM6))
3702 sse_ldxi_d(_XMM6_REGNO, _RSP_REGNO, 64);
3703 if (jit_regset_tstbit(&_jitc->function->regset, _R15))
3704 ldxi(_R15_REGNO, _RSP_REGNO, 56);
3705 if (jit_regset_tstbit(&_jitc->function->regset, _R14))
3706 ldxi(_R14_REGNO, _RSP_REGNO, 48);
3707 if (jit_regset_tstbit(&_jitc->function->regset, _R13))
3708 ldxi(_R13_REGNO, _RSP_REGNO, 40);
3709 if (jit_regset_tstbit(&_jitc->function->regset, _R12))
3710 ldxi(_R12_REGNO, _RSP_REGNO, 32);
3711 if (jit_regset_tstbit(&_jitc->function->regset, _RSI))
3712 ldxi(_RSI_REGNO, _RSP_REGNO, 24);
3713 if (jit_regset_tstbit(&_jitc->function->regset, _RDI))
3714 ldxi(_RDI_REGNO, _RSP_REGNO, 16);
3715 if (jit_regset_tstbit(&_jitc->function->regset, _RBX))
3716 ldxi(_RBX_REGNO, _RSP_REGNO, 8);
3718 if (jit_regset_tstbit(&_jitc->function->regset, _RBX))
3719 ldxi(_RBX_REGNO, _RSP_REGNO, 40);
3720 if (jit_regset_tstbit(&_jitc->function->regset, _R12))
3721 ldxi(_R12_REGNO, _RSP_REGNO, 32);
3722 if (jit_regset_tstbit(&_jitc->function->regset, _R13))
3723 ldxi(_R13_REGNO, _RSP_REGNO, 24);
3724 if (jit_regset_tstbit(&_jitc->function->regset, _R14))
3725 ldxi(_R14_REGNO, _RSP_REGNO, 16);
3726 if (jit_regset_tstbit(&_jitc->function->regset, _R15))
3727 ldxi(_R15_REGNO, _RSP_REGNO, 8);
3730 ldxi(_RBP_REGNO, _RSP_REGNO, 0);
3731 addi(_RSP_REGNO, _RSP_REGNO, stack_framesize - REAL_WORDSIZE);
3737 _vastart(jit_state_t *_jit, jit_int32_t r0)
3739 #if __X32 || __CYGWIN__ || _WIN32
3740 assert(_jitc->function->self.call & jit_call_varargs);
3741 addi(r0, _RBP_REGNO, _jitc->function->self.size);
3745 assert(_jitc->function->self.call & jit_call_varargs);
3747 /* Return jit_va_list_t in the register argument */
3748 addi(r0, _RBP_REGNO, _jitc->function->vaoff);
3749 reg = jit_get_reg(jit_class_gpr);
3751 /* Initialize gp offset in the save area. */
3752 movi(rn(reg), _jitc->function->vagp);
3753 stxi_i(offsetof(jit_va_list_t, gpoff), r0, rn(reg));
3755 /* Initialize fp offset in the save area. */
3756 movi(rn(reg), _jitc->function->vafp);
3757 stxi_i(offsetof(jit_va_list_t, fpoff), r0, rn(reg));
3759 /* Initialize overflow pointer to the first stack argument. */
3760 addi(rn(reg), _RBP_REGNO, _jitc->function->self.size);
3761 stxi(offsetof(jit_va_list_t, over), r0, rn(reg));
3763 /* Initialize register save area pointer. */
3764 addi(rn(reg), r0, first_gp_offset);
3765 stxi(offsetof(jit_va_list_t, save), r0, rn(reg));
3772 _vaarg(jit_state_t *_jit, jit_int32_t r0, jit_int32_t r1)
3774 #if __X32 || __CYGWIN__ || _WIN32
3775 assert(_jitc->function->self.call & jit_call_varargs);
3777 addi(r1, r1, va_gp_increment);
3784 assert(_jitc->function->self.call & jit_call_varargs);
3786 rg0 = jit_get_reg(jit_class_gpr);
3787 rg1 = jit_get_reg(jit_class_gpr);
3789 /* Load the gp offset in save area in the first temporary. */
3790 ldxi_i(rn(rg0), r1, offsetof(jit_va_list_t, gpoff));
3792 /* Jump over if there are no remaining arguments in the save area. */
3793 icmpi(rn(rg0), va_gp_max_offset);
3795 ge_code = _jit->pc.w;
3797 /* Load the save area pointer in the second temporary. */
3798 ldxi(rn(rg1), r1, offsetof(jit_va_list_t, save));
3800 /* Load the vararg argument in the first argument. */
3801 ldxr(r0, rn(rg1), rn(rg0));
3803 /* Update the gp offset. */
3804 addi(rn(rg0), rn(rg0), 8);
3805 stxi_i(offsetof(jit_va_list_t, gpoff), r1, rn(rg0));
3807 /* Will only need one temporary register below. */
3810 /* Jump over overflow code. */
3812 lt_code = _jit->pc.w;
3814 /* Where to land if argument is in overflow area. */
3815 patch_rel_char(ge_code, _jit->pc.w);
3817 /* Load overflow pointer. */
3818 ldxi(rn(rg0), r1, offsetof(jit_va_list_t, over));
3820 /* Load argument. */
3823 /* Update overflow pointer. */
3824 addi(rn(rg0), rn(rg0), va_gp_increment);
3825 stxi(offsetof(jit_va_list_t, over), r1, rn(rg0));
3827 /* Where to land if argument is in save area. */
3828 patch_rel_char(lt_code, _jit->pc.w);
3834 /* The x87 boolean argument tells if will put the result in a x87
3835 * register if non false, in a sse register otherwise. */
3837 _vaarg_d(jit_state_t *_jit, jit_int32_t r0, jit_int32_t r1, jit_bool_t x87)
3839 #if __X32 || __CYGWIN__ || _WIN32
3840 assert(_jitc->function->self.call & jit_call_varargs);
3852 assert(_jitc->function->self.call & jit_call_varargs);
3854 rg0 = jit_get_reg(jit_class_gpr);
3855 rg1 = jit_get_reg(jit_class_gpr);
3857 /* Load the fp offset in save area in the first temporary. */
3858 ldxi_i(rn(rg0), r1, offsetof(jit_va_list_t, fpoff));
3860 /* Jump over if there are no remaining arguments in the save area. */
3861 icmpi(rn(rg0), va_fp_max_offset);
3863 ge_code = _jit->pc.w;
3865 /* Load the save area pointer in the second temporary. */
3866 ldxi(rn(rg1), r1, offsetof(jit_va_list_t, save));
3868 /* Load the vararg argument in the first argument. */
3870 x87_ldxr_d(r0, rn(rg1), rn(rg0));
3872 sse_ldxr_d(r0, rn(rg1), rn(rg0));
3874 /* Update the fp offset. */
3875 addi(rn(rg0), rn(rg0), va_fp_increment);
3876 stxi_i(offsetof(jit_va_list_t, fpoff), r1, rn(rg0));
3878 /* Will only need one temporary register below. */
3881 /* Jump over overflow code. */
3883 lt_code = _jit->pc.w;
3885 /* Where to land if argument is in overflow area. */
3886 patch_rel_char(ge_code, _jit->pc.w);
3888 /* Load overflow pointer. */
3889 ldxi(rn(rg0), r1, offsetof(jit_va_list_t, over));
3891 /* Load argument. */
3893 x87_ldr_d(r0, rn(rg0));
3895 sse_ldr_d(r0, rn(rg0));
3897 /* Update overflow pointer. */
3898 addi(rn(rg0), rn(rg0), 8);
3899 stxi(offsetof(jit_va_list_t, over), r1, rn(rg0));
3901 /* Where to land if argument is in save area. */
3902 patch_rel_char(lt_code, _jit->pc.w);
3909 _patch_at(jit_state_t *_jit, jit_node_t *node,
3910 jit_word_t instr, jit_word_t label)
3912 switch (node->code) {
3914 case jit_code_calli:
3918 patch_abs(instr, label);
3921 patch_rel(instr, label);