2 * Copyright (C) 2012-2019 Free Software Foundation, Inc.
4 * This file is part of GNU lightning.
6 * GNU lightning is free software; you can redistribute it and/or modify it
7 * under the terms of the GNU Lesser General Public License as published
8 * by the Free Software Foundation; either version 3, or (at your option)
11 * GNU lightning is distributed in the hope that it will be useful, but
12 * WITHOUT ANY WARRANTY; without even the implied warranty of MERCHANTABILITY
13 * or FITNESS FOR A PARTICULAR PURPOSE. See the GNU Lesser General Public
14 * License for more details.
17 * Paulo Cesar Pereira de Andrade
20 /* avoid using it due to partial stalls */
24 # if __X32 || __X64_32
26 # define ldi(u, v) ldi_i(u, v)
27 # define ldr(u, v) ldr_i(u, v)
28 # define ldxr(u, v, w) ldxr_i(u, v, w)
29 # define ldxi(u, v, w) ldxi_i(u, v, w)
30 # define sti(u, v) sti_i(u, v)
31 # define stxi(u, v, w) stxi_i(u, v, w)
32 # define can_sign_extend_int_p(im) 1
33 # define can_zero_extend_int_p(im) 1
34 # define fits_uint32_p(im) 1
37 # define ldi(u, v) ldi_l(u, v)
38 # define ldr(u, v) ldr_l(u, v)
39 # define ldxr(u, v, w) ldxr_l(u, v, w)
40 # define ldxi(u, v, w) ldxi_l(u, v, w)
41 # define sti(u, v) sti_l(u, v)
42 # define stxi(u, v, w) stxi_l(u, v, w)
43 # define can_sign_extend_int_p(im) \
44 (((im) >= 0 && (long long)(im) <= 0x7fffffffLL) || \
45 ((im) < 0 && (long long)(im) > -0x80000000LL))
46 # define can_zero_extend_int_p(im) \
47 ((im) >= 0 && (im) < 0x80000000LL)
48 # define fits_uint32_p(im) (((im) & 0xffffffff00000000LL) == 0)
50 # if __X32 || __CYGWIN__ || __X64_32 || _WIN32
52 ((rn) >= _RAX_REGNO && (rn) <= _RBX_REGNO)
66 # define _R10_REGNO 10
67 # define _R11_REGNO 11
68 # define _R12_REGNO 12
69 # define _R13_REGNO 13
70 # define _R14_REGNO 14
71 # define _R15_REGNO 15
72 # define r7(reg) ((reg) & 7)
73 # define r8(reg) ((reg) & 15)
79 # define X86_OR 1 << 3
80 # define X86_ADC 2 << 3
81 # define X86_SBB 3 << 3
82 # define X86_AND 4 << 3
83 # define X86_SUB 5 << 3
84 # define X86_XOR 6 << 3
85 # define X86_CMP 7 << 3
100 # define X86_CC_NO 0x1
101 # define X86_CC_NAE 0x2
102 # define X86_CC_B 0x2
103 # define X86_CC_C 0x2
104 # define X86_CC_AE 0x3
105 # define X86_CC_NB 0x3
106 # define X86_CC_NC 0x3
107 # define X86_CC_E 0x4
108 # define X86_CC_Z 0x4
109 # define X86_CC_NE 0x5
110 # define X86_CC_NZ 0x5
111 # define X86_CC_BE 0x6
112 # define X86_CC_NA 0x6
113 # define X86_CC_A 0x7
114 # define X86_CC_NBE 0x7
115 # define X86_CC_S 0x8
116 # define X86_CC_NS 0x9
117 # define X86_CC_P 0xa
118 # define X86_CC_PE 0xa
119 # define X86_CC_NP 0xb
120 # define X86_CC_PO 0xb
121 # define X86_CC_L 0xc
122 # define X86_CC_NGE 0xc
123 # define X86_CC_GE 0xd
124 # define X86_CC_NL 0xd
125 # define X86_CC_LE 0xe
126 # define X86_CC_NG 0xe
127 # define X86_CC_G 0xf
128 # define X86_CC_NLE 0xf
129 # define mrm(md, r, m) *_jit->pc.uc++ = (md<<6) | (r<<3) | m
130 # define sib(sc, i, b) *_jit->pc.uc++ = (sc<<6) | (i<<3) | b
131 # define ic(c) *_jit->pc.uc++ = c
132 # define is(s) *_jit->pc.us++ = s
133 # define ii(i) *_jit->pc.ui++ = i
134 # if __X64 && !__X64_32
135 # define il(l) *_jit->pc.ul++ = l
139 # define patch_abs(instr, label) \
140 *(jit_word_t *)(instr - sizeof(jit_word_t)) = label
141 # define patch_rel(instr, label) \
142 *(jit_int32_t *)(instr - 4) = label - instr
143 # define patch_rel_char(instr, label) \
144 *(jit_int8_t *)(instr - 1) = label - instr
145 # define rex(l, w, r, x, b) _rex(_jit, l, w, r, x, b)
147 _rex(jit_state_t*,jit_int32_t,jit_int32_t,jit_int32_t,jit_int32_t,jit_int32_t);
148 # define rx(rd, md, rb, ri, ms) _rx(_jit, rd, md, rb, ri, ms)
150 _rx(jit_state_t*,jit_int32_t,jit_int32_t,jit_int32_t,jit_int32_t,jit_int32_t);
151 # define nop(n) _nop(_jit, n)
152 static void _nop(jit_state_t*, jit_int32_t);
153 # define emms() is(0x770f)
154 # define lea(md, rb, ri, ms, rd) _lea(_jit, md, rb, ri, ms, rd)
156 _lea(jit_state_t*,jit_int32_t,jit_int32_t,jit_int32_t,jit_int32_t,jit_int32_t);
157 # define pushr(r0) _pushr(_jit, r0)
158 static void _pushr(jit_state_t*, jit_int32_t) maybe_unused;
159 # define popr(r0) _popr(_jit, r0)
160 static void _popr(jit_state_t*, jit_int32_t) maybe_unused;
161 # define xchgr(r0, r1) _xchgr(_jit, r0, r1)
162 static void _xchgr(jit_state_t*, jit_int32_t, jit_int32_t);
163 # define testr(r0, r1) _testr(_jit, r0, r1)
164 static void _testr(jit_state_t*, jit_int32_t, jit_int32_t);
165 # define testi(r0, i0) _testi(_jit, r0, i0)
166 static void _testi(jit_state_t*, jit_int32_t, jit_word_t);
167 # define cc(code, r0) _cc(_jit, code, r0)
168 static void _cc(jit_state_t*, jit_int32_t, jit_int32_t);
169 # define icmpr(r0, r1) alur(X86_CMP, r0, r1)
170 # define alur(code, r0, r1) _alur(_jit, code, r0, r1)
171 static void _alur(jit_state_t*, jit_int32_t, jit_int32_t, jit_int32_t);
172 # define icmpi(r0, i0) alui(X86_CMP, r0, i0)
173 # define alui(code, r0, i0) _alui(_jit, code, r0, i0)
174 static void _alui(jit_state_t*, jit_int32_t, jit_int32_t, jit_word_t);
175 # define iaddr(r0, r1) alur(X86_ADD, r0, r1)
176 # define save(r0) _save(_jit, r0)
177 static void _save(jit_state_t*, jit_int32_t);
178 # define load(r0) _load(_jit, r0)
179 static void _load(jit_state_t*, jit_int32_t);
180 # define addr(r0, r1, r2) _addr(_jit, r0, r1, r2)
181 static void _addr(jit_state_t*, jit_int32_t, jit_int32_t, jit_int32_t);
182 # define iaddi(r0, i0) alui(X86_ADD, r0, i0)
183 # define addi(r0, r1, i0) _addi(_jit, r0, r1, i0)
184 static void _addi(jit_state_t*, jit_int32_t, jit_int32_t, jit_word_t);
185 #define addcr(r0, r1, r2) _addcr(_jit, r0, r1, r2)
186 static void _addcr(jit_state_t*, jit_int32_t, jit_int32_t, jit_int32_t);
187 #define addci(r0, r1, i0) _addci(_jit, r0, r1, i0)
188 static void _addci(jit_state_t*, jit_int32_t, jit_int32_t, jit_word_t);
189 # define iaddxr(r0, r1) alur(X86_ADC, r0, r1)
190 # define addxr(r0, r1, r2) _addxr(_jit, r0, r1, r2)
191 static void _addxr(jit_state_t*, jit_int32_t, jit_int32_t, jit_int32_t);
192 # define iaddxi(r0, i0) alui(X86_ADC, r0, i0)
193 # define addxi(r0, r1, i0) _addxi(_jit, r0, r1, i0)
194 static void _addxi(jit_state_t*, jit_int32_t, jit_int32_t, jit_word_t);
195 # define isubr(r0, r1) alur(X86_SUB, r0, r1)
196 # define subr(r0, r1, r2) _subr(_jit, r0, r1, r2)
197 static void _subr(jit_state_t*, jit_int32_t, jit_int32_t, jit_int32_t);
198 # define isubi(r0, i0) alui(X86_SUB, r0, i0)
199 # define subi(r0, r1, i0) _subi(_jit, r0, r1, i0)
200 static void _subi(jit_state_t*, jit_int32_t, jit_int32_t, jit_word_t);
201 # define subcr(r0, r1, r2) _subcr(_jit, r0, r1, r2)
202 static void _subcr(jit_state_t*,jit_int32_t,jit_int32_t,jit_int32_t);
203 # define subci(r0, r1, i0) _subci(_jit, r0, r1, i0)
204 static void _subci(jit_state_t*,jit_int32_t,jit_int32_t,jit_word_t);
205 # define isubxr(r0, r1) alur(X86_SBB, r0, r1)
206 # define subxr(r0, r1, r2) _subxr(_jit, r0, r1, r2)
207 static void _subxr(jit_state_t*,jit_int32_t,jit_int32_t,jit_int32_t);
208 # define isubxi(r0, i0) alui(X86_SBB, r0, i0)
209 # define subxi(r0, r1, i0) _subxi(_jit, r0, r1, i0)
210 static void _subxi(jit_state_t*,jit_int32_t,jit_int32_t,jit_word_t);
211 # define rsbi(r0, r1, i0) _rsbi(_jit, r0, r1, i0)
212 static void _rsbi(jit_state_t*,jit_int32_t,jit_int32_t,jit_word_t);
213 # define imulr(r0, r1) _imulr(_jit, r0, r1)
214 static void _imulr(jit_state_t*, jit_int32_t, jit_int32_t);
215 # define imuli(r0, r1, i0) _imuli(_jit, r0, r1, i0)
216 static void _imuli(jit_state_t*, jit_int32_t, jit_int32_t, jit_word_t);
217 # define mulr(r0, r1, r2) _mulr(_jit, r0, r1, r2)
218 static void _mulr(jit_state_t*, jit_int32_t, jit_int32_t, jit_int32_t);
219 # define muli(r0, r1, i0) _muli(_jit, r0, r1, i0)
220 static void _muli(jit_state_t*, jit_int32_t, jit_int32_t, jit_word_t);
221 # define umulr(r0) unr(X86_IMUL, r0)
222 # define umulr_u(r0) unr(X86_MUL, r0)
223 # define qmulr(r0, r1, r2, r3) _iqmulr(_jit, r0, r1, r2, r3, 1)
224 # define qmulr_u(r0, r1, r2, r3) _iqmulr(_jit, r0, r1, r2, r3, 0)
225 # define iqmulr(r0, r1, r2, r3, sign) _iqmulr(_jit, r0, r1, r2, r3, sign)
226 static void _iqmulr(jit_state_t*, jit_int32_t, jit_int32_t,
227 jit_int32_t,jit_int32_t, jit_bool_t);
228 # define qmuli(r0, r1, r2, i0) _iqmuli(_jit, r0, r1, r2, i0, 1)
229 # define qmuli_u(r0, r1, r2, i0) _iqmuli(_jit, r0, r1, r2, i0, 0)
230 # define iqmuli(r0, r1, r2, i0, sign) _iqmuli(_jit, r0, r1, r2, i0, sign)
231 static void _iqmuli(jit_state_t*, jit_int32_t, jit_int32_t,
232 jit_int32_t,jit_word_t, jit_bool_t);
233 # define sign_extend_rdx_rax() _sign_extend_rdx_rax(_jit)
234 static void _sign_extend_rdx_rax(jit_state_t*);
235 # define idivr(r0) unr(X86_IDIV, r0)
236 # define idivr_u(r0) unr(X86_DIV, r0)
237 # define divremr(r0, r1, r2, i0, i1) _divremr(_jit, r0, r1, r2, i0, i1)
239 _divremr(jit_state_t*,jit_int32_t,jit_int32_t,jit_int32_t,
240 jit_bool_t,jit_bool_t);
241 # define divremi(r0, r1, i0, i1, i2) _divremi(_jit, r0, r1, i0, i1, i2)
243 _divremi(jit_state_t*,jit_int32_t,jit_int32_t,jit_word_t,jit_bool_t,jit_bool_t);
244 # define divr(r0, r1, r2) divremr(r0, r1, r2, 1, 1)
245 # define divi(r0, r1, i0) divremi(r0, r1, i0, 1, 1)
246 # define divr_u(r0, r1, r2) divremr(r0, r1, r2, 0, 1)
247 # define divi_u(r0, r1, i0) divremi(r0, r1, i0, 0, 1)
248 # define qdivr(r0, r1, r2, r3) _iqdivr(_jit, r0, r1, r2, r3, 1)
249 # define qdivr_u(r0, r1, r2, r3) _iqdivr(_jit, r0, r1, r2, r3, 0)
250 # define iqdivr(r0, r1, r2, r3, sign) _iqdivr(_jit, r0, r1, r2, r3, sign)
251 static void _iqdivr(jit_state_t*, jit_int32_t, jit_int32_t,
252 jit_int32_t,jit_int32_t, jit_bool_t);
253 # define qdivi(r0, r1, r2, i0) _iqdivi(_jit, r0, r1, r2, i0, 1)
254 # define qdivi_u(r0, r1, r2, i0) _iqdivi(_jit, r0, r1, r2, i0, 0)
255 # define iqdivi(r0, r1, r2, i0, sign) _iqdivi(_jit, r0, r1, r2, i0, sign)
256 static void _iqdivi(jit_state_t*, jit_int32_t, jit_int32_t,
257 jit_int32_t,jit_word_t, jit_bool_t);
258 # define remr(r0, r1, r2) divremr(r0, r1, r2, 1, 0)
259 # define remi(r0, r1, i0) divremi(r0, r1, i0, 1, 0)
260 # define remr_u(r0, r1, r2) divremr(r0, r1, r2, 0, 0)
261 # define remi_u(r0, r1, i0) divremi(r0, r1, i0, 0, 0)
262 # define iandr(r0, r1) alur(X86_AND, r0, r1)
263 # define andr(r0, r1, r2) _andr(_jit, r0, r1, r2)
264 static void _andr(jit_state_t*,jit_int32_t,jit_int32_t,jit_int32_t);
265 # define iandi(r0, i0) alui(X86_AND, r0, i0)
266 # define andi(r0, r1, i0) _andi(_jit, r0, r1, i0)
267 static void _andi(jit_state_t*, jit_int32_t,jit_int32_t,jit_word_t);
268 # define iorr(r0, r1) alur(X86_OR, r0, r1)
269 # define orr(r0, r1, r2) _orr(_jit, r0, r1, r2)
270 static void _orr(jit_state_t*, jit_int32_t,jit_int32_t,jit_int32_t);
271 # define iori(r0, i0) alui(X86_OR, r0, i0)
272 # define ori(r0, r1, i0) _ori(_jit, r0, r1, i0)
273 static void _ori(jit_state_t*, jit_int32_t,jit_int32_t,jit_word_t);
274 # define ixorr(r0, r1) alur(X86_XOR, r0, r1)
275 # define xorr(r0, r1, r2) _xorr(_jit, r0, r1, r2)
276 static void _xorr(jit_state_t*, jit_int32_t,jit_int32_t,jit_int32_t);
277 # define ixori(r0, i0) alui(X86_XOR, r0, i0)
278 # define xori(r0, r1, i0) _xori(_jit, r0, r1, i0)
279 static void _xori(jit_state_t*, jit_int32_t,jit_int32_t,jit_word_t);
280 # define irotshr(code, r0) _irotshr(_jit, code, r0)
281 static void _irotshr(jit_state_t*, jit_int32_t, jit_int32_t);
282 # define rotshr(code, r0, r1, r2) _rotshr(_jit, code, r0, r1, r2)
284 _rotshr(jit_state_t*,jit_int32_t,jit_int32_t,jit_int32_t,jit_int32_t);
285 # define irotshi(code, r0, i0) _irotshi(_jit, code, r0, i0)
286 static void _irotshi(jit_state_t*, jit_int32_t, jit_int32_t, jit_word_t);
287 # define rotshi(code, r0, r1, i0) _rotshi(_jit, code, r0, r1, i0)
289 _rotshi(jit_state_t*,jit_int32_t,jit_int32_t,jit_int32_t,jit_word_t);
290 # define lshr(r0, r1, r2) rotshr(X86_SHL, r0, r1, r2)
291 # define lshi(r0, r1, i0) _lshi(_jit, r0, r1, i0)
292 static void _lshi(jit_state_t*, jit_int32_t, jit_int32_t, jit_word_t);
293 # define rshr(r0, r1, r2) rotshr(X86_SAR, r0, r1, r2)
294 # define rshi(r0, r1, i0) rotshi(X86_SAR, r0, r1, i0)
295 # define rshr_u(r0, r1, r2) rotshr(X86_SHR, r0, r1, r2)
296 # define rshi_u(r0, r1, i0) rotshi(X86_SHR, r0, r1, i0)
297 # define unr(code, r0) _unr(_jit, code, r0)
298 static void _unr(jit_state_t*, jit_int32_t, jit_int32_t);
299 # define inegr(r0) unr(X86_NEG, r0)
300 # define negr(r0, r1) _negr(_jit, r0, r1)
301 static void _negr(jit_state_t*, jit_int32_t, jit_int32_t);
302 # define icomr(r0) unr(X86_NOT, r0)
303 # define comr(r0, r1) _comr(_jit, r0, r1)
304 static void _comr(jit_state_t*, jit_int32_t, jit_int32_t);
306 # define incr(r0, r1) _incr(_jit, r0, r1)
307 static void _incr(jit_state_t*, jit_int32_t, jit_int32_t);
308 # define decr(r0, r1) _decr(_jit, r0, r1)
309 static void _decr(jit_state_t*, jit_int32_t, jit_int32_t);
311 # define cr(code, r0, r1, r2) _cr(_jit, code, r0, r1, r2)
313 _cr(jit_state_t*, jit_int32_t, jit_int32_t, jit_int32_t, jit_int32_t);
314 # define ci(code, r0, r1, i0) _ci(_jit, code, r0, r1, i0)
316 _ci(jit_state_t *_jit, jit_int32_t, jit_int32_t, jit_int32_t, jit_word_t);
317 # define ci0(code, r0, r1) _ci0(_jit, code, r0, r1)
318 static void _ci0(jit_state_t*, jit_int32_t, jit_int32_t, jit_int32_t);
319 # define ltr(r0, r1, r2) _ltr(_jit, r0, r1, r2)
320 static void _ltr(jit_state_t*, jit_int32_t, jit_int32_t, jit_int32_t);
321 # define lti(r0, r1, i0) _lti(_jit, r0, r1, i0)
322 static void _lti(jit_state_t*, jit_int32_t, jit_int32_t, jit_word_t);
323 # define ltr_u(r0, r1, r2) _ltr_u(_jit, r0, r1, r2)
324 static void _ltr_u(jit_state_t*, jit_int32_t, jit_int32_t, jit_int32_t);
325 # define lti_u(r0, r1, i0) ci(X86_CC_B, r0, r1, i0)
326 # define ler(r0, r1, r2) _ler(_jit, r0, r1, r2)
327 static void _ler(jit_state_t*, jit_int32_t, jit_int32_t, jit_int32_t);
328 # define lei(r0, r1, i0) ci(X86_CC_LE, r0, r1, i0)
329 # define ler_u(r0, r1, r2) _ler_u(_jit, r0, r1, r2)
330 static void _ler_u(jit_state_t*, jit_int32_t, jit_int32_t, jit_int32_t);
331 # define lei_u(r0, r1, i0) _lei_u(_jit, r0, r1, i0)
332 static void _lei_u(jit_state_t*, jit_int32_t, jit_int32_t, jit_word_t);
333 # define eqr(r0, r1, r2) _eqr(_jit, r0, r1, r2)
334 static void _eqr(jit_state_t*, jit_int32_t, jit_int32_t, jit_int32_t);
335 # define eqi(r0, r1, i0) _eqi(_jit, r0, r1, i0)
336 static void _eqi(jit_state_t*, jit_int32_t, jit_int32_t, jit_word_t);
337 # define ger(r0, r1, r2) _ger(_jit, r0, r1, r2)
338 static void _ger(jit_state_t*, jit_int32_t, jit_int32_t, jit_int32_t);
339 # define gei(r0, r1, i0) _gei(_jit, r0, r1, i0)
340 static void _gei(jit_state_t*, jit_int32_t, jit_int32_t, jit_word_t);
341 # define ger_u(r0, r1, r2) _ger_u(_jit, r0, r1, r2)
342 static void _ger_u(jit_state_t*, jit_int32_t, jit_int32_t, jit_int32_t);
343 # define gei_u(r0, r1, i0) _gei_u(_jit, r0, r1, i0)
344 static void _gei_u(jit_state_t*, jit_int32_t, jit_int32_t, jit_word_t);
345 # define gtr(r0, r1, r2) _gtr(_jit, r0, r1, r2)
346 static void _gtr(jit_state_t*, jit_int32_t, jit_int32_t, jit_int32_t);
347 # define gti(r0, r1, i0) _ci(_jit, X86_CC_G, r0, r1, i0)
348 # define gtr_u(r0, r1, r2) _gtr_u(_jit, r0, r1, r2)
349 static void _gtr_u(jit_state_t*, jit_int32_t, jit_int32_t, jit_int32_t);
350 # define gti_u(r0, r1, i0) _gti_u(_jit, r0, r1, i0)
351 static void _gti_u(jit_state_t*, jit_int32_t, jit_int32_t, jit_word_t);
352 # define ner(r0, r1, r2) _ner(_jit, r0, r1, r2)
353 static void _ner(jit_state_t*, jit_int32_t, jit_int32_t, jit_int32_t);
354 # define nei(r0, r1, i0) _nei(_jit, r0, r1, i0)
355 static void _nei(jit_state_t*, jit_int32_t, jit_int32_t, jit_word_t);
356 # define movr(r0, r1) _movr(_jit, r0, r1)
357 static void _movr(jit_state_t*, jit_int32_t, jit_int32_t);
358 # define imovi(r0, i0) _imovi(_jit, r0, i0)
359 static void _imovi(jit_state_t*, jit_int32_t, jit_word_t);
360 # define movi(r0, i0) _movi(_jit, r0, i0)
361 static void _movi(jit_state_t*, jit_int32_t, jit_word_t);
362 # define movi_p(r0, i0) _movi_p(_jit, r0, i0)
363 static jit_word_t _movi_p(jit_state_t*, jit_int32_t, jit_word_t);
364 # define movcr(r0, r1) _movcr(_jit, r0, r1)
365 static void _movcr(jit_state_t*,jit_int32_t,jit_int32_t);
366 # define movcr_u(r0, r1) _movcr_u(_jit, r0, r1)
367 static void _movcr_u(jit_state_t*,jit_int32_t,jit_int32_t);
368 # define movsr(r0, r1) _movsr(_jit, r0, r1)
369 static void _movsr(jit_state_t*,jit_int32_t,jit_int32_t);
370 # define movsr_u(r0, r1) _movsr_u(_jit, r0, r1)
371 static void _movsr_u(jit_state_t*,jit_int32_t,jit_int32_t);
372 # if __X64 && !__X64_32
373 # define movir(r0, r1) _movir(_jit, r0, r1)
374 static void _movir(jit_state_t*,jit_int32_t,jit_int32_t);
375 # define movir_u(r0, r1) _movir_u(_jit, r0, r1)
376 static void _movir_u(jit_state_t*,jit_int32_t,jit_int32_t);
378 # define htonr_us(r0, r1) _htonr_us(_jit, r0, r1)
379 static void _htonr_us(jit_state_t*,jit_int32_t,jit_int32_t);
380 # define htonr_ui(r0, r1) _htonr_ui(_jit, r0, r1)
381 static void _htonr_ui(jit_state_t*,jit_int32_t,jit_int32_t);
382 # if __X64 && !__X64_32
383 #define htonr_ul(r0, r1) _htonr_ul(_jit, r0, r1)
384 static void _htonr_ul(jit_state_t*,jit_int32_t,jit_int32_t);
386 # define extr_c(r0, r1) _extr_c(_jit, r0, r1)
387 static void _extr_c(jit_state_t*,jit_int32_t,jit_int32_t);
388 # define extr_uc(r0, r1) _extr_uc(_jit, r0, r1)
389 static void _extr_uc(jit_state_t*,jit_int32_t,jit_int32_t);
390 # define extr_s(r0, r1) movsr(r0, r1)
391 # define extr_us(r0, r1) movsr_u(r0, r1)
392 # if __X64 && !__X64_32
393 # define extr_i(r0, r1) movir(r0, r1)
394 # define extr_ui(r0, r1) movir_u(r0, r1)
396 # define ldr_c(r0, r1) _ldr_c(_jit, r0, r1)
397 static void _ldr_c(jit_state_t*, jit_int32_t, jit_int32_t);
398 # define ldi_c(r0, i0) _ldi_c(_jit, r0, i0)
399 static void _ldi_c(jit_state_t*, jit_int32_t, jit_word_t);
400 # define ldr_uc(r0, r1) _ldr_uc(_jit, r0, r1)
401 static void _ldr_uc(jit_state_t*, jit_int32_t, jit_int32_t);
402 # define ldi_uc(r0, i0) _ldi_uc(_jit, r0, i0)
403 static void _ldi_uc(jit_state_t*, jit_int32_t, jit_word_t);
404 # define ldr_s(r0, r1) _ldr_s(_jit, r0, r1)
405 static void _ldr_s(jit_state_t*, jit_int32_t, jit_int32_t);
406 # define ldi_s(r0, i0) _ldi_s(_jit, r0, i0)
407 static void _ldi_s(jit_state_t*, jit_int32_t, jit_word_t);
408 # define ldr_us(r0, r1) _ldr_us(_jit, r0, r1)
409 static void _ldr_us(jit_state_t*, jit_int32_t, jit_int32_t);
410 # define ldi_us(r0, i0) _ldi_us(_jit, r0, i0)
411 static void _ldi_us(jit_state_t*, jit_int32_t, jit_word_t);
412 # if __X32 || !__X64_32
413 # define ldr_i(r0, r1) _ldr_i(_jit, r0, r1)
414 static void _ldr_i(jit_state_t*, jit_int32_t, jit_int32_t);
415 # define ldi_i(r0, i0) _ldi_i(_jit, r0, i0)
416 static void _ldi_i(jit_state_t*, jit_int32_t, jit_word_t);
420 # define ldr_i(r0, r1) _ldr_ui(_jit, r0, r1)
421 # define ldi_i(r0, i0) _ldi_ui(_jit, r0, i0)
423 # define ldr_ui(r0, r1) _ldr_ui(_jit, r0, r1)
424 # define ldi_ui(r0, i0) _ldi_ui(_jit, r0, i0)
426 static void _ldr_ui(jit_state_t*, jit_int32_t, jit_int32_t);
427 static void _ldi_ui(jit_state_t*, jit_int32_t, jit_word_t);
429 # define ldr_l(r0, r1) _ldr_l(_jit, r0, r1)
430 static void _ldr_l(jit_state_t*, jit_int32_t, jit_int32_t);
431 # define ldi_l(r0, i0) _ldi_l(_jit, r0, i0)
432 static void _ldi_l(jit_state_t*, jit_int32_t, jit_word_t);
435 # define ldxr_c(r0, r1, r2) _ldxr_c(_jit, r0, r1, r2)
436 static void _ldxr_c(jit_state_t*, jit_int32_t, jit_int32_t, jit_int32_t);
437 # define ldxi_c(r0, r1, i0) _ldxi_c(_jit, r0, r1, i0)
438 static void _ldxi_c(jit_state_t*, jit_int32_t, jit_int32_t, jit_word_t);
439 # define ldxr_uc(r0, r1, r2) _ldxr_uc(_jit, r0, r1, r2)
440 static void _ldxr_uc(jit_state_t*, jit_int32_t, jit_int32_t, jit_int32_t);
441 # define ldxi_uc(r0, r1, i0) _ldxi_uc(_jit, r0, r1, i0)
442 static void _ldxi_uc(jit_state_t*, jit_int32_t, jit_int32_t, jit_word_t);
443 # define ldxr_s(r0, r1, r2) _ldxr_s(_jit, r0, r1, r2)
444 static void _ldxr_s(jit_state_t*, jit_int32_t, jit_int32_t, jit_int32_t);
445 # define ldxi_s(r0, r1, i0) _ldxi_s(_jit, r0, r1, i0)
446 static void _ldxi_s(jit_state_t*, jit_int32_t, jit_int32_t, jit_word_t);
447 # define ldxr_us(r0, r1, r2) _ldxr_us(_jit, r0, r1, r2)
448 static void _ldxr_us(jit_state_t*, jit_int32_t, jit_int32_t, jit_int32_t);
449 # define ldxi_us(r0, r1, i0) _ldxi_us(_jit, r0, r1, i0)
450 static void _ldxi_us(jit_state_t*, jit_int32_t, jit_int32_t, jit_word_t);
451 # if __X32 || !__X64_32
452 # define ldxr_i(r0, r1, r2) _ldxr_i(_jit, r0, r1, r2)
453 static void _ldxr_i(jit_state_t*, jit_int32_t, jit_int32_t, jit_int32_t);
454 # define ldxi_i(r0, r1, i0) _ldxi_i(_jit, r0, r1, i0)
455 static void _ldxi_i(jit_state_t*, jit_int32_t, jit_int32_t, jit_word_t);
459 # define ldxr_i(r0, r1, r2) _ldxr_ui(_jit, r0, r1, r2)
460 # define ldxi_i(r0, r1, i0) _ldxi_ui(_jit, r0, r1, i0)
462 # define ldxr_ui(r0, r1, r2) _ldxr_ui(_jit, r0, r1, r2)
463 # define ldxi_ui(r0, r1, i0) _ldxi_ui(_jit, r0, r1, i0)
465 static void _ldxr_ui(jit_state_t*, jit_int32_t, jit_int32_t, jit_int32_t);
466 static void _ldxi_ui(jit_state_t*, jit_int32_t, jit_int32_t, jit_word_t);
468 # define ldxr_l(r0, r1, r2) _ldxr_l(_jit, r0, r1, r2)
469 static void _ldxr_l(jit_state_t*, jit_int32_t, jit_int32_t, jit_int32_t);
470 # define ldxi_l(r0, r1, i0) _ldxi_l(_jit, r0, r1, i0)
471 static void _ldxi_l(jit_state_t*, jit_int32_t, jit_int32_t, jit_word_t);
474 # define str_c(r0, r1) _str_c(_jit, r0, r1)
475 static void _str_c(jit_state_t*, jit_int32_t, jit_int32_t);
476 # define sti_c(i0, r0) _sti_c(_jit, i0, r0)
477 static void _sti_c(jit_state_t*, jit_word_t, jit_int32_t);
478 # define str_s(r0, r1) _str_s(_jit, r0, r1)
479 static void _str_s(jit_state_t*, jit_int32_t, jit_int32_t);
480 # define sti_s(i0, r0) _sti_s(_jit, i0, r0)
481 static void _sti_s(jit_state_t*, jit_word_t, jit_int32_t);
482 # define str_i(r0, r1) _str_i(_jit, r0, r1)
483 static void _str_i(jit_state_t*, jit_int32_t, jit_int32_t);
484 # define sti_i(i0, r0) _sti_i(_jit, i0, r0)
485 static void _sti_i(jit_state_t*, jit_word_t, jit_int32_t);
486 # if __X64 && !__X64_32
487 # define str_l(r0, r1) _str_l(_jit, r0, r1)
488 static void _str_l(jit_state_t*, jit_int32_t, jit_int32_t);
489 # define sti_l(i0, r0) _sti_l(_jit, i0, r0)
490 static void _sti_l(jit_state_t*, jit_word_t, jit_int32_t);
492 # define stxr_c(r0, r1, r2) _stxr_c(_jit, r0, r1, r2)
493 static void _stxr_c(jit_state_t*, jit_int32_t, jit_int32_t, jit_int32_t);
494 # define stxi_c(i0, r0, r1) _stxi_c(_jit, i0, r0, r1)
495 static void _stxi_c(jit_state_t*, jit_word_t, jit_int32_t, jit_int32_t);
496 # define stxr_s(r0, r1, r2) _stxr_s(_jit, r0, r1, r2)
497 static void _stxr_s(jit_state_t*, jit_int32_t, jit_int32_t, jit_int32_t);
498 # define stxi_s(i0, r0, r1) _stxi_s(_jit, i0, r0, r1)
499 static void _stxi_s(jit_state_t*, jit_word_t, jit_int32_t, jit_int32_t);
500 # define stxr_i(r0, r1, r2) _stxr_i(_jit, r0, r1, r2)
501 static void _stxr_i(jit_state_t*, jit_int32_t, jit_int32_t, jit_int32_t);
502 # define stxi_i(i0, r0, r1) _stxi_i(_jit, i0, r0, r1)
503 static void _stxi_i(jit_state_t*, jit_word_t, jit_int32_t, jit_int32_t);
504 # if __X64 && !__X64_32
505 # define stxr_l(r0, r1, r2) _stxr_l(_jit, r0, r1, r2)
506 static void _stxr_l(jit_state_t*, jit_int32_t, jit_int32_t, jit_int32_t);
507 # define stxi_l(i0, r0, r1) _stxi_l(_jit, i0, r0, r1)
508 static void _stxi_l(jit_state_t*, jit_word_t, jit_int32_t, jit_int32_t);
510 # define jcc(code, i0) _jcc(_jit, code, i0)
511 # define jo(i0) jcc(X86_CC_O, i0)
512 # define jno(i0) jcc(X86_CC_NO, i0)
513 # define jnae(i0) jcc(X86_CC_NAE, i0)
514 # define jb(i0) jcc(X86_CC_B, i0)
515 # define jc(i0) jcc(X86_CC_C, i0)
516 # define jae(i0) jcc(X86_CC_AE, i0)
517 # define jnb(i0) jcc(X86_CC_NB, i0)
518 # define jnc(i0) jcc(X86_CC_NC, i0)
519 # define je(i0) jcc(X86_CC_E, i0)
520 # define jz(i0) jcc(X86_CC_Z, i0)
521 # define jne(i0) jcc(X86_CC_NE, i0)
522 # define jnz(i0) jcc(X86_CC_NZ, i0)
523 # define jbe(i0) jcc(X86_CC_BE, i0)
524 # define jna(i0) jcc(X86_CC_NA, i0)
525 # define ja(i0) jcc(X86_CC_A, i0)
526 # define jnbe(i0) jcc(X86_CC_NBE, i0)
527 # define js(i0) jcc(X86_CC_S, i0)
528 # define jns(i0) jcc(X86_CC_NS, i0)
529 # define jp(i0) jcc(X86_CC_P, i0)
530 # define jpe(i0) jcc(X86_CC_PE, i0)
531 # define jnp(i0) jcc(X86_CC_NP, i0)
532 # define jpo(i0) jcc(X86_CC_PO, i0)
533 # define jl(i0) jcc(X86_CC_L, i0)
534 # define jnge(i0) jcc(X86_CC_NGE, i0)
535 # define jge(i0) jcc(X86_CC_GE, i0)
536 # define jnl(i0) jcc(X86_CC_NL, i0)
537 # define jle(i0) jcc(X86_CC_LE, i0)
538 # define jng(i0) jcc(X86_CC_NG, i0)
539 # define jg(i0) jcc(X86_CC_G, i0)
540 # define jnle(i0) jcc(X86_CC_NLE, i0)
541 static void _jcc(jit_state_t*, jit_int32_t, jit_word_t);
542 # define jccs(code, i0) _jccs(_jit, code, i0)
543 # define jos(i0) jccs(X86_CC_O, i0)
544 # define jnos(i0) jccs(X86_CC_NO, i0)
545 # define jnaes(i0) jccs(X86_CC_NAE, i0)
546 # define jbs(i0) jccs(X86_CC_B, i0)
547 # define jcs(i0) jccs(X86_CC_C, i0)
548 # define jaes(i0) jccs(X86_CC_AE, i0)
549 # define jnbs(i0) jccs(X86_CC_NB, i0)
550 # define jncs(i0) jccs(X86_CC_NC, i0)
551 # define jes(i0) jccs(X86_CC_E, i0)
552 # define jzs(i0) jccs(X86_CC_Z, i0)
553 # define jnes(i0) jccs(X86_CC_NE, i0)
554 # define jnzs(i0) jccs(X86_CC_NZ, i0)
555 # define jbes(i0) jccs(X86_CC_BE, i0)
556 # define jnas(i0) jccs(X86_CC_NA, i0)
557 # define jas(i0) jccs(X86_CC_A, i0)
558 # define jnbes(i0) jccs(X86_CC_NBE, i0)
559 # define jss(i0) jccs(X86_CC_S, i0)
560 # define jnss(i0) jccs(X86_CC_NS, i0)
561 # define jps(i0) jccs(X86_CC_P, i0)
562 # define jpes(i0) jccs(X86_CC_PE, i0)
563 # define jnps(i0) jccs(X86_CC_NP, i0)
564 # define jpos(i0) jccs(X86_CC_PO, i0)
565 # define jls(i0) jccs(X86_CC_L, i0)
566 # define jnges(i0) jccs(X86_CC_NGE, i0)
567 # define jges(i0) jccs(X86_CC_GE, i0)
568 # define jnls(i0) jccs(X86_CC_NL, i0)
569 # define jles(i0) jccs(X86_CC_LE, i0)
570 # define jngs(i0) jccs(X86_CC_NG, i0)
571 # define jgs(i0) jccs(X86_CC_G, i0)
572 # define jnles(i0) jccs(X86_CC_NLE, i0)
573 static void _jccs(jit_state_t*, jit_int32_t, jit_word_t);
574 # define jcr(code, i0, r0, r1) _jcr(_jit, code, i0, r0, r1)
575 static void _jcr(jit_state_t*,jit_int32_t,jit_word_t,jit_int32_t,jit_int32_t);
576 # define jci(code, i0, r0, i1) _jci(_jit, code, i0, r0, i1)
577 static void _jci(jit_state_t*,jit_int32_t,jit_word_t,jit_int32_t,jit_word_t);
578 # define jci0(code, i0, r0) _jci0(_jit, code, i0, r0)
579 static void _jci0(jit_state_t*, jit_int32_t, jit_word_t, jit_int32_t);
580 # define bltr(i0, r0, r1) _bltr(_jit, i0, r0, r1)
581 static jit_word_t _bltr(jit_state_t*, jit_word_t, jit_int32_t, jit_int32_t);
582 # define blti(i0, r0, i1) _blti(_jit, i0, r0, i1)
583 static jit_word_t _blti(jit_state_t*, jit_word_t, jit_int32_t, jit_word_t);
584 # define bltr_u(i0, r0, r1) _bltr_u(_jit, i0, r0, r1)
585 static jit_word_t _bltr_u(jit_state_t*, jit_word_t, jit_int32_t, jit_int32_t);
586 # define blti_u(i0, r0, i1) _blti_u(_jit, i0, r0, i1)
587 static jit_word_t _blti_u(jit_state_t*, jit_word_t, jit_int32_t, jit_word_t);
588 # define bler(i0, r0, r1) _bler(_jit, i0, r0, r1)
589 static jit_word_t _bler(jit_state_t*, jit_word_t, jit_int32_t, jit_int32_t);
590 # define blei(i0, r0, i1) _blei(_jit, i0, r0, i1)
591 static jit_word_t _blei(jit_state_t*, jit_word_t, jit_int32_t, jit_word_t);
592 # define bler_u(i0, r0, r1) _bler_u(_jit, i0, r0, r1)
593 static jit_word_t _bler_u(jit_state_t*, jit_word_t, jit_int32_t, jit_int32_t);
594 # define blei_u(i0, r0, i1) _blei_u(_jit, i0, r0, i1)
595 static jit_word_t _blei_u(jit_state_t*, jit_word_t, jit_int32_t, jit_word_t);
596 # define beqr(i0, r0, r1) _beqr(_jit, i0, r0, r1)
597 static jit_word_t _beqr(jit_state_t*, jit_word_t, jit_int32_t, jit_int32_t);
598 # define beqi(i0, r0, i1) _beqi(_jit, i0, r0, i1)
599 static jit_word_t _beqi(jit_state_t*, jit_word_t, jit_int32_t, jit_word_t);
600 # define bger(i0, r0, r1) _bger(_jit, i0, r0, r1)
601 static jit_word_t _bger(jit_state_t*, jit_word_t, jit_int32_t, jit_int32_t);
602 # define bgei(i0, r0, i1) _bgei(_jit, i0, r0, i1)
603 static jit_word_t _bgei(jit_state_t*, jit_word_t, jit_int32_t, jit_word_t);
604 # define bger_u(i0, r0, r1) _bger_u(_jit, i0, r0, r1)
605 static jit_word_t _bger_u(jit_state_t*, jit_word_t, jit_int32_t, jit_int32_t);
606 # define bgei_u(i0, r0, i1) _bgei_u(_jit, i0, r0, i1)
607 static jit_word_t _bgei_u(jit_state_t*, jit_word_t, jit_int32_t, jit_word_t);
608 # define bgtr(i0, r0, r1) _bgtr(_jit, i0, r0, r1)
609 static jit_word_t _bgtr(jit_state_t*, jit_word_t, jit_int32_t, jit_int32_t);
610 # define bgti(i0, r0, i1) _bgti(_jit, i0, r0, i1)
611 static jit_word_t _bgti(jit_state_t*, jit_word_t, jit_int32_t, jit_word_t);
612 # define bgtr_u(i0, r0, r1) _bgtr_u(_jit, i0, r0, r1)
613 static jit_word_t _bgtr_u(jit_state_t*, jit_word_t, jit_int32_t, jit_int32_t);
614 # define bgti_u(i0, r0, i1) _bgti_u(_jit, i0, r0, i1)
615 static jit_word_t _bgti_u(jit_state_t*, jit_word_t, jit_int32_t, jit_word_t);
616 # define bner(i0, r0, r1) _bner(_jit, i0, r0, r1)
617 static jit_word_t _bner(jit_state_t*, jit_word_t, jit_int32_t, jit_int32_t);
618 # define bnei(i0, r0, i1) _bnei(_jit, i0, r0, i1)
619 static jit_word_t _bnei(jit_state_t*, jit_word_t, jit_int32_t, jit_word_t);
620 # define bmsr(i0, r0, r1) _bmsr(_jit, i0, r0, r1)
621 static jit_word_t _bmsr(jit_state_t*,jit_word_t,jit_int32_t,jit_int32_t);
622 # define bmsi(i0, r0, i1) _bmsi(_jit, i0, r0, i1)
623 static jit_word_t _bmsi(jit_state_t*,jit_word_t,jit_int32_t,jit_word_t);
624 # define bmcr(i0, r0, r1) _bmcr(_jit, i0, r0, r1)
625 static jit_word_t _bmcr(jit_state_t*,jit_word_t,jit_int32_t,jit_int32_t);
626 # define bmci(i0, r0, i1) _bmci(_jit, i0, r0, i1)
627 static jit_word_t _bmci(jit_state_t*,jit_word_t,jit_int32_t,jit_word_t);
628 # define boaddr(i0, r0, r1) _boaddr(_jit, i0, r0, r1)
629 static jit_word_t _boaddr(jit_state_t*,jit_word_t,jit_int32_t,jit_int32_t);
630 # define boaddi(i0, r0, i1) _boaddi(_jit, i0, r0, i1)
631 static jit_word_t _boaddi(jit_state_t*,jit_word_t,jit_int32_t,jit_word_t);
632 # define boaddr_u(i0, r0, r1) _boaddr_u(_jit, i0, r0, r1)
633 static jit_word_t _boaddr_u(jit_state_t*,jit_word_t,jit_int32_t,jit_int32_t);
634 # define boaddi_u(i0, r0, i1) _boaddi_u(_jit, i0, r0, i1)
635 static jit_word_t _boaddi_u(jit_state_t*,jit_word_t,jit_int32_t,jit_word_t);
636 # define bxaddr(i0, r0, r1) _bxaddr(_jit, i0, r0, r1)
637 static jit_word_t _bxaddr(jit_state_t*,jit_word_t,jit_int32_t,jit_int32_t);
638 # define bxaddi(i0, r0, i1) _bxaddi(_jit, i0, r0, i1)
639 static jit_word_t _bxaddi(jit_state_t*,jit_word_t,jit_int32_t,jit_word_t);
640 # define bxaddr_u(i0, r0, r1) _bxaddr_u(_jit, i0, r0, r1)
641 static jit_word_t _bxaddr_u(jit_state_t*,jit_word_t,jit_int32_t,jit_int32_t);
642 # define bxaddi_u(i0, r0, i1) _bxaddi_u(_jit, i0, r0, i1)
643 static jit_word_t _bxaddi_u(jit_state_t*,jit_word_t,jit_int32_t,jit_word_t);
644 # define bosubr(i0, r0, r1) _bosubr(_jit, i0, r0, r1)
645 static jit_word_t _bosubr(jit_state_t*,jit_word_t,jit_int32_t,jit_int32_t);
646 # define bosubi(i0, r0, i1) _bosubi(_jit, i0, r0, i1)
647 static jit_word_t _bosubi(jit_state_t*,jit_word_t,jit_int32_t,jit_word_t);
648 # define bosubr_u(i0, r0, r1) _bosubr_u(_jit, i0, r0, r1)
649 static jit_word_t _bosubr_u(jit_state_t*,jit_word_t,jit_int32_t,jit_int32_t);
650 # define bosubi_u(i0, r0, i1) _bosubi_u(_jit, i0, r0, i1)
651 static jit_word_t _bosubi_u(jit_state_t*,jit_word_t,jit_int32_t,jit_word_t);
652 # define bxsubr(i0, r0, r1) _bxsubr(_jit, i0, r0, r1)
653 static jit_word_t _bxsubr(jit_state_t*,jit_word_t,jit_int32_t,jit_int32_t);
654 # define bxsubi(i0, r0, i1) _bxsubi(_jit, i0, r0, i1)
655 static jit_word_t _bxsubi(jit_state_t*,jit_word_t,jit_int32_t,jit_word_t);
656 # define bxsubr_u(i0, r0, r1) _bxsubr_u(_jit, i0, r0, r1)
657 static jit_word_t _bxsubr_u(jit_state_t*,jit_word_t,jit_int32_t,jit_int32_t);
658 # define bxsubi_u(i0, r0, i1) _bxsubi_u(_jit, i0, r0, i1)
659 static jit_word_t _bxsubi_u(jit_state_t*,jit_word_t,jit_int32_t,jit_word_t);
660 # define callr(r0) _callr(_jit, r0)
661 static void _callr(jit_state_t*, jit_int32_t);
662 # define calli(i0) _calli(_jit, i0)
663 static jit_word_t _calli(jit_state_t*, jit_word_t);
665 # define calli_p(i0) _calli_p(_jit, i0)
666 static jit_word_t _calli_p(jit_state_t*, jit_word_t);
668 # define calli_p(i0) calli(i0)
670 # define jmpr(r0) _jmpr(_jit, r0)
671 static void _jmpr(jit_state_t*, jit_int32_t);
672 # define jmpi(i0) _jmpi(_jit, i0)
673 static jit_word_t _jmpi(jit_state_t*, jit_word_t);
675 # define jmpi_p(i0) _jmpi_p(_jit, i0)
676 static jit_word_t _jmpi_p(jit_state_t*, jit_word_t);
678 # define jmpi_p(i0) jmpi(i0)
680 # define jmpsi(i0) _jmpsi(_jit, i0)
681 static void _jmpsi(jit_state_t*, jit_uint8_t);
682 # define prolog(node) _prolog(_jit, node)
683 static void _prolog(jit_state_t*, jit_node_t*);
684 # define epilog(node) _epilog(_jit, node)
685 static void _epilog(jit_state_t*, jit_node_t*);
686 # define vastart(r0) _vastart(_jit, r0)
687 static void _vastart(jit_state_t*, jit_int32_t);
688 # define vaarg(r0, r1) _vaarg(_jit, r0, r1)
689 static void _vaarg(jit_state_t*, jit_int32_t, jit_int32_t);
690 # define vaarg_d(r0, r1, i0) _vaarg_d(_jit, r0, r1, i0)
691 static void _vaarg_d(jit_state_t*, jit_int32_t, jit_int32_t, jit_bool_t);
692 # define patch_at(node, instr, label) _patch_at(_jit, node, instr, label)
693 static void _patch_at(jit_state_t*, jit_node_t*, jit_word_t, jit_word_t);
694 # if !defined(HAVE_FFSL)
696 # define ffsl(i) __builtin_ffs(i)
698 # define ffsl(l) __builtin_ffsl(l)
705 _rex(jit_state_t *_jit, jit_int32_t l, jit_int32_t w,
706 jit_int32_t r, jit_int32_t x, jit_int32_t b)
709 jit_int32_t v = 0x40 | (w << 3);
723 _rx(jit_state_t *_jit, jit_int32_t rd, jit_int32_t md,
724 jit_int32_t rb, jit_int32_t ri, jit_int32_t ms)
729 mrm(0x00, r7(rd), 0x05);
731 mrm(0x00, r7(rd), 0x04);
732 sib(_SCL1, 0x04, 0x05);
736 else if (r7(rb) == _RSP_REGNO) {
738 mrm(0x00, r7(rd), 0x04);
741 else if ((jit_int8_t)md == md) {
742 mrm(0x01, r7(rd), 0x04);
747 mrm(0x02, r7(rd), 0x04);
753 if (md == 0 && r7(rb) != _RBP_REGNO)
754 mrm(0x00, r7(rd), r7(rb));
755 else if ((jit_int8_t)md == md) {
756 mrm(0x01, r7(rd), r7(rb));
760 mrm(0x02, r7(rd), r7(rb));
765 else if (rb == _NOREG) {
766 mrm(0x00, r7(rd), 0x04);
767 sib(ms, r7(ri), 0x05);
770 else if (r8(ri) != _RSP_REGNO) {
771 if (md == 0 && r7(rb) != _RBP_REGNO) {
772 mrm(0x00, r7(rd), 0x04);
773 sib(ms, r7(ri), r7(rb));
775 else if ((jit_int8_t)md == md) {
776 mrm(0x01, r7(rd), 0x04);
777 sib(ms, r7(ri), r7(rb));
781 mrm(0x02, r7(rd), 0x04);
782 sib(ms, r7(ri), r7(rb));
787 fprintf(stderr, "illegal index register");
793 _nop(jit_state_t *_jit, jit_int32_t count)
803 case 3: /* NOP DWORD ptr [EAX] */
804 ic(0x0f); ic(0x1f); ic(0x00);
806 case 4: /* NOP DWORD ptr [EAX + 00H] */
807 ic(0x0f); ic(0x1f); ic(0x40); ic(0x00);
809 case 5: /* NOP DWORD ptr [EAX + EAX*1 + 00H] */
810 ic(0x0f); ic(0x1f); ic(0x44); ic(0x00);
813 case 6: /* 66 NOP DWORD ptr [EAX + EAX*1 + 00H] */
814 ic(0x66); ic(0x0f); ic(0x1f); ic(0x44);
817 case 7: /* NOP DWORD ptr [EAX + 00000000H] */
818 ic(0x0f); ic(0x1f); ic(0x80); ii(0x0000);
820 case 8: /* NOP DWORD ptr [EAX + EAX*1 + 00000000H] */
821 ic(0x0f); ic(0x1f); ic(0x84); ic(0x00);
824 case 9: /* 66 NOP DWORD ptr [EAX + EAX*1 + 00000000H] */
825 ic(0x66); ic(0x0f); ic(0x1f); ic(0x84);
826 ic(0x00); ii(0x0000);
834 _lea(jit_state_t *_jit, jit_int32_t md, jit_int32_t rb,
835 jit_int32_t ri, jit_int32_t ms, jit_int32_t rd)
837 rex(0, WIDE, rd, ri, rb);
839 rx(rd, md, rb, ri, ms);
843 _pushr(jit_state_t *_jit, jit_int32_t r0)
845 rex(0, WIDE, 0, 0, r0);
850 _popr(jit_state_t *_jit, jit_int32_t r0)
852 rex(0, WIDE, 0, 0, r0);
857 _xchgr(jit_state_t *_jit, jit_int32_t r0, jit_int32_t r1)
859 rex(0, WIDE, r1, _NOREG, r0);
861 mrm(0x03, r7(r1), r7(r0));
865 _testr(jit_state_t *_jit, jit_int32_t r0, jit_int32_t r1)
867 rex(0, WIDE, r1, _NOREG, r0);
869 mrm(0x03, r7(r1), r7(r0));
873 _testi(jit_state_t *_jit, jit_int32_t r0, jit_word_t i0)
875 rex(0, WIDE, _NOREG, _NOREG, r0);
876 if (r0 == _RAX_REGNO)
880 mrm(0x03, 0x00, r7(r0));
886 _cc(jit_state_t *_jit, jit_int32_t code, jit_int32_t r0)
888 rex(0, 0, _NOREG, _NOREG, r0);
891 mrm(0x03, 0x00, r7(r0));
895 _alur(jit_state_t *_jit, jit_int32_t code, jit_int32_t r0, jit_int32_t r1)
897 rex(0, WIDE, r1, _NOREG, r0);
899 mrm(0x03, r7(r1), r7(r0));
903 _alui(jit_state_t *_jit, jit_int32_t code, jit_int32_t r0, jit_word_t i0)
906 if (can_sign_extend_int_p(i0)) {
907 rex(0, WIDE, _NOREG, _NOREG, r0);
908 if ((jit_int8_t)i0 == i0) {
910 ic(0xc0 | code | r7(r0));
914 if (r0 == _RAX_REGNO)
918 ic(0xc0 | code | r7(r0));
924 reg = jit_get_reg(jit_class_gpr);
926 alur(code, r0, rn(reg));
932 _save(jit_state_t *_jit, jit_int32_t r0)
934 if (!_jitc->function->regoff[r0]) {
935 _jitc->function->regoff[r0] = jit_allocai(sizeof(jit_word_t));
938 assert(!jit_regset_tstbit(&_jitc->regsav, r0));
939 jit_regset_setbit(&_jitc->regsav, r0);
940 stxi(_jitc->function->regoff[r0], _RBP_REGNO, r0);
944 _load(jit_state_t *_jit, jit_int32_t r0)
946 assert(_jitc->function->regoff[r0]);
947 assert(jit_regset_tstbit(&_jitc->regsav, r0));
948 jit_regset_clrbit(&_jitc->regsav, r0);
949 ldxi(r0, _RBP_REGNO, _jitc->function->regoff[r0]);
953 _addr(jit_state_t *_jit, jit_int32_t r0, jit_int32_t r1, jit_int32_t r2)
960 lea(0, r1, r2, _SCL1, r0);
964 _addi(jit_state_t *_jit, jit_int32_t r0, jit_int32_t r1, jit_word_t i0)
975 else if (can_sign_extend_int_p(i0)) {
979 lea(i0, r1, _NOREG, _SCL1, r0);
986 reg = jit_get_reg(jit_class_gpr);
994 _addcr(jit_state_t *_jit, jit_int32_t r0, jit_int32_t r1, jit_int32_t r2)
1005 _addci(jit_state_t *_jit, jit_int32_t r0, jit_int32_t r1, jit_word_t i0)
1008 if (can_sign_extend_int_p(i0)) {
1012 else if (r0 == r1) {
1013 reg = jit_get_reg(jit_class_gpr);
1025 _addxr(jit_state_t *_jit, jit_int32_t r0, jit_int32_t r1, jit_int32_t r2)
1036 _addxi(jit_state_t *_jit, jit_int32_t r0, jit_int32_t r1, jit_word_t i0)
1039 if (can_sign_extend_int_p(i0)) {
1043 else if (r0 == r1) {
1044 reg = jit_get_reg(jit_class_gpr);
1046 iaddxr(r0, rn(reg));
1056 _subr(jit_state_t *_jit, jit_int32_t r0, jit_int32_t r1, jit_int32_t r2)
1060 else if (r0 == r2) {
1071 _subi(jit_state_t *_jit, jit_int32_t r0, jit_int32_t r1, jit_word_t i0)
1082 else if (can_sign_extend_int_p(i0)) {
1086 lea(-i0, r1, _NOREG, _SCL1, r0);
1088 else if (r0 != r1) {
1093 reg = jit_get_reg(jit_class_gpr);
1101 _subcr(jit_state_t *_jit, jit_int32_t r0, jit_int32_t r1, jit_int32_t r2)
1104 if (r0 == r2 && r0 != r1) {
1105 reg = jit_get_reg(jit_class_gpr);
1118 _subci(jit_state_t *_jit, jit_int32_t r0, jit_int32_t r1, jit_word_t i0)
1122 if (can_sign_extend_int_p(i0))
1125 reg = jit_get_reg(jit_class_gpr);
1133 _subxr(jit_state_t *_jit, jit_int32_t r0, jit_int32_t r1, jit_int32_t r2)
1136 if (r0 == r2 && r0 != r1) {
1137 reg = jit_get_reg(jit_class_gpr);
1140 isubxr(r0, rn(reg));
1150 _subxi(jit_state_t *_jit, jit_int32_t r0, jit_int32_t r1, jit_word_t i0)
1154 if (can_sign_extend_int_p(i0))
1157 reg = jit_get_reg(jit_class_gpr);
1159 isubxr(r0, rn(reg));
1165 _rsbi(jit_state_t *_jit, jit_int32_t r0, jit_int32_t r1, jit_word_t i0)
1172 _imulr(jit_state_t *_jit, jit_int32_t r0, jit_int32_t r1)
1174 rex(0, WIDE, r0, _NOREG, r1);
1177 mrm(0x03, r7(r0), r7(r1));
1181 _imuli(jit_state_t *_jit, jit_int32_t r0, jit_int32_t r1, jit_word_t i0)
1184 if (can_sign_extend_int_p(i0)) {
1185 rex(0, WIDE, r0, _NOREG, r1);
1186 if ((jit_int8_t)i0 == i0) {
1188 mrm(0x03, r7(r0), r7(r1));
1193 mrm(0x03, r7(r0), r7(r1));
1198 reg = jit_get_reg(jit_class_gpr);
1206 _mulr(jit_state_t *_jit, jit_int32_t r0, jit_int32_t r1, jit_int32_t r2)
1219 _muli(jit_state_t *_jit, jit_int32_t r0, jit_int32_t r1, jit_word_t i0)
1232 lea(0, _NOREG, r1, _SCL2, r0);
1235 lea(0, _NOREG, r1, _SCL4, r0);
1238 lea(0, _NOREG, r1, _SCL8, r0);
1241 if (i0 > 0 && !(i0 & (i0 - 1)))
1242 lshi(r0, r1, ffsl(i0) - 1);
1243 else if (can_sign_extend_int_p(i0))
1245 else if (r0 != r1) {
1255 #define savset(rn) \
1258 if (r1 != rn && r2 != rn) \
1261 #define isavset(rn) \
1267 #define qsavset(rn) \
1268 if (r0 != rn && r1 != rn) { \
1270 if (r2 != rn && r3 != rn) \
1273 #define allocr(rn, rv) \
1274 if (set & (1 << rn)) \
1275 (void)jit_get_reg(rv|jit_class_gpr|jit_class_named); \
1276 if (sav & (1 << rn)) { \
1277 if ( jit_regset_tstbit(&_jitc->regsav, rv) || \
1278 !jit_regset_tstbit(&_jitc->reglive, rv)) \
1279 sav &= ~(1 << rn); \
1283 #define clear(rn, rv) \
1284 if (set & (1 << rn)) \
1285 jit_unget_reg(rv); \
1286 if (sav & (1 << rn)) \
1289 _iqmulr(jit_state_t *_jit, jit_int32_t r0, jit_int32_t r1,
1290 jit_int32_t r2, jit_int32_t r3, jit_bool_t sign)
1297 qsavset(_RDX_REGNO);
1298 qsavset(_RAX_REGNO);
1299 allocr(_RDX_REGNO, _RDX);
1300 allocr(_RAX_REGNO, _RAX);
1302 if (r3 == _RAX_REGNO)
1306 movr(_RAX_REGNO, r2);
1313 if (r0 == _RDX_REGNO && r1 == _RAX_REGNO)
1314 xchgr(_RAX_REGNO, _RDX_REGNO);
1316 if (r0 != _RDX_REGNO)
1317 movr(r0, _RAX_REGNO);
1318 movr(r1, _RDX_REGNO);
1319 if (r0 == _RDX_REGNO)
1320 movr(r0, _RAX_REGNO);
1323 clear(_RDX_REGNO, _RDX);
1324 clear(_RAX_REGNO, _RAX);
1328 _iqmuli(jit_state_t *_jit, jit_int32_t r0, jit_int32_t r1,
1329 jit_int32_t r2, jit_word_t i0, jit_bool_t sign)
1338 reg = jit_get_reg(jit_class_gpr);
1341 qmulr(r0, r1, r2, rn(reg));
1343 qmulr_u(r0, r1, r2, rn(reg));
1349 _sign_extend_rdx_rax(jit_state_t *_jit)
1351 rex(0, WIDE, 0, 0, 0);
1356 _divremr(jit_state_t *_jit, jit_int32_t r0, jit_int32_t r1, jit_int32_t r2,
1357 jit_bool_t sign, jit_bool_t divide)
1365 sav = set = use = 0;
1368 allocr(_RDX_REGNO, _RDX);
1369 allocr(_RAX_REGNO, _RAX);
1371 if (r2 == _RAX_REGNO) {
1372 if (r0 == _RAX_REGNO || r0 == _RDX_REGNO) {
1373 if ((reg = jit_get_reg(jit_class_gpr|jit_class_chk)) == JIT_NOREG)
1374 reg = jit_get_reg((r1 == _RCX_REGNO ? _RBX : _RCX) |
1375 jit_class_gpr|jit_class_named);
1378 movr(div, _RAX_REGNO);
1379 if (r1 != _RAX_REGNO)
1380 movr(_RAX_REGNO, r1);
1384 xchgr(r0, _RAX_REGNO);
1386 if (r0 != _RAX_REGNO)
1387 movr(r0, _RAX_REGNO);
1388 if (r1 != _RAX_REGNO)
1389 movr(_RAX_REGNO, r1);
1394 else if (r2 == _RDX_REGNO) {
1395 if (r0 == _RAX_REGNO || r0 == _RDX_REGNO) {
1396 if ((reg = jit_get_reg(jit_class_gpr|jit_class_chk)) == JIT_NOREG)
1397 reg = jit_get_reg((r1 == _RCX_REGNO ? _RBX : _RCX) |
1398 jit_class_gpr|jit_class_named);
1401 movr(div, _RDX_REGNO);
1402 if (r1 != _RAX_REGNO)
1403 movr(_RAX_REGNO, r1);
1406 if (r1 != _RAX_REGNO)
1407 movr(_RAX_REGNO, r1);
1408 movr(r0, _RDX_REGNO);
1413 if (r1 != _RAX_REGNO)
1414 movr(_RAX_REGNO, r1);
1419 sign_extend_rdx_rax();
1423 ixorr(_RDX_REGNO, _RDX_REGNO);
1431 movr(r0, _RAX_REGNO);
1433 movr(r0, _RDX_REGNO);
1435 clear(_RDX_REGNO, _RDX);
1436 clear(_RAX_REGNO, _RAX);
1440 _divremi(jit_state_t *_jit, jit_int32_t r0, jit_int32_t r1, jit_word_t i0,
1441 jit_bool_t sign, jit_bool_t divide)
1461 if (i0 > 0 && !(i0 & (i0 - 1))) {
1464 rshi(r0, r0, ffsl(i0) - 1);
1466 rshi_u(r0, r0, ffsl(i0) - 1);
1472 else if (i0 == 1 || (sign && i0 == -1)) {
1476 else if (!sign && i0 > 0 && !(i0 & (i0 - 1))) {
1477 if (can_sign_extend_int_p(i0)) {
1481 else if (r0 != r1) {
1486 reg = jit_get_reg(jit_class_gpr);
1487 movi(rn(reg), i0 - 1);
1494 sav = set = use = 0;
1495 isavset(_RDX_REGNO);
1496 isavset(_RAX_REGNO);
1497 allocr(_RDX_REGNO, _RDX);
1498 allocr(_RAX_REGNO, _RAX);
1500 if (r0 == _RAX_REGNO || r0 == _RDX_REGNO || r0 == r1) {
1501 if ((reg = jit_get_reg(jit_class_gpr|jit_class_chk)) == JIT_NOREG)
1502 reg = jit_get_reg((r1 == _RCX_REGNO ? _RBX : _RCX) |
1503 jit_class_gpr|jit_class_named);
1511 movr(_RAX_REGNO, r1);
1514 sign_extend_rdx_rax();
1518 ixorr(_RDX_REGNO, _RDX_REGNO);
1526 movr(r0, _RAX_REGNO);
1528 movr(r0, _RDX_REGNO);
1530 clear(_RDX_REGNO, _RDX);
1531 clear(_RAX_REGNO, _RAX);
1535 _iqdivr(jit_state_t *_jit, jit_int32_t r0, jit_int32_t r1,
1536 jit_int32_t r2, jit_int32_t r3, jit_bool_t sign)
1544 sav = set = use = 0;
1545 qsavset(_RDX_REGNO);
1546 qsavset(_RAX_REGNO);
1547 allocr(_RDX_REGNO, _RDX);
1548 allocr(_RAX_REGNO, _RAX);
1549 if (r3 == _RAX_REGNO) {
1550 if (r0 == _RAX_REGNO || r0 == _RDX_REGNO) {
1551 if ((reg = jit_get_reg(jit_class_gpr|jit_class_chk)) == JIT_NOREG)
1552 reg = jit_get_reg((r1 == _RCX_REGNO ? _RBX : _RCX) |
1553 jit_class_gpr|jit_class_named);
1556 movr(div, _RAX_REGNO);
1557 if (r2 != _RAX_REGNO)
1558 movr(_RAX_REGNO, r2);
1562 xchgr(r0, _RAX_REGNO);
1564 if (r0 != _RAX_REGNO)
1565 movr(r0, _RAX_REGNO);
1566 if (r2 != _RAX_REGNO)
1567 movr(_RAX_REGNO, r2);
1572 else if (r3 == _RDX_REGNO) {
1573 if (r0 == _RAX_REGNO || r0 == _RDX_REGNO) {
1574 if ((reg = jit_get_reg(jit_class_gpr|jit_class_chk)) == JIT_NOREG)
1575 reg = jit_get_reg((r1 == _RCX_REGNO ? _RBX : _RCX) |
1576 jit_class_gpr|jit_class_named);
1579 movr(div, _RDX_REGNO);
1580 if (r2 != _RAX_REGNO)
1581 movr(_RAX_REGNO, r2);
1584 if (r2 != _RAX_REGNO)
1585 movr(_RAX_REGNO, r2);
1586 movr(r0, _RDX_REGNO);
1591 if (r2 != _RAX_REGNO)
1592 movr(_RAX_REGNO, r2);
1596 sign_extend_rdx_rax();
1600 ixorr(_RDX_REGNO, _RDX_REGNO);
1606 if (r0 == _RDX_REGNO && r1 == _RAX_REGNO)
1607 xchgr(_RAX_REGNO, _RDX_REGNO);
1609 if (r0 != _RDX_REGNO)
1610 movr(r0, _RAX_REGNO);
1611 movr(r1, _RDX_REGNO);
1612 if (r0 == _RDX_REGNO)
1613 movr(r0, _RAX_REGNO);
1616 clear(_RDX_REGNO, _RDX);
1617 clear(_RAX_REGNO, _RAX);
1621 _iqdivi(jit_state_t *_jit, jit_int32_t r0, jit_int32_t r1,
1622 jit_int32_t r2, jit_word_t i0, jit_bool_t sign)
1626 reg = jit_get_reg(jit_class_gpr);
1629 qdivr(r0, r1, r2, rn(reg));
1631 qdivr_u(r0, r1, r2, rn(reg));
1639 _andr(jit_state_t *_jit, jit_int32_t r0, jit_int32_t r1, jit_int32_t r2)
1654 _andi(jit_state_t *_jit, jit_int32_t r0, jit_int32_t r1, jit_word_t i0)
1662 else if (r0 == r1) {
1663 if (can_sign_extend_int_p(i0))
1666 reg = jit_get_reg(jit_class_gpr);
1679 _orr(jit_state_t *_jit, jit_int32_t r0, jit_int32_t r1, jit_int32_t r2)
1694 _ori(jit_state_t *_jit, jit_int32_t r0, jit_int32_t r1, jit_word_t i0)
1701 else if (can_sign_extend_int_p(i0)) {
1705 else if (r0 != r1) {
1710 reg = jit_get_reg(jit_class_gpr);
1718 _xorr(jit_state_t *_jit, jit_int32_t r0, jit_int32_t r1, jit_int32_t r2)
1733 _xori(jit_state_t *_jit, jit_int32_t r0, jit_int32_t r1, jit_word_t i0)
1740 else if (can_sign_extend_int_p(i0)) {
1744 else if (r0 != r1) {
1749 reg = jit_get_reg(jit_class_gpr);
1757 _irotshr(jit_state_t *_jit, jit_int32_t code, jit_int32_t r0)
1759 rex(0, WIDE, _RCX_REGNO, _NOREG, r0);
1761 mrm(0x03, code, r7(r0));
1765 _rotshr(jit_state_t *_jit, jit_int32_t code,
1766 jit_int32_t r0, jit_int32_t r1, jit_int32_t r2)
1771 if (r0 == _RCX_REGNO) {
1772 reg = jit_get_reg(jit_class_gpr);
1774 if (r2 != _RCX_REGNO)
1775 movr(_RCX_REGNO, r2);
1776 irotshr(code, rn(reg));
1777 movr(_RCX_REGNO, rn(reg));
1780 else if (r2 != _RCX_REGNO) {
1781 use = !jit_reg_free_p(_RCX);
1783 reg = jit_get_reg(jit_class_gpr);
1784 movr(rn(reg), _RCX_REGNO);
1788 if (r1 == _RCX_REGNO) {
1790 xchgr(r0, _RCX_REGNO);
1793 movr(_RCX_REGNO, r2);
1797 movr(_RCX_REGNO, r2);
1802 movr(_RCX_REGNO, rn(reg));
1813 _irotshi(jit_state_t *_jit, jit_int32_t code, jit_int32_t r0, jit_word_t i0)
1815 rex(0, WIDE, _NOREG, _NOREG, r0);
1818 mrm(0x03, code, r7(r0));
1822 mrm(0x03, code, r7(r0));
1828 _rotshi(jit_state_t *_jit, jit_int32_t code,
1829 jit_int32_t r0, jit_int32_t r1, jit_word_t i0)
1833 irotshi(code, r0, i0);
1837 _lshi(jit_state_t *_jit, jit_int32_t r0, jit_int32_t r1, jit_word_t i0)
1842 lea(0, _NOREG, r1, i0 == 1 ? _SCL2 : i0 == 2 ? _SCL4 : _SCL8, r0);
1844 rotshi(X86_SHL, r0, r1, i0);
1848 _unr(jit_state_t *_jit, jit_int32_t code, jit_int32_t r0)
1850 rex(0, WIDE, _NOREG, _NOREG, r0);
1852 mrm(0x03, code, r7(r0));
1856 _negr(jit_state_t *_jit, jit_int32_t r0, jit_int32_t r1)
1867 _comr(jit_state_t *_jit, jit_int32_t r0, jit_int32_t r1)
1875 _incr(jit_state_t *_jit, jit_int32_t r0, jit_int32_t r1)
1879 rex(0, WIDE, _NOREG, _NOREG, r0);
1888 _decr(jit_state_t *_jit, jit_int32_t r0, jit_int32_t r1)
1892 rex(0, WIDE, _NOREG, _NOREG, r0);
1902 _cr(jit_state_t *_jit,
1903 jit_int32_t code, jit_int32_t r0, jit_int32_t r1, jit_int32_t r2)
1908 same = r0 == r1 || r0 == r2;
1917 reg = jit_get_reg(jit_class_gpr|jit_class_rg8);
1918 ixorr(rn(reg), rn(reg));
1927 _ci(jit_state_t *_jit,
1928 jit_int32_t code, jit_int32_t r0, jit_int32_t r1, jit_word_t i0)
1942 reg = jit_get_reg(jit_class_gpr|jit_class_rg8);
1943 ixorr(rn(reg), rn(reg));
1952 _ci0(jit_state_t *_jit, jit_int32_t code, jit_int32_t r0, jit_int32_t r1)
1966 reg = jit_get_reg(jit_class_gpr|jit_class_rg8);
1967 ixorr(rn(reg), rn(reg));
1976 _ltr(jit_state_t *_jit, jit_int32_t r0, jit_int32_t r1, jit_int32_t r2)
1981 cr(X86_CC_L, r0, r1, r2);
1985 _lti(jit_state_t *_jit, jit_int32_t r0, jit_int32_t r1, jit_word_t i0)
1988 ci(X86_CC_L, r0, r1, i0);
1990 ci0(X86_CC_S, r0, r1);
1994 _ltr_u(jit_state_t *_jit, jit_int32_t r0, jit_int32_t r1, jit_int32_t r2)
1999 cr(X86_CC_B, r0, r1, r2);
2003 _ler(jit_state_t *_jit, jit_int32_t r0, jit_int32_t r1, jit_int32_t r2)
2008 cr(X86_CC_LE, r0, r1, r2);
2012 _ler_u(jit_state_t *_jit, jit_int32_t r0, jit_int32_t r1, jit_int32_t r2)
2017 cr(X86_CC_BE, r0, r1, r2);
2021 _lei_u(jit_state_t *_jit, jit_int32_t r0, jit_int32_t r1, jit_word_t i0)
2024 ci(X86_CC_BE, r0, r1, i0);
2026 ci0(X86_CC_E, r0, r1);
2030 _eqr(jit_state_t *_jit, jit_int32_t r0, jit_int32_t r1, jit_int32_t r2)
2035 cr(X86_CC_E, r0, r1, r2);
2039 _eqi(jit_state_t *_jit, jit_int32_t r0, jit_int32_t r1, jit_word_t i0)
2042 ci(X86_CC_E, r0, r1, i0);
2044 ci0(X86_CC_E, r0, r1);
2048 _ger(jit_state_t *_jit, jit_int32_t r0, jit_int32_t r1, jit_int32_t r2)
2053 cr(X86_CC_GE, r0, r1, r2);
2057 _gei(jit_state_t *_jit, jit_int32_t r0, jit_int32_t r1, jit_word_t i0)
2060 ci(X86_CC_GE, r0, r1, i0);
2062 ci0(X86_CC_NS, r0, r1);
2066 _ger_u(jit_state_t *_jit, jit_int32_t r0, jit_int32_t r1, jit_int32_t r2)
2071 cr(X86_CC_AE, r0, r1, r2);
2075 _gei_u(jit_state_t *_jit, jit_int32_t r0, jit_int32_t r1, jit_word_t i0)
2078 ci(X86_CC_AE, r0, r1, i0);
2080 ci0(X86_CC_NB, r0, r1);
2084 _gtr(jit_state_t *_jit, jit_int32_t r0, jit_int32_t r1, jit_int32_t r2)
2089 cr(X86_CC_G, r0, r1, r2);
2093 _gtr_u(jit_state_t *_jit, jit_int32_t r0, jit_int32_t r1, jit_int32_t r2)
2098 cr(X86_CC_A, r0, r1, r2);
2102 _gti_u(jit_state_t *_jit, jit_int32_t r0, jit_int32_t r1, jit_word_t i0)
2105 ci(X86_CC_A, r0, r1, i0);
2107 ci0(X86_CC_NE, r0, r1);
2111 _ner(jit_state_t *_jit, jit_int32_t r0, jit_int32_t r1, jit_int32_t r2)
2116 cr(X86_CC_NE, r0, r1, r2);
2120 _nei(jit_state_t *_jit, jit_int32_t r0, jit_int32_t r1, jit_word_t i0)
2123 ci(X86_CC_NE, r0, r1, i0);
2125 ci0(X86_CC_NE, r0, r1);
2129 _movr(jit_state_t *_jit, jit_int32_t r0, jit_int32_t r1)
2132 rex(0, 1, r1, _NOREG, r0);
2134 ic(0xc0 | (r1 << 3) | r7(r0));
2139 _imovi(jit_state_t *_jit, jit_int32_t r0, jit_word_t i0)
2143 if (fits_uint32_p(i0)) {
2145 rex(0, 0, _NOREG, _NOREG, r0);
2151 rex(0, 1, _NOREG, _NOREG, r0);
2163 _movi(jit_state_t *_jit, jit_int32_t r0, jit_word_t i0)
2172 _movi_p(jit_state_t *_jit, jit_int32_t r0, jit_word_t i0)
2174 rex(0, WIDE, _NOREG, _NOREG, r0);
2177 return (_jit->pc.w);
2181 _movcr(jit_state_t *_jit, jit_int32_t r0, jit_int32_t r1)
2183 rex(0, WIDE, r0, _NOREG, r1);
2186 mrm(0x03, r7(r0), r7(r1));
2190 _movcr_u(jit_state_t *_jit, jit_int32_t r0, jit_int32_t r1)
2192 rex(0, WIDE, r0, _NOREG, r1);
2195 mrm(0x03, r7(r0), r7(r1));
2199 _movsr(jit_state_t *_jit, jit_int32_t r0, jit_int32_t r1)
2201 rex(0, WIDE, r0, _NOREG, r1);
2204 mrm(0x03, r7(r0), r7(r1));
2208 _movsr_u(jit_state_t *_jit, jit_int32_t r0, jit_int32_t r1)
2210 rex(0, WIDE, r0, _NOREG, r1);
2213 mrm(0x03, r7(r0), r7(r1));
2218 _movir(jit_state_t *_jit, jit_int32_t r0, jit_int32_t r1)
2220 rex(0, 1, r0, _NOREG, r1);
2222 mrm(0x03, r7(r0), r7(r1));
2226 _movir_u(jit_state_t *_jit, jit_int32_t r0, jit_int32_t r1)
2228 rex(0, 0, r1, _NOREG, r0);
2230 ic(0xc0 | (r1 << 3) | r7(r0));
2235 _htonr_us(jit_state_t *_jit, jit_int32_t r0, jit_int32_t r1)
2239 rex(0, 0, _NOREG, _NOREG, r0);
2241 mrm(0x03, X86_ROR, r7(r0));
2246 _htonr_ui(jit_state_t *_jit, jit_int32_t r0, jit_int32_t r1)
2249 rex(0, 0, _NOREG, _NOREG, r0);
2254 #if __X64 && !__X64_32
2256 _htonr_ul(jit_state_t *_jit, jit_int32_t r0, jit_int32_t r1)
2259 rex(0, 1, _NOREG, _NOREG, r0);
2266 _extr_c(jit_state_t *_jit, jit_int32_t r0, jit_int32_t r1)
2272 reg = jit_get_reg(jit_class_gpr|jit_class_rg8);
2280 _extr_uc(jit_state_t *_jit, jit_int32_t r0, jit_int32_t r1)
2286 reg = jit_get_reg(jit_class_gpr|jit_class_rg8);
2288 movcr_u(r0, rn(reg));
2294 _ldr_c(jit_state_t *_jit, jit_int32_t r0, jit_int32_t r1)
2296 rex(0, WIDE, r0, _NOREG, r1);
2299 rx(r0, 0, r1, _NOREG, _SCL1);
2303 _ldi_c(jit_state_t *_jit, jit_int32_t r0, jit_word_t i0)
2306 if (can_sign_extend_int_p(i0)) {
2307 rex(0, WIDE, r0, _NOREG, _NOREG);
2310 rx(r0, i0, _NOREG, _NOREG, _SCL1);
2313 reg = jit_get_reg(jit_class_gpr);
2321 _ldr_uc(jit_state_t *_jit, jit_int32_t r0, jit_int32_t r1)
2323 rex(0, WIDE, r0, _NOREG, r1);
2326 rx(r0, 0, r1, _NOREG, _SCL1);
2330 _ldi_uc(jit_state_t *_jit, jit_int32_t r0, jit_word_t i0)
2333 if (can_sign_extend_int_p(i0)) {
2334 rex(0, WIDE, r0, _NOREG, _NOREG);
2337 rx(r0, i0, _NOREG, _NOREG, _SCL1);
2340 reg = jit_get_reg(jit_class_gpr);
2342 ldr_uc(r0, rn(reg));
2348 _ldr_s(jit_state_t *_jit, jit_int32_t r0, jit_int32_t r1)
2350 rex(0, WIDE, r0, _NOREG, r1);
2353 rx(r0, 0, r1, _NOREG, _SCL1);
2357 _ldi_s(jit_state_t *_jit, jit_int32_t r0, jit_word_t i0)
2360 if (can_sign_extend_int_p(i0)) {
2361 rex(0, WIDE, r0, _NOREG, _NOREG);
2364 rx(r0, i0, _NOREG, _NOREG, _SCL1);
2367 reg = jit_get_reg(jit_class_gpr);
2375 _ldr_us(jit_state_t *_jit, jit_int32_t r0, jit_int32_t r1)
2377 rex(0, WIDE, r0, _NOREG, r1);
2380 rx(r0, 0, r1, _NOREG, _SCL1);
2384 _ldi_us(jit_state_t *_jit, jit_int32_t r0, jit_word_t i0)
2387 if (can_sign_extend_int_p(i0)) {
2388 rex(0, WIDE, r0, _NOREG, _NOREG);
2391 rx(r0, i0, _NOREG, _NOREG, _SCL1);
2394 reg = jit_get_reg(jit_class_gpr);
2396 ldr_us(r0, rn(reg));
2401 #if __X32 || !__X64_32
2403 _ldr_i(jit_state_t *_jit, jit_int32_t r0, jit_int32_t r1)
2406 rex(0, WIDE, r0, _NOREG, r1);
2411 rx(r0, 0, r1, _NOREG, _SCL1);
2415 _ldi_i(jit_state_t *_jit, jit_int32_t r0, jit_word_t i0)
2418 if (can_sign_extend_int_p(i0)) {
2420 rex(0, WIDE, r0, _NOREG, _NOREG);
2425 rx(r0, i0, _NOREG, _NOREG, _SCL1);
2428 reg = jit_get_reg(jit_class_gpr);
2438 _ldr_ui(jit_state_t *_jit, jit_int32_t r0, jit_int32_t r1)
2440 rex(0, 0, r0, _NOREG, r1);
2442 rx(r0, 0, r1, _NOREG, _SCL1);
2446 _ldi_ui(jit_state_t *_jit, jit_int32_t r0, jit_word_t i0)
2449 if (can_sign_extend_int_p(i0)) {
2450 rex(0, 0, r0, _NOREG, _NOREG);
2452 rx(r0, i0, _NOREG, _NOREG, _SCL1);
2455 reg = jit_get_reg(jit_class_gpr);
2457 ldr_ui(r0, rn(reg));
2464 _ldr_l(jit_state_t *_jit, jit_int32_t r0, jit_int32_t r1)
2466 rex(0, 1, r0, _NOREG, r1);
2468 rx(r0, 0, r1, _NOREG, _SCL1);
2472 _ldi_l(jit_state_t *_jit, jit_int32_t r0, jit_word_t i0)
2475 if (can_sign_extend_int_p(i0)) {
2476 rex(0, 1, r0, _NOREG, _NOREG);
2478 rx(r0, i0, _NOREG, _NOREG, _SCL1);
2481 reg = jit_get_reg(jit_class_gpr);
2491 _ldxr_c(jit_state_t *_jit, jit_int32_t r0, jit_int32_t r1, jit_int32_t r2)
2497 rex(0, WIDE, r0, r1, r2);
2500 rx(r0, 0, r2, r1, _SCL1);
2505 _ldxi_c(jit_state_t *_jit, jit_int32_t r0, jit_int32_t r1, jit_word_t i0)
2508 if (can_sign_extend_int_p(i0)) {
2509 rex(0, WIDE, r0, _NOREG, r1);
2512 rx(r0, i0, r1, _NOREG, _SCL1);
2515 reg = jit_get_reg(jit_class_gpr);
2517 ldxr_c(r0, r1, rn(reg));
2523 _ldxr_uc(jit_state_t *_jit, jit_int32_t r0, jit_int32_t r1, jit_int32_t r2)
2529 rex(0, WIDE, r0, r1, r2);
2532 rx(r0, 0, r2, r1, _SCL1);
2537 _ldxi_uc(jit_state_t *_jit, jit_int32_t r0, jit_int32_t r1, jit_word_t i0)
2540 if (can_sign_extend_int_p(i0)) {
2541 rex(0, WIDE, r0, _NOREG, r1);
2544 rx(r0, i0, r1, _NOREG, _SCL1);
2547 reg = jit_get_reg(jit_class_gpr);
2549 ldxr_uc(r0, r1, rn(reg));
2555 _ldxr_s(jit_state_t *_jit, jit_int32_t r0, jit_int32_t r1, jit_int32_t r2)
2561 rex(0, WIDE, r0, r1, r2);
2564 rx(r0, 0, r2, r1, _SCL1);
2569 _ldxi_s(jit_state_t *_jit, jit_int32_t r0, jit_int32_t r1, jit_word_t i0)
2572 if (can_sign_extend_int_p(i0)) {
2573 rex(0, WIDE, r0, _NOREG, r1);
2576 rx(r0, i0, r1, _NOREG, _SCL1);
2579 reg = jit_get_reg(jit_class_gpr);
2581 ldxr_s(r0, r1, rn(reg));
2587 _ldxr_us(jit_state_t *_jit, jit_int32_t r0, jit_int32_t r1, jit_int32_t r2)
2593 rex(0, WIDE, r0, r1, r2);
2596 rx(r0, 0, r2, r1, _SCL1);
2601 _ldxi_us(jit_state_t *_jit, jit_int32_t r0, jit_int32_t r1, jit_word_t i0)
2604 if (can_sign_extend_int_p(i0)) {
2605 rex(0, WIDE, r0, _NOREG, r1);
2608 rx(r0, i0, r1, _NOREG, _SCL1);
2611 reg = jit_get_reg(jit_class_gpr);
2613 ldxr_us(r0, r1, rn(reg));
2618 #if __X64 || !__X64_32
2620 _ldxr_i(jit_state_t *_jit, jit_int32_t r0, jit_int32_t r1, jit_int32_t r2)
2623 rex(0, WIDE, r0, r1, r2);
2628 rx(r0, 0, r2, r1, _SCL1);
2632 _ldxi_i(jit_state_t *_jit, jit_int32_t r0, jit_int32_t r1, jit_word_t i0)
2635 if (can_sign_extend_int_p(i0)) {
2637 rex(0, WIDE, r0, _NOREG, r1);
2642 rx(r0, i0, r1, _NOREG, _SCL1);
2645 reg = jit_get_reg(jit_class_gpr);
2647 ldxr_i(r0, r1, rn(reg));
2655 _ldxr_ui(jit_state_t *_jit, jit_int32_t r0, jit_int32_t r1, jit_int32_t r2)
2659 /* to avoid confusion with macro renames */
2660 _ldr_ui(_jit, r0, r0);
2662 rex(0, 0, r0, r1, r2);
2664 rx(r0, 0, r2, r1, _SCL1);
2669 _ldxi_ui(jit_state_t *_jit, jit_int32_t r0, jit_int32_t r1, jit_word_t i0)
2672 if (can_sign_extend_int_p(i0)) {
2673 rex(0, 0, r0, _NOREG, r1);
2675 rx(r0, i0, r1, _NOREG, _SCL1);
2678 reg = jit_get_reg(jit_class_gpr);
2680 ldxr_ui(r0, r1, rn(reg));
2687 _ldxr_l(jit_state_t *_jit, jit_int32_t r0, jit_int32_t r1, jit_int32_t r2)
2689 rex(0, 1, r0, r1, r2);
2691 rx(r0, 0, r2, r1, _SCL1);
2695 _ldxi_l(jit_state_t *_jit, jit_int32_t r0, jit_int32_t r1, jit_word_t i0)
2698 if (can_sign_extend_int_p(i0)) {
2699 rex(0, 1, r0, _NOREG, r1);
2701 rx(r0, i0, r1, _NOREG, _SCL1);
2704 reg = jit_get_reg(jit_class_gpr);
2706 ldxr_l(r0, r1, rn(reg));
2714 _str_c(jit_state_t *_jit, jit_int32_t r0, jit_int32_t r1)
2718 rex(0, 0, r1, _NOREG, r0);
2720 rx(r1, 0, r0, _NOREG, _SCL1);
2723 reg = jit_get_reg(jit_class_gpr|jit_class_rg8);
2725 rex(0, 0, rn(reg), _NOREG, r0);
2727 rx(rn(reg), 0, r0, _NOREG, _SCL1);
2733 _sti_c(jit_state_t *_jit, jit_word_t i0, jit_int32_t r0)
2736 if (can_sign_extend_int_p(i0)) {
2738 rex(0, 0, r0, _NOREG, _NOREG);
2740 rx(r0, i0, _NOREG, _NOREG, _SCL1);
2743 reg = jit_get_reg(jit_class_gpr|jit_class_rg8);
2745 rex(0, 0, rn(reg), _NOREG, _NOREG);
2747 rx(rn(reg), i0, _NOREG, _NOREG, _SCL1);
2752 reg = jit_get_reg(jit_class_gpr);
2760 _str_s(jit_state_t *_jit, jit_int32_t r0, jit_int32_t r1)
2763 rex(0, 0, r1, _NOREG, r0);
2765 rx(r1, 0, r0, _NOREG, _SCL1);
2769 _sti_s(jit_state_t *_jit, jit_word_t i0, jit_int32_t r0)
2772 if (can_sign_extend_int_p(i0)) {
2774 rex(0, 0, r0, _NOREG, _NOREG);
2776 rx(r0, i0, _NOREG, _NOREG, _SCL1);
2779 reg = jit_get_reg(jit_class_gpr);
2787 _str_i(jit_state_t *_jit, jit_int32_t r0, jit_int32_t r1)
2789 rex(0, 0, r1, _NOREG, r0);
2791 rx(r1, 0, r0, _NOREG, _SCL1);
2795 _sti_i(jit_state_t *_jit, jit_word_t i0, jit_int32_t r0)
2798 if (can_sign_extend_int_p(i0)) {
2799 rex(0, 0, r0, _NOREG, _NOREG);
2801 rx(r0, i0, _NOREG, _NOREG, _SCL1);
2804 reg = jit_get_reg(jit_class_gpr);
2811 #if __X64 && !__X64_32
2813 _str_l(jit_state_t *_jit, jit_int32_t r0, jit_int32_t r1)
2815 rex(0, 1, r1, _NOREG, r0);
2817 rx(r1, 0, r0, _NOREG, _SCL1);
2821 _sti_l(jit_state_t *_jit, jit_word_t i0, jit_int32_t r0)
2824 if (can_sign_extend_int_p(i0)) {
2825 rex(0, 1, r0, _NOREG, _NOREG);
2827 rx(r0, i0, _NOREG, _NOREG, _SCL1);
2830 reg = jit_get_reg(jit_class_gpr);
2839 _stxr_c(jit_state_t *_jit, jit_int32_t r0, jit_int32_t r1, jit_int32_t r2)
2843 reg = jit_get_reg(jit_class_gpr);
2844 addr(rn(reg), r0, r1);
2849 rex(0, 0, r2, r1, r0);
2851 rx(r2, 0, r0, r1, _SCL1);
2854 reg = jit_get_reg(jit_class_gpr|jit_class_rg8);
2856 rex(0, 0, rn(reg), r1, r0);
2858 rx(rn(reg), 0, r0, r1, _SCL1);
2865 _stxi_c(jit_state_t *_jit, jit_word_t i0, jit_int32_t r0, jit_int32_t r1)
2868 if (can_sign_extend_int_p(i0)) {
2870 rex(0, 0, r1, _NOREG, r0);
2872 rx(r1, i0, r0, _NOREG, _SCL1);
2875 reg = jit_get_reg(jit_class_gpr|jit_class_rg8);
2877 rex(0, 0, rn(reg), _NOREG, r0);
2879 rx(rn(reg), i0, r0, _NOREG, _SCL1);
2884 reg = jit_get_reg(jit_class_gpr);
2886 stxr_c(rn(reg), r0, r1);
2892 _stxr_s(jit_state_t *_jit, jit_int32_t r0, jit_int32_t r1, jit_int32_t r2)
2896 reg = jit_get_reg(jit_class_gpr);
2897 addr(rn(reg), r0, r1);
2902 rex(0, 0, r2, r1, r0);
2904 rx(r2, 0, r0, r1, _SCL1);
2909 _stxi_s(jit_state_t *_jit, jit_word_t i0, jit_int32_t r0, jit_int32_t r1)
2912 if (can_sign_extend_int_p(i0)) {
2914 rex(0, 0, r1, _NOREG, r0);
2916 rx(r1, i0, r0, _NOREG, _SCL1);
2919 reg = jit_get_reg(jit_class_gpr);
2921 stxr_s(rn(reg), r0, r1);
2927 _stxr_i(jit_state_t *_jit, jit_int32_t r0, jit_int32_t r1, jit_int32_t r2)
2931 reg = jit_get_reg(jit_class_gpr);
2932 addr(rn(reg), r0, r1);
2936 rex(0, 0, r2, r1, r0);
2938 rx(r2, 0, r0, r1, _SCL1);
2943 _stxi_i(jit_state_t *_jit, jit_word_t i0, jit_int32_t r0, jit_int32_t r1)
2946 if (can_sign_extend_int_p(i0)) {
2947 rex(0, 0, r1, _NOREG, r0);
2949 rx(r1, i0, r0, _NOREG, _SCL1);
2952 reg = jit_get_reg(jit_class_gpr);
2954 stxr_i(rn(reg), r0, r1);
2959 #if __X64 && !__X64_32
2961 _stxr_l(jit_state_t *_jit, jit_int32_t r0, jit_int32_t r1, jit_int32_t r2)
2963 rex(0, 1, r2, r1, r0);
2965 rx(r2, 0, r0, r1, _SCL1);
2969 _stxi_l(jit_state_t *_jit, jit_word_t i0, jit_int32_t r0, jit_int32_t r1)
2972 if (can_sign_extend_int_p(i0)) {
2973 rex(0, 1, r1, _NOREG, r0);
2975 rx(r1, i0, r0, _NOREG, _SCL1);
2978 reg = jit_get_reg(jit_class_gpr);
2980 stxr_l(rn(reg), r0, r1);
2987 _jccs(jit_state_t *_jit, jit_int32_t code, jit_word_t i0)
2991 w = i0 - (_jit->pc.w + 1);
2996 _jcc(jit_state_t *_jit, jit_int32_t code, jit_word_t i0)
3001 w = i0 - (_jit->pc.w + 4);
3006 _jcr(jit_state_t *_jit,
3007 jit_int32_t code, jit_word_t i0, jit_int32_t r0, jit_int32_t r1)
3009 alur(X86_CMP, r0, r1);
3014 _jci(jit_state_t *_jit,
3015 jit_int32_t code, jit_word_t i0, jit_int32_t r0, jit_word_t i1)
3017 alui(X86_CMP, r0, i1);
3022 _jci0(jit_state_t *_jit, jit_int32_t code, jit_word_t i0, jit_int32_t r0)
3029 _bltr(jit_state_t *_jit, jit_word_t i0, jit_int32_t r0, jit_int32_t r1)
3031 jcr(X86_CC_L, i0, r0, r1);
3032 return (_jit->pc.w);
3036 _blti(jit_state_t *_jit, jit_word_t i0, jit_int32_t r0, jit_word_t i1)
3038 if (i1) jci (X86_CC_L, i0, r0, i1);
3039 else jci0(X86_CC_S, i0, r0);
3040 return (_jit->pc.w);
3044 _bltr_u(jit_state_t *_jit, jit_word_t i0, jit_int32_t r0, jit_int32_t r1)
3046 jcr(X86_CC_B, i0, r0, r1);
3047 return (_jit->pc.w);
3051 _blti_u(jit_state_t *_jit, jit_word_t i0, jit_int32_t r0, jit_word_t i1)
3053 if (i1) jci (X86_CC_B, i0, r0, i1);
3054 else jci0(X86_CC_B, i0, r0);
3055 return (_jit->pc.w);
3059 _bler(jit_state_t *_jit, jit_word_t i0, jit_int32_t r0, jit_int32_t r1)
3061 if (r0 == r1) jmpi(i0);
3062 else jcr (X86_CC_LE, i0, r0, r1);
3063 return (_jit->pc.w);
3067 _blei(jit_state_t *_jit, jit_word_t i0, jit_int32_t r0, jit_word_t i1)
3069 if (i1) jci (X86_CC_LE, i0, r0, i1);
3070 else jci0(X86_CC_LE, i0, r0);
3071 return (_jit->pc.w);
3075 _bler_u(jit_state_t *_jit, jit_word_t i0, jit_int32_t r0, jit_int32_t r1)
3077 if (r0 == r1) jmpi(i0);
3078 else jcr (X86_CC_BE, i0, r0, r1);
3079 return (_jit->pc.w);
3083 _blei_u(jit_state_t *_jit, jit_word_t i0, jit_int32_t r0, jit_word_t i1)
3085 if (i1) jci (X86_CC_BE, i0, r0, i1);
3086 else jci0(X86_CC_BE, i0, r0);
3087 return (_jit->pc.w);
3091 _beqr(jit_state_t *_jit, jit_word_t i0, jit_int32_t r0, jit_int32_t r1)
3093 if (r0 == r1) jmpi(i0);
3094 else jcr (X86_CC_E, i0, r0, r1);
3095 return (_jit->pc.w);
3099 _beqi(jit_state_t *_jit, jit_word_t i0, jit_int32_t r0, jit_word_t i1)
3101 if (i1) jci (X86_CC_E, i0, r0, i1);
3102 else jci0(X86_CC_E, i0, r0);
3103 return (_jit->pc.w);
3107 _bger(jit_state_t *_jit, jit_word_t i0, jit_int32_t r0, jit_int32_t r1)
3109 if (r0 == r1) jmpi(i0);
3110 else jcr (X86_CC_GE, i0, r0, r1);
3111 return (_jit->pc.w);
3115 _bgei(jit_state_t *_jit, jit_word_t i0, jit_int32_t r0, jit_word_t i1)
3117 if (i1) jci (X86_CC_GE, i0, r0, i1);
3118 else jci0(X86_CC_NS, i0, r0);
3119 return (_jit->pc.w);
3123 _bger_u(jit_state_t *_jit, jit_word_t i0, jit_int32_t r0, jit_int32_t r1)
3125 if (r0 == r1) jmpi(i0);
3126 else jcr (X86_CC_AE, i0, r0, r1);
3127 return (_jit->pc.w);
3131 _bgei_u(jit_state_t *_jit, jit_word_t i0, jit_int32_t r0, jit_word_t i1)
3133 if (i1) jci (X86_CC_AE, i0, r0, i1);
3135 return (_jit->pc.w);
3139 _bgtr(jit_state_t *_jit, jit_word_t i0, jit_int32_t r0, jit_int32_t r1)
3141 jcr(X86_CC_G, i0, r0, r1);
3142 return (_jit->pc.w);
3146 _bgti(jit_state_t *_jit, jit_word_t i0, jit_int32_t r0, jit_word_t i1)
3148 jci(X86_CC_G, i0, r0, i1);
3149 return (_jit->pc.w);
3153 _bgtr_u(jit_state_t *_jit, jit_word_t i0, jit_int32_t r0, jit_int32_t r1)
3155 jcr(X86_CC_A, i0, r0, r1);
3156 return (_jit->pc.w);
3160 _bgti_u(jit_state_t *_jit, jit_word_t i0, jit_int32_t r0, jit_word_t i1)
3162 if (i1) jci (X86_CC_A, i0, r0, i1);
3163 else jci0(X86_CC_NE, i0, r0);
3164 return (_jit->pc.w);
3168 _bner(jit_state_t *_jit, jit_word_t i0, jit_int32_t r0, jit_int32_t r1)
3170 jcr(X86_CC_NE, i0, r0, r1);
3171 return (_jit->pc.w);
3175 _bnei(jit_state_t *_jit, jit_word_t i0, jit_int32_t r0, jit_word_t i1)
3177 if (i1) jci (X86_CC_NE, i0, r0, i1);
3178 else jci0(X86_CC_NE, i0, r0);
3179 return (_jit->pc.w);
3183 _bmsr(jit_state_t *_jit, jit_word_t i0, jit_int32_t r0, jit_int32_t r1)
3187 return (_jit->pc.w);
3191 _bmsi(jit_state_t *_jit, jit_word_t i0, jit_int32_t r0, jit_word_t i1)
3194 if (can_zero_extend_int_p(i1))
3197 reg = jit_get_reg(jit_class_gpr);
3203 return (_jit->pc.w);
3207 _bmcr(jit_state_t *_jit, jit_word_t i0, jit_int32_t r0, jit_int32_t r1)
3211 return (_jit->pc.w);
3215 _bmci(jit_state_t *_jit, jit_word_t i0, jit_int32_t r0, jit_word_t i1)
3218 if (can_zero_extend_int_p(i1))
3221 reg = jit_get_reg(jit_class_gpr);
3227 return (_jit->pc.w);
3231 _boaddr(jit_state_t *_jit, jit_word_t i0, jit_int32_t r0, jit_int32_t r1)
3235 return (_jit->pc.w);
3239 _boaddi(jit_state_t *_jit, jit_word_t i0, jit_int32_t r0, jit_word_t i1)
3242 if (can_sign_extend_int_p(i1)) {
3245 return (_jit->pc.w);
3247 reg = jit_get_reg(jit_class_gpr|jit_class_nospill);
3250 return (boaddr(i0, r0, rn(reg)));
3254 _boaddr_u(jit_state_t *_jit, jit_word_t i0, jit_int32_t r0, jit_int32_t r1)
3258 return (_jit->pc.w);
3262 _boaddi_u(jit_state_t *_jit, jit_word_t i0, jit_int32_t r0, jit_word_t i1)
3265 if (can_sign_extend_int_p(i1)) {
3268 return (_jit->pc.w);
3270 reg = jit_get_reg(jit_class_gpr|jit_class_nospill);
3273 return (boaddr_u(i0, r0, rn(reg)));
3277 _bxaddr(jit_state_t *_jit, jit_word_t i0, jit_int32_t r0, jit_int32_t r1)
3281 return (_jit->pc.w);
3285 _bxaddi(jit_state_t *_jit, jit_word_t i0, jit_int32_t r0, jit_word_t i1)
3288 if (can_sign_extend_int_p(i1)) {
3291 return (_jit->pc.w);
3293 reg = jit_get_reg(jit_class_gpr|jit_class_nospill);
3296 return (bxaddr(i0, r0, rn(reg)));
3300 _bxaddr_u(jit_state_t *_jit, jit_word_t i0, jit_int32_t r0, jit_int32_t r1)
3304 return (_jit->pc.w);
3308 _bxaddi_u(jit_state_t *_jit, jit_word_t i0, jit_int32_t r0, jit_word_t i1)
3311 if (can_sign_extend_int_p(i1)) {
3314 return (_jit->pc.w);
3316 reg = jit_get_reg(jit_class_gpr|jit_class_nospill);
3319 return (bxaddr_u(i0, r0, rn(reg)));
3323 _bosubr(jit_state_t *_jit, jit_word_t i0, jit_int32_t r0, jit_int32_t r1)
3327 return (_jit->pc.w);
3331 _bosubi(jit_state_t *_jit, jit_word_t i0, jit_int32_t r0, jit_word_t i1)
3334 if (can_sign_extend_int_p(i1)) {
3337 return (_jit->pc.w);
3339 reg = jit_get_reg(jit_class_gpr|jit_class_nospill);
3342 return (bosubr(i0, r0, rn(reg)));
3346 _bosubr_u(jit_state_t *_jit, jit_word_t i0, jit_int32_t r0, jit_int32_t r1)
3350 return (_jit->pc.w);
3354 _bosubi_u(jit_state_t *_jit, jit_word_t i0, jit_int32_t r0, jit_word_t i1)
3357 if (can_sign_extend_int_p(i1)) {
3360 return (_jit->pc.w);
3362 reg = jit_get_reg(jit_class_gpr|jit_class_nospill);
3365 return (bosubr_u(i0, r0, rn(reg)));
3369 _bxsubr(jit_state_t *_jit, jit_word_t i0, jit_int32_t r0, jit_int32_t r1)
3373 return (_jit->pc.w);
3377 _bxsubi(jit_state_t *_jit, jit_word_t i0, jit_int32_t r0, jit_word_t i1)
3380 if (can_sign_extend_int_p(i1)) {
3383 return (_jit->pc.w);
3385 reg = jit_get_reg(jit_class_gpr|jit_class_nospill);
3388 return (bxsubr(i0, r0, rn(reg)));
3392 _bxsubr_u(jit_state_t *_jit, jit_word_t i0, jit_int32_t r0, jit_int32_t r1)
3396 return (_jit->pc.w);
3400 _bxsubi_u(jit_state_t *_jit, jit_word_t i0, jit_int32_t r0, jit_word_t i1)
3403 if (can_sign_extend_int_p(i1)) {
3406 return (_jit->pc.w);
3408 reg = jit_get_reg(jit_class_gpr|jit_class_nospill);
3411 return (bxsubr_u(i0, r0, rn(reg)));
3415 _callr(jit_state_t *_jit, jit_int32_t r0)
3417 rex(0, 0, _NOREG, _NOREG, r0);
3419 mrm(0x03, 0x02, r7(r0));
3423 _calli(jit_state_t *_jit, jit_word_t i0)
3428 w = i0 - (_jit->pc.w + 5);
3429 if ((jit_int32_t)w == w) {
3432 w = i0 - (_jit->pc.w + 4);
3445 _calli_p(jit_state_t *_jit, jit_word_t i0)
3449 reg = jit_get_reg(jit_class_gpr);
3450 word = movi_p(rn(reg), i0);
3458 _jmpr(jit_state_t *_jit, jit_int32_t r0)
3460 rex(0, 0, _NOREG, _NOREG, r0);
3462 mrm(0x03, 0x04, r7(r0));
3466 _jmpi(jit_state_t *_jit, jit_word_t i0)
3471 w = i0 - (_jit->pc.w + 5);
3472 if ((jit_int32_t)w == w) {
3475 w = i0 - (_jit->pc.w + 4);
3488 _jmpi_p(jit_state_t *_jit, jit_word_t i0)
3492 reg = jit_get_reg(jit_class_gpr|jit_class_nospill);
3493 word = movi_p(rn(reg), i0);
3501 _jmpsi(jit_state_t *_jit, jit_uint8_t i0)
3508 _prolog(jit_state_t *_jit, jit_node_t *node)
3511 if (_jitc->function->define_frame || _jitc->function->assume_frame) {
3512 jit_int32_t frame = -_jitc->function->frame;
3513 assert(_jitc->function->self.aoff >= frame);
3514 if (_jitc->function->assume_frame)
3516 _jitc->function->self.aoff = frame;
3518 if (_jitc->function->allocar)
3519 _jitc->function->self.aoff &= -16;
3520 #if __X64 && (__CYGWIN__ || _WIN32)
3521 _jitc->function->stack = (((/* first 32 bytes must be allocated */
3522 (_jitc->function->self.alen > 32 ?
3523 _jitc->function->self.alen : 32) -
3524 /* align stack at 16 bytes */
3525 _jitc->function->self.aoff) + 15) & -16) +
3528 _jitc->function->stack = (((_jitc->function->self.alen -
3529 _jitc->function->self.aoff) + 15) & -16) +
3532 subi(_RSP_REGNO, _RSP_REGNO, stack_framesize - REAL_WORDSIZE);
3533 /* callee save registers */
3535 if (jit_regset_tstbit(&_jitc->function->regset, _RDI))
3536 stxi(12, _RSP_REGNO, _RDI_REGNO);
3537 if (jit_regset_tstbit(&_jitc->function->regset, _RSI))
3538 stxi( 8, _RSP_REGNO, _RSI_REGNO);
3539 if (jit_regset_tstbit(&_jitc->function->regset, _RBX))
3540 stxi( 4, _RSP_REGNO, _RBX_REGNO);
3542 # if __CYGWIN__ || _WIN32
3543 if (jit_regset_tstbit(&_jitc->function->regset, _XMM15))
3544 sse_stxi_d(136, _RSP_REGNO, _XMM15_REGNO);
3545 if (jit_regset_tstbit(&_jitc->function->regset, _XMM14))
3546 sse_stxi_d(128, _RSP_REGNO, _XMM14_REGNO);
3547 if (jit_regset_tstbit(&_jitc->function->regset, _XMM13))
3548 sse_stxi_d(120, _RSP_REGNO, _XMM13_REGNO);
3549 if (jit_regset_tstbit(&_jitc->function->regset, _XMM12))
3550 sse_stxi_d(112, _RSP_REGNO, _XMM12_REGNO);
3551 if (jit_regset_tstbit(&_jitc->function->regset, _XMM11))
3552 sse_stxi_d(104, _RSP_REGNO, _XMM11_REGNO);
3553 if (jit_regset_tstbit(&_jitc->function->regset, _XMM10))
3554 sse_stxi_d(96, _RSP_REGNO, _XMM10_REGNO);
3555 if (jit_regset_tstbit(&_jitc->function->regset, _XMM9))
3556 sse_stxi_d(88, _RSP_REGNO, _XMM9_REGNO);
3557 if (jit_regset_tstbit(&_jitc->function->regset, _XMM8))
3558 sse_stxi_d(80, _RSP_REGNO, _XMM8_REGNO);
3559 if (jit_regset_tstbit(&_jitc->function->regset, _XMM7))
3560 sse_stxi_d(72, _RSP_REGNO, _XMM7_REGNO);
3561 if (jit_regset_tstbit(&_jitc->function->regset, _XMM6))
3562 sse_stxi_d(64, _RSP_REGNO, _XMM6_REGNO);
3563 if (jit_regset_tstbit(&_jitc->function->regset, _R15))
3564 stxi(56, _RSP_REGNO, _R15_REGNO);
3565 if (jit_regset_tstbit(&_jitc->function->regset, _R14))
3566 stxi(48, _RSP_REGNO, _R14_REGNO);
3567 if (jit_regset_tstbit(&_jitc->function->regset, _R13))
3568 stxi(40, _RSP_REGNO, _R13_REGNO);
3569 if (jit_regset_tstbit(&_jitc->function->regset, _R12))
3570 stxi(32, _RSP_REGNO, _R12_REGNO);
3571 if (jit_regset_tstbit(&_jitc->function->regset, _RSI))
3572 stxi(24, _RSP_REGNO, _RSI_REGNO);
3573 if (jit_regset_tstbit(&_jitc->function->regset, _RDI))
3574 stxi(16, _RSP_REGNO, _RDI_REGNO);
3575 if (jit_regset_tstbit(&_jitc->function->regset, _RBX))
3576 stxi( 8, _RSP_REGNO, _RBX_REGNO);
3578 if (jit_regset_tstbit(&_jitc->function->regset, _RBX))
3579 stxi(40, _RSP_REGNO, _RBX_REGNO);
3580 if (jit_regset_tstbit(&_jitc->function->regset, _R12))
3581 stxi(32, _RSP_REGNO, _R12_REGNO);
3582 if (jit_regset_tstbit(&_jitc->function->regset, _R13))
3583 stxi(24, _RSP_REGNO, _R13_REGNO);
3584 if (jit_regset_tstbit(&_jitc->function->regset, _R14))
3585 stxi(16, _RSP_REGNO, _R14_REGNO);
3586 if (jit_regset_tstbit(&_jitc->function->regset, _R15))
3587 stxi( 8, _RSP_REGNO, _R15_REGNO);
3590 stxi(0, _RSP_REGNO, _RBP_REGNO);
3591 movr(_RBP_REGNO, _RSP_REGNO);
3594 subi(_RSP_REGNO, _RSP_REGNO, _jitc->function->stack);
3595 if (_jitc->function->allocar) {
3596 reg = jit_get_reg(jit_class_gpr);
3597 movi(rn(reg), _jitc->function->self.aoff);
3598 stxi_i(_jitc->function->aoffoff, _RBP_REGNO, rn(reg));
3602 #if __X64 && !(__CYGWIN__ || _WIN32)
3603 if (_jitc->function->self.call & jit_call_varargs) {
3604 jit_word_t nofp_code;
3606 /* Save gp registers in the save area, if any is a vararg */
3607 for (reg = first_gp_from_offset(_jitc->function->vagp);
3608 jit_arg_reg_p(reg); ++reg)
3609 stxi(_jitc->function->vaoff + first_gp_offset +
3610 reg * 8, _RBP_REGNO, rn(JIT_RA0 - reg));
3612 reg = first_fp_from_offset(_jitc->function->vafp);
3613 if (jit_arg_f_reg_p(reg)) {
3614 /* Skip over if no float registers were passed as argument */
3619 nofp_code = _jit->pc.w;
3621 /* Save fp registers in the save area, if any is a vararg */
3622 /* Note that the full 16 byte xmm is not saved, because
3623 * lightning only handles float and double, and, while
3624 * attempting to provide a va_list compatible pointer as
3625 * jit_va_start return, does not guarantee it (on all ports). */
3626 for (; jit_arg_f_reg_p(reg); ++reg)
3627 sse_stxi_d(_jitc->function->vaoff + first_fp_offset +
3628 reg * va_fp_increment, _RBP_REGNO, rn(_XMM0 - reg));
3630 patch_rel_char(nofp_code, _jit->pc.w);
3637 _epilog(jit_state_t *_jit, jit_node_t *node)
3639 if (_jitc->function->assume_frame)
3641 /* callee save registers */
3642 movr(_RSP_REGNO, _RBP_REGNO);
3644 if (jit_regset_tstbit(&_jitc->function->regset, _RDI))
3645 ldxi(_RDI_REGNO, _RSP_REGNO, 12);
3646 if (jit_regset_tstbit(&_jitc->function->regset, _RSI))
3647 ldxi(_RSI_REGNO, _RSP_REGNO, 8);
3648 if (jit_regset_tstbit(&_jitc->function->regset, _RBX))
3649 ldxi(_RBX_REGNO, _RSP_REGNO, 4);
3651 # if __CYGWIN__ || _WIN32
3652 if (jit_regset_tstbit(&_jitc->function->regset, _XMM15))
3653 sse_ldxi_d(_XMM15_REGNO, _RSP_REGNO, 136);
3654 if (jit_regset_tstbit(&_jitc->function->regset, _XMM14))
3655 sse_ldxi_d(_XMM14_REGNO, _RSP_REGNO, 128);
3656 if (jit_regset_tstbit(&_jitc->function->regset, _XMM13))
3657 sse_ldxi_d(_XMM13_REGNO, _RSP_REGNO, 120);
3658 if (jit_regset_tstbit(&_jitc->function->regset, _XMM12))
3659 sse_ldxi_d(_XMM12_REGNO, _RSP_REGNO, 112);
3660 if (jit_regset_tstbit(&_jitc->function->regset, _XMM11))
3661 sse_ldxi_d(_XMM11_REGNO, _RSP_REGNO, 104);
3662 if (jit_regset_tstbit(&_jitc->function->regset, _XMM10))
3663 sse_ldxi_d(_XMM10_REGNO, _RSP_REGNO, 96);
3664 if (jit_regset_tstbit(&_jitc->function->regset, _XMM9))
3665 sse_ldxi_d(_XMM9_REGNO, _RSP_REGNO, 88);
3666 if (jit_regset_tstbit(&_jitc->function->regset, _XMM8))
3667 sse_ldxi_d(_XMM8_REGNO, _RSP_REGNO, 80);
3668 if (jit_regset_tstbit(&_jitc->function->regset, _XMM7))
3669 sse_ldxi_d(_XMM7_REGNO, _RSP_REGNO, 72);
3670 if (jit_regset_tstbit(&_jitc->function->regset, _XMM6))
3671 sse_ldxi_d(_XMM6_REGNO, _RSP_REGNO, 64);
3672 if (jit_regset_tstbit(&_jitc->function->regset, _R15))
3673 ldxi(_R15_REGNO, _RSP_REGNO, 56);
3674 if (jit_regset_tstbit(&_jitc->function->regset, _R14))
3675 ldxi(_R14_REGNO, _RSP_REGNO, 48);
3676 if (jit_regset_tstbit(&_jitc->function->regset, _R13))
3677 ldxi(_R13_REGNO, _RSP_REGNO, 40);
3678 if (jit_regset_tstbit(&_jitc->function->regset, _R12))
3679 ldxi(_R12_REGNO, _RSP_REGNO, 32);
3680 if (jit_regset_tstbit(&_jitc->function->regset, _RSI))
3681 ldxi(_RSI_REGNO, _RSP_REGNO, 24);
3682 if (jit_regset_tstbit(&_jitc->function->regset, _RDI))
3683 ldxi(_RDI_REGNO, _RSP_REGNO, 16);
3684 if (jit_regset_tstbit(&_jitc->function->regset, _RBX))
3685 ldxi(_RBX_REGNO, _RSP_REGNO, 8);
3687 if (jit_regset_tstbit(&_jitc->function->regset, _RBX))
3688 ldxi(_RBX_REGNO, _RSP_REGNO, 40);
3689 if (jit_regset_tstbit(&_jitc->function->regset, _R12))
3690 ldxi(_R12_REGNO, _RSP_REGNO, 32);
3691 if (jit_regset_tstbit(&_jitc->function->regset, _R13))
3692 ldxi(_R13_REGNO, _RSP_REGNO, 24);
3693 if (jit_regset_tstbit(&_jitc->function->regset, _R14))
3694 ldxi(_R14_REGNO, _RSP_REGNO, 16);
3695 if (jit_regset_tstbit(&_jitc->function->regset, _R15))
3696 ldxi(_R15_REGNO, _RSP_REGNO, 8);
3699 ldxi(_RBP_REGNO, _RSP_REGNO, 0);
3700 addi(_RSP_REGNO, _RSP_REGNO, stack_framesize - REAL_WORDSIZE);
3706 _vastart(jit_state_t *_jit, jit_int32_t r0)
3708 #if __X32 || __CYGWIN__ || _WIN32
3709 assert(_jitc->function->self.call & jit_call_varargs);
3710 addi(r0, _RBP_REGNO, _jitc->function->self.size);
3714 assert(_jitc->function->self.call & jit_call_varargs);
3716 /* Return jit_va_list_t in the register argument */
3717 addi(r0, _RBP_REGNO, _jitc->function->vaoff);
3718 reg = jit_get_reg(jit_class_gpr);
3720 /* Initialize gp offset in the save area. */
3721 movi(rn(reg), _jitc->function->vagp);
3722 stxi_i(offsetof(jit_va_list_t, gpoff), r0, rn(reg));
3724 /* Initialize fp offset in the save area. */
3725 movi(rn(reg), _jitc->function->vafp);
3726 stxi_i(offsetof(jit_va_list_t, fpoff), r0, rn(reg));
3728 /* Initialize overflow pointer to the first stack argument. */
3729 addi(rn(reg), _RBP_REGNO, _jitc->function->self.size);
3730 stxi(offsetof(jit_va_list_t, over), r0, rn(reg));
3732 /* Initialize register save area pointer. */
3733 addi(rn(reg), r0, first_gp_offset);
3734 stxi(offsetof(jit_va_list_t, save), r0, rn(reg));
3741 _vaarg(jit_state_t *_jit, jit_int32_t r0, jit_int32_t r1)
3743 #if __X32 || __CYGWIN__ || _WIN32
3744 assert(_jitc->function->self.call & jit_call_varargs);
3746 addi(r1, r1, va_gp_increment);
3753 assert(_jitc->function->self.call & jit_call_varargs);
3755 rg0 = jit_get_reg(jit_class_gpr);
3756 rg1 = jit_get_reg(jit_class_gpr);
3758 /* Load the gp offset in save area in the first temporary. */
3759 ldxi_i(rn(rg0), r1, offsetof(jit_va_list_t, gpoff));
3761 /* Jump over if there are no remaining arguments in the save area. */
3762 icmpi(rn(rg0), va_gp_max_offset);
3764 ge_code = _jit->pc.w;
3766 /* Load the save area pointer in the second temporary. */
3767 ldxi(rn(rg1), r1, offsetof(jit_va_list_t, save));
3769 /* Load the vararg argument in the first argument. */
3770 ldxr(r0, rn(rg1), rn(rg0));
3772 /* Update the gp offset. */
3773 addi(rn(rg0), rn(rg0), 8);
3774 stxi_i(offsetof(jit_va_list_t, gpoff), r1, rn(rg0));
3776 /* Will only need one temporary register below. */
3779 /* Jump over overflow code. */
3781 lt_code = _jit->pc.w;
3783 /* Where to land if argument is in overflow area. */
3784 patch_rel_char(ge_code, _jit->pc.w);
3786 /* Load overflow pointer. */
3787 ldxi(rn(rg0), r1, offsetof(jit_va_list_t, over));
3789 /* Load argument. */
3792 /* Update overflow pointer. */
3793 addi(rn(rg0), rn(rg0), va_gp_increment);
3794 stxi(offsetof(jit_va_list_t, over), r1, rn(rg0));
3796 /* Where to land if argument is in save area. */
3797 patch_rel_char(lt_code, _jit->pc.w);
3803 /* The x87 boolean argument tells if will put the result in a x87
3804 * register if non false, in a sse register otherwise. */
3806 _vaarg_d(jit_state_t *_jit, jit_int32_t r0, jit_int32_t r1, jit_bool_t x87)
3808 #if __X32 || __CYGWIN__ || _WIN32
3809 assert(_jitc->function->self.call & jit_call_varargs);
3821 assert(_jitc->function->self.call & jit_call_varargs);
3823 rg0 = jit_get_reg(jit_class_gpr);
3824 rg1 = jit_get_reg(jit_class_gpr);
3826 /* Load the fp offset in save area in the first temporary. */
3827 ldxi_i(rn(rg0), r1, offsetof(jit_va_list_t, fpoff));
3829 /* Jump over if there are no remaining arguments in the save area. */
3830 icmpi(rn(rg0), va_fp_max_offset);
3832 ge_code = _jit->pc.w;
3834 /* Load the save area pointer in the second temporary. */
3835 ldxi(rn(rg1), r1, offsetof(jit_va_list_t, save));
3837 /* Load the vararg argument in the first argument. */
3839 x87_ldxr_d(r0, rn(rg1), rn(rg0));
3841 sse_ldxr_d(r0, rn(rg1), rn(rg0));
3843 /* Update the fp offset. */
3844 addi(rn(rg0), rn(rg0), va_fp_increment);
3845 stxi_i(offsetof(jit_va_list_t, fpoff), r1, rn(rg0));
3847 /* Will only need one temporary register below. */
3850 /* Jump over overflow code. */
3852 lt_code = _jit->pc.w;
3854 /* Where to land if argument is in overflow area. */
3855 patch_rel_char(ge_code, _jit->pc.w);
3857 /* Load overflow pointer. */
3858 ldxi(rn(rg0), r1, offsetof(jit_va_list_t, over));
3860 /* Load argument. */
3862 x87_ldr_d(r0, rn(rg0));
3864 sse_ldr_d(r0, rn(rg0));
3866 /* Update overflow pointer. */
3867 addi(rn(rg0), rn(rg0), 8);
3868 stxi(offsetof(jit_va_list_t, over), r1, rn(rg0));
3870 /* Where to land if argument is in save area. */
3871 patch_rel_char(lt_code, _jit->pc.w);
3878 _patch_at(jit_state_t *_jit, jit_node_t *node,
3879 jit_word_t instr, jit_word_t label)
3881 switch (node->code) {
3883 case jit_code_calli:
3887 patch_abs(instr, label);
3890 patch_rel(instr, label);