2 * Copyright (C) 2012-2023 Free Software Foundation, Inc.
4 * This file is part of GNU lightning.
6 * GNU lightning is free software; you can redistribute it and/or modify it
7 * under the terms of the GNU Lesser General Public License as published
8 * by the Free Software Foundation; either version 3, or (at your option)
11 * GNU lightning is distributed in the hope that it will be useful, but
12 * WITHOUT ANY WARRANTY; without even the implied warranty of MERCHANTABILITY
13 * or FITNESS FOR A PARTICULAR PURPOSE. See the GNU Lesser General Public
14 * License for more details.
17 * Paulo Cesar Pereira de Andrade
20 /* avoid using it due to partial stalls */
24 # if __X32 || __X64_32
26 # define ldi(u, v) ldi_i(u, v)
27 # define ldr(u, v) ldr_i(u, v)
28 # define ldxr(u, v, w) ldxr_i(u, v, w)
29 # define ldxi(u, v, w) ldxi_i(u, v, w)
30 # define sti(u, v) sti_i(u, v)
31 # define stxi(u, v, w) stxi_i(u, v, w)
32 # define can_sign_extend_int_p(im) 1
33 # define can_zero_extend_int_p(im) 1
34 # define fits_uint32_p(im) 1
37 # define ldi(u, v) ldi_l(u, v)
38 # define ldr(u, v) ldr_l(u, v)
39 # define ldxr(u, v, w) ldxr_l(u, v, w)
40 # define ldxi(u, v, w) ldxi_l(u, v, w)
41 # define sti(u, v) sti_l(u, v)
42 # define stxi(u, v, w) stxi_l(u, v, w)
43 # define can_sign_extend_int_p(im) \
44 (((im) >= 0 && (long long)(im) <= 0x7fffffffLL) || \
45 ((im) < 0 && (long long)(im) > -0x80000000LL))
46 # define can_zero_extend_int_p(im) \
47 ((im) >= 0 && (im) < 0x80000000LL)
48 # define fits_uint32_p(im) (((im) & 0xffffffff00000000LL) == 0)
50 # if __X32 || __CYGWIN__ || __X64_32 || _WIN32
52 ((rn) >= _RAX_REGNO && (rn) <= _RBX_REGNO)
66 # define _R10_REGNO 10
67 # define _R11_REGNO 11
68 # define _R12_REGNO 12
69 # define _R13_REGNO 13
70 # define _R14_REGNO 14
71 # define _R15_REGNO 15
72 # define r7(reg) ((reg) & 7)
73 # define r8(reg) ((reg) & 15)
79 # define X86_OR 1 << 3
80 # define X86_ADC 2 << 3
81 # define X86_SBB 3 << 3
82 # define X86_AND 4 << 3
83 # define X86_SUB 5 << 3
84 # define X86_XOR 6 << 3
85 # define X86_CMP 7 << 3
100 # define X86_CC_NO 0x1
101 # define X86_CC_NAE 0x2
102 # define X86_CC_B 0x2
103 # define X86_CC_C 0x2
104 # define X86_CC_AE 0x3
105 # define X86_CC_NB 0x3
106 # define X86_CC_NC 0x3
107 # define X86_CC_E 0x4
108 # define X86_CC_Z 0x4
109 # define X86_CC_NE 0x5
110 # define X86_CC_NZ 0x5
111 # define X86_CC_BE 0x6
112 # define X86_CC_NA 0x6
113 # define X86_CC_A 0x7
114 # define X86_CC_NBE 0x7
115 # define X86_CC_S 0x8
116 # define X86_CC_NS 0x9
117 # define X86_CC_P 0xa
118 # define X86_CC_PE 0xa
119 # define X86_CC_NP 0xb
120 # define X86_CC_PO 0xb
121 # define X86_CC_L 0xc
122 # define X86_CC_NGE 0xc
123 # define X86_CC_GE 0xd
124 # define X86_CC_NL 0xd
125 # define X86_CC_LE 0xe
126 # define X86_CC_NG 0xe
127 # define X86_CC_G 0xf
128 # define X86_CC_NLE 0xf
129 # define mrm(md, r, m) *_jit->pc.uc++ = (md<<6) | (r<<3) | m
130 # define sib(sc, i, b) *_jit->pc.uc++ = (sc<<6) | (i<<3) | b
131 # define ic(c) *_jit->pc.uc++ = c
132 # define is(s) *_jit->pc.us++ = s
133 # define ii(i) *_jit->pc.ui++ = i
134 # if __X64 && !__X64_32
135 # define il(l) *_jit->pc.ul++ = l
139 # define rex(l, w, r, x, b) _rex(_jit, l, w, r, x, b)
141 _rex(jit_state_t*,jit_int32_t,jit_int32_t,jit_int32_t,jit_int32_t,jit_int32_t);
142 # define rx(rd, md, rb, ri, ms) _rx(_jit, rd, md, rb, ri, ms)
144 _rx(jit_state_t*,jit_int32_t,jit_int32_t,jit_int32_t,jit_int32_t,jit_int32_t);
145 # define nop(n) _nop(_jit, n)
146 static void _nop(jit_state_t*, jit_int32_t);
147 # define emms() is(0x770f)
148 # define lea(md, rb, ri, ms, rd) _lea(_jit, md, rb, ri, ms, rd)
150 _lea(jit_state_t*,jit_int32_t,jit_int32_t,jit_int32_t,jit_int32_t,jit_int32_t);
151 # define pushr(r0) _pushr(_jit, r0)
152 static void _pushr(jit_state_t*, jit_int32_t) maybe_unused;
153 # define popr(r0) _popr(_jit, r0)
154 static void _popr(jit_state_t*, jit_int32_t) maybe_unused;
155 # define xchgr(r0, r1) _xchgr(_jit, r0, r1)
156 static void _xchgr(jit_state_t*, jit_int32_t, jit_int32_t);
157 # define testr(r0, r1) _testr(_jit, r0, r1)
158 static void _testr(jit_state_t*, jit_int32_t, jit_int32_t);
159 # define testi(r0, i0) _testi(_jit, r0, i0)
160 static void _testi(jit_state_t*, jit_int32_t, jit_word_t);
161 # define cc(code, r0) _cc(_jit, code, r0)
162 static void _cc(jit_state_t*, jit_int32_t, jit_int32_t);
163 # define icmpr(r0, r1) alur(X86_CMP, r0, r1)
164 # define alur(code, r0, r1) _alur(_jit, code, r0, r1)
165 static void _alur(jit_state_t*, jit_int32_t, jit_int32_t, jit_int32_t);
166 # define icmpi(r0, i0) alui(X86_CMP, r0, i0)
167 # define alui(code, r0, i0) _alui(_jit, code, r0, i0)
168 static void _alui(jit_state_t*, jit_int32_t, jit_int32_t, jit_word_t);
169 # define iaddr(r0, r1) alur(X86_ADD, r0, r1)
170 # define save(r0) _save(_jit, r0)
171 static void _save(jit_state_t*, jit_int32_t);
172 # define load(r0) _load(_jit, r0)
173 static void _load(jit_state_t*, jit_int32_t);
174 # define addr(r0, r1, r2) _addr(_jit, r0, r1, r2)
175 static void _addr(jit_state_t*, jit_int32_t, jit_int32_t, jit_int32_t);
176 # define iaddi(r0, i0) alui(X86_ADD, r0, i0)
177 # define addi(r0, r1, i0) _addi(_jit, r0, r1, i0)
178 static void _addi(jit_state_t*, jit_int32_t, jit_int32_t, jit_word_t);
179 #define addcr(r0, r1, r2) _addcr(_jit, r0, r1, r2)
180 static void _addcr(jit_state_t*, jit_int32_t, jit_int32_t, jit_int32_t);
181 #define addci(r0, r1, i0) _addci(_jit, r0, r1, i0)
182 static void _addci(jit_state_t*, jit_int32_t, jit_int32_t, jit_word_t);
183 # define iaddxr(r0, r1) _iaddxr(_jit, r0, r1)
184 static void _iaddxr(jit_state_t*, jit_int32_t, jit_int32_t);
185 # define addxr(r0, r1, r2) _addxr(_jit, r0, r1, r2)
186 static void _addxr(jit_state_t*, jit_int32_t, jit_int32_t, jit_int32_t);
187 # define iaddxi(r0, i0) alui(X86_ADC, r0, i0)
188 # define addxi(r0, r1, i0) _addxi(_jit, r0, r1, i0)
189 static void _addxi(jit_state_t*, jit_int32_t, jit_int32_t, jit_word_t);
190 # define isubr(r0, r1) alur(X86_SUB, r0, r1)
191 # define subr(r0, r1, r2) _subr(_jit, r0, r1, r2)
192 static void _subr(jit_state_t*, jit_int32_t, jit_int32_t, jit_int32_t);
193 # define isubi(r0, i0) alui(X86_SUB, r0, i0)
194 # define subi(r0, r1, i0) _subi(_jit, r0, r1, i0)
195 static void _subi(jit_state_t*, jit_int32_t, jit_int32_t, jit_word_t);
196 # define subcr(r0, r1, r2) _subcr(_jit, r0, r1, r2)
197 static void _subcr(jit_state_t*,jit_int32_t,jit_int32_t,jit_int32_t);
198 # define subci(r0, r1, i0) _subci(_jit, r0, r1, i0)
199 static void _subci(jit_state_t*,jit_int32_t,jit_int32_t,jit_word_t);
200 # define isubxr(r0, r1) alur(X86_SBB, r0, r1)
201 # define subxr(r0, r1, r2) _subxr(_jit, r0, r1, r2)
202 static void _subxr(jit_state_t*,jit_int32_t,jit_int32_t,jit_int32_t);
203 # define isubxi(r0, i0) alui(X86_SBB, r0, i0)
204 # define subxi(r0, r1, i0) _subxi(_jit, r0, r1, i0)
205 static void _subxi(jit_state_t*,jit_int32_t,jit_int32_t,jit_word_t);
206 # define rsbi(r0, r1, i0) _rsbi(_jit, r0, r1, i0)
207 static void _rsbi(jit_state_t*,jit_int32_t,jit_int32_t,jit_word_t);
208 # define imulr(r0, r1) _imulr(_jit, r0, r1)
209 static void _imulr(jit_state_t*, jit_int32_t, jit_int32_t);
210 # define imuli(r0, r1, i0) _imuli(_jit, r0, r1, i0)
211 static void _imuli(jit_state_t*, jit_int32_t, jit_int32_t, jit_word_t);
212 # define mulr(r0, r1, r2) _mulr(_jit, r0, r1, r2)
213 static void _mulr(jit_state_t*, jit_int32_t, jit_int32_t, jit_int32_t);
214 # define muli(r0, r1, i0) _muli(_jit, r0, r1, i0)
215 static void _muli(jit_state_t*, jit_int32_t, jit_int32_t, jit_word_t);
216 # define umulr(r0) unr(X86_IMUL, r0)
217 # define umulr_u(r0) unr(X86_MUL, r0)
218 # define qmulr(r0, r1, r2, r3) _iqmulr(_jit, r0, r1, r2, r3, 1)
219 # define qmulr_u(r0, r1, r2, r3) _iqmulr(_jit, r0, r1, r2, r3, 0)
220 # define iqmulr(r0, r1, r2, r3, sign) _iqmulr(_jit, r0, r1, r2, r3, sign)
221 static void _iqmulr(jit_state_t*, jit_int32_t, jit_int32_t,
222 jit_int32_t,jit_int32_t, jit_bool_t);
223 # define qmuli(r0, r1, r2, i0) _iqmuli(_jit, r0, r1, r2, i0, 1)
224 # define qmuli_u(r0, r1, r2, i0) _iqmuli(_jit, r0, r1, r2, i0, 0)
225 # define iqmuli(r0, r1, r2, i0, sign) _iqmuli(_jit, r0, r1, r2, i0, sign)
226 static void _iqmuli(jit_state_t*, jit_int32_t, jit_int32_t,
227 jit_int32_t,jit_word_t, jit_bool_t);
228 # define sign_extend_rdx_rax() _sign_extend_rdx_rax(_jit)
229 static void _sign_extend_rdx_rax(jit_state_t*);
230 # define idivr(r0) unr(X86_IDIV, r0)
231 # define idivr_u(r0) unr(X86_DIV, r0)
232 # define divremr(r0, r1, r2, i0, i1) _divremr(_jit, r0, r1, r2, i0, i1)
234 _divremr(jit_state_t*,jit_int32_t,jit_int32_t,jit_int32_t,
235 jit_bool_t,jit_bool_t);
236 # define divremi(r0, r1, i0, i1, i2) _divremi(_jit, r0, r1, i0, i1, i2)
238 _divremi(jit_state_t*,jit_int32_t,jit_int32_t,jit_word_t,jit_bool_t,jit_bool_t);
239 # define divr(r0, r1, r2) divremr(r0, r1, r2, 1, 1)
240 # define divi(r0, r1, i0) divremi(r0, r1, i0, 1, 1)
241 # define divr_u(r0, r1, r2) divremr(r0, r1, r2, 0, 1)
242 # define divi_u(r0, r1, i0) divremi(r0, r1, i0, 0, 1)
243 # define qdivr(r0, r1, r2, r3) _iqdivr(_jit, r0, r1, r2, r3, 1)
244 # define qdivr_u(r0, r1, r2, r3) _iqdivr(_jit, r0, r1, r2, r3, 0)
245 # define iqdivr(r0, r1, r2, r3, sign) _iqdivr(_jit, r0, r1, r2, r3, sign)
246 static void _iqdivr(jit_state_t*, jit_int32_t, jit_int32_t,
247 jit_int32_t,jit_int32_t, jit_bool_t);
248 # define qdivi(r0, r1, r2, i0) _iqdivi(_jit, r0, r1, r2, i0, 1)
249 # define qdivi_u(r0, r1, r2, i0) _iqdivi(_jit, r0, r1, r2, i0, 0)
250 # define iqdivi(r0, r1, r2, i0, sign) _iqdivi(_jit, r0, r1, r2, i0, sign)
251 static void _iqdivi(jit_state_t*, jit_int32_t, jit_int32_t,
252 jit_int32_t,jit_word_t, jit_bool_t);
253 # define remr(r0, r1, r2) divremr(r0, r1, r2, 1, 0)
254 # define remi(r0, r1, i0) divremi(r0, r1, i0, 1, 0)
255 # define remr_u(r0, r1, r2) divremr(r0, r1, r2, 0, 0)
256 # define remi_u(r0, r1, i0) divremi(r0, r1, i0, 0, 0)
257 # define iandr(r0, r1) alur(X86_AND, r0, r1)
258 # define andr(r0, r1, r2) _andr(_jit, r0, r1, r2)
259 static void _andr(jit_state_t*,jit_int32_t,jit_int32_t,jit_int32_t);
260 # define iandi(r0, i0) alui(X86_AND, r0, i0)
261 # define andi(r0, r1, i0) _andi(_jit, r0, r1, i0)
262 static void _andi(jit_state_t*, jit_int32_t,jit_int32_t,jit_word_t);
263 # define iorr(r0, r1) alur(X86_OR, r0, r1)
264 # define orr(r0, r1, r2) _orr(_jit, r0, r1, r2)
265 static void _orr(jit_state_t*, jit_int32_t,jit_int32_t,jit_int32_t);
266 # define iori(r0, i0) alui(X86_OR, r0, i0)
267 # define ori(r0, r1, i0) _ori(_jit, r0, r1, i0)
268 static void _ori(jit_state_t*, jit_int32_t,jit_int32_t,jit_word_t);
269 # define ixorr(r0, r1) alur(X86_XOR, r0, r1)
270 # define xorr(r0, r1, r2) _xorr(_jit, r0, r1, r2)
271 static void _xorr(jit_state_t*, jit_int32_t,jit_int32_t,jit_int32_t);
272 # define ixori(r0, i0) alui(X86_XOR, r0, i0)
273 # define xori(r0, r1, i0) _xori(_jit, r0, r1, i0)
274 static void _xori(jit_state_t*, jit_int32_t,jit_int32_t,jit_word_t);
275 # define irotshr(code, r0) _irotshr(_jit, code, r0)
276 static void _irotshr(jit_state_t*, jit_int32_t, jit_int32_t);
277 # define rotshr(code, r0, r1, r2) _rotshr(_jit, code, r0, r1, r2)
279 _rotshr(jit_state_t*,jit_int32_t,jit_int32_t,jit_int32_t,jit_int32_t);
280 # define irotshi(code, r0, i0) _irotshi(_jit, code, r0, i0)
281 static void _irotshi(jit_state_t*, jit_int32_t, jit_int32_t, jit_word_t);
282 # define rotshi(code, r0, r1, i0) _rotshi(_jit, code, r0, r1, i0)
284 _rotshi(jit_state_t*,jit_int32_t,jit_int32_t,jit_int32_t,jit_word_t);
285 # define lshr(r0, r1, r2) rotshr(X86_SHL, r0, r1, r2)
286 # define lshi(r0, r1, i0) _lshi(_jit, r0, r1, i0)
287 static void _lshi(jit_state_t*, jit_int32_t, jit_int32_t, jit_word_t);
288 # define rshr(r0, r1, r2) rotshr(X86_SAR, r0, r1, r2)
289 # define rshi(r0, r1, i0) rotshi(X86_SAR, r0, r1, i0)
290 # define rshr_u(r0, r1, r2) rotshr(X86_SHR, r0, r1, r2)
291 # define rshi_u(r0, r1, i0) rotshi(X86_SHR, r0, r1, i0)
292 # define unr(code, r0) _unr(_jit, code, r0)
293 static void _unr(jit_state_t*, jit_int32_t, jit_int32_t);
294 # define inegr(r0) unr(X86_NEG, r0)
295 # define negr(r0, r1) _negr(_jit, r0, r1)
296 static void _negr(jit_state_t*, jit_int32_t, jit_int32_t);
297 # define icomr(r0) unr(X86_NOT, r0)
298 # define comr(r0, r1) _comr(_jit, r0, r1)
299 static void _comr(jit_state_t*, jit_int32_t, jit_int32_t);
301 # define incr(r0, r1) _incr(_jit, r0, r1)
302 static void _incr(jit_state_t*, jit_int32_t, jit_int32_t);
303 # define decr(r0, r1) _decr(_jit, r0, r1)
304 static void _decr(jit_state_t*, jit_int32_t, jit_int32_t);
306 # define clor(r0, r1) _clor(_jit, r0, r1)
307 static void _clor(jit_state_t*, jit_int32_t, jit_int32_t);
308 # define clzr(r0, r1) _clzr(_jit, r0, r1)
309 static void _clzr(jit_state_t*, jit_int32_t, jit_int32_t);
310 # define ctor(r0, r1) _ctor(_jit, r0, r1)
311 static void _ctor(jit_state_t*, jit_int32_t, jit_int32_t);
312 # define ctzr(r0, r1) _ctzr(_jit, r0, r1)
313 static void _ctzr(jit_state_t*, jit_int32_t, jit_int32_t);
314 # define cr(code, r0, r1, r2) _cr(_jit, code, r0, r1, r2)
316 _cr(jit_state_t*, jit_int32_t, jit_int32_t, jit_int32_t, jit_int32_t);
317 # define ci(code, r0, r1, i0) _ci(_jit, code, r0, r1, i0)
319 _ci(jit_state_t *_jit, jit_int32_t, jit_int32_t, jit_int32_t, jit_word_t);
320 # define ci0(code, r0, r1) _ci0(_jit, code, r0, r1)
321 static void _ci0(jit_state_t*, jit_int32_t, jit_int32_t, jit_int32_t);
322 # define ltr(r0, r1, r2) _ltr(_jit, r0, r1, r2)
323 static void _ltr(jit_state_t*, jit_int32_t, jit_int32_t, jit_int32_t);
324 # define lti(r0, r1, i0) _lti(_jit, r0, r1, i0)
325 static void _lti(jit_state_t*, jit_int32_t, jit_int32_t, jit_word_t);
326 # define ltr_u(r0, r1, r2) _ltr_u(_jit, r0, r1, r2)
327 static void _ltr_u(jit_state_t*, jit_int32_t, jit_int32_t, jit_int32_t);
328 # define lti_u(r0, r1, i0) ci(X86_CC_B, r0, r1, i0)
329 # define ler(r0, r1, r2) _ler(_jit, r0, r1, r2)
330 static void _ler(jit_state_t*, jit_int32_t, jit_int32_t, jit_int32_t);
331 # define lei(r0, r1, i0) ci(X86_CC_LE, r0, r1, i0)
332 # define ler_u(r0, r1, r2) _ler_u(_jit, r0, r1, r2)
333 static void _ler_u(jit_state_t*, jit_int32_t, jit_int32_t, jit_int32_t);
334 # define lei_u(r0, r1, i0) _lei_u(_jit, r0, r1, i0)
335 static void _lei_u(jit_state_t*, jit_int32_t, jit_int32_t, jit_word_t);
336 # define eqr(r0, r1, r2) _eqr(_jit, r0, r1, r2)
337 static void _eqr(jit_state_t*, jit_int32_t, jit_int32_t, jit_int32_t);
338 # define eqi(r0, r1, i0) _eqi(_jit, r0, r1, i0)
339 static void _eqi(jit_state_t*, jit_int32_t, jit_int32_t, jit_word_t);
340 # define ger(r0, r1, r2) _ger(_jit, r0, r1, r2)
341 static void _ger(jit_state_t*, jit_int32_t, jit_int32_t, jit_int32_t);
342 # define gei(r0, r1, i0) _gei(_jit, r0, r1, i0)
343 static void _gei(jit_state_t*, jit_int32_t, jit_int32_t, jit_word_t);
344 # define ger_u(r0, r1, r2) _ger_u(_jit, r0, r1, r2)
345 static void _ger_u(jit_state_t*, jit_int32_t, jit_int32_t, jit_int32_t);
346 # define gei_u(r0, r1, i0) _gei_u(_jit, r0, r1, i0)
347 static void _gei_u(jit_state_t*, jit_int32_t, jit_int32_t, jit_word_t);
348 # define gtr(r0, r1, r2) _gtr(_jit, r0, r1, r2)
349 static void _gtr(jit_state_t*, jit_int32_t, jit_int32_t, jit_int32_t);
350 # define gti(r0, r1, i0) _ci(_jit, X86_CC_G, r0, r1, i0)
351 # define gtr_u(r0, r1, r2) _gtr_u(_jit, r0, r1, r2)
352 static void _gtr_u(jit_state_t*, jit_int32_t, jit_int32_t, jit_int32_t);
353 # define gti_u(r0, r1, i0) _gti_u(_jit, r0, r1, i0)
354 static void _gti_u(jit_state_t*, jit_int32_t, jit_int32_t, jit_word_t);
355 # define ner(r0, r1, r2) _ner(_jit, r0, r1, r2)
356 static void _ner(jit_state_t*, jit_int32_t, jit_int32_t, jit_int32_t);
357 # define nei(r0, r1, i0) _nei(_jit, r0, r1, i0)
358 static void _nei(jit_state_t*, jit_int32_t, jit_int32_t, jit_word_t);
359 # define movr(r0, r1) _movr(_jit, r0, r1)
360 static void _movr(jit_state_t*, jit_int32_t, jit_int32_t);
361 # define imovi(r0, i0) _imovi(_jit, r0, i0)
362 static void _imovi(jit_state_t*, jit_int32_t, jit_word_t);
363 # define movi(r0, i0) _movi(_jit, r0, i0)
370 _movi(jit_state_t*, jit_int32_t, jit_word_t);
371 # define movi_p(r0, i0) _movi_p(_jit, r0, i0)
372 static jit_word_t _movi_p(jit_state_t*, jit_int32_t, jit_word_t);
373 # define movcr(r0, r1) _movcr(_jit, r0, r1)
374 static void _movcr(jit_state_t*,jit_int32_t,jit_int32_t);
375 # define movcr_u(r0, r1) _movcr_u(_jit, r0, r1)
376 static void _movcr_u(jit_state_t*,jit_int32_t,jit_int32_t);
377 # define movsr(r0, r1) _movsr(_jit, r0, r1)
378 static void _movsr(jit_state_t*,jit_int32_t,jit_int32_t);
379 # define movsr_u(r0, r1) _movsr_u(_jit, r0, r1)
380 static void _movsr_u(jit_state_t*,jit_int32_t,jit_int32_t);
381 # define casx(r0, r1, r2, r3, i0) _casx(_jit, r0, r1, r2, r3, i0)
382 static void _casx(jit_state_t *_jit,jit_int32_t,jit_int32_t,
383 jit_int32_t,jit_int32_t,jit_word_t);
384 #define casr(r0, r1, r2, r3) casx(r0, r1, r2, r3, 0)
385 #define casi(r0, i0, r1, r2) casx(r0, _NOREG, r1, r2, i0)
386 #define movnr(r0, r1, r2) _movnr(_jit, r0, r1, r2)
387 static void _movnr(jit_state_t*, jit_int32_t, jit_int32_t, jit_int32_t);
388 #define movzr(r0, r1, r2) _movzr(_jit, r0, r1, r2)
389 static void _movzr(jit_state_t*, jit_int32_t, jit_int32_t, jit_int32_t);
390 # if __X64 && !__X64_32
391 # define movir(r0, r1) _movir(_jit, r0, r1)
392 static void _movir(jit_state_t*,jit_int32_t,jit_int32_t);
393 # define movir_u(r0, r1) _movir_u(_jit, r0, r1)
394 static void _movir_u(jit_state_t*,jit_int32_t,jit_int32_t);
396 # define bswapr_us(r0, r1) _bswapr_us(_jit, r0, r1)
397 static void _bswapr_us(jit_state_t*,jit_int32_t,jit_int32_t);
398 # define bswapr_ui(r0, r1) _bswapr_ui(_jit, r0, r1)
399 static void _bswapr_ui(jit_state_t*,jit_int32_t,jit_int32_t);
400 # if __X64 && !__X64_32
401 #define bswapr_ul(r0, r1) _bswapr_ul(_jit, r0, r1)
402 static void _bswapr_ul(jit_state_t*,jit_int32_t,jit_int32_t);
404 # define extr_c(r0, r1) _extr_c(_jit, r0, r1)
405 static void _extr_c(jit_state_t*,jit_int32_t,jit_int32_t);
406 # define extr_uc(r0, r1) _extr_uc(_jit, r0, r1)
407 static void _extr_uc(jit_state_t*,jit_int32_t,jit_int32_t);
408 # define extr_s(r0, r1) movsr(r0, r1)
409 # define extr_us(r0, r1) movsr_u(r0, r1)
410 # if __X64 && !__X64_32
411 # define extr_i(r0, r1) movir(r0, r1)
412 # define extr_ui(r0, r1) movir_u(r0, r1)
414 # define ldr_c(r0, r1) _ldr_c(_jit, r0, r1)
415 static void _ldr_c(jit_state_t*, jit_int32_t, jit_int32_t);
416 # define ldi_c(r0, i0) _ldi_c(_jit, r0, i0)
417 static void _ldi_c(jit_state_t*, jit_int32_t, jit_word_t);
418 # define ldr_uc(r0, r1) _ldr_uc(_jit, r0, r1)
419 static void _ldr_uc(jit_state_t*, jit_int32_t, jit_int32_t);
420 # define ldi_uc(r0, i0) _ldi_uc(_jit, r0, i0)
421 static void _ldi_uc(jit_state_t*, jit_int32_t, jit_word_t);
422 # define ldr_s(r0, r1) _ldr_s(_jit, r0, r1)
423 static void _ldr_s(jit_state_t*, jit_int32_t, jit_int32_t);
424 # define ldi_s(r0, i0) _ldi_s(_jit, r0, i0)
425 static void _ldi_s(jit_state_t*, jit_int32_t, jit_word_t);
426 # define ldr_us(r0, r1) _ldr_us(_jit, r0, r1)
427 static void _ldr_us(jit_state_t*, jit_int32_t, jit_int32_t);
428 # define ldi_us(r0, i0) _ldi_us(_jit, r0, i0)
429 static void _ldi_us(jit_state_t*, jit_int32_t, jit_word_t);
430 # if __X32 || !__X64_32
431 # define ldr_i(r0, r1) _ldr_i(_jit, r0, r1)
432 static void _ldr_i(jit_state_t*, jit_int32_t, jit_int32_t);
433 # define ldi_i(r0, i0) _ldi_i(_jit, r0, i0)
434 static void _ldi_i(jit_state_t*, jit_int32_t, jit_word_t);
438 # define ldr_i(r0, r1) _ldr_ui(_jit, r0, r1)
439 # define ldi_i(r0, i0) _ldi_ui(_jit, r0, i0)
441 # define ldr_ui(r0, r1) _ldr_ui(_jit, r0, r1)
442 # define ldi_ui(r0, i0) _ldi_ui(_jit, r0, i0)
444 static void _ldr_ui(jit_state_t*, jit_int32_t, jit_int32_t);
445 static void _ldi_ui(jit_state_t*, jit_int32_t, jit_word_t);
447 # define ldr_l(r0, r1) _ldr_l(_jit, r0, r1)
448 static void _ldr_l(jit_state_t*, jit_int32_t, jit_int32_t);
449 # define ldi_l(r0, i0) _ldi_l(_jit, r0, i0)
450 static void _ldi_l(jit_state_t*, jit_int32_t, jit_word_t);
453 # define ldxr_c(r0, r1, r2) _ldxr_c(_jit, r0, r1, r2)
454 static void _ldxr_c(jit_state_t*, jit_int32_t, jit_int32_t, jit_int32_t);
455 # define ldxi_c(r0, r1, i0) _ldxi_c(_jit, r0, r1, i0)
456 static void _ldxi_c(jit_state_t*, jit_int32_t, jit_int32_t, jit_word_t);
457 # define ldxr_uc(r0, r1, r2) _ldxr_uc(_jit, r0, r1, r2)
458 static void _ldxr_uc(jit_state_t*, jit_int32_t, jit_int32_t, jit_int32_t);
459 # define ldxi_uc(r0, r1, i0) _ldxi_uc(_jit, r0, r1, i0)
460 static void _ldxi_uc(jit_state_t*, jit_int32_t, jit_int32_t, jit_word_t);
461 # define ldxr_s(r0, r1, r2) _ldxr_s(_jit, r0, r1, r2)
462 static void _ldxr_s(jit_state_t*, jit_int32_t, jit_int32_t, jit_int32_t);
463 # define ldxi_s(r0, r1, i0) _ldxi_s(_jit, r0, r1, i0)
464 static void _ldxi_s(jit_state_t*, jit_int32_t, jit_int32_t, jit_word_t);
465 # define ldxr_us(r0, r1, r2) _ldxr_us(_jit, r0, r1, r2)
466 static void _ldxr_us(jit_state_t*, jit_int32_t, jit_int32_t, jit_int32_t);
467 # define ldxi_us(r0, r1, i0) _ldxi_us(_jit, r0, r1, i0)
468 static void _ldxi_us(jit_state_t*, jit_int32_t, jit_int32_t, jit_word_t);
469 # if __X32 || !__X64_32
470 # define ldxr_i(r0, r1, r2) _ldxr_i(_jit, r0, r1, r2)
471 static void _ldxr_i(jit_state_t*, jit_int32_t, jit_int32_t, jit_int32_t);
472 # define ldxi_i(r0, r1, i0) _ldxi_i(_jit, r0, r1, i0)
473 static void _ldxi_i(jit_state_t*, jit_int32_t, jit_int32_t, jit_word_t);
477 # define ldxr_i(r0, r1, r2) _ldxr_ui(_jit, r0, r1, r2)
478 # define ldxi_i(r0, r1, i0) _ldxi_ui(_jit, r0, r1, i0)
480 # define ldxr_ui(r0, r1, r2) _ldxr_ui(_jit, r0, r1, r2)
481 # define ldxi_ui(r0, r1, i0) _ldxi_ui(_jit, r0, r1, i0)
483 static void _ldxr_ui(jit_state_t*, jit_int32_t, jit_int32_t, jit_int32_t);
484 static void _ldxi_ui(jit_state_t*, jit_int32_t, jit_int32_t, jit_word_t);
486 # define ldxr_l(r0, r1, r2) _ldxr_l(_jit, r0, r1, r2)
487 static void _ldxr_l(jit_state_t*, jit_int32_t, jit_int32_t, jit_int32_t);
488 # define ldxi_l(r0, r1, i0) _ldxi_l(_jit, r0, r1, i0)
489 static void _ldxi_l(jit_state_t*, jit_int32_t, jit_int32_t, jit_word_t);
492 # define str_c(r0, r1) _str_c(_jit, r0, r1)
493 static void _str_c(jit_state_t*, jit_int32_t, jit_int32_t);
494 # define sti_c(i0, r0) _sti_c(_jit, i0, r0)
495 static void _sti_c(jit_state_t*, jit_word_t, jit_int32_t);
496 # define str_s(r0, r1) _str_s(_jit, r0, r1)
497 static void _str_s(jit_state_t*, jit_int32_t, jit_int32_t);
498 # define sti_s(i0, r0) _sti_s(_jit, i0, r0)
499 static void _sti_s(jit_state_t*, jit_word_t, jit_int32_t);
500 # define str_i(r0, r1) _str_i(_jit, r0, r1)
501 static void _str_i(jit_state_t*, jit_int32_t, jit_int32_t);
502 # define sti_i(i0, r0) _sti_i(_jit, i0, r0)
503 static void _sti_i(jit_state_t*, jit_word_t, jit_int32_t);
504 # if __X64 && !__X64_32
505 # define str_l(r0, r1) _str_l(_jit, r0, r1)
506 static void _str_l(jit_state_t*, jit_int32_t, jit_int32_t);
507 # define sti_l(i0, r0) _sti_l(_jit, i0, r0)
508 static void _sti_l(jit_state_t*, jit_word_t, jit_int32_t);
510 # define stxr_c(r0, r1, r2) _stxr_c(_jit, r0, r1, r2)
511 static void _stxr_c(jit_state_t*, jit_int32_t, jit_int32_t, jit_int32_t);
512 # define stxi_c(i0, r0, r1) _stxi_c(_jit, i0, r0, r1)
513 static void _stxi_c(jit_state_t*, jit_word_t, jit_int32_t, jit_int32_t);
514 # define stxr_s(r0, r1, r2) _stxr_s(_jit, r0, r1, r2)
515 static void _stxr_s(jit_state_t*, jit_int32_t, jit_int32_t, jit_int32_t);
516 # define stxi_s(i0, r0, r1) _stxi_s(_jit, i0, r0, r1)
517 static void _stxi_s(jit_state_t*, jit_word_t, jit_int32_t, jit_int32_t);
518 # define stxr_i(r0, r1, r2) _stxr_i(_jit, r0, r1, r2)
519 static void _stxr_i(jit_state_t*, jit_int32_t, jit_int32_t, jit_int32_t);
520 # define stxi_i(i0, r0, r1) _stxi_i(_jit, i0, r0, r1)
521 static void _stxi_i(jit_state_t*, jit_word_t, jit_int32_t, jit_int32_t);
522 # if __X64 && !__X64_32
523 # define stxr_l(r0, r1, r2) _stxr_l(_jit, r0, r1, r2)
524 static void _stxr_l(jit_state_t*, jit_int32_t, jit_int32_t, jit_int32_t);
525 # define stxi_l(i0, r0, r1) _stxi_l(_jit, i0, r0, r1)
526 static void _stxi_l(jit_state_t*, jit_word_t, jit_int32_t, jit_int32_t);
528 # define jcc(code, i0) _jcc(_jit, code, i0)
529 # define jo(i0) jcc(X86_CC_O, i0)
530 # define jno(i0) jcc(X86_CC_NO, i0)
531 # define jnae(i0) jcc(X86_CC_NAE, i0)
532 # define jb(i0) jcc(X86_CC_B, i0)
533 # define jc(i0) jcc(X86_CC_C, i0)
534 # define jae(i0) jcc(X86_CC_AE, i0)
535 # define jnb(i0) jcc(X86_CC_NB, i0)
536 # define jnc(i0) jcc(X86_CC_NC, i0)
537 # define je(i0) jcc(X86_CC_E, i0)
538 # define jz(i0) jcc(X86_CC_Z, i0)
539 # define jne(i0) jcc(X86_CC_NE, i0)
540 # define jnz(i0) jcc(X86_CC_NZ, i0)
541 # define jbe(i0) jcc(X86_CC_BE, i0)
542 # define jna(i0) jcc(X86_CC_NA, i0)
543 # define ja(i0) jcc(X86_CC_A, i0)
544 # define jnbe(i0) jcc(X86_CC_NBE, i0)
545 # define js(i0) jcc(X86_CC_S, i0)
546 # define jns(i0) jcc(X86_CC_NS, i0)
547 # define jp(i0) jcc(X86_CC_P, i0)
548 # define jpe(i0) jcc(X86_CC_PE, i0)
549 # define jnp(i0) jcc(X86_CC_NP, i0)
550 # define jpo(i0) jcc(X86_CC_PO, i0)
551 # define jl(i0) jcc(X86_CC_L, i0)
552 # define jnge(i0) jcc(X86_CC_NGE, i0)
553 # define jge(i0) jcc(X86_CC_GE, i0)
554 # define jnl(i0) jcc(X86_CC_NL, i0)
555 # define jle(i0) jcc(X86_CC_LE, i0)
556 # define jng(i0) jcc(X86_CC_NG, i0)
557 # define jg(i0) jcc(X86_CC_G, i0)
558 # define jnle(i0) jcc(X86_CC_NLE, i0)
559 static jit_word_t _jcc(jit_state_t*, jit_int32_t, jit_word_t);
560 # define jccs(code, i0) _jccs(_jit, code, i0)
561 # define jos(i0) jccs(X86_CC_O, i0)
562 # define jnos(i0) jccs(X86_CC_NO, i0)
563 # define jnaes(i0) jccs(X86_CC_NAE, i0)
564 # define jbs(i0) jccs(X86_CC_B, i0)
565 # define jcs(i0) jccs(X86_CC_C, i0)
566 # define jaes(i0) jccs(X86_CC_AE, i0)
567 # define jnbs(i0) jccs(X86_CC_NB, i0)
568 # define jncs(i0) jccs(X86_CC_NC, i0)
569 # define jes(i0) jccs(X86_CC_E, i0)
570 # define jzs(i0) jccs(X86_CC_Z, i0)
571 # define jnes(i0) jccs(X86_CC_NE, i0)
572 # define jnzs(i0) jccs(X86_CC_NZ, i0)
573 # define jbes(i0) jccs(X86_CC_BE, i0)
574 # define jnas(i0) jccs(X86_CC_NA, i0)
575 # define jas(i0) jccs(X86_CC_A, i0)
576 # define jnbes(i0) jccs(X86_CC_NBE, i0)
577 # define jss(i0) jccs(X86_CC_S, i0)
578 # define jnss(i0) jccs(X86_CC_NS, i0)
579 # define jps(i0) jccs(X86_CC_P, i0)
580 # define jpes(i0) jccs(X86_CC_PE, i0)
581 # define jnps(i0) jccs(X86_CC_NP, i0)
582 # define jpos(i0) jccs(X86_CC_PO, i0)
583 # define jls(i0) jccs(X86_CC_L, i0)
584 # define jnges(i0) jccs(X86_CC_NGE, i0)
585 # define jges(i0) jccs(X86_CC_GE, i0)
586 # define jnls(i0) jccs(X86_CC_NL, i0)
587 # define jles(i0) jccs(X86_CC_LE, i0)
588 # define jngs(i0) jccs(X86_CC_NG, i0)
589 # define jgs(i0) jccs(X86_CC_G, i0)
590 # define jnles(i0) jccs(X86_CC_NLE, i0)
591 static jit_word_t _jccs(jit_state_t*, jit_int32_t, jit_word_t);
592 # define jcr(code, i0, r0, r1) _jcr(_jit, code, i0, r0, r1)
593 static jit_word_t _jcr(jit_state_t*,
594 jit_int32_t,jit_word_t,jit_int32_t,jit_int32_t);
595 # define jci(code, i0, r0, i1) _jci(_jit, code, i0, r0, i1)
596 static jit_word_t _jci(jit_state_t*,
597 jit_int32_t,jit_word_t,jit_int32_t,jit_word_t);
598 # define jci0(code, i0, r0) _jci0(_jit, code, i0, r0)
599 static jit_word_t _jci0(jit_state_t*, jit_int32_t, jit_word_t, jit_int32_t);
600 # define bltr(i0, r0, r1) _bltr(_jit, i0, r0, r1)
601 static jit_word_t _bltr(jit_state_t*, jit_word_t, jit_int32_t, jit_int32_t);
602 # define blti(i0, r0, i1) _blti(_jit, i0, r0, i1)
603 static jit_word_t _blti(jit_state_t*, jit_word_t, jit_int32_t, jit_word_t);
604 # define bltr_u(i0, r0, r1) _bltr_u(_jit, i0, r0, r1)
605 static jit_word_t _bltr_u(jit_state_t*, jit_word_t, jit_int32_t, jit_int32_t);
606 # define blti_u(i0, r0, i1) _blti_u(_jit, i0, r0, i1)
607 static jit_word_t _blti_u(jit_state_t*, jit_word_t, jit_int32_t, jit_word_t);
608 # define bler(i0, r0, r1) _bler(_jit, i0, r0, r1)
609 static jit_word_t _bler(jit_state_t*, jit_word_t, jit_int32_t, jit_int32_t);
610 # define blei(i0, r0, i1) _blei(_jit, i0, r0, i1)
611 static jit_word_t _blei(jit_state_t*, jit_word_t, jit_int32_t, jit_word_t);
612 # define bler_u(i0, r0, r1) _bler_u(_jit, i0, r0, r1)
613 static jit_word_t _bler_u(jit_state_t*, jit_word_t, jit_int32_t, jit_int32_t);
614 # define blei_u(i0, r0, i1) _blei_u(_jit, i0, r0, i1)
615 static jit_word_t _blei_u(jit_state_t*, jit_word_t, jit_int32_t, jit_word_t);
616 # define beqr(i0, r0, r1) _beqr(_jit, i0, r0, r1)
617 static jit_word_t _beqr(jit_state_t*, jit_word_t, jit_int32_t, jit_int32_t);
618 # define beqi(i0, r0, i1) _beqi(_jit, i0, r0, i1)
619 static jit_word_t _beqi(jit_state_t*, jit_word_t, jit_int32_t, jit_word_t);
620 # define bger(i0, r0, r1) _bger(_jit, i0, r0, r1)
621 static jit_word_t _bger(jit_state_t*, jit_word_t, jit_int32_t, jit_int32_t);
622 # define bgei(i0, r0, i1) _bgei(_jit, i0, r0, i1)
623 static jit_word_t _bgei(jit_state_t*, jit_word_t, jit_int32_t, jit_word_t);
624 # define bger_u(i0, r0, r1) _bger_u(_jit, i0, r0, r1)
625 static jit_word_t _bger_u(jit_state_t*, jit_word_t, jit_int32_t, jit_int32_t);
626 # define bgei_u(i0, r0, i1) _bgei_u(_jit, i0, r0, i1)
627 static jit_word_t _bgei_u(jit_state_t*, jit_word_t, jit_int32_t, jit_word_t);
628 # define bgtr(i0, r0, r1) _bgtr(_jit, i0, r0, r1)
629 static jit_word_t _bgtr(jit_state_t*, jit_word_t, jit_int32_t, jit_int32_t);
630 # define bgti(i0, r0, i1) _bgti(_jit, i0, r0, i1)
631 static jit_word_t _bgti(jit_state_t*, jit_word_t, jit_int32_t, jit_word_t);
632 # define bgtr_u(i0, r0, r1) _bgtr_u(_jit, i0, r0, r1)
633 static jit_word_t _bgtr_u(jit_state_t*, jit_word_t, jit_int32_t, jit_int32_t);
634 # define bgti_u(i0, r0, i1) _bgti_u(_jit, i0, r0, i1)
635 static jit_word_t _bgti_u(jit_state_t*, jit_word_t, jit_int32_t, jit_word_t);
636 # define bner(i0, r0, r1) _bner(_jit, i0, r0, r1)
637 static jit_word_t _bner(jit_state_t*, jit_word_t, jit_int32_t, jit_int32_t);
638 # define bnei(i0, r0, i1) _bnei(_jit, i0, r0, i1)
639 static jit_word_t _bnei(jit_state_t*, jit_word_t, jit_int32_t, jit_word_t);
640 # define bmsr(i0, r0, r1) _bmsr(_jit, i0, r0, r1)
641 static jit_word_t _bmsr(jit_state_t*,jit_word_t,jit_int32_t,jit_int32_t);
642 # define bmsi(i0, r0, i1) _bmsi(_jit, i0, r0, i1)
643 static jit_word_t _bmsi(jit_state_t*,jit_word_t,jit_int32_t,jit_word_t);
644 # define bmcr(i0, r0, r1) _bmcr(_jit, i0, r0, r1)
645 static jit_word_t _bmcr(jit_state_t*,jit_word_t,jit_int32_t,jit_int32_t);
646 # define bmci(i0, r0, i1) _bmci(_jit, i0, r0, i1)
647 static jit_word_t _bmci(jit_state_t*,jit_word_t,jit_int32_t,jit_word_t);
648 # define boaddr(i0, r0, r1) _boaddr(_jit, i0, r0, r1)
649 static jit_word_t _boaddr(jit_state_t*,jit_word_t,jit_int32_t,jit_int32_t);
650 # define boaddi(i0, r0, i1) _boaddi(_jit, i0, r0, i1)
651 static jit_word_t _boaddi(jit_state_t*,jit_word_t,jit_int32_t,jit_word_t);
652 # define boaddr_u(i0, r0, r1) _boaddr_u(_jit, i0, r0, r1)
653 static jit_word_t _boaddr_u(jit_state_t*,jit_word_t,jit_int32_t,jit_int32_t);
654 # define boaddi_u(i0, r0, i1) _boaddi_u(_jit, i0, r0, i1)
655 static jit_word_t _boaddi_u(jit_state_t*,jit_word_t,jit_int32_t,jit_word_t);
656 # define bxaddr(i0, r0, r1) _bxaddr(_jit, i0, r0, r1)
657 static jit_word_t _bxaddr(jit_state_t*,jit_word_t,jit_int32_t,jit_int32_t);
658 # define bxaddi(i0, r0, i1) _bxaddi(_jit, i0, r0, i1)
659 static jit_word_t _bxaddi(jit_state_t*,jit_word_t,jit_int32_t,jit_word_t);
660 # define bxaddr_u(i0, r0, r1) _bxaddr_u(_jit, i0, r0, r1)
661 static jit_word_t _bxaddr_u(jit_state_t*,jit_word_t,jit_int32_t,jit_int32_t);
662 # define bxaddi_u(i0, r0, i1) _bxaddi_u(_jit, i0, r0, i1)
663 static jit_word_t _bxaddi_u(jit_state_t*,jit_word_t,jit_int32_t,jit_word_t);
664 # define bosubr(i0, r0, r1) _bosubr(_jit, i0, r0, r1)
665 static jit_word_t _bosubr(jit_state_t*,jit_word_t,jit_int32_t,jit_int32_t);
666 # define bosubi(i0, r0, i1) _bosubi(_jit, i0, r0, i1)
667 static jit_word_t _bosubi(jit_state_t*,jit_word_t,jit_int32_t,jit_word_t);
668 # define bosubr_u(i0, r0, r1) _bosubr_u(_jit, i0, r0, r1)
669 static jit_word_t _bosubr_u(jit_state_t*,jit_word_t,jit_int32_t,jit_int32_t);
670 # define bosubi_u(i0, r0, i1) _bosubi_u(_jit, i0, r0, i1)
671 static jit_word_t _bosubi_u(jit_state_t*,jit_word_t,jit_int32_t,jit_word_t);
672 # define bxsubr(i0, r0, r1) _bxsubr(_jit, i0, r0, r1)
673 static jit_word_t _bxsubr(jit_state_t*,jit_word_t,jit_int32_t,jit_int32_t);
674 # define bxsubi(i0, r0, i1) _bxsubi(_jit, i0, r0, i1)
675 static jit_word_t _bxsubi(jit_state_t*,jit_word_t,jit_int32_t,jit_word_t);
676 # define bxsubr_u(i0, r0, r1) _bxsubr_u(_jit, i0, r0, r1)
677 static jit_word_t _bxsubr_u(jit_state_t*,jit_word_t,jit_int32_t,jit_int32_t);
678 # define bxsubi_u(i0, r0, i1) _bxsubi_u(_jit, i0, r0, i1)
679 static jit_word_t _bxsubi_u(jit_state_t*,jit_word_t,jit_int32_t,jit_word_t);
680 # define callr(r0) _callr(_jit, r0)
681 static void _callr(jit_state_t*, jit_int32_t);
682 # define calli(i0) _calli(_jit, i0)
683 static jit_word_t _calli(jit_state_t*, jit_word_t);
685 # define calli_p(i0) _calli_p(_jit, i0)
686 static jit_word_t _calli_p(jit_state_t*, jit_word_t);
688 # define calli_p(i0) calli(i0)
690 # define jmpr(r0) _jmpr(_jit, r0)
691 static void _jmpr(jit_state_t*, jit_int32_t);
692 # define jmpi(i0) _jmpi(_jit, i0)
693 static jit_word_t _jmpi(jit_state_t*, jit_word_t);
695 # define jmpi_p(i0) _jmpi_p(_jit, i0)
696 static jit_word_t _jmpi_p(jit_state_t*, jit_word_t);
698 # define jmpi_p(i0) jmpi(i0)
700 # define jmpsi(i0) _jmpsi(_jit, i0)
701 static jit_word_t _jmpsi(jit_state_t*, jit_uint8_t);
702 # define prolog(node) _prolog(_jit, node)
703 static void _prolog(jit_state_t*, jit_node_t*);
704 # define epilog(node) _epilog(_jit, node)
705 static void _epilog(jit_state_t*, jit_node_t*);
706 # define vastart(r0) _vastart(_jit, r0)
707 static void _vastart(jit_state_t*, jit_int32_t);
708 # define vaarg(r0, r1) _vaarg(_jit, r0, r1)
709 static void _vaarg(jit_state_t*, jit_int32_t, jit_int32_t);
710 # define vaarg_d(r0, r1, i0) _vaarg_d(_jit, r0, r1, i0)
711 static void _vaarg_d(jit_state_t*, jit_int32_t, jit_int32_t, jit_bool_t);
712 # define patch_at(instr, label) _patch_at(_jit, instr, label)
713 static void _patch_at(jit_state_t*, jit_word_t, jit_word_t);
714 # if !defined(HAVE_FFSL)
716 # define ffsl(i) __builtin_ffs(i)
718 # define ffsl(l) __builtin_ffsl(l)
721 # define jit_cmov_p() jit_cpu.cmov
726 _rex(jit_state_t *_jit, jit_int32_t l, jit_int32_t w,
727 jit_int32_t r, jit_int32_t x, jit_int32_t b)
730 jit_int32_t v = 0x40 | (w << 3);
744 _rx(jit_state_t *_jit, jit_int32_t rd, jit_int32_t md,
745 jit_int32_t rb, jit_int32_t ri, jit_int32_t ms)
749 /* Use ms == _SCL8 to tell it is a %rip relative displacement */
753 mrm(0x00, r7(rd), 0x05);
756 mrm(0x00, r7(rd), 0x04);
757 sib(_SCL1, 0x04, 0x05);
762 else if (r7(rb) == _RSP_REGNO) {
764 mrm(0x00, r7(rd), 0x04);
767 else if ((jit_int8_t)md == md) {
768 mrm(0x01, r7(rd), 0x04);
773 mrm(0x02, r7(rd), 0x04);
779 if (md == 0 && r7(rb) != _RBP_REGNO)
780 mrm(0x00, r7(rd), r7(rb));
781 else if ((jit_int8_t)md == md) {
782 mrm(0x01, r7(rd), r7(rb));
786 mrm(0x02, r7(rd), r7(rb));
791 else if (rb == _NOREG) {
792 mrm(0x00, r7(rd), 0x04);
793 sib(ms, r7(ri), 0x05);
796 else if (r8(ri) != _RSP_REGNO) {
797 if (md == 0 && r7(rb) != _RBP_REGNO) {
798 mrm(0x00, r7(rd), 0x04);
799 sib(ms, r7(ri), r7(rb));
801 else if ((jit_int8_t)md == md) {
802 mrm(0x01, r7(rd), 0x04);
803 sib(ms, r7(ri), r7(rb));
807 mrm(0x02, r7(rd), 0x04);
808 sib(ms, r7(ri), r7(rb));
813 fprintf(stderr, "illegal index register");
819 _nop(jit_state_t *_jit, jit_int32_t count)
835 case 3: /* NOP DWORD ptr [EAX] */
836 ic(0x0f); ic(0x1f); ic(0x00);
838 case 4: /* NOP DWORD ptr [EAX + 00H] */
839 ic(0x0f); ic(0x1f); ic(0x40); ic(0x00);
841 case 5: /* NOP DWORD ptr [EAX + EAX*1 + 00H] */
842 ic(0x0f); ic(0x1f); ic(0x44); ic(0x00);
845 case 6: /* 66 NOP DWORD ptr [EAX + EAX*1 + 00H] */
846 ic(0x66); ic(0x0f); ic(0x1f); ic(0x44);
849 case 7: /* NOP DWORD ptr [EAX + 00000000H] */
850 ic(0x0f); ic(0x1f); ic(0x80); ii(0x0000);
852 case 8: /* NOP DWORD ptr [EAX + EAX*1 + 00000000H] */
853 ic(0x0f); ic(0x1f); ic(0x84); ic(0x00);
856 case 9: /* 66 NOP DWORD ptr [EAX + EAX*1 + 00000000H] */
857 ic(0x66); ic(0x0f); ic(0x1f); ic(0x84);
858 ic(0x00); ii(0x0000);
865 _lea(jit_state_t *_jit, jit_int32_t md, jit_int32_t rb,
866 jit_int32_t ri, jit_int32_t ms, jit_int32_t rd)
868 rex(0, WIDE, rd, ri, rb);
870 rx(rd, md, rb, ri, ms);
874 _pushr(jit_state_t *_jit, jit_int32_t r0)
876 rex(0, WIDE, 0, 0, r0);
881 _popr(jit_state_t *_jit, jit_int32_t r0)
883 rex(0, WIDE, 0, 0, r0);
888 _xchgr(jit_state_t *_jit, jit_int32_t r0, jit_int32_t r1)
890 rex(0, WIDE, r1, _NOREG, r0);
892 mrm(0x03, r7(r1), r7(r0));
896 _testr(jit_state_t *_jit, jit_int32_t r0, jit_int32_t r1)
898 rex(0, WIDE, r1, _NOREG, r0);
900 mrm(0x03, r7(r1), r7(r0));
904 _testi(jit_state_t *_jit, jit_int32_t r0, jit_word_t i0)
906 rex(0, WIDE, _NOREG, _NOREG, r0);
907 if (r0 == _RAX_REGNO)
911 mrm(0x03, 0x00, r7(r0));
917 _cc(jit_state_t *_jit, jit_int32_t code, jit_int32_t r0)
919 rex(0, 0, _NOREG, _NOREG, r0);
922 mrm(0x03, 0x00, r7(r0));
926 _alur(jit_state_t *_jit, jit_int32_t code, jit_int32_t r0, jit_int32_t r1)
928 rex(0, WIDE, r1, _NOREG, r0);
930 mrm(0x03, r7(r1), r7(r0));
934 _alui(jit_state_t *_jit, jit_int32_t code, jit_int32_t r0, jit_word_t i0)
937 if (can_sign_extend_int_p(i0)) {
938 rex(0, WIDE, _NOREG, _NOREG, r0);
939 if ((jit_int8_t)i0 == i0) {
941 ic(0xc0 | code | r7(r0));
945 if (r0 == _RAX_REGNO)
949 ic(0xc0 | code | r7(r0));
955 reg = jit_get_reg(jit_class_gpr);
957 alur(code, r0, rn(reg));
963 _save(jit_state_t *_jit, jit_int32_t r0)
965 if (!_jitc->function->regoff[r0]) {
966 _jitc->function->regoff[r0] = jit_allocai(sizeof(jit_word_t));
969 assert(!jit_regset_tstbit(&_jitc->regsav, r0));
970 jit_regset_setbit(&_jitc->regsav, r0);
971 stxi(_jitc->function->regoff[r0], _RBP_REGNO, r0);
975 _load(jit_state_t *_jit, jit_int32_t r0)
977 assert(_jitc->function->regoff[r0]);
978 assert(jit_regset_tstbit(&_jitc->regsav, r0));
979 jit_regset_clrbit(&_jitc->regsav, r0);
980 ldxi(r0, _RBP_REGNO, _jitc->function->regoff[r0]);
984 _addr(jit_state_t *_jit, jit_int32_t r0, jit_int32_t r1, jit_int32_t r2)
991 lea(0, r1, r2, _SCL1, r0);
995 _addi(jit_state_t *_jit, jit_int32_t r0, jit_int32_t r1, jit_word_t i0)
1006 else if (can_sign_extend_int_p(i0)) {
1010 lea(i0, r1, _NOREG, _SCL1, r0);
1012 else if (r0 != r1) {
1017 reg = jit_get_reg(jit_class_gpr);
1025 _addcr(jit_state_t *_jit, jit_int32_t r0, jit_int32_t r1, jit_int32_t r2)
1036 _addci(jit_state_t *_jit, jit_int32_t r0, jit_int32_t r1, jit_word_t i0)
1039 if (can_sign_extend_int_p(i0)) {
1043 else if (r0 == r1) {
1044 reg = jit_get_reg(jit_class_gpr);
1056 _iaddxr(jit_state_t *_jit, jit_int32_t r0, jit_int32_t r1)
1058 /* FIXME: this is not doing what I did expect for the simple test case:
1059 * mov $0xffffffffffffffff, %rax -- rax = 0xffffffffffffffff (-1)
1060 * mov $0xffffffffffffffff, %r10 -- r10 = 0xffffffffffffffff (-1)
1061 * mov $0x1, %r11d -- r11 = 1
1062 * xor %rbx, %rbx -- rbx = 0
1065 * add %r11, %rax -- r11 = 0x10000000000000000 (0)
1066 * does not fit in 64 bit ^
1068 * $2 = [ CF PF AF ZF IF ]
1069 * adcx %r10, %rbx -- r10 = 0xffffffffffffffff (-1)
1071 * $3 = [ CF PF AF ZF IF ]
1073 * $4 = 0xffffffffffffffff
1074 * but, r10 should be zero, as it is:
1075 * -1 (%r10) + 0 (%rbx) + carry (!!eflags.CF)
1076 * FIXME: maybe should only use ADCX in the third operation onward, that
1077 * is, after the first ADC? In either case, the add -1+0+carry should
1078 * have used and consumed the carry? At least this is what is expected
1082 /* Significantly longer instruction, but avoid cpu stalls as only
1083 * the carry flag is used in a sequence. */
1087 rex(0, WIDE, r1, _NOREG, r0);
1091 mrm(0x03, r7(r1), r7(r0));
1095 alur(X86_ADC, r0, r1);
1099 _addxr(jit_state_t *_jit, jit_int32_t r0, jit_int32_t r1, jit_int32_t r2)
1110 _addxi(jit_state_t *_jit, jit_int32_t r0, jit_int32_t r1, jit_word_t i0)
1115 /* Do not mix ADC and ADCX */
1118 can_sign_extend_int_p(i0)) {
1122 else if (r0 == r1) {
1123 reg = jit_get_reg(jit_class_gpr);
1125 iaddxr(r0, rn(reg));
1135 _subr(jit_state_t *_jit, jit_int32_t r0, jit_int32_t r1, jit_int32_t r2)
1139 else if (r0 == r2) {
1150 _subi(jit_state_t *_jit, jit_int32_t r0, jit_int32_t r1, jit_word_t i0)
1161 else if (can_sign_extend_int_p(i0)) {
1165 lea(-i0, r1, _NOREG, _SCL1, r0);
1167 else if (r0 != r1) {
1172 reg = jit_get_reg(jit_class_gpr);
1180 _subcr(jit_state_t *_jit, jit_int32_t r0, jit_int32_t r1, jit_int32_t r2)
1183 if (r0 == r2 && r0 != r1) {
1184 reg = jit_get_reg(jit_class_gpr);
1197 _subci(jit_state_t *_jit, jit_int32_t r0, jit_int32_t r1, jit_word_t i0)
1201 if (can_sign_extend_int_p(i0))
1204 reg = jit_get_reg(jit_class_gpr);
1212 _subxr(jit_state_t *_jit, jit_int32_t r0, jit_int32_t r1, jit_int32_t r2)
1215 if (r0 == r2 && r0 != r1) {
1216 reg = jit_get_reg(jit_class_gpr);
1219 isubxr(r0, rn(reg));
1229 _subxi(jit_state_t *_jit, jit_int32_t r0, jit_int32_t r1, jit_word_t i0)
1233 if (can_sign_extend_int_p(i0))
1236 reg = jit_get_reg(jit_class_gpr);
1238 isubxr(r0, rn(reg));
1244 _rsbi(jit_state_t *_jit, jit_int32_t r0, jit_int32_t r1, jit_word_t i0)
1251 _imulr(jit_state_t *_jit, jit_int32_t r0, jit_int32_t r1)
1253 rex(0, WIDE, r0, _NOREG, r1);
1256 mrm(0x03, r7(r0), r7(r1));
1260 _imuli(jit_state_t *_jit, jit_int32_t r0, jit_int32_t r1, jit_word_t i0)
1263 if (can_sign_extend_int_p(i0)) {
1264 rex(0, WIDE, r0, _NOREG, r1);
1265 if ((jit_int8_t)i0 == i0) {
1267 mrm(0x03, r7(r0), r7(r1));
1272 mrm(0x03, r7(r0), r7(r1));
1277 reg = jit_get_reg(jit_class_gpr);
1285 _mulr(jit_state_t *_jit, jit_int32_t r0, jit_int32_t r1, jit_int32_t r2)
1298 _muli(jit_state_t *_jit, jit_int32_t r0, jit_int32_t r1, jit_word_t i0)
1311 lea(0, _NOREG, r1, _SCL2, r0);
1314 lea(0, _NOREG, r1, _SCL4, r0);
1317 lea(0, _NOREG, r1, _SCL8, r0);
1320 if (i0 > 0 && !(i0 & (i0 - 1)))
1321 lshi(r0, r1, ffsl(i0) - 1);
1322 else if (can_sign_extend_int_p(i0))
1324 else if (r0 != r1) {
1334 #define savset(rn) \
1337 if (r1 != rn && r2 != rn) \
1340 #define isavset(rn) \
1346 #define qsavset(rn) \
1347 if (r0 != rn && r1 != rn) { \
1349 if (r2 != rn && r3 != rn) \
1352 #define allocr(rn, rv) \
1353 if (set & (1 << rn)) \
1354 (void)jit_get_reg(rv|jit_class_gpr|jit_class_named); \
1355 if (sav & (1 << rn)) { \
1356 if ( jit_regset_tstbit(&_jitc->regsav, rv) || \
1357 !jit_regset_tstbit(&_jitc->reglive, rv)) \
1358 sav &= ~(1 << rn); \
1362 #define clear(rn, rv) \
1363 if (set & (1 << rn)) \
1364 jit_unget_reg(rv); \
1365 if (sav & (1 << rn)) \
1368 _iqmulr(jit_state_t *_jit, jit_int32_t r0, jit_int32_t r1,
1369 jit_int32_t r2, jit_int32_t r3, jit_bool_t sign)
1376 qsavset(_RDX_REGNO);
1377 qsavset(_RAX_REGNO);
1378 allocr(_RDX_REGNO, _RDX);
1379 allocr(_RAX_REGNO, _RAX);
1381 if (r3 == _RAX_REGNO)
1385 movr(_RAX_REGNO, r2);
1392 if (r0 == _RDX_REGNO && r1 == _RAX_REGNO)
1393 xchgr(_RAX_REGNO, _RDX_REGNO);
1395 if (r0 != _RDX_REGNO)
1396 movr(r0, _RAX_REGNO);
1397 movr(r1, _RDX_REGNO);
1398 if (r0 == _RDX_REGNO)
1399 movr(r0, _RAX_REGNO);
1402 clear(_RDX_REGNO, _RDX);
1403 clear(_RAX_REGNO, _RAX);
1407 _iqmuli(jit_state_t *_jit, jit_int32_t r0, jit_int32_t r1,
1408 jit_int32_t r2, jit_word_t i0, jit_bool_t sign)
1417 reg = jit_get_reg(jit_class_gpr);
1420 qmulr(r0, r1, r2, rn(reg));
1422 qmulr_u(r0, r1, r2, rn(reg));
1428 _sign_extend_rdx_rax(jit_state_t *_jit)
1430 rex(0, WIDE, 0, 0, 0);
1435 _divremr(jit_state_t *_jit, jit_int32_t r0, jit_int32_t r1, jit_int32_t r2,
1436 jit_bool_t sign, jit_bool_t divide)
1444 sav = set = use = 0;
1447 allocr(_RDX_REGNO, _RDX);
1448 allocr(_RAX_REGNO, _RAX);
1450 if (r2 == _RAX_REGNO) {
1451 if (r0 == _RAX_REGNO || r0 == _RDX_REGNO) {
1452 if ((reg = jit_get_reg(jit_class_gpr|jit_class_chk)) == JIT_NOREG)
1453 reg = jit_get_reg((r1 == _RCX_REGNO ? _RBX : _RCX) |
1454 jit_class_gpr|jit_class_named);
1457 movr(div, _RAX_REGNO);
1458 if (r1 != _RAX_REGNO)
1459 movr(_RAX_REGNO, r1);
1463 xchgr(r0, _RAX_REGNO);
1465 if (r0 != _RAX_REGNO)
1466 movr(r0, _RAX_REGNO);
1467 if (r1 != _RAX_REGNO)
1468 movr(_RAX_REGNO, r1);
1473 else if (r2 == _RDX_REGNO) {
1474 if (r0 == _RAX_REGNO || r0 == _RDX_REGNO) {
1475 if ((reg = jit_get_reg(jit_class_gpr|jit_class_chk)) == JIT_NOREG)
1476 reg = jit_get_reg((r1 == _RCX_REGNO ? _RBX : _RCX) |
1477 jit_class_gpr|jit_class_named);
1480 movr(div, _RDX_REGNO);
1481 if (r1 != _RAX_REGNO)
1482 movr(_RAX_REGNO, r1);
1485 if (r1 != _RAX_REGNO)
1486 movr(_RAX_REGNO, r1);
1487 movr(r0, _RDX_REGNO);
1492 if (r1 != _RAX_REGNO)
1493 movr(_RAX_REGNO, r1);
1498 sign_extend_rdx_rax();
1502 ixorr(_RDX_REGNO, _RDX_REGNO);
1510 movr(r0, _RAX_REGNO);
1512 movr(r0, _RDX_REGNO);
1514 clear(_RDX_REGNO, _RDX);
1515 clear(_RAX_REGNO, _RAX);
1519 _divremi(jit_state_t *_jit, jit_int32_t r0, jit_int32_t r1, jit_word_t i0,
1520 jit_bool_t sign, jit_bool_t divide)
1540 if (i0 > 0 && !(i0 & (i0 - 1))) {
1543 rshi(r0, r0, ffsl(i0) - 1);
1545 rshi_u(r0, r0, ffsl(i0) - 1);
1551 else if (i0 == 1 || (sign && i0 == -1)) {
1555 else if (!sign && i0 > 0 && !(i0 & (i0 - 1))) {
1556 if (can_sign_extend_int_p(i0)) {
1560 else if (r0 != r1) {
1565 reg = jit_get_reg(jit_class_gpr);
1566 movi(rn(reg), i0 - 1);
1573 sav = set = use = 0;
1574 isavset(_RDX_REGNO);
1575 isavset(_RAX_REGNO);
1576 allocr(_RDX_REGNO, _RDX);
1577 allocr(_RAX_REGNO, _RAX);
1579 if (r0 == _RAX_REGNO || r0 == _RDX_REGNO || r0 == r1) {
1580 if ((reg = jit_get_reg(jit_class_gpr|jit_class_chk)) == JIT_NOREG)
1581 reg = jit_get_reg((r1 == _RCX_REGNO ? _RBX : _RCX) |
1582 jit_class_gpr|jit_class_named);
1590 movr(_RAX_REGNO, r1);
1593 sign_extend_rdx_rax();
1597 ixorr(_RDX_REGNO, _RDX_REGNO);
1605 movr(r0, _RAX_REGNO);
1607 movr(r0, _RDX_REGNO);
1609 clear(_RDX_REGNO, _RDX);
1610 clear(_RAX_REGNO, _RAX);
1614 _iqdivr(jit_state_t *_jit, jit_int32_t r0, jit_int32_t r1,
1615 jit_int32_t r2, jit_int32_t r3, jit_bool_t sign)
1623 sav = set = use = 0;
1624 qsavset(_RDX_REGNO);
1625 qsavset(_RAX_REGNO);
1626 allocr(_RDX_REGNO, _RDX);
1627 allocr(_RAX_REGNO, _RAX);
1628 if (r3 == _RAX_REGNO) {
1629 if (r0 == _RAX_REGNO || r0 == _RDX_REGNO) {
1630 if ((reg = jit_get_reg(jit_class_gpr|jit_class_chk)) == JIT_NOREG)
1631 reg = jit_get_reg((r1 == _RCX_REGNO ? _RBX : _RCX) |
1632 jit_class_gpr|jit_class_named);
1635 movr(div, _RAX_REGNO);
1636 if (r2 != _RAX_REGNO)
1637 movr(_RAX_REGNO, r2);
1641 xchgr(r0, _RAX_REGNO);
1643 if (r0 != _RAX_REGNO)
1644 movr(r0, _RAX_REGNO);
1645 if (r2 != _RAX_REGNO)
1646 movr(_RAX_REGNO, r2);
1651 else if (r3 == _RDX_REGNO) {
1652 if (r0 == _RAX_REGNO || r0 == _RDX_REGNO) {
1653 if ((reg = jit_get_reg(jit_class_gpr|jit_class_chk)) == JIT_NOREG)
1654 reg = jit_get_reg((r1 == _RCX_REGNO ? _RBX : _RCX) |
1655 jit_class_gpr|jit_class_named);
1658 movr(div, _RDX_REGNO);
1659 if (r2 != _RAX_REGNO)
1660 movr(_RAX_REGNO, r2);
1663 if (r2 != _RAX_REGNO)
1664 movr(_RAX_REGNO, r2);
1665 movr(r0, _RDX_REGNO);
1670 if (r2 != _RAX_REGNO)
1671 movr(_RAX_REGNO, r2);
1675 sign_extend_rdx_rax();
1679 ixorr(_RDX_REGNO, _RDX_REGNO);
1685 if (r0 == _RDX_REGNO && r1 == _RAX_REGNO)
1686 xchgr(_RAX_REGNO, _RDX_REGNO);
1688 if (r0 != _RDX_REGNO)
1689 movr(r0, _RAX_REGNO);
1690 movr(r1, _RDX_REGNO);
1691 if (r0 == _RDX_REGNO)
1692 movr(r0, _RAX_REGNO);
1695 clear(_RDX_REGNO, _RDX);
1696 clear(_RAX_REGNO, _RAX);
1700 _iqdivi(jit_state_t *_jit, jit_int32_t r0, jit_int32_t r1,
1701 jit_int32_t r2, jit_word_t i0, jit_bool_t sign)
1705 reg = jit_get_reg(jit_class_gpr);
1708 qdivr(r0, r1, r2, rn(reg));
1710 qdivr_u(r0, r1, r2, rn(reg));
1718 _andr(jit_state_t *_jit, jit_int32_t r0, jit_int32_t r1, jit_int32_t r2)
1733 _andi(jit_state_t *_jit, jit_int32_t r0, jit_int32_t r1, jit_word_t i0)
1741 else if (r0 == r1) {
1742 if (can_sign_extend_int_p(i0))
1745 reg = jit_get_reg(jit_class_gpr);
1758 _orr(jit_state_t *_jit, jit_int32_t r0, jit_int32_t r1, jit_int32_t r2)
1773 _ori(jit_state_t *_jit, jit_int32_t r0, jit_int32_t r1, jit_word_t i0)
1780 else if (can_sign_extend_int_p(i0)) {
1784 else if (r0 != r1) {
1789 reg = jit_get_reg(jit_class_gpr);
1797 _xorr(jit_state_t *_jit, jit_int32_t r0, jit_int32_t r1, jit_int32_t r2)
1812 _xori(jit_state_t *_jit, jit_int32_t r0, jit_int32_t r1, jit_word_t i0)
1819 else if (can_sign_extend_int_p(i0)) {
1823 else if (r0 != r1) {
1828 reg = jit_get_reg(jit_class_gpr);
1836 _irotshr(jit_state_t *_jit, jit_int32_t code, jit_int32_t r0)
1838 rex(0, WIDE, _RCX_REGNO, _NOREG, r0);
1840 mrm(0x03, code, r7(r0));
1844 _rotshr(jit_state_t *_jit, jit_int32_t code,
1845 jit_int32_t r0, jit_int32_t r1, jit_int32_t r2)
1850 if (r0 == _RCX_REGNO) {
1851 reg = jit_get_reg(jit_class_gpr);
1853 if (r2 != _RCX_REGNO)
1854 movr(_RCX_REGNO, r2);
1855 irotshr(code, rn(reg));
1856 movr(_RCX_REGNO, rn(reg));
1859 else if (r2 != _RCX_REGNO) {
1860 use = !jit_reg_free_p(_RCX);
1862 reg = jit_get_reg(jit_class_gpr);
1863 movr(rn(reg), _RCX_REGNO);
1867 if (r1 == _RCX_REGNO) {
1869 xchgr(r0, _RCX_REGNO);
1872 movr(_RCX_REGNO, r2);
1876 movr(_RCX_REGNO, r2);
1881 movr(_RCX_REGNO, rn(reg));
1892 _irotshi(jit_state_t *_jit, jit_int32_t code, jit_int32_t r0, jit_word_t i0)
1894 rex(0, WIDE, _NOREG, _NOREG, r0);
1897 mrm(0x03, code, r7(r0));
1901 mrm(0x03, code, r7(r0));
1907 _rotshi(jit_state_t *_jit, jit_int32_t code,
1908 jit_int32_t r0, jit_int32_t r1, jit_word_t i0)
1912 irotshi(code, r0, i0);
1916 _lshi(jit_state_t *_jit, jit_int32_t r0, jit_int32_t r1, jit_word_t i0)
1921 lea(0, _NOREG, r1, i0 == 1 ? _SCL2 : i0 == 2 ? _SCL4 : _SCL8, r0);
1923 rotshi(X86_SHL, r0, r1, i0);
1927 _unr(jit_state_t *_jit, jit_int32_t code, jit_int32_t r0)
1929 rex(0, WIDE, _NOREG, _NOREG, r0);
1931 mrm(0x03, code, r7(r0));
1935 _negr(jit_state_t *_jit, jit_int32_t r0, jit_int32_t r1)
1946 _comr(jit_state_t *_jit, jit_int32_t r0, jit_int32_t r1)
1954 _incr(jit_state_t *_jit, jit_int32_t r0, jit_int32_t r1)
1958 rex(0, WIDE, _NOREG, _NOREG, r0);
1967 _decr(jit_state_t *_jit, jit_int32_t r0, jit_int32_t r1)
1971 rex(0, WIDE, _NOREG, _NOREG, r0);
1981 _clor(jit_state_t *_jit, jit_int32_t r0, jit_int32_t r1)
1988 _clzr(jit_state_t *_jit, jit_int32_t r0, jit_int32_t r1)
1995 rex(0, WIDE, r0, _NOREG, r1);
1998 mrm(0x3, r7(r0), r7(r1));
2000 /* jump if undefined: r1 == 0 */
2001 w = jccs(X86_CC_E, _jit->pc.w);
2002 /* count leading zeros */
2003 rsbi(r0, r0, __WORDSIZE - 1);
2005 x = jmpsi(_jit->pc.w);
2007 patch_at(w, _jit->pc.w);
2008 movi(r0, __WORDSIZE);
2010 patch_at(x, _jit->pc.w);
2012 /* LZCNT has defined behavior for value zero and count leading zeros */
2016 _ctor(jit_state_t *_jit, jit_int32_t r0, jit_int32_t r1)
2023 _ctzr(jit_state_t *_jit, jit_int32_t r0, jit_int32_t r1)
2029 t0 = jit_get_reg(jit_class_gpr|jit_class_nospill|jit_class_chk);
2033 movi(rn(t0), __WORDSIZE);
2039 rex(0, WIDE, r0, _NOREG, r1);
2042 mrm(0x3, r7(r0), r7(r1));
2044 /* No conditional move or need spill/reload a temporary */
2046 w = jccs(X86_CC_E, _jit->pc.w);
2047 movi(r0, __WORDSIZE);
2048 patch_at(w, _jit->pc.w);
2052 rex(0, WIDE, r0, _NOREG, rn(t0));
2055 mrm(0x3, r7(r0), r7(rn(t0)));
2059 /* TZCNT has defined behavior for value zero */
2063 _cr(jit_state_t *_jit,
2064 jit_int32_t code, jit_int32_t r0, jit_int32_t r1, jit_int32_t r2)
2069 same = r0 == r1 || r0 == r2;
2078 reg = jit_get_reg(jit_class_gpr|jit_class_rg8);
2079 ixorr(rn(reg), rn(reg));
2088 _ci(jit_state_t *_jit,
2089 jit_int32_t code, jit_int32_t r0, jit_int32_t r1, jit_word_t i0)
2103 reg = jit_get_reg(jit_class_gpr|jit_class_rg8);
2104 ixorr(rn(reg), rn(reg));
2113 _ci0(jit_state_t *_jit, jit_int32_t code, jit_int32_t r0, jit_int32_t r1)
2127 reg = jit_get_reg(jit_class_gpr|jit_class_rg8);
2128 ixorr(rn(reg), rn(reg));
2137 _ltr(jit_state_t *_jit, jit_int32_t r0, jit_int32_t r1, jit_int32_t r2)
2142 cr(X86_CC_L, r0, r1, r2);
2146 _lti(jit_state_t *_jit, jit_int32_t r0, jit_int32_t r1, jit_word_t i0)
2149 ci(X86_CC_L, r0, r1, i0);
2151 ci0(X86_CC_S, r0, r1);
2155 _ltr_u(jit_state_t *_jit, jit_int32_t r0, jit_int32_t r1, jit_int32_t r2)
2160 cr(X86_CC_B, r0, r1, r2);
2164 _ler(jit_state_t *_jit, jit_int32_t r0, jit_int32_t r1, jit_int32_t r2)
2169 cr(X86_CC_LE, r0, r1, r2);
2173 _ler_u(jit_state_t *_jit, jit_int32_t r0, jit_int32_t r1, jit_int32_t r2)
2178 cr(X86_CC_BE, r0, r1, r2);
2182 _lei_u(jit_state_t *_jit, jit_int32_t r0, jit_int32_t r1, jit_word_t i0)
2185 ci(X86_CC_BE, r0, r1, i0);
2187 ci0(X86_CC_E, r0, r1);
2191 _eqr(jit_state_t *_jit, jit_int32_t r0, jit_int32_t r1, jit_int32_t r2)
2196 cr(X86_CC_E, r0, r1, r2);
2200 _eqi(jit_state_t *_jit, jit_int32_t r0, jit_int32_t r1, jit_word_t i0)
2203 ci(X86_CC_E, r0, r1, i0);
2205 ci0(X86_CC_E, r0, r1);
2209 _ger(jit_state_t *_jit, jit_int32_t r0, jit_int32_t r1, jit_int32_t r2)
2214 cr(X86_CC_GE, r0, r1, r2);
2218 _gei(jit_state_t *_jit, jit_int32_t r0, jit_int32_t r1, jit_word_t i0)
2221 ci(X86_CC_GE, r0, r1, i0);
2223 ci0(X86_CC_NS, r0, r1);
2227 _ger_u(jit_state_t *_jit, jit_int32_t r0, jit_int32_t r1, jit_int32_t r2)
2232 cr(X86_CC_AE, r0, r1, r2);
2236 _gei_u(jit_state_t *_jit, jit_int32_t r0, jit_int32_t r1, jit_word_t i0)
2239 ci(X86_CC_AE, r0, r1, i0);
2241 ci0(X86_CC_NB, r0, r1);
2245 _gtr(jit_state_t *_jit, jit_int32_t r0, jit_int32_t r1, jit_int32_t r2)
2250 cr(X86_CC_G, r0, r1, r2);
2254 _gtr_u(jit_state_t *_jit, jit_int32_t r0, jit_int32_t r1, jit_int32_t r2)
2259 cr(X86_CC_A, r0, r1, r2);
2263 _gti_u(jit_state_t *_jit, jit_int32_t r0, jit_int32_t r1, jit_word_t i0)
2266 ci(X86_CC_A, r0, r1, i0);
2268 ci0(X86_CC_NE, r0, r1);
2272 _ner(jit_state_t *_jit, jit_int32_t r0, jit_int32_t r1, jit_int32_t r2)
2277 cr(X86_CC_NE, r0, r1, r2);
2281 _nei(jit_state_t *_jit, jit_int32_t r0, jit_int32_t r1, jit_word_t i0)
2284 ci(X86_CC_NE, r0, r1, i0);
2286 ci0(X86_CC_NE, r0, r1);
2290 _movr(jit_state_t *_jit, jit_int32_t r0, jit_int32_t r1)
2293 rex(0, 1, r1, _NOREG, r0);
2295 ic(0xc0 | (r1 << 3) | r7(r0));
2300 _imovi(jit_state_t *_jit, jit_int32_t r0, jit_word_t i0)
2304 if (fits_uint32_p(i0)) {
2306 rex(0, 0, _NOREG, _NOREG, r0);
2311 else if (can_sign_extend_int_p(i0)) {
2312 rex(0, 1, _NOREG, _NOREG, r0);
2318 rex(0, 1, _NOREG, _NOREG, r0);
2334 _movi(jit_state_t *_jit, jit_int32_t r0, jit_word_t i0)
2340 rel = rel < 0 ? rel - 8 : rel + 8;
2341 if (can_sign_extend_int_p(rel)) {
2342 /* lea rel(%rip), %r0 */
2343 rex(0, WIDE, r0, _NOREG, _NOREG);
2346 rx(r0, i0 - (_jit->pc.w + 5), _NOREG, _NOREG, _SCL8);
2360 _movi_p(jit_state_t *_jit, jit_int32_t r0, jit_word_t i0)
2363 rex(0, WIDE, _NOREG, _NOREG, r0);
2371 _movcr(jit_state_t *_jit, jit_int32_t r0, jit_int32_t r1)
2373 rex(0, WIDE, r0, _NOREG, r1);
2376 mrm(0x03, r7(r0), r7(r1));
2380 _movcr_u(jit_state_t *_jit, jit_int32_t r0, jit_int32_t r1)
2382 rex(0, WIDE, r0, _NOREG, r1);
2385 mrm(0x03, r7(r0), r7(r1));
2389 _movsr(jit_state_t *_jit, jit_int32_t r0, jit_int32_t r1)
2391 rex(0, WIDE, r0, _NOREG, r1);
2394 mrm(0x03, r7(r0), r7(r1));
2398 _movsr_u(jit_state_t *_jit, jit_int32_t r0, jit_int32_t r1)
2400 rex(0, WIDE, r0, _NOREG, r1);
2403 mrm(0x03, r7(r0), r7(r1));
2407 _casx(jit_state_t *_jit, jit_int32_t r0, jit_int32_t r1,
2408 jit_int32_t r2, jit_int32_t r3, jit_word_t i0)
2410 jit_int32_t save_rax, restore_rax;
2411 jit_int32_t ascasr_reg, ascasr_use;
2412 if (r0 != _RAX_REGNO) { /* result not in %rax */
2413 if (r2 != _RAX_REGNO) { /* old value not in %rax */
2414 save_rax = jit_get_reg(jit_class_gpr);
2415 movr(rn(save_rax), _RAX_REGNO);
2423 if (r2 != _RAX_REGNO)
2424 movr(_RAX_REGNO, r2);
2425 if (r1 == _NOREG) { /* using immediate address */
2426 if (!can_sign_extend_int_p(i0)) {
2427 ascasr_reg = jit_get_reg(jit_class_gpr);
2428 if (ascasr_reg == _RAX) {
2429 ascasr_reg = jit_get_reg(jit_class_gpr);
2430 jit_unget_reg(_RAX);
2433 movi(rn(ascasr_reg), i0);
2440 ic(0xf0); /* lock */
2442 rex(0, WIDE, r3, _NOREG, rn(ascasr_reg));
2444 rex(0, WIDE, r3, _NOREG, r1);
2447 if (r1 != _NOREG) /* casr */
2448 rx(r3, 0, r1, _NOREG, _SCL1);
2451 rx(r3, 0, rn(ascasr_reg), _NOREG, _SCL1); /* address in reg */
2453 rx(r3, i0, _NOREG, _NOREG, _SCL1); /* address in offset */
2456 if (r0 != _RAX_REGNO)
2457 movr(r0, _RAX_REGNO);
2459 movr(_RAX_REGNO, rn(save_rax));
2460 jit_unget_reg(save_rax);
2463 jit_unget_reg(ascasr_reg);
2467 _movnr(jit_state_t *_jit, jit_int32_t r0, jit_int32_t r1, jit_int32_t r2)
2469 assert(jit_cmov_p());
2473 rex(0, WIDE, r0, _NOREG, r1);
2476 mrm(0x03, r7(r0), r7(r1));
2480 _movzr(jit_state_t *_jit, jit_int32_t r0, jit_int32_t r1, jit_int32_t r2)
2482 assert(jit_cmov_p());
2486 rex(0, WIDE, r0, _NOREG, r1);
2489 mrm(0x03, r7(r0), r7(r1));
2494 _movir(jit_state_t *_jit, jit_int32_t r0, jit_int32_t r1)
2496 rex(0, 1, r0, _NOREG, r1);
2498 mrm(0x03, r7(r0), r7(r1));
2502 _movir_u(jit_state_t *_jit, jit_int32_t r0, jit_int32_t r1)
2504 rex(0, 0, r1, _NOREG, r0);
2506 ic(0xc0 | (r1 << 3) | r7(r0));
2511 _bswapr_us(jit_state_t *_jit, jit_int32_t r0, jit_int32_t r1)
2515 rex(0, 0, _NOREG, _NOREG, r0);
2517 mrm(0x03, X86_ROR, r7(r0));
2522 _bswapr_ui(jit_state_t *_jit, jit_int32_t r0, jit_int32_t r1)
2525 rex(0, 0, _NOREG, _NOREG, r0);
2530 #if __X64 && !__X64_32
2532 _bswapr_ul(jit_state_t *_jit, jit_int32_t r0, jit_int32_t r1)
2535 rex(0, 1, _NOREG, _NOREG, r0);
2542 _extr_c(jit_state_t *_jit, jit_int32_t r0, jit_int32_t r1)
2548 reg = jit_get_reg(jit_class_gpr|jit_class_rg8);
2556 _extr_uc(jit_state_t *_jit, jit_int32_t r0, jit_int32_t r1)
2562 reg = jit_get_reg(jit_class_gpr|jit_class_rg8);
2564 movcr_u(r0, rn(reg));
2570 _ldr_c(jit_state_t *_jit, jit_int32_t r0, jit_int32_t r1)
2572 rex(0, WIDE, r0, _NOREG, r1);
2575 rx(r0, 0, r1, _NOREG, _SCL1);
2579 _ldi_c(jit_state_t *_jit, jit_int32_t r0, jit_word_t i0)
2583 jit_word_t rel = i0 - _jit->pc.w;
2584 rel = rel < 0 ? rel - 8 : rel + 8;
2585 if (can_sign_extend_int_p(rel)) {
2586 rex(0, WIDE, r0, _NOREG, _NOREG);
2589 rx(r0, i0 - (_jit->pc.w + 5), _NOREG, _NOREG, _SCL8);
2593 if (address_p(i0)) {
2594 rex(0, WIDE, r0, _NOREG, _NOREG);
2597 rx(r0, i0, _NOREG, _NOREG, _SCL1);
2600 reg = jit_get_reg(jit_class_gpr);
2608 _ldr_uc(jit_state_t *_jit, jit_int32_t r0, jit_int32_t r1)
2610 rex(0, WIDE, r0, _NOREG, r1);
2613 rx(r0, 0, r1, _NOREG, _SCL1);
2617 _ldi_uc(jit_state_t *_jit, jit_int32_t r0, jit_word_t i0)
2621 jit_word_t rel = i0 - _jit->pc.w;
2622 rel = rel < 0 ? rel - 8 : rel + 8;
2623 if (can_sign_extend_int_p(rel)) {
2624 rex(0, WIDE, r0, _NOREG, _NOREG);
2627 rx(r0, i0 - (_jit->pc.w + 5), _NOREG, _NOREG, _SCL8);
2631 if (address_p(i0)) {
2632 rex(0, WIDE, r0, _NOREG, _NOREG);
2635 rx(r0, i0, _NOREG, _NOREG, _SCL1);
2638 reg = jit_get_reg(jit_class_gpr);
2640 ldr_uc(r0, rn(reg));
2646 _ldr_s(jit_state_t *_jit, jit_int32_t r0, jit_int32_t r1)
2648 rex(0, WIDE, r0, _NOREG, r1);
2651 rx(r0, 0, r1, _NOREG, _SCL1);
2655 _ldi_s(jit_state_t *_jit, jit_int32_t r0, jit_word_t i0)
2659 jit_word_t rel = i0 - _jit->pc.w;
2660 rel = rel < 0 ? rel - 8 : rel + 8;
2661 if (can_sign_extend_int_p(rel)) {
2662 rex(0, WIDE, r0, _NOREG, _NOREG);
2665 rx(r0, i0 - (_jit->pc.w + 5), _NOREG, _NOREG, _SCL8);
2669 if (address_p(i0)) {
2670 rex(0, WIDE, r0, _NOREG, _NOREG);
2673 rx(r0, i0, _NOREG, _NOREG, _SCL1);
2676 reg = jit_get_reg(jit_class_gpr);
2684 _ldr_us(jit_state_t *_jit, jit_int32_t r0, jit_int32_t r1)
2686 rex(0, WIDE, r0, _NOREG, r1);
2689 rx(r0, 0, r1, _NOREG, _SCL1);
2693 _ldi_us(jit_state_t *_jit, jit_int32_t r0, jit_word_t i0)
2697 jit_word_t rel = i0 - _jit->pc.w;
2698 rel = rel < 0 ? rel - 8 : rel + 8;
2699 if (can_sign_extend_int_p(rel)) {
2700 rex(0, WIDE, r0, _NOREG, _NOREG);
2703 rx(r0, i0 - (_jit->pc.w + 5), _NOREG, _NOREG, _SCL8);
2707 if (address_p(i0)) {
2708 rex(0, WIDE, r0, _NOREG, _NOREG);
2711 rx(r0, i0, _NOREG, _NOREG, _SCL1);
2714 reg = jit_get_reg(jit_class_gpr);
2716 ldr_us(r0, rn(reg));
2721 #if __X32 || !__X64_32
2723 _ldr_i(jit_state_t *_jit, jit_int32_t r0, jit_int32_t r1)
2726 rex(0, WIDE, r0, _NOREG, r1);
2731 rx(r0, 0, r1, _NOREG, _SCL1);
2735 _ldi_i(jit_state_t *_jit, jit_int32_t r0, jit_word_t i0)
2739 jit_word_t rel = i0 - _jit->pc.w;
2740 rel = rel < 0 ? rel - 8 : rel + 8;
2741 if (can_sign_extend_int_p(rel)) {
2742 rex(0, WIDE, r0, _NOREG, _NOREG);
2744 rx(r0, i0 - (_jit->pc.w + 5), _NOREG, _NOREG, _SCL8);
2748 if (address_p(i0)) {
2750 rex(0, WIDE, r0, _NOREG, _NOREG);
2755 rx(r0, i0, _NOREG, _NOREG, _SCL1);
2758 reg = jit_get_reg(jit_class_gpr);
2768 _ldr_ui(jit_state_t *_jit, jit_int32_t r0, jit_int32_t r1)
2770 rex(0, 0, r0, _NOREG, r1);
2772 rx(r0, 0, r1, _NOREG, _SCL1);
2776 _ldi_ui(jit_state_t *_jit, jit_int32_t r0, jit_word_t i0)
2780 jit_word_t rel = i0 - _jit->pc.w;
2781 rel = rel < 0 ? rel - 8 : rel + 8;
2782 if (can_sign_extend_int_p(rel)) {
2783 rex(0, 0, r0, _NOREG, _NOREG);
2785 rx(r0, i0 - (_jit->pc.w + 5), _NOREG, _NOREG, _SCL8);
2789 if (address_p(i0)) {
2790 rex(0, 0, r0, _NOREG, _NOREG);
2792 rx(r0, i0, _NOREG, _NOREG, _SCL1);
2795 reg = jit_get_reg(jit_class_gpr);
2800 ldr_ui(r0, rn(reg));
2808 _ldr_l(jit_state_t *_jit, jit_int32_t r0, jit_int32_t r1)
2810 rex(0, 1, r0, _NOREG, r1);
2812 rx(r0, 0, r1, _NOREG, _SCL1);
2816 _ldi_l(jit_state_t *_jit, jit_int32_t r0, jit_word_t i0)
2819 jit_word_t rel = i0 - _jit->pc.w;
2820 rel = rel < 0 ? rel - 8 : rel + 8;
2821 if (can_sign_extend_int_p(rel)) {
2822 rex(0, WIDE, r0, _NOREG, _NOREG);
2824 rx(r0, i0 - (_jit->pc.w + 5), _NOREG, _NOREG, _SCL8);
2826 else if (can_sign_extend_int_p(i0)) {
2827 rex(0, WIDE, r0, _NOREG, _NOREG);
2829 rx(r0, i0, _NOREG, _NOREG, _SCL1);
2832 reg = jit_get_reg(jit_class_gpr);
2842 _ldxr_c(jit_state_t *_jit, jit_int32_t r0, jit_int32_t r1, jit_int32_t r2)
2848 rex(0, WIDE, r0, r1, r2);
2851 rx(r0, 0, r2, r1, _SCL1);
2856 _ldxi_c(jit_state_t *_jit, jit_int32_t r0, jit_int32_t r1, jit_word_t i0)
2859 if (can_sign_extend_int_p(i0)) {
2860 rex(0, WIDE, r0, _NOREG, r1);
2863 rx(r0, i0, r1, _NOREG, _SCL1);
2866 reg = jit_get_reg(jit_class_gpr);
2868 ldxr_c(r0, r1, rn(reg));
2874 _ldxr_uc(jit_state_t *_jit, jit_int32_t r0, jit_int32_t r1, jit_int32_t r2)
2880 rex(0, WIDE, r0, r1, r2);
2883 rx(r0, 0, r2, r1, _SCL1);
2888 _ldxi_uc(jit_state_t *_jit, jit_int32_t r0, jit_int32_t r1, jit_word_t i0)
2891 if (can_sign_extend_int_p(i0)) {
2892 rex(0, WIDE, r0, _NOREG, r1);
2895 rx(r0, i0, r1, _NOREG, _SCL1);
2898 reg = jit_get_reg(jit_class_gpr);
2900 ldxr_uc(r0, r1, rn(reg));
2906 _ldxr_s(jit_state_t *_jit, jit_int32_t r0, jit_int32_t r1, jit_int32_t r2)
2912 rex(0, WIDE, r0, r1, r2);
2915 rx(r0, 0, r2, r1, _SCL1);
2920 _ldxi_s(jit_state_t *_jit, jit_int32_t r0, jit_int32_t r1, jit_word_t i0)
2923 if (can_sign_extend_int_p(i0)) {
2924 rex(0, WIDE, r0, _NOREG, r1);
2927 rx(r0, i0, r1, _NOREG, _SCL1);
2930 reg = jit_get_reg(jit_class_gpr);
2932 ldxr_s(r0, r1, rn(reg));
2938 _ldxr_us(jit_state_t *_jit, jit_int32_t r0, jit_int32_t r1, jit_int32_t r2)
2944 rex(0, WIDE, r0, r1, r2);
2947 rx(r0, 0, r2, r1, _SCL1);
2952 _ldxi_us(jit_state_t *_jit, jit_int32_t r0, jit_int32_t r1, jit_word_t i0)
2955 if (can_sign_extend_int_p(i0)) {
2956 rex(0, WIDE, r0, _NOREG, r1);
2959 rx(r0, i0, r1, _NOREG, _SCL1);
2962 reg = jit_get_reg(jit_class_gpr);
2964 ldxr_us(r0, r1, rn(reg));
2969 #if __X64 || !__X64_32
2971 _ldxr_i(jit_state_t *_jit, jit_int32_t r0, jit_int32_t r1, jit_int32_t r2)
2974 rex(0, WIDE, r0, r1, r2);
2979 rx(r0, 0, r2, r1, _SCL1);
2983 _ldxi_i(jit_state_t *_jit, jit_int32_t r0, jit_int32_t r1, jit_word_t i0)
2986 if (can_sign_extend_int_p(i0)) {
2988 rex(0, WIDE, r0, _NOREG, r1);
2993 rx(r0, i0, r1, _NOREG, _SCL1);
2996 reg = jit_get_reg(jit_class_gpr);
2998 ldxr_i(r0, r1, rn(reg));
3006 _ldxr_ui(jit_state_t *_jit, jit_int32_t r0, jit_int32_t r1, jit_int32_t r2)
3010 /* to avoid confusion with macro renames */
3011 _ldr_ui(_jit, r0, r0);
3013 rex(0, 0, r0, r1, r2);
3015 rx(r0, 0, r2, r1, _SCL1);
3020 _ldxi_ui(jit_state_t *_jit, jit_int32_t r0, jit_int32_t r1, jit_word_t i0)
3023 if (can_sign_extend_int_p(i0)) {
3024 rex(0, 0, r0, _NOREG, r1);
3026 rx(r0, i0, r1, _NOREG, _SCL1);
3029 reg = jit_get_reg(jit_class_gpr);
3032 ldxr_i(r0, r1, rn(reg));
3034 ldxr_ui(r0, r1, rn(reg));
3042 _ldxr_l(jit_state_t *_jit, jit_int32_t r0, jit_int32_t r1, jit_int32_t r2)
3044 rex(0, 1, r0, r1, r2);
3046 rx(r0, 0, r2, r1, _SCL1);
3050 _ldxi_l(jit_state_t *_jit, jit_int32_t r0, jit_int32_t r1, jit_word_t i0)
3053 if (can_sign_extend_int_p(i0)) {
3054 rex(0, 1, r0, _NOREG, r1);
3056 rx(r0, i0, r1, _NOREG, _SCL1);
3059 reg = jit_get_reg(jit_class_gpr);
3061 ldxr_l(r0, r1, rn(reg));
3069 _str_c(jit_state_t *_jit, jit_int32_t r0, jit_int32_t r1)
3073 rex(0, 0, r1, _NOREG, r0);
3075 rx(r1, 0, r0, _NOREG, _SCL1);
3078 reg = jit_get_reg(jit_class_gpr|jit_class_rg8);
3080 rex(0, 0, rn(reg), _NOREG, r0);
3082 rx(rn(reg), 0, r0, _NOREG, _SCL1);
3088 _sti_c(jit_state_t *_jit, jit_word_t i0, jit_int32_t r0)
3092 jit_word_t rel = i0 - _jit->pc.w;
3093 rel = rel < 0 ? rel - 16 : rel + 16;
3094 if (can_sign_extend_int_p(rel)) {
3096 rex(0, 0, r0, _NOREG, _NOREG);
3098 rx(r0, i0 - (_jit->pc.w + 5), _NOREG, _NOREG, _SCL8);
3101 reg = jit_get_reg(jit_class_gpr|jit_class_rg8);
3103 rex(0, 0, rn(reg), _NOREG, _NOREG);
3105 rx(rn(reg), i0 - (_jit->pc.w + 5), _NOREG, _NOREG, _SCL8);
3111 if (address_p(i0)) {
3113 rex(0, 0, r0, _NOREG, _NOREG);
3115 rx(r0, i0, _NOREG, _NOREG, _SCL1);
3118 reg = jit_get_reg(jit_class_gpr|jit_class_rg8);
3120 rex(0, 0, rn(reg), _NOREG, _NOREG);
3122 rx(rn(reg), i0, _NOREG, _NOREG, _SCL1);
3127 reg = jit_get_reg(jit_class_gpr);
3135 _str_s(jit_state_t *_jit, jit_int32_t r0, jit_int32_t r1)
3138 rex(0, 0, r1, _NOREG, r0);
3140 rx(r1, 0, r0, _NOREG, _SCL1);
3144 _sti_s(jit_state_t *_jit, jit_word_t i0, jit_int32_t r0)
3148 jit_word_t rel = i0 - _jit->pc.w;
3149 rel = rel < 0 ? rel - 8 : rel + 8;
3150 if (can_sign_extend_int_p(rel)) {
3152 rex(0, 0, r0, _NOREG, _NOREG);
3154 rx(r0, i0 - (_jit->pc.w + 5), _NOREG, _NOREG, _SCL8);
3158 if (address_p(i0)) {
3160 rex(0, 0, r0, _NOREG, _NOREG);
3162 rx(r0, i0, _NOREG, _NOREG, _SCL1);
3165 reg = jit_get_reg(jit_class_gpr);
3173 _str_i(jit_state_t *_jit, jit_int32_t r0, jit_int32_t r1)
3175 rex(0, 0, r1, _NOREG, r0);
3177 rx(r1, 0, r0, _NOREG, _SCL1);
3181 _sti_i(jit_state_t *_jit, jit_word_t i0, jit_int32_t r0)
3185 jit_word_t rel = i0 - _jit->pc.w;
3186 rel = rel < 0 ? rel - 8 : rel + 8;
3187 if (can_sign_extend_int_p(rel)) {
3188 rex(0, 0, r0, _NOREG, _NOREG);
3190 rx(r0, i0 - (_jit->pc.w + 5), _NOREG, _NOREG, _SCL8);
3194 if (address_p(i0)) {
3195 rex(0, 0, r0, _NOREG, _NOREG);
3197 rx(r0, i0, _NOREG, _NOREG, _SCL1);
3200 reg = jit_get_reg(jit_class_gpr);
3207 #if __X64 && !__X64_32
3209 _str_l(jit_state_t *_jit, jit_int32_t r0, jit_int32_t r1)
3211 rex(0, 1, r1, _NOREG, r0);
3213 rx(r1, 0, r0, _NOREG, _SCL1);
3217 _sti_l(jit_state_t *_jit, jit_word_t i0, jit_int32_t r0)
3221 jit_word_t rel = i0 - _jit->pc.w;
3222 rel = rel < 0 ? rel - 8 : rel + 8;
3223 if (can_sign_extend_int_p(rel)) {
3224 rex(0, WIDE, r0, _NOREG, _NOREG);
3226 rx(r0, i0 - (_jit->pc.w + 5), _NOREG, _NOREG, _SCL8);
3230 if (can_sign_extend_int_p(i0)) {
3231 rex(0, WIDE, r0, _NOREG, _NOREG);
3233 rx(r0, i0, _NOREG, _NOREG, _SCL1);
3236 reg = jit_get_reg(jit_class_gpr);
3245 _stxr_c(jit_state_t *_jit, jit_int32_t r0, jit_int32_t r1, jit_int32_t r2)
3249 reg = jit_get_reg(jit_class_gpr);
3250 addr(rn(reg), r0, r1);
3255 rex(0, 0, r2, r1, r0);
3257 rx(r2, 0, r0, r1, _SCL1);
3260 reg = jit_get_reg(jit_class_gpr|jit_class_rg8);
3262 rex(0, 0, rn(reg), r1, r0);
3264 rx(rn(reg), 0, r0, r1, _SCL1);
3271 _stxi_c(jit_state_t *_jit, jit_word_t i0, jit_int32_t r0, jit_int32_t r1)
3274 if (can_sign_extend_int_p(i0)) {
3276 rex(0, 0, r1, _NOREG, r0);
3278 rx(r1, i0, r0, _NOREG, _SCL1);
3281 reg = jit_get_reg(jit_class_gpr|jit_class_rg8);
3283 rex(0, 0, rn(reg), _NOREG, r0);
3285 rx(rn(reg), i0, r0, _NOREG, _SCL1);
3290 reg = jit_get_reg(jit_class_gpr);
3292 stxr_c(rn(reg), r0, r1);
3298 _stxr_s(jit_state_t *_jit, jit_int32_t r0, jit_int32_t r1, jit_int32_t r2)
3302 reg = jit_get_reg(jit_class_gpr);
3303 addr(rn(reg), r0, r1);
3308 rex(0, 0, r2, r1, r0);
3310 rx(r2, 0, r0, r1, _SCL1);
3315 _stxi_s(jit_state_t *_jit, jit_word_t i0, jit_int32_t r0, jit_int32_t r1)
3318 if (can_sign_extend_int_p(i0)) {
3320 rex(0, 0, r1, _NOREG, r0);
3322 rx(r1, i0, r0, _NOREG, _SCL1);
3325 reg = jit_get_reg(jit_class_gpr);
3327 stxr_s(rn(reg), r0, r1);
3333 _stxr_i(jit_state_t *_jit, jit_int32_t r0, jit_int32_t r1, jit_int32_t r2)
3337 reg = jit_get_reg(jit_class_gpr);
3338 addr(rn(reg), r0, r1);
3342 rex(0, 0, r2, r1, r0);
3344 rx(r2, 0, r0, r1, _SCL1);
3349 _stxi_i(jit_state_t *_jit, jit_word_t i0, jit_int32_t r0, jit_int32_t r1)
3352 if (can_sign_extend_int_p(i0)) {
3353 rex(0, 0, r1, _NOREG, r0);
3355 rx(r1, i0, r0, _NOREG, _SCL1);
3358 reg = jit_get_reg(jit_class_gpr);
3360 stxr_i(rn(reg), r0, r1);
3365 #if __X64 && !__X64_32
3367 _stxr_l(jit_state_t *_jit, jit_int32_t r0, jit_int32_t r1, jit_int32_t r2)
3369 rex(0, 1, r2, r1, r0);
3371 rx(r2, 0, r0, r1, _SCL1);
3375 _stxi_l(jit_state_t *_jit, jit_word_t i0, jit_int32_t r0, jit_int32_t r1)
3378 if (can_sign_extend_int_p(i0)) {
3379 rex(0, 1, r1, _NOREG, r0);
3381 rx(r1, i0, r0, _NOREG, _SCL1);
3384 reg = jit_get_reg(jit_class_gpr);
3386 stxr_l(rn(reg), r0, r1);
3393 _jccs(jit_state_t *_jit, jit_int32_t code, jit_word_t i0)
3405 _jcc(jit_state_t *_jit, jit_int32_t code, jit_word_t i0)
3418 _jcr(jit_state_t *_jit,
3419 jit_int32_t code, jit_word_t i0, jit_int32_t r0, jit_int32_t r1)
3421 alur(X86_CMP, r0, r1);
3422 return (jcc(code, i0));
3426 _jci(jit_state_t *_jit,
3427 jit_int32_t code, jit_word_t i0, jit_int32_t r0, jit_word_t i1)
3429 alui(X86_CMP, r0, i1);
3430 return (jcc(code, i0));
3434 _jci0(jit_state_t *_jit, jit_int32_t code, jit_word_t i0, jit_int32_t r0)
3437 return (jcc(code, i0));
3441 _bltr(jit_state_t *_jit, jit_word_t i0, jit_int32_t r0, jit_int32_t r1)
3443 return (jcr(X86_CC_L, i0, r0, r1));
3447 _blti(jit_state_t *_jit, jit_word_t i0, jit_int32_t r0, jit_word_t i1)
3450 if (i1) w = jci (X86_CC_L, i0, r0, i1);
3451 else w = jci0(X86_CC_S, i0, r0);
3456 _bltr_u(jit_state_t *_jit, jit_word_t i0, jit_int32_t r0, jit_int32_t r1)
3458 return (jcr(X86_CC_B, i0, r0, r1));
3462 _blti_u(jit_state_t *_jit, jit_word_t i0, jit_int32_t r0, jit_word_t i1)
3465 if (i1) w = jci (X86_CC_B, i0, r0, i1);
3466 else w = jci0(X86_CC_B, i0, r0);
3471 _bler(jit_state_t *_jit, jit_word_t i0, jit_int32_t r0, jit_int32_t r1)
3474 if (r0 == r1) w = jmpi(i0);
3475 else w = jcr (X86_CC_LE, i0, r0, r1);
3480 _blei(jit_state_t *_jit, jit_word_t i0, jit_int32_t r0, jit_word_t i1)
3483 if (i1) w = jci (X86_CC_LE, i0, r0, i1);
3484 else w = jci0(X86_CC_LE, i0, r0);
3489 _bler_u(jit_state_t *_jit, jit_word_t i0, jit_int32_t r0, jit_int32_t r1)
3492 if (r0 == r1) w = jmpi(i0);
3493 else w = jcr (X86_CC_BE, i0, r0, r1);
3498 _blei_u(jit_state_t *_jit, jit_word_t i0, jit_int32_t r0, jit_word_t i1)
3501 if (i1) w = jci (X86_CC_BE, i0, r0, i1);
3502 else w = jci0(X86_CC_BE, i0, r0);
3507 _beqr(jit_state_t *_jit, jit_word_t i0, jit_int32_t r0, jit_int32_t r1)
3510 if (r0 == r1) w = jmpi(i0);
3511 else w = jcr (X86_CC_E, i0, r0, r1);
3516 _beqi(jit_state_t *_jit, jit_word_t i0, jit_int32_t r0, jit_word_t i1)
3519 if (i1) w = jci (X86_CC_E, i0, r0, i1);
3520 else w = jci0(X86_CC_E, i0, r0);
3525 _bger(jit_state_t *_jit, jit_word_t i0, jit_int32_t r0, jit_int32_t r1)
3528 if (r0 == r1) w = jmpi(i0);
3529 else w = jcr (X86_CC_GE, i0, r0, r1);
3534 _bgei(jit_state_t *_jit, jit_word_t i0, jit_int32_t r0, jit_word_t i1)
3537 if (i1) w = jci (X86_CC_GE, i0, r0, i1);
3538 else w = jci0(X86_CC_NS, i0, r0);
3543 _bger_u(jit_state_t *_jit, jit_word_t i0, jit_int32_t r0, jit_int32_t r1)
3546 if (r0 == r1) w = jmpi(i0);
3547 else w = jcr (X86_CC_AE, i0, r0, r1);
3552 _bgei_u(jit_state_t *_jit, jit_word_t i0, jit_int32_t r0, jit_word_t i1)
3555 if (i1) w = jci (X86_CC_AE, i0, r0, i1);
3561 _bgtr(jit_state_t *_jit, jit_word_t i0, jit_int32_t r0, jit_int32_t r1)
3563 return (jcr(X86_CC_G, i0, r0, r1));
3567 _bgti(jit_state_t *_jit, jit_word_t i0, jit_int32_t r0, jit_word_t i1)
3569 return (jci(X86_CC_G, i0, r0, i1));
3573 _bgtr_u(jit_state_t *_jit, jit_word_t i0, jit_int32_t r0, jit_int32_t r1)
3575 return (jcr(X86_CC_A, i0, r0, r1));
3579 _bgti_u(jit_state_t *_jit, jit_word_t i0, jit_int32_t r0, jit_word_t i1)
3582 if (i1) w = jci (X86_CC_A, i0, r0, i1);
3583 else w = jci0(X86_CC_NE, i0, r0);
3588 _bner(jit_state_t *_jit, jit_word_t i0, jit_int32_t r0, jit_int32_t r1)
3590 return (jcr(X86_CC_NE, i0, r0, r1));
3594 _bnei(jit_state_t *_jit, jit_word_t i0, jit_int32_t r0, jit_word_t i1)
3597 if (i1) w = jci (X86_CC_NE, i0, r0, i1);
3598 else w = jci0(X86_CC_NE, i0, r0);
3603 _bmsr(jit_state_t *_jit, jit_word_t i0, jit_int32_t r0, jit_int32_t r1)
3610 _bmsi(jit_state_t *_jit, jit_word_t i0, jit_int32_t r0, jit_word_t i1)
3613 if (can_zero_extend_int_p(i1))
3616 reg = jit_get_reg(jit_class_gpr);
3625 _bmcr(jit_state_t *_jit, jit_word_t i0, jit_int32_t r0, jit_int32_t r1)
3632 _bmci(jit_state_t *_jit, jit_word_t i0, jit_int32_t r0, jit_word_t i1)
3635 if (can_zero_extend_int_p(i1))
3638 reg = jit_get_reg(jit_class_gpr);
3647 _boaddr(jit_state_t *_jit, jit_word_t i0, jit_int32_t r0, jit_int32_t r1)
3654 _boaddi(jit_state_t *_jit, jit_word_t i0, jit_int32_t r0, jit_word_t i1)
3657 if (can_sign_extend_int_p(i1)) {
3661 reg = jit_get_reg(jit_class_gpr|jit_class_nospill);
3664 return (boaddr(i0, r0, rn(reg)));
3668 _boaddr_u(jit_state_t *_jit, jit_word_t i0, jit_int32_t r0, jit_int32_t r1)
3675 _boaddi_u(jit_state_t *_jit, jit_word_t i0, jit_int32_t r0, jit_word_t i1)
3678 if (can_sign_extend_int_p(i1)) {
3682 reg = jit_get_reg(jit_class_gpr|jit_class_nospill);
3685 return (boaddr_u(i0, r0, rn(reg)));
3689 _bxaddr(jit_state_t *_jit, jit_word_t i0, jit_int32_t r0, jit_int32_t r1)
3696 _bxaddi(jit_state_t *_jit, jit_word_t i0, jit_int32_t r0, jit_word_t i1)
3699 if (can_sign_extend_int_p(i1)) {
3703 reg = jit_get_reg(jit_class_gpr|jit_class_nospill);
3706 return (bxaddr(i0, r0, rn(reg)));
3710 _bxaddr_u(jit_state_t *_jit, jit_word_t i0, jit_int32_t r0, jit_int32_t r1)
3717 _bxaddi_u(jit_state_t *_jit, jit_word_t i0, jit_int32_t r0, jit_word_t i1)
3720 if (can_sign_extend_int_p(i1)) {
3724 reg = jit_get_reg(jit_class_gpr|jit_class_nospill);
3727 return (bxaddr_u(i0, r0, rn(reg)));
3731 _bosubr(jit_state_t *_jit, jit_word_t i0, jit_int32_t r0, jit_int32_t r1)
3738 _bosubi(jit_state_t *_jit, jit_word_t i0, jit_int32_t r0, jit_word_t i1)
3741 if (can_sign_extend_int_p(i1)) {
3745 reg = jit_get_reg(jit_class_gpr|jit_class_nospill);
3748 return (bosubr(i0, r0, rn(reg)));
3752 _bosubr_u(jit_state_t *_jit, jit_word_t i0, jit_int32_t r0, jit_int32_t r1)
3759 _bosubi_u(jit_state_t *_jit, jit_word_t i0, jit_int32_t r0, jit_word_t i1)
3762 if (can_sign_extend_int_p(i1)) {
3766 reg = jit_get_reg(jit_class_gpr|jit_class_nospill);
3769 return (bosubr_u(i0, r0, rn(reg)));
3773 _bxsubr(jit_state_t *_jit, jit_word_t i0, jit_int32_t r0, jit_int32_t r1)
3780 _bxsubi(jit_state_t *_jit, jit_word_t i0, jit_int32_t r0, jit_word_t i1)
3783 if (can_sign_extend_int_p(i1)) {
3787 reg = jit_get_reg(jit_class_gpr|jit_class_nospill);
3790 return (bxsubr(i0, r0, rn(reg)));
3794 _bxsubr_u(jit_state_t *_jit, jit_word_t i0, jit_int32_t r0, jit_int32_t r1)
3801 _bxsubi_u(jit_state_t *_jit, jit_word_t i0, jit_int32_t r0, jit_word_t i1)
3804 if (can_sign_extend_int_p(i1)) {
3808 reg = jit_get_reg(jit_class_gpr|jit_class_nospill);
3811 return (bxsubr_u(i0, r0, rn(reg)));
3815 _callr(jit_state_t *_jit, jit_int32_t r0)
3817 rex(0, 0, _NOREG, _NOREG, r0);
3819 mrm(0x03, 0x02, r7(r0));
3823 _calli(jit_state_t *_jit, jit_word_t i0)
3827 jit_word_t l = _jit->pc.w + 5;
3832 !((d < 0) ^ (l < 0)) &&
3834 (jit_int32_t)d == d) {
3849 _calli_p(jit_state_t *_jit, jit_word_t i0)
3853 reg = jit_get_reg(jit_class_gpr);
3854 w = movi_p(rn(reg), i0);
3862 _jmpr(jit_state_t *_jit, jit_int32_t r0)
3864 rex(0, 0, _NOREG, _NOREG, r0);
3866 mrm(0x03, 0x04, r7(r0));
3870 _jmpi(jit_state_t *_jit, jit_word_t i0)
3874 jit_word_t l = _jit->pc.w + 5;
3879 !((d < 0) ^ (l < 0)) &&
3881 (jit_int32_t)d == d) {
3896 _jmpi_p(jit_state_t *_jit, jit_word_t i0)
3900 reg = jit_get_reg(jit_class_gpr|jit_class_nospill);
3901 w = movi_p(rn(reg), i0);
3909 _jmpsi(jit_state_t *_jit, jit_uint8_t i0)
3911 jit_word_t w = _jit->pc.w;
3918 _prolog(jit_state_t *_jit, jit_node_t *node)
3920 jit_int32_t reg, offs;
3921 if (_jitc->function->define_frame || _jitc->function->assume_frame) {
3922 jit_int32_t frame = -_jitc->function->frame;
3924 assert(_jitc->function->self.aoff >= frame);
3925 if (_jitc->function->assume_frame)
3927 _jitc->function->self.aoff = frame;
3929 if (_jitc->function->allocar)
3930 _jitc->function->self.aoff &= -16;
3931 #if __X64 && (__CYGWIN__ || _WIN32)
3932 _jitc->function->stack = (((/* first 32 bytes must be allocated */
3933 (_jitc->function->self.alen > 32 ?
3934 _jitc->function->self.alen : 32) -
3935 /* align stack at 16 bytes */
3936 _jitc->function->self.aoff) + 15) & -16);
3938 _jitc->function->stack = (((_jitc->function->self.alen -
3939 _jitc->function->self.aoff) + 15) & -16);
3942 if (_jitc->function->stack)
3943 _jitc->function->need_stack = 1;
3945 if (!_jitc->function->need_frame && !_jitc->function->need_stack) {
3946 /* check if any callee save register needs to be saved */
3947 for (reg = 0; reg < _jitc->reglen; ++reg)
3948 if (jit_regset_tstbit(&_jitc->function->regset, reg) &&
3949 (_rvs[reg].spec & jit_class_sav)) {
3950 _jitc->function->need_stack = 1;
3955 if (_jitc->function->need_frame || _jitc->function->need_stack)
3956 subi(_RSP_REGNO, _RSP_REGNO, jit_framesize());
3957 /* callee save registers */
3958 for (reg = 0, offs = REAL_WORDSIZE; reg < jit_size(iregs); reg++) {
3959 if (jit_regset_tstbit(&_jitc->function->regset, iregs[reg])) {
3960 stxi(offs, _RSP_REGNO, rn(iregs[reg]));
3961 offs += REAL_WORDSIZE;
3964 #if __X64 && (__CYGWIN__ || _WIN32)
3965 for (reg = 0; reg < jit_size(fregs); reg++) {
3966 if (jit_regset_tstbit(&_jitc->function->regset, fregs[reg])) {
3967 sse_stxi_d(offs, _RSP_REGNO, rn(fregs[reg]));
3968 offs += sizeof(jit_float64_t);
3973 if (_jitc->function->need_frame) {
3974 stxi(0, _RSP_REGNO, _RBP_REGNO);
3975 movr(_RBP_REGNO, _RSP_REGNO);
3979 if (_jitc->function->stack)
3980 subi(_RSP_REGNO, _RSP_REGNO, _jitc->function->stack);
3981 if (_jitc->function->allocar) {
3982 reg = jit_get_reg(jit_class_gpr);
3983 movi(rn(reg), _jitc->function->self.aoff);
3984 stxi_i(_jitc->function->aoffoff, _RBP_REGNO, rn(reg));
3988 #if __X64 && !(__CYGWIN__ || _WIN32)
3989 if (_jitc->function->self.call & jit_call_varargs) {
3990 jit_word_t nofp_code;
3992 /* Save gp registers in the save area, if any is a vararg */
3993 for (reg = first_gp_from_offset(_jitc->function->vagp);
3994 jit_arg_reg_p(reg); ++reg)
3995 stxi(_jitc->function->vaoff + first_gp_offset +
3996 reg * 8, _RBP_REGNO, rn(JIT_RA0 - reg));
3998 reg = first_fp_from_offset(_jitc->function->vafp);
3999 if (jit_arg_f_reg_p(reg)) {
4000 /* Skip over if no float registers were passed as argument */
4006 /* Save fp registers in the save area, if any is a vararg */
4007 /* Note that the full 16 byte xmm is not saved, because
4008 * lightning only handles float and double, and, while
4009 * attempting to provide a va_list compatible pointer as
4010 * jit_va_start return, does not guarantee it (on all ports). */
4011 for (; jit_arg_f_reg_p(reg); ++reg)
4012 sse_stxi_d(_jitc->function->vaoff + first_fp_offset +
4013 reg * va_fp_increment, _RBP_REGNO, rn(_XMM0 - reg));
4015 patch_at(nofp_code, _jit->pc.w);
4022 _epilog(jit_state_t *_jit, jit_node_t *node)
4024 jit_int32_t reg, offs;
4025 if (_jitc->function->assume_frame)
4027 if (_jitc->function->need_frame)
4028 movr(_RSP_REGNO, _RBP_REGNO);
4030 /* callee save registers */
4031 for (reg = 0, offs = REAL_WORDSIZE; reg < jit_size(iregs); reg++) {
4032 if (jit_regset_tstbit(&_jitc->function->regset, iregs[reg])) {
4033 ldxi(rn(iregs[reg]), _RSP_REGNO, offs);
4034 offs += REAL_WORDSIZE;
4037 #if __X64 && (__CYGWIN__ || _WIN32)
4038 for (reg = 0; reg < jit_size(fregs); reg++) {
4039 if (jit_regset_tstbit(&_jitc->function->regset, fregs[reg])) {
4040 sse_ldxi_d(rn(fregs[reg]), _RSP_REGNO, offs);
4041 offs += sizeof(jit_float64_t);
4046 if (_jitc->function->need_frame) {
4047 ldxi(_RBP_REGNO, _RSP_REGNO, 0);
4048 addi(_RSP_REGNO, _RSP_REGNO, jit_framesize());
4050 /* This condition does not happen as much as expected because
4051 * it is not safe to not create a frame pointer if any function
4052 * is called, even jit functions, as those might call external
4054 else if (_jitc->function->need_stack)
4055 addi(_RSP_REGNO, _RSP_REGNO, jit_framesize());
4061 _vastart(jit_state_t *_jit, jit_int32_t r0)
4063 #if __X32 || __CYGWIN__ || _WIN32
4064 assert(_jitc->function->self.call & jit_call_varargs);
4065 addi(r0, _RBP_REGNO, jit_selfsize());
4069 assert(_jitc->function->self.call & jit_call_varargs);
4071 /* Return jit_va_list_t in the register argument */
4072 addi(r0, _RBP_REGNO, _jitc->function->vaoff);
4073 reg = jit_get_reg(jit_class_gpr);
4075 /* Initialize gp offset in the save area. */
4076 movi(rn(reg), _jitc->function->vagp);
4077 stxi_i(offsetof(jit_va_list_t, gpoff), r0, rn(reg));
4079 /* Initialize fp offset in the save area. */
4080 movi(rn(reg), _jitc->function->vafp);
4081 stxi_i(offsetof(jit_va_list_t, fpoff), r0, rn(reg));
4083 /* Initialize overflow pointer to the first stack argument. */
4084 addi(rn(reg), _RBP_REGNO, jit_selfsize());
4085 stxi(offsetof(jit_va_list_t, over), r0, rn(reg));
4087 /* Initialize register save area pointer. */
4088 addi(rn(reg), r0, first_gp_offset);
4089 stxi(offsetof(jit_va_list_t, save), r0, rn(reg));
4096 _vaarg(jit_state_t *_jit, jit_int32_t r0, jit_int32_t r1)
4098 #if __X32 || __CYGWIN__ || _WIN32
4099 assert(_jitc->function->self.call & jit_call_varargs);
4101 addi(r1, r1, va_gp_increment);
4108 assert(_jitc->function->self.call & jit_call_varargs);
4110 rg0 = jit_get_reg(jit_class_gpr);
4111 rg1 = jit_get_reg(jit_class_gpr);
4113 /* Load the gp offset in save area in the first temporary. */
4114 ldxi_i(rn(rg0), r1, offsetof(jit_va_list_t, gpoff));
4116 /* Jump over if there are no remaining arguments in the save area. */
4117 icmpi(rn(rg0), va_gp_max_offset);
4120 /* Load the save area pointer in the second temporary. */
4121 ldxi(rn(rg1), r1, offsetof(jit_va_list_t, save));
4123 /* Load the vararg argument in the first argument. */
4124 ldxr(r0, rn(rg1), rn(rg0));
4126 /* Update the gp offset. */
4127 addi(rn(rg0), rn(rg0), 8);
4128 stxi_i(offsetof(jit_va_list_t, gpoff), r1, rn(rg0));
4130 /* Will only need one temporary register below. */
4133 /* Jump over overflow code. */
4136 /* Where to land if argument is in overflow area. */
4137 patch_at(ge_code, _jit->pc.w);
4139 /* Load overflow pointer. */
4140 ldxi(rn(rg0), r1, offsetof(jit_va_list_t, over));
4142 /* Load argument. */
4145 /* Update overflow pointer. */
4146 addi(rn(rg0), rn(rg0), va_gp_increment);
4147 stxi(offsetof(jit_va_list_t, over), r1, rn(rg0));
4149 /* Where to land if argument is in save area. */
4150 patch_at(lt_code, _jit->pc.w);
4156 /* The x87 boolean argument tells if will put the result in a x87
4157 * register if non false, in a sse register otherwise. */
4159 _vaarg_d(jit_state_t *_jit, jit_int32_t r0, jit_int32_t r1, jit_bool_t x87)
4161 #if __X32 || __CYGWIN__ || _WIN32
4162 assert(_jitc->function->self.call & jit_call_varargs);
4174 assert(_jitc->function->self.call & jit_call_varargs);
4176 rg0 = jit_get_reg(jit_class_gpr);
4177 rg1 = jit_get_reg(jit_class_gpr);
4179 /* Load the fp offset in save area in the first temporary. */
4180 ldxi_i(rn(rg0), r1, offsetof(jit_va_list_t, fpoff));
4182 /* Jump over if there are no remaining arguments in the save area. */
4183 icmpi(rn(rg0), va_fp_max_offset);
4186 /* Load the save area pointer in the second temporary. */
4187 ldxi(rn(rg1), r1, offsetof(jit_va_list_t, save));
4189 /* Load the vararg argument in the first argument. */
4191 x87_ldxr_d(r0, rn(rg1), rn(rg0));
4193 sse_ldxr_d(r0, rn(rg1), rn(rg0));
4195 /* Update the fp offset. */
4196 addi(rn(rg0), rn(rg0), va_fp_increment);
4197 stxi_i(offsetof(jit_va_list_t, fpoff), r1, rn(rg0));
4199 /* Will only need one temporary register below. */
4202 /* Jump over overflow code. */
4205 /* Where to land if argument is in overflow area. */
4206 patch_at(ge_code, _jit->pc.w);
4208 /* Load overflow pointer. */
4209 ldxi(rn(rg0), r1, offsetof(jit_va_list_t, over));
4211 /* Load argument. */
4213 x87_ldr_d(r0, rn(rg0));
4215 sse_ldr_d(r0, rn(rg0));
4217 /* Update overflow pointer. */
4218 addi(rn(rg0), rn(rg0), 8);
4219 stxi(offsetof(jit_va_list_t, over), r1, rn(rg0));
4221 /* Where to land if argument is in save area. */
4222 patch_at(lt_code, _jit->pc.w);
4229 _patch_at(jit_state_t *_jit, jit_word_t instr, jit_word_t label)
4232 jit_uint8_t *code = (jit_uint8_t *)instr;
4237 *(jit_word_t *)instr = label;
4239 /* forward pc relative address known to be in range */
4249 if (code[1] < 0x80 || code[1] > 0x8f)
4258 disp = label - (instr + 4);
4259 assert((jit_int32_t)disp == disp);
4260 *(jit_int32_t *)instr = disp;
4266 disp = label - (instr + 1);
4267 assert((jit_int8_t)disp == disp);
4268 *(jit_int8_t *)instr = disp;