2 * Copyright (C) 2012-2019 Free Software Foundation, Inc.
4 * This file is part of GNU lightning.
6 * GNU lightning is free software; you can redistribute it and/or modify it
7 * under the terms of the GNU Lesser General Public License as published
8 * by the Free Software Foundation; either version 3, or (at your option)
11 * GNU lightning is distributed in the hope that it will be useful, but
12 * WITHOUT ANY WARRANTY; without even the implied warranty of MERCHANTABILITY
13 * or FITNESS FOR A PARTICULAR PURPOSE. See the GNU Lesser General Public
14 * License for more details.
17 * Paulo Cesar Pereira de Andrade
20 /* avoid using it due to partial stalls */
24 # if __X32 || __X64_32
26 # define ldi(u, v) ldi_i(u, v)
27 # define ldr(u, v) ldr_i(u, v)
28 # define ldxr(u, v, w) ldxr_i(u, v, w)
29 # define ldxi(u, v, w) ldxi_i(u, v, w)
30 # define sti(u, v) sti_i(u, v)
31 # define stxi(u, v, w) stxi_i(u, v, w)
32 # define can_sign_extend_int_p(im) 1
33 # define can_zero_extend_int_p(im) 1
34 # define fits_uint32_p(im) 1
37 # define ldi(u, v) ldi_l(u, v)
38 # define ldr(u, v) ldr_l(u, v)
39 # define ldxr(u, v, w) ldxr_l(u, v, w)
40 # define ldxi(u, v, w) ldxi_l(u, v, w)
41 # define sti(u, v) sti_l(u, v)
42 # define stxi(u, v, w) stxi_l(u, v, w)
43 # define can_sign_extend_int_p(im) \
44 (((im) >= 0 && (long long)(im) <= 0x7fffffffLL) || \
45 ((im) < 0 && (long long)(im) > -0x80000000LL))
46 # define can_zero_extend_int_p(im) \
47 ((im) >= 0 && (im) < 0x80000000LL)
48 # define fits_uint32_p(im) (((im) & 0xffffffff00000000LL) == 0)
50 # if __X32 || __CYGWIN__ || __X64_32 || _WIN32
52 ((rn) >= _RAX_REGNO && (rn) <= _RBX_REGNO)
66 # define _R10_REGNO 10
67 # define _R11_REGNO 11
68 # define _R12_REGNO 12
69 # define _R13_REGNO 13
70 # define _R14_REGNO 14
71 # define _R15_REGNO 15
72 # define r7(reg) ((reg) & 7)
73 # define r8(reg) ((reg) & 15)
79 # define X86_OR 1 << 3
80 # define X86_ADC 2 << 3
81 # define X86_SBB 3 << 3
82 # define X86_AND 4 << 3
83 # define X86_SUB 5 << 3
84 # define X86_XOR 6 << 3
85 # define X86_CMP 7 << 3
100 # define X86_CC_NO 0x1
101 # define X86_CC_NAE 0x2
102 # define X86_CC_B 0x2
103 # define X86_CC_C 0x2
104 # define X86_CC_AE 0x3
105 # define X86_CC_NB 0x3
106 # define X86_CC_NC 0x3
107 # define X86_CC_E 0x4
108 # define X86_CC_Z 0x4
109 # define X86_CC_NE 0x5
110 # define X86_CC_NZ 0x5
111 # define X86_CC_BE 0x6
112 # define X86_CC_NA 0x6
113 # define X86_CC_A 0x7
114 # define X86_CC_NBE 0x7
115 # define X86_CC_S 0x8
116 # define X86_CC_NS 0x9
117 # define X86_CC_P 0xa
118 # define X86_CC_PE 0xa
119 # define X86_CC_NP 0xb
120 # define X86_CC_PO 0xb
121 # define X86_CC_L 0xc
122 # define X86_CC_NGE 0xc
123 # define X86_CC_GE 0xd
124 # define X86_CC_NL 0xd
125 # define X86_CC_LE 0xe
126 # define X86_CC_NG 0xe
127 # define X86_CC_G 0xf
128 # define X86_CC_NLE 0xf
129 # define mrm(md, r, m) *_jit->pc.uc++ = (md<<6) | (r<<3) | m
130 # define sib(sc, i, b) *_jit->pc.uc++ = (sc<<6) | (i<<3) | b
131 # define ic(c) *_jit->pc.uc++ = c
132 # define is(s) *_jit->pc.us++ = s
133 # define ii(i) *_jit->pc.ui++ = i
134 # if __X64 && !__X64_32
135 # define il(l) *_jit->pc.ul++ = l
139 # define patch_abs(instr, label) \
140 *(jit_word_t *)(instr - sizeof(jit_word_t)) = label
141 # define patch_rel(instr, label) \
142 *(jit_int32_t *)(instr - 4) = label - instr
143 # define patch_rel_char(instr, label) \
144 *(jit_int8_t *)(instr - 1) = label - instr
145 # define rex(l, w, r, x, b) _rex(_jit, l, w, r, x, b)
147 _rex(jit_state_t*,jit_int32_t,jit_int32_t,jit_int32_t,jit_int32_t,jit_int32_t);
148 # define rx(rd, md, rb, ri, ms) _rx(_jit, rd, md, rb, ri, ms)
150 _rx(jit_state_t*,jit_int32_t,jit_int32_t,jit_int32_t,jit_int32_t,jit_int32_t);
151 # define nop(n) _nop(_jit, n)
152 static void _nop(jit_state_t*, jit_int32_t);
153 # define emms() is(0x770f)
154 # define lea(md, rb, ri, ms, rd) _lea(_jit, md, rb, ri, ms, rd)
156 _lea(jit_state_t*,jit_int32_t,jit_int32_t,jit_int32_t,jit_int32_t,jit_int32_t);
157 # define pushr(r0) _pushr(_jit, r0)
158 static void _pushr(jit_state_t*, jit_int32_t) maybe_unused;
159 # define popr(r0) _popr(_jit, r0)
160 static void _popr(jit_state_t*, jit_int32_t) maybe_unused;
161 # define xchgr(r0, r1) _xchgr(_jit, r0, r1)
162 static void _xchgr(jit_state_t*, jit_int32_t, jit_int32_t);
163 # define testr(r0, r1) _testr(_jit, r0, r1)
164 static void _testr(jit_state_t*, jit_int32_t, jit_int32_t);
165 # define testi(r0, i0) _testi(_jit, r0, i0)
166 static void _testi(jit_state_t*, jit_int32_t, jit_word_t);
167 # define cc(code, r0) _cc(_jit, code, r0)
168 static void _cc(jit_state_t*, jit_int32_t, jit_int32_t);
169 # define icmpr(r0, r1) alur(X86_CMP, r0, r1)
170 # define alur(code, r0, r1) _alur(_jit, code, r0, r1)
171 static void _alur(jit_state_t*, jit_int32_t, jit_int32_t, jit_int32_t);
172 # define icmpi(r0, i0) alui(X86_CMP, r0, i0)
173 # define alui(code, r0, i0) _alui(_jit, code, r0, i0)
174 static void _alui(jit_state_t*, jit_int32_t, jit_int32_t, jit_word_t);
175 # define iaddr(r0, r1) alur(X86_ADD, r0, r1)
176 # define save(r0) _save(_jit, r0)
177 static void _save(jit_state_t*, jit_int32_t);
178 # define load(r0) _load(_jit, r0)
179 static void _load(jit_state_t*, jit_int32_t);
180 # define addr(r0, r1, r2) _addr(_jit, r0, r1, r2)
181 static void _addr(jit_state_t*, jit_int32_t, jit_int32_t, jit_int32_t);
182 # define iaddi(r0, i0) alui(X86_ADD, r0, i0)
183 # define addi(r0, r1, i0) _addi(_jit, r0, r1, i0)
184 static void _addi(jit_state_t*, jit_int32_t, jit_int32_t, jit_word_t);
185 #define addcr(r0, r1, r2) _addcr(_jit, r0, r1, r2)
186 static void _addcr(jit_state_t*, jit_int32_t, jit_int32_t, jit_int32_t);
187 #define addci(r0, r1, i0) _addci(_jit, r0, r1, i0)
188 static void _addci(jit_state_t*, jit_int32_t, jit_int32_t, jit_word_t);
189 # define iaddxr(r0, r1) alur(X86_ADC, r0, r1)
190 # define addxr(r0, r1, r2) _addxr(_jit, r0, r1, r2)
191 static void _addxr(jit_state_t*, jit_int32_t, jit_int32_t, jit_int32_t);
192 # define iaddxi(r0, i0) alui(X86_ADC, r0, i0)
193 # define addxi(r0, r1, i0) _addxi(_jit, r0, r1, i0)
194 static void _addxi(jit_state_t*, jit_int32_t, jit_int32_t, jit_word_t);
195 # define isubr(r0, r1) alur(X86_SUB, r0, r1)
196 # define subr(r0, r1, r2) _subr(_jit, r0, r1, r2)
197 static void _subr(jit_state_t*, jit_int32_t, jit_int32_t, jit_int32_t);
198 # define isubi(r0, i0) alui(X86_SUB, r0, i0)
199 # define subi(r0, r1, i0) _subi(_jit, r0, r1, i0)
200 static void _subi(jit_state_t*, jit_int32_t, jit_int32_t, jit_word_t);
201 # define subcr(r0, r1, r2) _subcr(_jit, r0, r1, r2)
202 static void _subcr(jit_state_t*,jit_int32_t,jit_int32_t,jit_int32_t);
203 # define subci(r0, r1, i0) _subci(_jit, r0, r1, i0)
204 static void _subci(jit_state_t*,jit_int32_t,jit_int32_t,jit_word_t);
205 # define isubxr(r0, r1) alur(X86_SBB, r0, r1)
206 # define subxr(r0, r1, r2) _subxr(_jit, r0, r1, r2)
207 static void _subxr(jit_state_t*,jit_int32_t,jit_int32_t,jit_int32_t);
208 # define isubxi(r0, i0) alui(X86_SBB, r0, i0)
209 # define subxi(r0, r1, i0) _subxi(_jit, r0, r1, i0)
210 static void _subxi(jit_state_t*,jit_int32_t,jit_int32_t,jit_word_t);
211 # define rsbi(r0, r1, i0) _rsbi(_jit, r0, r1, i0)
212 static void _rsbi(jit_state_t*,jit_int32_t,jit_int32_t,jit_word_t);
213 # define imulr(r0, r1) _imulr(_jit, r0, r1)
214 static void _imulr(jit_state_t*, jit_int32_t, jit_int32_t);
215 # define imuli(r0, r1, i0) _imuli(_jit, r0, r1, i0)
216 static void _imuli(jit_state_t*, jit_int32_t, jit_int32_t, jit_word_t);
217 # define mulr(r0, r1, r2) _mulr(_jit, r0, r1, r2)
218 static void _mulr(jit_state_t*, jit_int32_t, jit_int32_t, jit_int32_t);
219 # define muli(r0, r1, i0) _muli(_jit, r0, r1, i0)
220 static void _muli(jit_state_t*, jit_int32_t, jit_int32_t, jit_word_t);
221 # define umulr(r0) unr(X86_IMUL, r0)
222 # define umulr_u(r0) unr(X86_MUL, r0)
223 # define qmulr(r0, r1, r2, r3) _iqmulr(_jit, r0, r1, r2, r3, 1)
224 # define qmulr_u(r0, r1, r2, r3) _iqmulr(_jit, r0, r1, r2, r3, 0)
225 # define iqmulr(r0, r1, r2, r3, sign) _iqmulr(_jit, r0, r1, r2, r3, sign)
226 static void _iqmulr(jit_state_t*, jit_int32_t, jit_int32_t,
227 jit_int32_t,jit_int32_t, jit_bool_t);
228 # define qmuli(r0, r1, r2, i0) _iqmuli(_jit, r0, r1, r2, i0, 1)
229 # define qmuli_u(r0, r1, r2, i0) _iqmuli(_jit, r0, r1, r2, i0, 0)
230 # define iqmuli(r0, r1, r2, i0, sign) _iqmuli(_jit, r0, r1, r2, i0, sign)
231 static void _iqmuli(jit_state_t*, jit_int32_t, jit_int32_t,
232 jit_int32_t,jit_word_t, jit_bool_t);
233 # define sign_extend_rdx_rax() _sign_extend_rdx_rax(_jit)
234 static void _sign_extend_rdx_rax(jit_state_t*);
235 # define idivr(r0) unr(X86_IDIV, r0)
236 # define idivr_u(r0) unr(X86_DIV, r0)
237 # define divremr(r0, r1, r2, i0, i1) _divremr(_jit, r0, r1, r2, i0, i1)
239 _divremr(jit_state_t*,jit_int32_t,jit_int32_t,jit_int32_t,
240 jit_bool_t,jit_bool_t);
241 # define divremi(r0, r1, i0, i1, i2) _divremi(_jit, r0, r1, i0, i1, i2)
243 _divremi(jit_state_t*,jit_int32_t,jit_int32_t,jit_word_t,jit_bool_t,jit_bool_t);
244 # define divr(r0, r1, r2) divremr(r0, r1, r2, 1, 1)
245 # define divi(r0, r1, i0) divremi(r0, r1, i0, 1, 1)
246 # define divr_u(r0, r1, r2) divremr(r0, r1, r2, 0, 1)
247 # define divi_u(r0, r1, i0) divremi(r0, r1, i0, 0, 1)
248 # define qdivr(r0, r1, r2, r3) _iqdivr(_jit, r0, r1, r2, r3, 1)
249 # define qdivr_u(r0, r1, r2, r3) _iqdivr(_jit, r0, r1, r2, r3, 0)
250 # define iqdivr(r0, r1, r2, r3, sign) _iqdivr(_jit, r0, r1, r2, r3, sign)
251 static void _iqdivr(jit_state_t*, jit_int32_t, jit_int32_t,
252 jit_int32_t,jit_int32_t, jit_bool_t);
253 # define qdivi(r0, r1, r2, i0) _iqdivi(_jit, r0, r1, r2, i0, 1)
254 # define qdivi_u(r0, r1, r2, i0) _iqdivi(_jit, r0, r1, r2, i0, 0)
255 # define iqdivi(r0, r1, r2, i0, sign) _iqdivi(_jit, r0, r1, r2, i0, sign)
256 static void _iqdivi(jit_state_t*, jit_int32_t, jit_int32_t,
257 jit_int32_t,jit_word_t, jit_bool_t);
258 # define remr(r0, r1, r2) divremr(r0, r1, r2, 1, 0)
259 # define remi(r0, r1, i0) divremi(r0, r1, i0, 1, 0)
260 # define remr_u(r0, r1, r2) divremr(r0, r1, r2, 0, 0)
261 # define remi_u(r0, r1, i0) divremi(r0, r1, i0, 0, 0)
262 # define iandr(r0, r1) alur(X86_AND, r0, r1)
263 # define andr(r0, r1, r2) _andr(_jit, r0, r1, r2)
264 static void _andr(jit_state_t*,jit_int32_t,jit_int32_t,jit_int32_t);
265 # define iandi(r0, i0) alui(X86_AND, r0, i0)
266 # define andi(r0, r1, i0) _andi(_jit, r0, r1, i0)
267 static void _andi(jit_state_t*, jit_int32_t,jit_int32_t,jit_word_t);
268 # define iorr(r0, r1) alur(X86_OR, r0, r1)
269 # define orr(r0, r1, r2) _orr(_jit, r0, r1, r2)
270 static void _orr(jit_state_t*, jit_int32_t,jit_int32_t,jit_int32_t);
271 # define iori(r0, i0) alui(X86_OR, r0, i0)
272 # define ori(r0, r1, i0) _ori(_jit, r0, r1, i0)
273 static void _ori(jit_state_t*, jit_int32_t,jit_int32_t,jit_word_t);
274 # define ixorr(r0, r1) alur(X86_XOR, r0, r1)
275 # define xorr(r0, r1, r2) _xorr(_jit, r0, r1, r2)
276 static void _xorr(jit_state_t*, jit_int32_t,jit_int32_t,jit_int32_t);
277 # define ixori(r0, i0) alui(X86_XOR, r0, i0)
278 # define xori(r0, r1, i0) _xori(_jit, r0, r1, i0)
279 static void _xori(jit_state_t*, jit_int32_t,jit_int32_t,jit_word_t);
280 # define irotshr(code, r0) _irotshr(_jit, code, r0)
281 static void _irotshr(jit_state_t*, jit_int32_t, jit_int32_t);
282 # define rotshr(code, r0, r1, r2) _rotshr(_jit, code, r0, r1, r2)
284 _rotshr(jit_state_t*,jit_int32_t,jit_int32_t,jit_int32_t,jit_int32_t);
285 # define irotshi(code, r0, i0) _irotshi(_jit, code, r0, i0)
286 static void _irotshi(jit_state_t*, jit_int32_t, jit_int32_t, jit_word_t);
287 # define rotshi(code, r0, r1, i0) _rotshi(_jit, code, r0, r1, i0)
289 _rotshi(jit_state_t*,jit_int32_t,jit_int32_t,jit_int32_t,jit_word_t);
290 # define lshr(r0, r1, r2) rotshr(X86_SHL, r0, r1, r2)
291 # define lshi(r0, r1, i0) _lshi(_jit, r0, r1, i0)
292 static void _lshi(jit_state_t*, jit_int32_t, jit_int32_t, jit_word_t);
293 # define rshr(r0, r1, r2) rotshr(X86_SAR, r0, r1, r2)
294 # define rshi(r0, r1, i0) rotshi(X86_SAR, r0, r1, i0)
295 # define rshr_u(r0, r1, r2) rotshr(X86_SHR, r0, r1, r2)
296 # define rshi_u(r0, r1, i0) rotshi(X86_SHR, r0, r1, i0)
297 # define unr(code, r0) _unr(_jit, code, r0)
298 static void _unr(jit_state_t*, jit_int32_t, jit_int32_t);
299 # define inegr(r0) unr(X86_NEG, r0)
300 # define negr(r0, r1) _negr(_jit, r0, r1)
301 static void _negr(jit_state_t*, jit_int32_t, jit_int32_t);
302 # define icomr(r0) unr(X86_NOT, r0)
303 # define comr(r0, r1) _comr(_jit, r0, r1)
304 static void _comr(jit_state_t*, jit_int32_t, jit_int32_t);
306 # define incr(r0, r1) _incr(_jit, r0, r1)
307 static void _incr(jit_state_t*, jit_int32_t, jit_int32_t);
308 # define decr(r0, r1) _decr(_jit, r0, r1)
309 static void _decr(jit_state_t*, jit_int32_t, jit_int32_t);
311 # define cr(code, r0, r1, r2) _cr(_jit, code, r0, r1, r2)
313 _cr(jit_state_t*, jit_int32_t, jit_int32_t, jit_int32_t, jit_int32_t);
314 # define ci(code, r0, r1, i0) _ci(_jit, code, r0, r1, i0)
316 _ci(jit_state_t *_jit, jit_int32_t, jit_int32_t, jit_int32_t, jit_word_t);
317 # define ci0(code, r0, r1) _ci0(_jit, code, r0, r1)
318 static void _ci0(jit_state_t*, jit_int32_t, jit_int32_t, jit_int32_t);
319 # define ltr(r0, r1, r2) _ltr(_jit, r0, r1, r2)
320 static void _ltr(jit_state_t*, jit_int32_t, jit_int32_t, jit_int32_t);
321 # define lti(r0, r1, i0) _lti(_jit, r0, r1, i0)
322 static void _lti(jit_state_t*, jit_int32_t, jit_int32_t, jit_word_t);
323 # define ltr_u(r0, r1, r2) _ltr_u(_jit, r0, r1, r2)
324 static void _ltr_u(jit_state_t*, jit_int32_t, jit_int32_t, jit_int32_t);
325 # define lti_u(r0, r1, i0) ci(X86_CC_B, r0, r1, i0)
326 # define ler(r0, r1, r2) _ler(_jit, r0, r1, r2)
327 static void _ler(jit_state_t*, jit_int32_t, jit_int32_t, jit_int32_t);
328 # define lei(r0, r1, i0) ci(X86_CC_LE, r0, r1, i0)
329 # define ler_u(r0, r1, r2) _ler_u(_jit, r0, r1, r2)
330 static void _ler_u(jit_state_t*, jit_int32_t, jit_int32_t, jit_int32_t);
331 # define lei_u(r0, r1, i0) _lei_u(_jit, r0, r1, i0)
332 static void _lei_u(jit_state_t*, jit_int32_t, jit_int32_t, jit_word_t);
333 # define eqr(r0, r1, r2) _eqr(_jit, r0, r1, r2)
334 static void _eqr(jit_state_t*, jit_int32_t, jit_int32_t, jit_int32_t);
335 # define eqi(r0, r1, i0) _eqi(_jit, r0, r1, i0)
336 static void _eqi(jit_state_t*, jit_int32_t, jit_int32_t, jit_word_t);
337 # define ger(r0, r1, r2) _ger(_jit, r0, r1, r2)
338 static void _ger(jit_state_t*, jit_int32_t, jit_int32_t, jit_int32_t);
339 # define gei(r0, r1, i0) _gei(_jit, r0, r1, i0)
340 static void _gei(jit_state_t*, jit_int32_t, jit_int32_t, jit_word_t);
341 # define ger_u(r0, r1, r2) _ger_u(_jit, r0, r1, r2)
342 static void _ger_u(jit_state_t*, jit_int32_t, jit_int32_t, jit_int32_t);
343 # define gei_u(r0, r1, i0) _gei_u(_jit, r0, r1, i0)
344 static void _gei_u(jit_state_t*, jit_int32_t, jit_int32_t, jit_word_t);
345 # define gtr(r0, r1, r2) _gtr(_jit, r0, r1, r2)
346 static void _gtr(jit_state_t*, jit_int32_t, jit_int32_t, jit_int32_t);
347 # define gti(r0, r1, i0) _ci(_jit, X86_CC_G, r0, r1, i0)
348 # define gtr_u(r0, r1, r2) _gtr_u(_jit, r0, r1, r2)
349 static void _gtr_u(jit_state_t*, jit_int32_t, jit_int32_t, jit_int32_t);
350 # define gti_u(r0, r1, i0) _gti_u(_jit, r0, r1, i0)
351 static void _gti_u(jit_state_t*, jit_int32_t, jit_int32_t, jit_word_t);
352 # define ner(r0, r1, r2) _ner(_jit, r0, r1, r2)
353 static void _ner(jit_state_t*, jit_int32_t, jit_int32_t, jit_int32_t);
354 # define nei(r0, r1, i0) _nei(_jit, r0, r1, i0)
355 static void _nei(jit_state_t*, jit_int32_t, jit_int32_t, jit_word_t);
356 # define movr(r0, r1) _movr(_jit, r0, r1)
357 static void _movr(jit_state_t*, jit_int32_t, jit_int32_t);
358 # define imovi(r0, i0) _imovi(_jit, r0, i0)
359 static void _imovi(jit_state_t*, jit_int32_t, jit_word_t);
360 # define movi(r0, i0) _movi(_jit, r0, i0)
361 static void _movi(jit_state_t*, jit_int32_t, jit_word_t);
362 # define movi_p(r0, i0) _movi_p(_jit, r0, i0)
363 static jit_word_t _movi_p(jit_state_t*, jit_int32_t, jit_word_t);
364 # define movcr(r0, r1) _movcr(_jit, r0, r1)
365 static void _movcr(jit_state_t*,jit_int32_t,jit_int32_t);
366 # define movcr_u(r0, r1) _movcr_u(_jit, r0, r1)
367 static void _movcr_u(jit_state_t*,jit_int32_t,jit_int32_t);
368 # define movsr(r0, r1) _movsr(_jit, r0, r1)
369 static void _movsr(jit_state_t*,jit_int32_t,jit_int32_t);
370 # define movsr_u(r0, r1) _movsr_u(_jit, r0, r1)
371 static void _movsr_u(jit_state_t*,jit_int32_t,jit_int32_t);
372 # if __X64 && !__X64_32
373 # define movir(r0, r1) _movir(_jit, r0, r1)
374 static void _movir(jit_state_t*,jit_int32_t,jit_int32_t);
375 # define movir_u(r0, r1) _movir_u(_jit, r0, r1)
376 static void _movir_u(jit_state_t*,jit_int32_t,jit_int32_t);
378 # define htonr_us(r0, r1) _htonr_us(_jit, r0, r1)
379 static void _htonr_us(jit_state_t*,jit_int32_t,jit_int32_t);
380 # define htonr_ui(r0, r1) _htonr_ui(_jit, r0, r1)
381 static void _htonr_ui(jit_state_t*,jit_int32_t,jit_int32_t);
382 # if __X64 && !__X64_32
383 #define htonr_ul(r0, r1) _htonr_ul(_jit, r0, r1)
384 static void _htonr_ul(jit_state_t*,jit_int32_t,jit_int32_t);
386 # define extr_c(r0, r1) _extr_c(_jit, r0, r1)
387 static void _extr_c(jit_state_t*,jit_int32_t,jit_int32_t);
388 # define extr_uc(r0, r1) _extr_uc(_jit, r0, r1)
389 static void _extr_uc(jit_state_t*,jit_int32_t,jit_int32_t);
390 # define extr_s(r0, r1) movsr(r0, r1)
391 # define extr_us(r0, r1) movsr_u(r0, r1)
392 # if __X64 && !__X64_32
393 # define extr_i(r0, r1) movir(r0, r1)
394 # define extr_ui(r0, r1) movir_u(r0, r1)
396 # define ldr_c(r0, r1) _ldr_c(_jit, r0, r1)
397 static void _ldr_c(jit_state_t*, jit_int32_t, jit_int32_t);
398 # define ldi_c(r0, i0) _ldi_c(_jit, r0, i0)
399 static void _ldi_c(jit_state_t*, jit_int32_t, jit_word_t);
400 # define ldr_uc(r0, r1) _ldr_uc(_jit, r0, r1)
401 static void _ldr_uc(jit_state_t*, jit_int32_t, jit_int32_t);
402 # define ldi_uc(r0, i0) _ldi_uc(_jit, r0, i0)
403 static void _ldi_uc(jit_state_t*, jit_int32_t, jit_word_t);
404 # define ldr_s(r0, r1) _ldr_s(_jit, r0, r1)
405 static void _ldr_s(jit_state_t*, jit_int32_t, jit_int32_t);
406 # define ldi_s(r0, i0) _ldi_s(_jit, r0, i0)
407 static void _ldi_s(jit_state_t*, jit_int32_t, jit_word_t);
408 # define ldr_us(r0, r1) _ldr_us(_jit, r0, r1)
409 static void _ldr_us(jit_state_t*, jit_int32_t, jit_int32_t);
410 # define ldi_us(r0, i0) _ldi_us(_jit, r0, i0)
411 static void _ldi_us(jit_state_t*, jit_int32_t, jit_word_t);
412 # if __X32 || !__X64_32
413 # define ldr_i(r0, r1) _ldr_i(_jit, r0, r1)
414 static void _ldr_i(jit_state_t*, jit_int32_t, jit_int32_t);
415 # define ldi_i(r0, i0) _ldi_i(_jit, r0, i0)
416 static void _ldi_i(jit_state_t*, jit_int32_t, jit_word_t);
420 # define ldr_i(r0, r1) _ldr_ui(_jit, r0, r1)
421 # define ldi_i(r0, i0) _ldi_ui(_jit, r0, i0)
423 # define ldr_ui(r0, r1) _ldr_ui(_jit, r0, r1)
424 # define ldi_ui(r0, i0) _ldi_ui(_jit, r0, i0)
426 static void _ldr_ui(jit_state_t*, jit_int32_t, jit_int32_t);
427 static void _ldi_ui(jit_state_t*, jit_int32_t, jit_word_t);
429 # define ldr_l(r0, r1) _ldr_l(_jit, r0, r1)
430 static void _ldr_l(jit_state_t*, jit_int32_t, jit_int32_t);
431 # define ldi_l(r0, i0) _ldi_l(_jit, r0, i0)
432 static void _ldi_l(jit_state_t*, jit_int32_t, jit_word_t);
435 # define ldxr_c(r0, r1, r2) _ldxr_c(_jit, r0, r1, r2)
436 static void _ldxr_c(jit_state_t*, jit_int32_t, jit_int32_t, jit_int32_t);
437 # define ldxi_c(r0, r1, i0) _ldxi_c(_jit, r0, r1, i0)
438 static void _ldxi_c(jit_state_t*, jit_int32_t, jit_int32_t, jit_word_t);
439 # define ldxr_uc(r0, r1, r2) _ldxr_uc(_jit, r0, r1, r2)
440 static void _ldxr_uc(jit_state_t*, jit_int32_t, jit_int32_t, jit_int32_t);
441 # define ldxi_uc(r0, r1, i0) _ldxi_uc(_jit, r0, r1, i0)
442 static void _ldxi_uc(jit_state_t*, jit_int32_t, jit_int32_t, jit_word_t);
443 # define ldxr_s(r0, r1, r2) _ldxr_s(_jit, r0, r1, r2)
444 static void _ldxr_s(jit_state_t*, jit_int32_t, jit_int32_t, jit_int32_t);
445 # define ldxi_s(r0, r1, i0) _ldxi_s(_jit, r0, r1, i0)
446 static void _ldxi_s(jit_state_t*, jit_int32_t, jit_int32_t, jit_word_t);
447 # define ldxr_us(r0, r1, r2) _ldxr_us(_jit, r0, r1, r2)
448 static void _ldxr_us(jit_state_t*, jit_int32_t, jit_int32_t, jit_int32_t);
449 # define ldxi_us(r0, r1, i0) _ldxi_us(_jit, r0, r1, i0)
450 static void _ldxi_us(jit_state_t*, jit_int32_t, jit_int32_t, jit_word_t);
451 # if __X32 || !__X64_32
452 # define ldxr_i(r0, r1, r2) _ldxr_i(_jit, r0, r1, r2)
453 static void _ldxr_i(jit_state_t*, jit_int32_t, jit_int32_t, jit_int32_t);
454 # define ldxi_i(r0, r1, i0) _ldxi_i(_jit, r0, r1, i0)
455 static void _ldxi_i(jit_state_t*, jit_int32_t, jit_int32_t, jit_word_t);
459 # define ldxr_i(r0, r1, r2) _ldxr_ui(_jit, r0, r1, r2)
460 # define ldxi_i(r0, r1, i0) _ldxi_ui(_jit, r0, r1, i0)
462 # define ldxr_ui(r0, r1, r2) _ldxr_ui(_jit, r0, r1, r2)
463 # define ldxi_ui(r0, r1, i0) _ldxi_ui(_jit, r0, r1, i0)
465 static void _ldxr_ui(jit_state_t*, jit_int32_t, jit_int32_t, jit_int32_t);
466 static void _ldxi_ui(jit_state_t*, jit_int32_t, jit_int32_t, jit_word_t);
468 # define ldxr_l(r0, r1, r2) _ldxr_l(_jit, r0, r1, r2)
469 static void _ldxr_l(jit_state_t*, jit_int32_t, jit_int32_t, jit_int32_t);
470 # define ldxi_l(r0, r1, i0) _ldxi_l(_jit, r0, r1, i0)
471 static void _ldxi_l(jit_state_t*, jit_int32_t, jit_int32_t, jit_word_t);
474 # define str_c(r0, r1) _str_c(_jit, r0, r1)
475 static void _str_c(jit_state_t*, jit_int32_t, jit_int32_t);
476 # define sti_c(i0, r0) _sti_c(_jit, i0, r0)
477 static void _sti_c(jit_state_t*, jit_word_t, jit_int32_t);
478 # define str_s(r0, r1) _str_s(_jit, r0, r1)
479 static void _str_s(jit_state_t*, jit_int32_t, jit_int32_t);
480 # define sti_s(i0, r0) _sti_s(_jit, i0, r0)
481 static void _sti_s(jit_state_t*, jit_word_t, jit_int32_t);
482 # define str_i(r0, r1) _str_i(_jit, r0, r1)
483 static void _str_i(jit_state_t*, jit_int32_t, jit_int32_t);
484 # define sti_i(i0, r0) _sti_i(_jit, i0, r0)
485 static void _sti_i(jit_state_t*, jit_word_t, jit_int32_t);
486 # if __X64 && !__X64_32
487 # define str_l(r0, r1) _str_l(_jit, r0, r1)
488 static void _str_l(jit_state_t*, jit_int32_t, jit_int32_t);
489 # define sti_l(i0, r0) _sti_l(_jit, i0, r0)
490 static void _sti_l(jit_state_t*, jit_word_t, jit_int32_t);
492 # define stxr_c(r0, r1, r2) _stxr_c(_jit, r0, r1, r2)
493 static void _stxr_c(jit_state_t*, jit_int32_t, jit_int32_t, jit_int32_t);
494 # define stxi_c(i0, r0, r1) _stxi_c(_jit, i0, r0, r1)
495 static void _stxi_c(jit_state_t*, jit_word_t, jit_int32_t, jit_int32_t);
496 # define stxr_s(r0, r1, r2) _stxr_s(_jit, r0, r1, r2)
497 static void _stxr_s(jit_state_t*, jit_int32_t, jit_int32_t, jit_int32_t);
498 # define stxi_s(i0, r0, r1) _stxi_s(_jit, i0, r0, r1)
499 static void _stxi_s(jit_state_t*, jit_word_t, jit_int32_t, jit_int32_t);
500 # define stxr_i(r0, r1, r2) _stxr_i(_jit, r0, r1, r2)
501 static void _stxr_i(jit_state_t*, jit_int32_t, jit_int32_t, jit_int32_t);
502 # define stxi_i(i0, r0, r1) _stxi_i(_jit, i0, r0, r1)
503 static void _stxi_i(jit_state_t*, jit_word_t, jit_int32_t, jit_int32_t);
504 # if __X64 && !__X64_32
505 # define stxr_l(r0, r1, r2) _stxr_l(_jit, r0, r1, r2)
506 static void _stxr_l(jit_state_t*, jit_int32_t, jit_int32_t, jit_int32_t);
507 # define stxi_l(i0, r0, r1) _stxi_l(_jit, i0, r0, r1)
508 static void _stxi_l(jit_state_t*, jit_word_t, jit_int32_t, jit_int32_t);
510 # define jcc(code, i0) _jcc(_jit, code, i0)
511 # define jo(i0) jcc(X86_CC_O, i0)
512 # define jno(i0) jcc(X86_CC_NO, i0)
513 # define jnae(i0) jcc(X86_CC_NAE, i0)
514 # define jb(i0) jcc(X86_CC_B, i0)
515 # define jc(i0) jcc(X86_CC_C, i0)
516 # define jae(i0) jcc(X86_CC_AE, i0)
517 # define jnb(i0) jcc(X86_CC_NB, i0)
518 # define jnc(i0) jcc(X86_CC_NC, i0)
519 # define je(i0) jcc(X86_CC_E, i0)
520 # define jz(i0) jcc(X86_CC_Z, i0)
521 # define jne(i0) jcc(X86_CC_NE, i0)
522 # define jnz(i0) jcc(X86_CC_NZ, i0)
523 # define jbe(i0) jcc(X86_CC_BE, i0)
524 # define jna(i0) jcc(X86_CC_NA, i0)
525 # define ja(i0) jcc(X86_CC_A, i0)
526 # define jnbe(i0) jcc(X86_CC_NBE, i0)
527 # define js(i0) jcc(X86_CC_S, i0)
528 # define jns(i0) jcc(X86_CC_NS, i0)
529 # define jp(i0) jcc(X86_CC_P, i0)
530 # define jpe(i0) jcc(X86_CC_PE, i0)
531 # define jnp(i0) jcc(X86_CC_NP, i0)
532 # define jpo(i0) jcc(X86_CC_PO, i0)
533 # define jl(i0) jcc(X86_CC_L, i0)
534 # define jnge(i0) jcc(X86_CC_NGE, i0)
535 # define jge(i0) jcc(X86_CC_GE, i0)
536 # define jnl(i0) jcc(X86_CC_NL, i0)
537 # define jle(i0) jcc(X86_CC_LE, i0)
538 # define jng(i0) jcc(X86_CC_NG, i0)
539 # define jg(i0) jcc(X86_CC_G, i0)
540 # define jnle(i0) jcc(X86_CC_NLE, i0)
541 static void _jcc(jit_state_t*, jit_int32_t, jit_word_t);
542 # define jccs(code, i0) _jccs(_jit, code, i0)
543 # define jos(i0) jccs(X86_CC_O, i0)
544 # define jnos(i0) jccs(X86_CC_NO, i0)
545 # define jnaes(i0) jccs(X86_CC_NAE, i0)
546 # define jbs(i0) jccs(X86_CC_B, i0)
547 # define jcs(i0) jccs(X86_CC_C, i0)
548 # define jaes(i0) jccs(X86_CC_AE, i0)
549 # define jnbs(i0) jccs(X86_CC_NB, i0)
550 # define jncs(i0) jccs(X86_CC_NC, i0)
551 # define jes(i0) jccs(X86_CC_E, i0)
552 # define jzs(i0) jccs(X86_CC_Z, i0)
553 # define jnes(i0) jccs(X86_CC_NE, i0)
554 # define jnzs(i0) jccs(X86_CC_NZ, i0)
555 # define jbes(i0) jccs(X86_CC_BE, i0)
556 # define jnas(i0) jccs(X86_CC_NA, i0)
557 # define jas(i0) jccs(X86_CC_A, i0)
558 # define jnbes(i0) jccs(X86_CC_NBE, i0)
559 # define jss(i0) jccs(X86_CC_S, i0)
560 # define jnss(i0) jccs(X86_CC_NS, i0)
561 # define jps(i0) jccs(X86_CC_P, i0)
562 # define jpes(i0) jccs(X86_CC_PE, i0)
563 # define jnps(i0) jccs(X86_CC_NP, i0)
564 # define jpos(i0) jccs(X86_CC_PO, i0)
565 # define jls(i0) jccs(X86_CC_L, i0)
566 # define jnges(i0) jccs(X86_CC_NGE, i0)
567 # define jges(i0) jccs(X86_CC_GE, i0)
568 # define jnls(i0) jccs(X86_CC_NL, i0)
569 # define jles(i0) jccs(X86_CC_LE, i0)
570 # define jngs(i0) jccs(X86_CC_NG, i0)
571 # define jgs(i0) jccs(X86_CC_G, i0)
572 # define jnles(i0) jccs(X86_CC_NLE, i0)
573 static void _jccs(jit_state_t*, jit_int32_t, jit_word_t);
574 # define jcr(code, i0, r0, r1) _jcr(_jit, code, i0, r0, r1)
575 static void _jcr(jit_state_t*,jit_int32_t,jit_word_t,jit_int32_t,jit_int32_t);
576 # define jci(code, i0, r0, i1) _jci(_jit, code, i0, r0, i1)
577 static void _jci(jit_state_t*,jit_int32_t,jit_word_t,jit_int32_t,jit_word_t);
578 # define jci0(code, i0, r0) _jci0(_jit, code, i0, r0)
579 static void _jci0(jit_state_t*, jit_int32_t, jit_word_t, jit_int32_t);
580 # define bltr(i0, r0, r1) _bltr(_jit, i0, r0, r1)
581 static jit_word_t _bltr(jit_state_t*, jit_word_t, jit_int32_t, jit_int32_t);
582 # define blti(i0, r0, i1) _blti(_jit, i0, r0, i1)
583 static jit_word_t _blti(jit_state_t*, jit_word_t, jit_int32_t, jit_word_t);
584 # define bltr_u(i0, r0, r1) _bltr_u(_jit, i0, r0, r1)
585 static jit_word_t _bltr_u(jit_state_t*, jit_word_t, jit_int32_t, jit_int32_t);
586 # define blti_u(i0, r0, i1) _blti_u(_jit, i0, r0, i1)
587 static jit_word_t _blti_u(jit_state_t*, jit_word_t, jit_int32_t, jit_word_t);
588 # define bler(i0, r0, r1) _bler(_jit, i0, r0, r1)
589 static jit_word_t _bler(jit_state_t*, jit_word_t, jit_int32_t, jit_int32_t);
590 # define blei(i0, r0, i1) _blei(_jit, i0, r0, i1)
591 static jit_word_t _blei(jit_state_t*, jit_word_t, jit_int32_t, jit_word_t);
592 # define bler_u(i0, r0, r1) _bler_u(_jit, i0, r0, r1)
593 static jit_word_t _bler_u(jit_state_t*, jit_word_t, jit_int32_t, jit_int32_t);
594 # define blei_u(i0, r0, i1) _blei_u(_jit, i0, r0, i1)
595 static jit_word_t _blei_u(jit_state_t*, jit_word_t, jit_int32_t, jit_word_t);
596 # define beqr(i0, r0, r1) _beqr(_jit, i0, r0, r1)
597 static jit_word_t _beqr(jit_state_t*, jit_word_t, jit_int32_t, jit_int32_t);
598 # define beqi(i0, r0, i1) _beqi(_jit, i0, r0, i1)
599 static jit_word_t _beqi(jit_state_t*, jit_word_t, jit_int32_t, jit_word_t);
600 # define bger(i0, r0, r1) _bger(_jit, i0, r0, r1)
601 static jit_word_t _bger(jit_state_t*, jit_word_t, jit_int32_t, jit_int32_t);
602 # define bgei(i0, r0, i1) _bgei(_jit, i0, r0, i1)
603 static jit_word_t _bgei(jit_state_t*, jit_word_t, jit_int32_t, jit_word_t);
604 # define bger_u(i0, r0, r1) _bger_u(_jit, i0, r0, r1)
605 static jit_word_t _bger_u(jit_state_t*, jit_word_t, jit_int32_t, jit_int32_t);
606 # define bgei_u(i0, r0, i1) _bgei_u(_jit, i0, r0, i1)
607 static jit_word_t _bgei_u(jit_state_t*, jit_word_t, jit_int32_t, jit_word_t);
608 # define bgtr(i0, r0, r1) _bgtr(_jit, i0, r0, r1)
609 static jit_word_t _bgtr(jit_state_t*, jit_word_t, jit_int32_t, jit_int32_t);
610 # define bgti(i0, r0, i1) _bgti(_jit, i0, r0, i1)
611 static jit_word_t _bgti(jit_state_t*, jit_word_t, jit_int32_t, jit_word_t);
612 # define bgtr_u(i0, r0, r1) _bgtr_u(_jit, i0, r0, r1)
613 static jit_word_t _bgtr_u(jit_state_t*, jit_word_t, jit_int32_t, jit_int32_t);
614 # define bgti_u(i0, r0, i1) _bgti_u(_jit, i0, r0, i1)
615 static jit_word_t _bgti_u(jit_state_t*, jit_word_t, jit_int32_t, jit_word_t);
616 # define bner(i0, r0, r1) _bner(_jit, i0, r0, r1)
617 static jit_word_t _bner(jit_state_t*, jit_word_t, jit_int32_t, jit_int32_t);
618 # define bnei(i0, r0, i1) _bnei(_jit, i0, r0, i1)
619 static jit_word_t _bnei(jit_state_t*, jit_word_t, jit_int32_t, jit_word_t);
620 # define bmsr(i0, r0, r1) _bmsr(_jit, i0, r0, r1)
621 static jit_word_t _bmsr(jit_state_t*,jit_word_t,jit_int32_t,jit_int32_t);
622 # define bmsi(i0, r0, i1) _bmsi(_jit, i0, r0, i1)
623 static jit_word_t _bmsi(jit_state_t*,jit_word_t,jit_int32_t,jit_word_t);
624 # define bmcr(i0, r0, r1) _bmcr(_jit, i0, r0, r1)
625 static jit_word_t _bmcr(jit_state_t*,jit_word_t,jit_int32_t,jit_int32_t);
626 # define bmci(i0, r0, i1) _bmci(_jit, i0, r0, i1)
627 static jit_word_t _bmci(jit_state_t*,jit_word_t,jit_int32_t,jit_word_t);
628 # define boaddr(i0, r0, r1) _boaddr(_jit, i0, r0, r1)
629 static jit_word_t _boaddr(jit_state_t*,jit_word_t,jit_int32_t,jit_int32_t);
630 # define boaddi(i0, r0, i1) _boaddi(_jit, i0, r0, i1)
631 static jit_word_t _boaddi(jit_state_t*,jit_word_t,jit_int32_t,jit_word_t);
632 # define boaddr_u(i0, r0, r1) _boaddr_u(_jit, i0, r0, r1)
633 static jit_word_t _boaddr_u(jit_state_t*,jit_word_t,jit_int32_t,jit_int32_t);
634 # define boaddi_u(i0, r0, i1) _boaddi_u(_jit, i0, r0, i1)
635 static jit_word_t _boaddi_u(jit_state_t*,jit_word_t,jit_int32_t,jit_word_t);
636 # define bxaddr(i0, r0, r1) _bxaddr(_jit, i0, r0, r1)
637 static jit_word_t _bxaddr(jit_state_t*,jit_word_t,jit_int32_t,jit_int32_t);
638 # define bxaddi(i0, r0, i1) _bxaddi(_jit, i0, r0, i1)
639 static jit_word_t _bxaddi(jit_state_t*,jit_word_t,jit_int32_t,jit_word_t);
640 # define bxaddr_u(i0, r0, r1) _bxaddr_u(_jit, i0, r0, r1)
641 static jit_word_t _bxaddr_u(jit_state_t*,jit_word_t,jit_int32_t,jit_int32_t);
642 # define bxaddi_u(i0, r0, i1) _bxaddi_u(_jit, i0, r0, i1)
643 static jit_word_t _bxaddi_u(jit_state_t*,jit_word_t,jit_int32_t,jit_word_t);
644 # define bosubr(i0, r0, r1) _bosubr(_jit, i0, r0, r1)
645 static jit_word_t _bosubr(jit_state_t*,jit_word_t,jit_int32_t,jit_int32_t);
646 # define bosubi(i0, r0, i1) _bosubi(_jit, i0, r0, i1)
647 static jit_word_t _bosubi(jit_state_t*,jit_word_t,jit_int32_t,jit_word_t);
648 # define bosubr_u(i0, r0, r1) _bosubr_u(_jit, i0, r0, r1)
649 static jit_word_t _bosubr_u(jit_state_t*,jit_word_t,jit_int32_t,jit_int32_t);
650 # define bosubi_u(i0, r0, i1) _bosubi_u(_jit, i0, r0, i1)
651 static jit_word_t _bosubi_u(jit_state_t*,jit_word_t,jit_int32_t,jit_word_t);
652 # define bxsubr(i0, r0, r1) _bxsubr(_jit, i0, r0, r1)
653 static jit_word_t _bxsubr(jit_state_t*,jit_word_t,jit_int32_t,jit_int32_t);
654 # define bxsubi(i0, r0, i1) _bxsubi(_jit, i0, r0, i1)
655 static jit_word_t _bxsubi(jit_state_t*,jit_word_t,jit_int32_t,jit_word_t);
656 # define bxsubr_u(i0, r0, r1) _bxsubr_u(_jit, i0, r0, r1)
657 static jit_word_t _bxsubr_u(jit_state_t*,jit_word_t,jit_int32_t,jit_int32_t);
658 # define bxsubi_u(i0, r0, i1) _bxsubi_u(_jit, i0, r0, i1)
659 static jit_word_t _bxsubi_u(jit_state_t*,jit_word_t,jit_int32_t,jit_word_t);
660 # define callr(r0) _callr(_jit, r0)
661 static void _callr(jit_state_t*, jit_int32_t);
662 # define calli(i0) _calli(_jit, i0)
663 static jit_word_t _calli(jit_state_t*, jit_word_t);
664 # define jmpr(r0) _jmpr(_jit, r0)
665 static void _jmpr(jit_state_t*, jit_int32_t);
666 # define jmpi(i0) _jmpi(_jit, i0)
667 static jit_word_t _jmpi(jit_state_t*, jit_word_t);
668 # define jmpsi(i0) _jmpsi(_jit, i0)
669 static void _jmpsi(jit_state_t*, jit_uint8_t);
670 # define prolog(node) _prolog(_jit, node)
671 static void _prolog(jit_state_t*, jit_node_t*);
672 # define epilog(node) _epilog(_jit, node)
673 static void _epilog(jit_state_t*, jit_node_t*);
674 # define vastart(r0) _vastart(_jit, r0)
675 static void _vastart(jit_state_t*, jit_int32_t);
676 # define vaarg(r0, r1) _vaarg(_jit, r0, r1)
677 static void _vaarg(jit_state_t*, jit_int32_t, jit_int32_t);
678 # define vaarg_d(r0, r1, i0) _vaarg_d(_jit, r0, r1, i0)
679 static void _vaarg_d(jit_state_t*, jit_int32_t, jit_int32_t, jit_bool_t);
680 # define patch_at(node, instr, label) _patch_at(_jit, node, instr, label)
681 static void _patch_at(jit_state_t*, jit_node_t*, jit_word_t, jit_word_t);
682 # if !defined(HAVE_FFSL)
684 # define ffsl(i) __builtin_ffs(i)
686 # define ffsl(l) __builtin_ffsl(l)
693 _rex(jit_state_t *_jit, jit_int32_t l, jit_int32_t w,
694 jit_int32_t r, jit_int32_t x, jit_int32_t b)
697 jit_int32_t v = 0x40 | (w << 3);
711 _rx(jit_state_t *_jit, jit_int32_t rd, jit_int32_t md,
712 jit_int32_t rb, jit_int32_t ri, jit_int32_t ms)
717 mrm(0x00, r7(rd), 0x05);
719 mrm(0x00, r7(rd), 0x04);
720 sib(_SCL1, 0x04, 0x05);
724 else if (r7(rb) == _RSP_REGNO) {
726 mrm(0x00, r7(rd), 0x04);
729 else if ((jit_int8_t)md == md) {
730 mrm(0x01, r7(rd), 0x04);
735 mrm(0x02, r7(rd), 0x04);
741 if (md == 0 && r7(rb) != _RBP_REGNO)
742 mrm(0x00, r7(rd), r7(rb));
743 else if ((jit_int8_t)md == md) {
744 mrm(0x01, r7(rd), r7(rb));
748 mrm(0x02, r7(rd), r7(rb));
753 else if (rb == _NOREG) {
754 mrm(0x00, r7(rd), 0x04);
755 sib(ms, r7(ri), 0x05);
758 else if (r8(ri) != _RSP_REGNO) {
759 if (md == 0 && r7(rb) != _RBP_REGNO) {
760 mrm(0x00, r7(rd), 0x04);
761 sib(ms, r7(ri), r7(rb));
763 else if ((jit_int8_t)md == md) {
764 mrm(0x01, r7(rd), 0x04);
765 sib(ms, r7(ri), r7(rb));
769 mrm(0x02, r7(rd), 0x04);
770 sib(ms, r7(ri), r7(rb));
775 fprintf(stderr, "illegal index register");
781 _nop(jit_state_t *_jit, jit_int32_t count)
791 case 3: /* NOP DWORD ptr [EAX] */
792 ic(0x0f); ic(0x1f); ic(0x00);
794 case 4: /* NOP DWORD ptr [EAX + 00H] */
795 ic(0x0f); ic(0x1f); ic(0x40); ic(0x00);
797 case 5: /* NOP DWORD ptr [EAX + EAX*1 + 00H] */
798 ic(0x0f); ic(0x1f); ic(0x44); ic(0x00);
801 case 6: /* 66 NOP DWORD ptr [EAX + EAX*1 + 00H] */
802 ic(0x66); ic(0x0f); ic(0x1f); ic(0x44);
805 case 7: /* NOP DWORD ptr [EAX + 00000000H] */
806 ic(0x0f); ic(0x1f); ic(0x80); ii(0x0000);
808 case 8: /* NOP DWORD ptr [EAX + EAX*1 + 00000000H] */
809 ic(0x0f); ic(0x1f); ic(0x84); ic(0x00);
812 case 9: /* 66 NOP DWORD ptr [EAX + EAX*1 + 00000000H] */
813 ic(0x66); ic(0x0f); ic(0x1f); ic(0x84);
814 ic(0x00); ii(0x0000);
822 _lea(jit_state_t *_jit, jit_int32_t md, jit_int32_t rb,
823 jit_int32_t ri, jit_int32_t ms, jit_int32_t rd)
825 rex(0, WIDE, rd, ri, rb);
827 rx(rd, md, rb, ri, ms);
831 _pushr(jit_state_t *_jit, jit_int32_t r0)
833 rex(0, WIDE, 0, 0, r0);
838 _popr(jit_state_t *_jit, jit_int32_t r0)
840 rex(0, WIDE, 0, 0, r0);
845 _xchgr(jit_state_t *_jit, jit_int32_t r0, jit_int32_t r1)
847 rex(0, WIDE, r1, _NOREG, r0);
849 mrm(0x03, r7(r1), r7(r0));
853 _testr(jit_state_t *_jit, jit_int32_t r0, jit_int32_t r1)
855 rex(0, WIDE, r1, _NOREG, r0);
857 mrm(0x03, r7(r1), r7(r0));
861 _testi(jit_state_t *_jit, jit_int32_t r0, jit_word_t i0)
863 rex(0, WIDE, _NOREG, _NOREG, r0);
864 if (r0 == _RAX_REGNO)
868 mrm(0x03, 0x00, r7(r0));
874 _cc(jit_state_t *_jit, jit_int32_t code, jit_int32_t r0)
876 rex(0, 0, _NOREG, _NOREG, r0);
879 mrm(0x03, 0x00, r7(r0));
883 _alur(jit_state_t *_jit, jit_int32_t code, jit_int32_t r0, jit_int32_t r1)
885 rex(0, WIDE, r1, _NOREG, r0);
887 mrm(0x03, r7(r1), r7(r0));
891 _alui(jit_state_t *_jit, jit_int32_t code, jit_int32_t r0, jit_word_t i0)
894 if (can_sign_extend_int_p(i0)) {
895 rex(0, WIDE, _NOREG, _NOREG, r0);
896 if ((jit_int8_t)i0 == i0) {
898 ic(0xc0 | code | r7(r0));
902 if (r0 == _RAX_REGNO)
906 ic(0xc0 | code | r7(r0));
912 reg = jit_get_reg(jit_class_gpr);
914 alur(code, r0, rn(reg));
920 _save(jit_state_t *_jit, jit_int32_t r0)
922 if (!_jitc->function->regoff[r0]) {
923 _jitc->function->regoff[r0] = jit_allocai(sizeof(jit_word_t));
926 assert(!jit_regset_tstbit(&_jitc->regsav, r0));
927 jit_regset_setbit(&_jitc->regsav, r0);
928 stxi(_jitc->function->regoff[r0], _RBP_REGNO, r0);
932 _load(jit_state_t *_jit, jit_int32_t r0)
934 assert(_jitc->function->regoff[r0]);
935 assert(jit_regset_tstbit(&_jitc->regsav, r0));
936 jit_regset_clrbit(&_jitc->regsav, r0);
937 ldxi(r0, _RBP_REGNO, _jitc->function->regoff[r0]);
941 _addr(jit_state_t *_jit, jit_int32_t r0, jit_int32_t r1, jit_int32_t r2)
948 lea(0, r1, r2, _SCL1, r0);
952 _addi(jit_state_t *_jit, jit_int32_t r0, jit_int32_t r1, jit_word_t i0)
963 else if (can_sign_extend_int_p(i0)) {
967 lea(i0, r1, _NOREG, _SCL1, r0);
974 reg = jit_get_reg(jit_class_gpr);
982 _addcr(jit_state_t *_jit, jit_int32_t r0, jit_int32_t r1, jit_int32_t r2)
993 _addci(jit_state_t *_jit, jit_int32_t r0, jit_int32_t r1, jit_word_t i0)
996 if (can_sign_extend_int_p(i0)) {
1000 else if (r0 == r1) {
1001 reg = jit_get_reg(jit_class_gpr);
1013 _addxr(jit_state_t *_jit, jit_int32_t r0, jit_int32_t r1, jit_int32_t r2)
1024 _addxi(jit_state_t *_jit, jit_int32_t r0, jit_int32_t r1, jit_word_t i0)
1027 if (can_sign_extend_int_p(i0)) {
1031 else if (r0 == r1) {
1032 reg = jit_get_reg(jit_class_gpr);
1034 iaddxr(r0, rn(reg));
1044 _subr(jit_state_t *_jit, jit_int32_t r0, jit_int32_t r1, jit_int32_t r2)
1048 else if (r0 == r2) {
1059 _subi(jit_state_t *_jit, jit_int32_t r0, jit_int32_t r1, jit_word_t i0)
1070 else if (can_sign_extend_int_p(i0)) {
1074 lea(-i0, r1, _NOREG, _SCL1, r0);
1076 else if (r0 != r1) {
1081 reg = jit_get_reg(jit_class_gpr);
1089 _subcr(jit_state_t *_jit, jit_int32_t r0, jit_int32_t r1, jit_int32_t r2)
1092 if (r0 == r2 && r0 != r1) {
1093 reg = jit_get_reg(jit_class_gpr);
1106 _subci(jit_state_t *_jit, jit_int32_t r0, jit_int32_t r1, jit_word_t i0)
1110 if (can_sign_extend_int_p(i0))
1113 reg = jit_get_reg(jit_class_gpr);
1121 _subxr(jit_state_t *_jit, jit_int32_t r0, jit_int32_t r1, jit_int32_t r2)
1124 if (r0 == r2 && r0 != r1) {
1125 reg = jit_get_reg(jit_class_gpr);
1128 isubxr(r0, rn(reg));
1138 _subxi(jit_state_t *_jit, jit_int32_t r0, jit_int32_t r1, jit_word_t i0)
1142 if (can_sign_extend_int_p(i0))
1145 reg = jit_get_reg(jit_class_gpr);
1147 isubxr(r0, rn(reg));
1153 _rsbi(jit_state_t *_jit, jit_int32_t r0, jit_int32_t r1, jit_word_t i0)
1160 _imulr(jit_state_t *_jit, jit_int32_t r0, jit_int32_t r1)
1162 rex(0, WIDE, r0, _NOREG, r1);
1165 mrm(0x03, r7(r0), r7(r1));
1169 _imuli(jit_state_t *_jit, jit_int32_t r0, jit_int32_t r1, jit_word_t i0)
1172 if (can_sign_extend_int_p(i0)) {
1173 rex(0, WIDE, r0, _NOREG, r1);
1174 if ((jit_int8_t)i0 == i0) {
1176 mrm(0x03, r7(r0), r7(r1));
1181 mrm(0x03, r7(r0), r7(r1));
1186 reg = jit_get_reg(jit_class_gpr);
1194 _mulr(jit_state_t *_jit, jit_int32_t r0, jit_int32_t r1, jit_int32_t r2)
1207 _muli(jit_state_t *_jit, jit_int32_t r0, jit_int32_t r1, jit_word_t i0)
1220 lea(0, _NOREG, r1, _SCL2, r0);
1223 lea(0, _NOREG, r1, _SCL4, r0);
1226 lea(0, _NOREG, r1, _SCL8, r0);
1229 if (i0 > 0 && !(i0 & (i0 - 1)))
1230 lshi(r0, r1, ffsl(i0) - 1);
1231 else if (can_sign_extend_int_p(i0))
1233 else if (r0 != r1) {
1243 #define savset(rn) \
1246 if (r1 != rn && r2 != rn) \
1249 #define isavset(rn) \
1255 #define qsavset(rn) \
1256 if (r0 != rn && r1 != rn) { \
1258 if (r2 != rn && r3 != rn) \
1261 #define allocr(rn, rv) \
1262 if (set & (1 << rn)) \
1263 (void)jit_get_reg(rv|jit_class_gpr|jit_class_named); \
1264 if (sav & (1 << rn)) { \
1265 if ( jit_regset_tstbit(&_jitc->regsav, rv) || \
1266 !jit_regset_tstbit(&_jitc->reglive, rv)) \
1267 sav &= ~(1 << rn); \
1271 #define clear(rn, rv) \
1272 if (set & (1 << rn)) \
1273 jit_unget_reg(rv); \
1274 if (sav & (1 << rn)) \
1277 _iqmulr(jit_state_t *_jit, jit_int32_t r0, jit_int32_t r1,
1278 jit_int32_t r2, jit_int32_t r3, jit_bool_t sign)
1285 qsavset(_RDX_REGNO);
1286 qsavset(_RAX_REGNO);
1287 allocr(_RDX_REGNO, _RDX);
1288 allocr(_RAX_REGNO, _RAX);
1290 if (r3 == _RAX_REGNO)
1294 movr(_RAX_REGNO, r2);
1301 if (r0 == _RDX_REGNO && r1 == _RAX_REGNO)
1302 xchgr(_RAX_REGNO, _RDX_REGNO);
1304 if (r0 != _RDX_REGNO)
1305 movr(r0, _RAX_REGNO);
1306 movr(r1, _RDX_REGNO);
1307 if (r0 == _RDX_REGNO)
1308 movr(r0, _RAX_REGNO);
1311 clear(_RDX_REGNO, _RDX);
1312 clear(_RAX_REGNO, _RAX);
1316 _iqmuli(jit_state_t *_jit, jit_int32_t r0, jit_int32_t r1,
1317 jit_int32_t r2, jit_word_t i0, jit_bool_t sign)
1326 reg = jit_get_reg(jit_class_gpr);
1329 qmulr(r0, r1, r2, rn(reg));
1331 qmulr_u(r0, r1, r2, rn(reg));
1337 _sign_extend_rdx_rax(jit_state_t *_jit)
1339 rex(0, WIDE, 0, 0, 0);
1344 _divremr(jit_state_t *_jit, jit_int32_t r0, jit_int32_t r1, jit_int32_t r2,
1345 jit_bool_t sign, jit_bool_t divide)
1353 sav = set = use = 0;
1356 allocr(_RDX_REGNO, _RDX);
1357 allocr(_RAX_REGNO, _RAX);
1359 if (r2 == _RAX_REGNO) {
1360 if (r0 == _RAX_REGNO || r0 == _RDX_REGNO) {
1361 if ((reg = jit_get_reg(jit_class_gpr|jit_class_chk)) == JIT_NOREG)
1362 reg = jit_get_reg((r1 == _RCX_REGNO ? _RBX : _RCX) |
1363 jit_class_gpr|jit_class_named);
1366 movr(div, _RAX_REGNO);
1367 if (r1 != _RAX_REGNO)
1368 movr(_RAX_REGNO, r1);
1372 xchgr(r0, _RAX_REGNO);
1374 if (r0 != _RAX_REGNO)
1375 movr(r0, _RAX_REGNO);
1376 if (r1 != _RAX_REGNO)
1377 movr(_RAX_REGNO, r1);
1382 else if (r2 == _RDX_REGNO) {
1383 if (r0 == _RAX_REGNO || r0 == _RDX_REGNO) {
1384 if ((reg = jit_get_reg(jit_class_gpr|jit_class_chk)) == JIT_NOREG)
1385 reg = jit_get_reg((r1 == _RCX_REGNO ? _RBX : _RCX) |
1386 jit_class_gpr|jit_class_named);
1389 movr(div, _RDX_REGNO);
1390 if (r1 != _RAX_REGNO)
1391 movr(_RAX_REGNO, r1);
1394 if (r1 != _RAX_REGNO)
1395 movr(_RAX_REGNO, r1);
1396 movr(r0, _RDX_REGNO);
1401 if (r1 != _RAX_REGNO)
1402 movr(_RAX_REGNO, r1);
1407 sign_extend_rdx_rax();
1411 ixorr(_RDX_REGNO, _RDX_REGNO);
1419 movr(r0, _RAX_REGNO);
1421 movr(r0, _RDX_REGNO);
1423 clear(_RDX_REGNO, _RDX);
1424 clear(_RAX_REGNO, _RAX);
1428 _divremi(jit_state_t *_jit, jit_int32_t r0, jit_int32_t r1, jit_word_t i0,
1429 jit_bool_t sign, jit_bool_t divide)
1449 if (i0 > 0 && !(i0 & (i0 - 1))) {
1452 rshi(r0, r0, ffsl(i0) - 1);
1454 rshi_u(r0, r0, ffsl(i0) - 1);
1460 else if (i0 == 1 || (sign && i0 == -1)) {
1464 else if (!sign && i0 > 0 && !(i0 & (i0 - 1))) {
1465 if (can_sign_extend_int_p(i0)) {
1469 else if (r0 != r1) {
1474 reg = jit_get_reg(jit_class_gpr);
1475 movi(rn(reg), i0 - 1);
1482 sav = set = use = 0;
1483 isavset(_RDX_REGNO);
1484 isavset(_RAX_REGNO);
1485 allocr(_RDX_REGNO, _RDX);
1486 allocr(_RAX_REGNO, _RAX);
1488 if (r0 == _RAX_REGNO || r0 == _RDX_REGNO || r0 == r1) {
1489 if ((reg = jit_get_reg(jit_class_gpr|jit_class_chk)) == JIT_NOREG)
1490 reg = jit_get_reg((r1 == _RCX_REGNO ? _RBX : _RCX) |
1491 jit_class_gpr|jit_class_named);
1499 movr(_RAX_REGNO, r1);
1502 sign_extend_rdx_rax();
1506 ixorr(_RDX_REGNO, _RDX_REGNO);
1514 movr(r0, _RAX_REGNO);
1516 movr(r0, _RDX_REGNO);
1518 clear(_RDX_REGNO, _RDX);
1519 clear(_RAX_REGNO, _RAX);
1523 _iqdivr(jit_state_t *_jit, jit_int32_t r0, jit_int32_t r1,
1524 jit_int32_t r2, jit_int32_t r3, jit_bool_t sign)
1532 sav = set = use = 0;
1533 qsavset(_RDX_REGNO);
1534 qsavset(_RAX_REGNO);
1535 allocr(_RDX_REGNO, _RDX);
1536 allocr(_RAX_REGNO, _RAX);
1537 if (r3 == _RAX_REGNO) {
1538 if (r0 == _RAX_REGNO || r0 == _RDX_REGNO) {
1539 if ((reg = jit_get_reg(jit_class_gpr|jit_class_chk)) == JIT_NOREG)
1540 reg = jit_get_reg((r1 == _RCX_REGNO ? _RBX : _RCX) |
1541 jit_class_gpr|jit_class_named);
1544 movr(div, _RAX_REGNO);
1545 if (r2 != _RAX_REGNO)
1546 movr(_RAX_REGNO, r2);
1550 xchgr(r0, _RAX_REGNO);
1552 if (r0 != _RAX_REGNO)
1553 movr(r0, _RAX_REGNO);
1554 if (r2 != _RAX_REGNO)
1555 movr(_RAX_REGNO, r2);
1560 else if (r3 == _RDX_REGNO) {
1561 if (r0 == _RAX_REGNO || r0 == _RDX_REGNO) {
1562 if ((reg = jit_get_reg(jit_class_gpr|jit_class_chk)) == JIT_NOREG)
1563 reg = jit_get_reg((r1 == _RCX_REGNO ? _RBX : _RCX) |
1564 jit_class_gpr|jit_class_named);
1567 movr(div, _RDX_REGNO);
1568 if (r2 != _RAX_REGNO)
1569 movr(_RAX_REGNO, r2);
1572 if (r2 != _RAX_REGNO)
1573 movr(_RAX_REGNO, r2);
1574 movr(r0, _RDX_REGNO);
1579 if (r2 != _RAX_REGNO)
1580 movr(_RAX_REGNO, r2);
1584 sign_extend_rdx_rax();
1588 ixorr(_RDX_REGNO, _RDX_REGNO);
1594 if (r0 == _RDX_REGNO && r1 == _RAX_REGNO)
1595 xchgr(_RAX_REGNO, _RDX_REGNO);
1597 if (r0 != _RDX_REGNO)
1598 movr(r0, _RAX_REGNO);
1599 movr(r1, _RDX_REGNO);
1600 if (r0 == _RDX_REGNO)
1601 movr(r0, _RAX_REGNO);
1604 clear(_RDX_REGNO, _RDX);
1605 clear(_RAX_REGNO, _RAX);
1609 _iqdivi(jit_state_t *_jit, jit_int32_t r0, jit_int32_t r1,
1610 jit_int32_t r2, jit_word_t i0, jit_bool_t sign)
1614 reg = jit_get_reg(jit_class_gpr);
1617 qdivr(r0, r1, r2, rn(reg));
1619 qdivr_u(r0, r1, r2, rn(reg));
1627 _andr(jit_state_t *_jit, jit_int32_t r0, jit_int32_t r1, jit_int32_t r2)
1642 _andi(jit_state_t *_jit, jit_int32_t r0, jit_int32_t r1, jit_word_t i0)
1650 else if (r0 == r1) {
1651 if (can_sign_extend_int_p(i0))
1654 reg = jit_get_reg(jit_class_gpr);
1667 _orr(jit_state_t *_jit, jit_int32_t r0, jit_int32_t r1, jit_int32_t r2)
1682 _ori(jit_state_t *_jit, jit_int32_t r0, jit_int32_t r1, jit_word_t i0)
1689 else if (can_sign_extend_int_p(i0)) {
1693 else if (r0 != r1) {
1698 reg = jit_get_reg(jit_class_gpr);
1706 _xorr(jit_state_t *_jit, jit_int32_t r0, jit_int32_t r1, jit_int32_t r2)
1721 _xori(jit_state_t *_jit, jit_int32_t r0, jit_int32_t r1, jit_word_t i0)
1728 else if (can_sign_extend_int_p(i0)) {
1732 else if (r0 != r1) {
1737 reg = jit_get_reg(jit_class_gpr);
1745 _irotshr(jit_state_t *_jit, jit_int32_t code, jit_int32_t r0)
1747 rex(0, WIDE, _RCX_REGNO, _NOREG, r0);
1749 mrm(0x03, code, r7(r0));
1753 _rotshr(jit_state_t *_jit, jit_int32_t code,
1754 jit_int32_t r0, jit_int32_t r1, jit_int32_t r2)
1759 if (r0 == _RCX_REGNO) {
1760 reg = jit_get_reg(jit_class_gpr);
1762 if (r2 != _RCX_REGNO)
1763 movr(_RCX_REGNO, r2);
1764 irotshr(code, rn(reg));
1765 movr(_RCX_REGNO, rn(reg));
1768 else if (r2 != _RCX_REGNO) {
1769 use = !jit_reg_free_p(_RCX);
1771 reg = jit_get_reg(jit_class_gpr);
1772 movr(rn(reg), _RCX_REGNO);
1776 if (r1 == _RCX_REGNO) {
1778 xchgr(r0, _RCX_REGNO);
1781 movr(_RCX_REGNO, r2);
1785 movr(_RCX_REGNO, r2);
1790 movr(_RCX_REGNO, rn(reg));
1801 _irotshi(jit_state_t *_jit, jit_int32_t code, jit_int32_t r0, jit_word_t i0)
1803 rex(0, WIDE, _NOREG, _NOREG, r0);
1806 mrm(0x03, code, r7(r0));
1810 mrm(0x03, code, r7(r0));
1816 _rotshi(jit_state_t *_jit, jit_int32_t code,
1817 jit_int32_t r0, jit_int32_t r1, jit_word_t i0)
1821 irotshi(code, r0, i0);
1825 _lshi(jit_state_t *_jit, jit_int32_t r0, jit_int32_t r1, jit_word_t i0)
1830 lea(0, _NOREG, r1, i0 == 1 ? _SCL2 : i0 == 2 ? _SCL4 : _SCL8, r0);
1832 rotshi(X86_SHL, r0, r1, i0);
1836 _unr(jit_state_t *_jit, jit_int32_t code, jit_int32_t r0)
1838 rex(0, WIDE, _NOREG, _NOREG, r0);
1840 mrm(0x03, code, r7(r0));
1844 _negr(jit_state_t *_jit, jit_int32_t r0, jit_int32_t r1)
1855 _comr(jit_state_t *_jit, jit_int32_t r0, jit_int32_t r1)
1863 _incr(jit_state_t *_jit, jit_int32_t r0, jit_int32_t r1)
1867 rex(0, WIDE, _NOREG, _NOREG, r0);
1876 _decr(jit_state_t *_jit, jit_int32_t r0, jit_int32_t r1)
1880 rex(0, WIDE, _NOREG, _NOREG, r0);
1890 _cr(jit_state_t *_jit,
1891 jit_int32_t code, jit_int32_t r0, jit_int32_t r1, jit_int32_t r2)
1896 same = r0 == r1 || r0 == r2;
1905 reg = jit_get_reg(jit_class_gpr|jit_class_rg8);
1906 ixorr(rn(reg), rn(reg));
1915 _ci(jit_state_t *_jit,
1916 jit_int32_t code, jit_int32_t r0, jit_int32_t r1, jit_word_t i0)
1930 reg = jit_get_reg(jit_class_gpr|jit_class_rg8);
1931 ixorr(rn(reg), rn(reg));
1940 _ci0(jit_state_t *_jit, jit_int32_t code, jit_int32_t r0, jit_int32_t r1)
1954 reg = jit_get_reg(jit_class_gpr|jit_class_rg8);
1955 ixorr(rn(reg), rn(reg));
1964 _ltr(jit_state_t *_jit, jit_int32_t r0, jit_int32_t r1, jit_int32_t r2)
1969 cr(X86_CC_L, r0, r1, r2);
1973 _lti(jit_state_t *_jit, jit_int32_t r0, jit_int32_t r1, jit_word_t i0)
1976 ci(X86_CC_L, r0, r1, i0);
1978 ci0(X86_CC_S, r0, r1);
1982 _ltr_u(jit_state_t *_jit, jit_int32_t r0, jit_int32_t r1, jit_int32_t r2)
1987 cr(X86_CC_B, r0, r1, r2);
1991 _ler(jit_state_t *_jit, jit_int32_t r0, jit_int32_t r1, jit_int32_t r2)
1996 cr(X86_CC_LE, r0, r1, r2);
2000 _ler_u(jit_state_t *_jit, jit_int32_t r0, jit_int32_t r1, jit_int32_t r2)
2005 cr(X86_CC_BE, r0, r1, r2);
2009 _lei_u(jit_state_t *_jit, jit_int32_t r0, jit_int32_t r1, jit_word_t i0)
2012 ci(X86_CC_BE, r0, r1, i0);
2014 ci0(X86_CC_E, r0, r1);
2018 _eqr(jit_state_t *_jit, jit_int32_t r0, jit_int32_t r1, jit_int32_t r2)
2023 cr(X86_CC_E, r0, r1, r2);
2027 _eqi(jit_state_t *_jit, jit_int32_t r0, jit_int32_t r1, jit_word_t i0)
2030 ci(X86_CC_E, r0, r1, i0);
2032 ci0(X86_CC_E, r0, r1);
2036 _ger(jit_state_t *_jit, jit_int32_t r0, jit_int32_t r1, jit_int32_t r2)
2041 cr(X86_CC_GE, r0, r1, r2);
2045 _gei(jit_state_t *_jit, jit_int32_t r0, jit_int32_t r1, jit_word_t i0)
2048 ci(X86_CC_GE, r0, r1, i0);
2050 ci0(X86_CC_NS, r0, r1);
2054 _ger_u(jit_state_t *_jit, jit_int32_t r0, jit_int32_t r1, jit_int32_t r2)
2059 cr(X86_CC_AE, r0, r1, r2);
2063 _gei_u(jit_state_t *_jit, jit_int32_t r0, jit_int32_t r1, jit_word_t i0)
2066 ci(X86_CC_AE, r0, r1, i0);
2068 ci0(X86_CC_NB, r0, r1);
2072 _gtr(jit_state_t *_jit, jit_int32_t r0, jit_int32_t r1, jit_int32_t r2)
2077 cr(X86_CC_G, r0, r1, r2);
2081 _gtr_u(jit_state_t *_jit, jit_int32_t r0, jit_int32_t r1, jit_int32_t r2)
2086 cr(X86_CC_A, r0, r1, r2);
2090 _gti_u(jit_state_t *_jit, jit_int32_t r0, jit_int32_t r1, jit_word_t i0)
2093 ci(X86_CC_A, r0, r1, i0);
2095 ci0(X86_CC_NE, r0, r1);
2099 _ner(jit_state_t *_jit, jit_int32_t r0, jit_int32_t r1, jit_int32_t r2)
2104 cr(X86_CC_NE, r0, r1, r2);
2108 _nei(jit_state_t *_jit, jit_int32_t r0, jit_int32_t r1, jit_word_t i0)
2111 ci(X86_CC_NE, r0, r1, i0);
2113 ci0(X86_CC_NE, r0, r1);
2117 _movr(jit_state_t *_jit, jit_int32_t r0, jit_int32_t r1)
2120 rex(0, 1, r1, _NOREG, r0);
2122 ic(0xc0 | (r1 << 3) | r7(r0));
2127 _imovi(jit_state_t *_jit, jit_int32_t r0, jit_word_t i0)
2131 if (fits_uint32_p(i0)) {
2133 rex(0, 0, _NOREG, _NOREG, r0);
2139 rex(0, 1, _NOREG, _NOREG, r0);
2151 _movi(jit_state_t *_jit, jit_int32_t r0, jit_word_t i0)
2160 _movi_p(jit_state_t *_jit, jit_int32_t r0, jit_word_t i0)
2162 rex(0, WIDE, _NOREG, _NOREG, r0);
2165 return (_jit->pc.w);
2169 _movcr(jit_state_t *_jit, jit_int32_t r0, jit_int32_t r1)
2171 rex(0, WIDE, r0, _NOREG, r1);
2174 mrm(0x03, r7(r0), r7(r1));
2178 _movcr_u(jit_state_t *_jit, jit_int32_t r0, jit_int32_t r1)
2180 rex(0, WIDE, r0, _NOREG, r1);
2183 mrm(0x03, r7(r0), r7(r1));
2187 _movsr(jit_state_t *_jit, jit_int32_t r0, jit_int32_t r1)
2189 rex(0, WIDE, r0, _NOREG, r1);
2192 mrm(0x03, r7(r0), r7(r1));
2196 _movsr_u(jit_state_t *_jit, jit_int32_t r0, jit_int32_t r1)
2198 rex(0, WIDE, r0, _NOREG, r1);
2201 mrm(0x03, r7(r0), r7(r1));
2206 _movir(jit_state_t *_jit, jit_int32_t r0, jit_int32_t r1)
2208 rex(0, 1, r0, _NOREG, r1);
2210 mrm(0x03, r7(r0), r7(r1));
2214 _movir_u(jit_state_t *_jit, jit_int32_t r0, jit_int32_t r1)
2216 rex(0, 0, r1, _NOREG, r0);
2218 ic(0xc0 | (r1 << 3) | r7(r0));
2223 _htonr_us(jit_state_t *_jit, jit_int32_t r0, jit_int32_t r1)
2227 rex(0, 0, _NOREG, _NOREG, r0);
2229 mrm(0x03, X86_ROR, r7(r0));
2234 _htonr_ui(jit_state_t *_jit, jit_int32_t r0, jit_int32_t r1)
2237 rex(0, 0, _NOREG, _NOREG, r0);
2242 #if __X64 && !__X64_32
2244 _htonr_ul(jit_state_t *_jit, jit_int32_t r0, jit_int32_t r1)
2247 rex(0, 1, _NOREG, _NOREG, r0);
2254 _extr_c(jit_state_t *_jit, jit_int32_t r0, jit_int32_t r1)
2260 reg = jit_get_reg(jit_class_gpr|jit_class_rg8);
2268 _extr_uc(jit_state_t *_jit, jit_int32_t r0, jit_int32_t r1)
2274 reg = jit_get_reg(jit_class_gpr|jit_class_rg8);
2276 movcr_u(r0, rn(reg));
2282 _ldr_c(jit_state_t *_jit, jit_int32_t r0, jit_int32_t r1)
2284 rex(0, WIDE, r0, _NOREG, r1);
2287 rx(r0, 0, r1, _NOREG, _SCL1);
2291 _ldi_c(jit_state_t *_jit, jit_int32_t r0, jit_word_t i0)
2294 if (can_sign_extend_int_p(i0)) {
2295 rex(0, WIDE, r0, _NOREG, _NOREG);
2298 rx(r0, i0, _NOREG, _NOREG, _SCL1);
2301 reg = jit_get_reg(jit_class_gpr);
2309 _ldr_uc(jit_state_t *_jit, jit_int32_t r0, jit_int32_t r1)
2311 rex(0, WIDE, r0, _NOREG, r1);
2314 rx(r0, 0, r1, _NOREG, _SCL1);
2318 _ldi_uc(jit_state_t *_jit, jit_int32_t r0, jit_word_t i0)
2321 if (can_sign_extend_int_p(i0)) {
2322 rex(0, WIDE, r0, _NOREG, _NOREG);
2325 rx(r0, i0, _NOREG, _NOREG, _SCL1);
2328 reg = jit_get_reg(jit_class_gpr);
2330 ldr_uc(r0, rn(reg));
2336 _ldr_s(jit_state_t *_jit, jit_int32_t r0, jit_int32_t r1)
2338 rex(0, WIDE, r0, _NOREG, r1);
2341 rx(r0, 0, r1, _NOREG, _SCL1);
2345 _ldi_s(jit_state_t *_jit, jit_int32_t r0, jit_word_t i0)
2348 if (can_sign_extend_int_p(i0)) {
2349 rex(0, WIDE, r0, _NOREG, _NOREG);
2352 rx(r0, i0, _NOREG, _NOREG, _SCL1);
2355 reg = jit_get_reg(jit_class_gpr);
2363 _ldr_us(jit_state_t *_jit, jit_int32_t r0, jit_int32_t r1)
2365 rex(0, WIDE, r0, _NOREG, r1);
2368 rx(r0, 0, r1, _NOREG, _SCL1);
2372 _ldi_us(jit_state_t *_jit, jit_int32_t r0, jit_word_t i0)
2375 if (can_sign_extend_int_p(i0)) {
2376 rex(0, WIDE, r0, _NOREG, _NOREG);
2379 rx(r0, i0, _NOREG, _NOREG, _SCL1);
2382 reg = jit_get_reg(jit_class_gpr);
2384 ldr_us(r0, rn(reg));
2389 #if __X32 || !__X64_32
2391 _ldr_i(jit_state_t *_jit, jit_int32_t r0, jit_int32_t r1)
2394 rex(0, WIDE, r0, _NOREG, r1);
2399 rx(r0, 0, r1, _NOREG, _SCL1);
2403 _ldi_i(jit_state_t *_jit, jit_int32_t r0, jit_word_t i0)
2406 if (can_sign_extend_int_p(i0)) {
2408 rex(0, WIDE, r0, _NOREG, _NOREG);
2413 rx(r0, i0, _NOREG, _NOREG, _SCL1);
2416 reg = jit_get_reg(jit_class_gpr);
2426 _ldr_ui(jit_state_t *_jit, jit_int32_t r0, jit_int32_t r1)
2428 rex(0, 0, r0, _NOREG, r1);
2430 rx(r0, 0, r1, _NOREG, _SCL1);
2434 _ldi_ui(jit_state_t *_jit, jit_int32_t r0, jit_word_t i0)
2437 if (can_sign_extend_int_p(i0)) {
2438 rex(0, 0, r0, _NOREG, _NOREG);
2440 rx(r0, i0, _NOREG, _NOREG, _SCL1);
2443 reg = jit_get_reg(jit_class_gpr);
2445 ldr_ui(r0, rn(reg));
2452 _ldr_l(jit_state_t *_jit, jit_int32_t r0, jit_int32_t r1)
2454 rex(0, 1, r0, _NOREG, r1);
2456 rx(r0, 0, r1, _NOREG, _SCL1);
2460 _ldi_l(jit_state_t *_jit, jit_int32_t r0, jit_word_t i0)
2463 if (can_sign_extend_int_p(i0)) {
2464 rex(0, 1, r0, _NOREG, _NOREG);
2466 rx(r0, i0, _NOREG, _NOREG, _SCL1);
2469 reg = jit_get_reg(jit_class_gpr);
2479 _ldxr_c(jit_state_t *_jit, jit_int32_t r0, jit_int32_t r1, jit_int32_t r2)
2485 rex(0, WIDE, r0, r1, r2);
2488 rx(r0, 0, r2, r1, _SCL1);
2493 _ldxi_c(jit_state_t *_jit, jit_int32_t r0, jit_int32_t r1, jit_word_t i0)
2496 if (can_sign_extend_int_p(i0)) {
2497 rex(0, WIDE, r0, _NOREG, r1);
2500 rx(r0, i0, r1, _NOREG, _SCL1);
2503 reg = jit_get_reg(jit_class_gpr);
2505 ldxr_c(r0, r1, rn(reg));
2511 _ldxr_uc(jit_state_t *_jit, jit_int32_t r0, jit_int32_t r1, jit_int32_t r2)
2517 rex(0, WIDE, r0, r1, r2);
2520 rx(r0, 0, r2, r1, _SCL1);
2525 _ldxi_uc(jit_state_t *_jit, jit_int32_t r0, jit_int32_t r1, jit_word_t i0)
2528 if (can_sign_extend_int_p(i0)) {
2529 rex(0, WIDE, r0, _NOREG, r1);
2532 rx(r0, i0, r1, _NOREG, _SCL1);
2535 reg = jit_get_reg(jit_class_gpr);
2537 ldxr_uc(r0, r1, rn(reg));
2543 _ldxr_s(jit_state_t *_jit, jit_int32_t r0, jit_int32_t r1, jit_int32_t r2)
2549 rex(0, WIDE, r0, r1, r2);
2552 rx(r0, 0, r2, r1, _SCL1);
2557 _ldxi_s(jit_state_t *_jit, jit_int32_t r0, jit_int32_t r1, jit_word_t i0)
2560 if (can_sign_extend_int_p(i0)) {
2561 rex(0, WIDE, r0, _NOREG, r1);
2564 rx(r0, i0, r1, _NOREG, _SCL1);
2567 reg = jit_get_reg(jit_class_gpr);
2569 ldxr_s(r0, r1, rn(reg));
2575 _ldxr_us(jit_state_t *_jit, jit_int32_t r0, jit_int32_t r1, jit_int32_t r2)
2581 rex(0, WIDE, r0, r1, r2);
2584 rx(r0, 0, r2, r1, _SCL1);
2589 _ldxi_us(jit_state_t *_jit, jit_int32_t r0, jit_int32_t r1, jit_word_t i0)
2592 if (can_sign_extend_int_p(i0)) {
2593 rex(0, WIDE, r0, _NOREG, r1);
2596 rx(r0, i0, r1, _NOREG, _SCL1);
2599 reg = jit_get_reg(jit_class_gpr);
2601 ldxr_us(r0, r1, rn(reg));
2606 #if __X64 || !__X64_32
2608 _ldxr_i(jit_state_t *_jit, jit_int32_t r0, jit_int32_t r1, jit_int32_t r2)
2611 rex(0, WIDE, r0, r1, r2);
2616 rx(r0, 0, r2, r1, _SCL1);
2620 _ldxi_i(jit_state_t *_jit, jit_int32_t r0, jit_int32_t r1, jit_word_t i0)
2623 if (can_sign_extend_int_p(i0)) {
2625 rex(0, WIDE, r0, _NOREG, r1);
2630 rx(r0, i0, r1, _NOREG, _SCL1);
2633 reg = jit_get_reg(jit_class_gpr);
2635 ldxr_i(r0, r1, rn(reg));
2643 _ldxr_ui(jit_state_t *_jit, jit_int32_t r0, jit_int32_t r1, jit_int32_t r2)
2647 /* to avoid confusion with macro renames */
2648 _ldr_ui(_jit, r0, r0);
2650 rex(0, 0, r0, r1, r2);
2652 rx(r0, 0, r2, r1, _SCL1);
2657 _ldxi_ui(jit_state_t *_jit, jit_int32_t r0, jit_int32_t r1, jit_word_t i0)
2660 if (can_sign_extend_int_p(i0)) {
2661 rex(0, 0, r0, _NOREG, r1);
2663 rx(r0, i0, r1, _NOREG, _SCL1);
2666 reg = jit_get_reg(jit_class_gpr);
2668 ldxr_ui(r0, r1, rn(reg));
2675 _ldxr_l(jit_state_t *_jit, jit_int32_t r0, jit_int32_t r1, jit_int32_t r2)
2677 rex(0, 1, r0, r1, r2);
2679 rx(r0, 0, r2, r1, _SCL1);
2683 _ldxi_l(jit_state_t *_jit, jit_int32_t r0, jit_int32_t r1, jit_word_t i0)
2686 if (can_sign_extend_int_p(i0)) {
2687 rex(0, 1, r0, _NOREG, r1);
2689 rx(r0, i0, r1, _NOREG, _SCL1);
2692 reg = jit_get_reg(jit_class_gpr);
2694 ldxr_l(r0, r1, rn(reg));
2702 _str_c(jit_state_t *_jit, jit_int32_t r0, jit_int32_t r1)
2706 rex(0, 0, r1, _NOREG, r0);
2708 rx(r1, 0, r0, _NOREG, _SCL1);
2711 reg = jit_get_reg(jit_class_gpr|jit_class_rg8);
2713 rex(0, 0, rn(reg), _NOREG, r0);
2715 rx(rn(reg), 0, r0, _NOREG, _SCL1);
2721 _sti_c(jit_state_t *_jit, jit_word_t i0, jit_int32_t r0)
2724 if (can_sign_extend_int_p(i0)) {
2726 rex(0, 0, r0, _NOREG, _NOREG);
2728 rx(r0, i0, _NOREG, _NOREG, _SCL1);
2731 reg = jit_get_reg(jit_class_gpr|jit_class_rg8);
2733 rex(0, 0, rn(reg), _NOREG, _NOREG);
2735 rx(rn(reg), i0, _NOREG, _NOREG, _SCL1);
2740 reg = jit_get_reg(jit_class_gpr);
2748 _str_s(jit_state_t *_jit, jit_int32_t r0, jit_int32_t r1)
2751 rex(0, 0, r1, _NOREG, r0);
2753 rx(r1, 0, r0, _NOREG, _SCL1);
2757 _sti_s(jit_state_t *_jit, jit_word_t i0, jit_int32_t r0)
2760 if (can_sign_extend_int_p(i0)) {
2762 rex(0, 0, r0, _NOREG, _NOREG);
2764 rx(r0, i0, _NOREG, _NOREG, _SCL1);
2767 reg = jit_get_reg(jit_class_gpr);
2775 _str_i(jit_state_t *_jit, jit_int32_t r0, jit_int32_t r1)
2777 rex(0, 0, r1, _NOREG, r0);
2779 rx(r1, 0, r0, _NOREG, _SCL1);
2783 _sti_i(jit_state_t *_jit, jit_word_t i0, jit_int32_t r0)
2786 if (can_sign_extend_int_p(i0)) {
2787 rex(0, 0, r0, _NOREG, _NOREG);
2789 rx(r0, i0, _NOREG, _NOREG, _SCL1);
2792 reg = jit_get_reg(jit_class_gpr);
2799 #if __X64 && !__X64_32
2801 _str_l(jit_state_t *_jit, jit_int32_t r0, jit_int32_t r1)
2803 rex(0, 1, r1, _NOREG, r0);
2805 rx(r1, 0, r0, _NOREG, _SCL1);
2809 _sti_l(jit_state_t *_jit, jit_word_t i0, jit_int32_t r0)
2812 if (can_sign_extend_int_p(i0)) {
2813 rex(0, 1, r0, _NOREG, _NOREG);
2815 rx(r0, i0, _NOREG, _NOREG, _SCL1);
2818 reg = jit_get_reg(jit_class_gpr);
2827 _stxr_c(jit_state_t *_jit, jit_int32_t r0, jit_int32_t r1, jit_int32_t r2)
2831 reg = jit_get_reg(jit_class_gpr);
2832 addr(rn(reg), r0, r1);
2837 rex(0, 0, r2, r1, r0);
2839 rx(r2, 0, r0, r1, _SCL1);
2842 reg = jit_get_reg(jit_class_gpr|jit_class_rg8);
2844 rex(0, 0, rn(reg), r1, r0);
2846 rx(rn(reg), 0, r0, r1, _SCL1);
2853 _stxi_c(jit_state_t *_jit, jit_word_t i0, jit_int32_t r0, jit_int32_t r1)
2856 if (can_sign_extend_int_p(i0)) {
2858 rex(0, 0, r1, _NOREG, r0);
2860 rx(r1, i0, r0, _NOREG, _SCL1);
2863 reg = jit_get_reg(jit_class_gpr|jit_class_rg8);
2865 rex(0, 0, rn(reg), _NOREG, r0);
2867 rx(rn(reg), i0, r0, _NOREG, _SCL1);
2872 reg = jit_get_reg(jit_class_gpr);
2874 stxr_c(rn(reg), r0, r1);
2880 _stxr_s(jit_state_t *_jit, jit_int32_t r0, jit_int32_t r1, jit_int32_t r2)
2884 reg = jit_get_reg(jit_class_gpr);
2885 addr(rn(reg), r0, r1);
2890 rex(0, 0, r2, r1, r0);
2892 rx(r2, 0, r0, r1, _SCL1);
2897 _stxi_s(jit_state_t *_jit, jit_word_t i0, jit_int32_t r0, jit_int32_t r1)
2900 if (can_sign_extend_int_p(i0)) {
2902 rex(0, 0, r1, _NOREG, r0);
2904 rx(r1, i0, r0, _NOREG, _SCL1);
2907 reg = jit_get_reg(jit_class_gpr);
2909 stxr_s(rn(reg), r0, r1);
2915 _stxr_i(jit_state_t *_jit, jit_int32_t r0, jit_int32_t r1, jit_int32_t r2)
2919 reg = jit_get_reg(jit_class_gpr);
2920 addr(rn(reg), r0, r1);
2924 rex(0, 0, r2, r1, r0);
2926 rx(r2, 0, r0, r1, _SCL1);
2931 _stxi_i(jit_state_t *_jit, jit_word_t i0, jit_int32_t r0, jit_int32_t r1)
2934 if (can_sign_extend_int_p(i0)) {
2935 rex(0, 0, r1, _NOREG, r0);
2937 rx(r1, i0, r0, _NOREG, _SCL1);
2940 reg = jit_get_reg(jit_class_gpr);
2942 stxr_i(rn(reg), r0, r1);
2947 #if __X64 && !__X64_32
2949 _stxr_l(jit_state_t *_jit, jit_int32_t r0, jit_int32_t r1, jit_int32_t r2)
2951 rex(0, 1, r2, r1, r0);
2953 rx(r2, 0, r0, r1, _SCL1);
2957 _stxi_l(jit_state_t *_jit, jit_word_t i0, jit_int32_t r0, jit_int32_t r1)
2960 if (can_sign_extend_int_p(i0)) {
2961 rex(0, 1, r1, _NOREG, r0);
2963 rx(r1, i0, r0, _NOREG, _SCL1);
2966 reg = jit_get_reg(jit_class_gpr);
2968 stxr_l(rn(reg), r0, r1);
2975 _jccs(jit_state_t *_jit, jit_int32_t code, jit_word_t i0)
2979 w = i0 - (_jit->pc.w + 1);
2984 _jcc(jit_state_t *_jit, jit_int32_t code, jit_word_t i0)
2989 w = i0 - (_jit->pc.w + 4);
2994 _jcr(jit_state_t *_jit,
2995 jit_int32_t code, jit_word_t i0, jit_int32_t r0, jit_int32_t r1)
2997 alur(X86_CMP, r0, r1);
3002 _jci(jit_state_t *_jit,
3003 jit_int32_t code, jit_word_t i0, jit_int32_t r0, jit_word_t i1)
3005 alui(X86_CMP, r0, i1);
3010 _jci0(jit_state_t *_jit, jit_int32_t code, jit_word_t i0, jit_int32_t r0)
3017 _bltr(jit_state_t *_jit, jit_word_t i0, jit_int32_t r0, jit_int32_t r1)
3019 jcr(X86_CC_L, i0, r0, r1);
3020 return (_jit->pc.w);
3024 _blti(jit_state_t *_jit, jit_word_t i0, jit_int32_t r0, jit_word_t i1)
3026 if (i1) jci (X86_CC_L, i0, r0, i1);
3027 else jci0(X86_CC_S, i0, r0);
3028 return (_jit->pc.w);
3032 _bltr_u(jit_state_t *_jit, jit_word_t i0, jit_int32_t r0, jit_int32_t r1)
3034 jcr(X86_CC_B, i0, r0, r1);
3035 return (_jit->pc.w);
3039 _blti_u(jit_state_t *_jit, jit_word_t i0, jit_int32_t r0, jit_word_t i1)
3041 if (i1) jci (X86_CC_B, i0, r0, i1);
3042 else jci0(X86_CC_B, i0, r0);
3043 return (_jit->pc.w);
3047 _bler(jit_state_t *_jit, jit_word_t i0, jit_int32_t r0, jit_int32_t r1)
3049 if (r0 == r1) jmpi(i0);
3050 else jcr (X86_CC_LE, i0, r0, r1);
3051 return (_jit->pc.w);
3055 _blei(jit_state_t *_jit, jit_word_t i0, jit_int32_t r0, jit_word_t i1)
3057 if (i1) jci (X86_CC_LE, i0, r0, i1);
3058 else jci0(X86_CC_LE, i0, r0);
3059 return (_jit->pc.w);
3063 _bler_u(jit_state_t *_jit, jit_word_t i0, jit_int32_t r0, jit_int32_t r1)
3065 if (r0 == r1) jmpi(i0);
3066 else jcr (X86_CC_BE, i0, r0, r1);
3067 return (_jit->pc.w);
3071 _blei_u(jit_state_t *_jit, jit_word_t i0, jit_int32_t r0, jit_word_t i1)
3073 if (i1) jci (X86_CC_BE, i0, r0, i1);
3074 else jci0(X86_CC_BE, i0, r0);
3075 return (_jit->pc.w);
3079 _beqr(jit_state_t *_jit, jit_word_t i0, jit_int32_t r0, jit_int32_t r1)
3081 if (r0 == r1) jmpi(i0);
3082 else jcr (X86_CC_E, i0, r0, r1);
3083 return (_jit->pc.w);
3087 _beqi(jit_state_t *_jit, jit_word_t i0, jit_int32_t r0, jit_word_t i1)
3089 if (i1) jci (X86_CC_E, i0, r0, i1);
3090 else jci0(X86_CC_E, i0, r0);
3091 return (_jit->pc.w);
3095 _bger(jit_state_t *_jit, jit_word_t i0, jit_int32_t r0, jit_int32_t r1)
3097 if (r0 == r1) jmpi(i0);
3098 else jcr (X86_CC_GE, i0, r0, r1);
3099 return (_jit->pc.w);
3103 _bgei(jit_state_t *_jit, jit_word_t i0, jit_int32_t r0, jit_word_t i1)
3105 if (i1) jci (X86_CC_GE, i0, r0, i1);
3106 else jci0(X86_CC_NS, i0, r0);
3107 return (_jit->pc.w);
3111 _bger_u(jit_state_t *_jit, jit_word_t i0, jit_int32_t r0, jit_int32_t r1)
3113 if (r0 == r1) jmpi(i0);
3114 else jcr (X86_CC_AE, i0, r0, r1);
3115 return (_jit->pc.w);
3119 _bgei_u(jit_state_t *_jit, jit_word_t i0, jit_int32_t r0, jit_word_t i1)
3121 if (i1) jci (X86_CC_AE, i0, r0, i1);
3123 return (_jit->pc.w);
3127 _bgtr(jit_state_t *_jit, jit_word_t i0, jit_int32_t r0, jit_int32_t r1)
3129 jcr(X86_CC_G, i0, r0, r1);
3130 return (_jit->pc.w);
3134 _bgti(jit_state_t *_jit, jit_word_t i0, jit_int32_t r0, jit_word_t i1)
3136 jci(X86_CC_G, i0, r0, i1);
3137 return (_jit->pc.w);
3141 _bgtr_u(jit_state_t *_jit, jit_word_t i0, jit_int32_t r0, jit_int32_t r1)
3143 jcr(X86_CC_A, i0, r0, r1);
3144 return (_jit->pc.w);
3148 _bgti_u(jit_state_t *_jit, jit_word_t i0, jit_int32_t r0, jit_word_t i1)
3150 if (i1) jci (X86_CC_A, i0, r0, i1);
3151 else jci0(X86_CC_NE, i0, r0);
3152 return (_jit->pc.w);
3156 _bner(jit_state_t *_jit, jit_word_t i0, jit_int32_t r0, jit_int32_t r1)
3158 jcr(X86_CC_NE, i0, r0, r1);
3159 return (_jit->pc.w);
3163 _bnei(jit_state_t *_jit, jit_word_t i0, jit_int32_t r0, jit_word_t i1)
3165 if (i1) jci (X86_CC_NE, i0, r0, i1);
3166 else jci0(X86_CC_NE, i0, r0);
3167 return (_jit->pc.w);
3171 _bmsr(jit_state_t *_jit, jit_word_t i0, jit_int32_t r0, jit_int32_t r1)
3175 return (_jit->pc.w);
3179 _bmsi(jit_state_t *_jit, jit_word_t i0, jit_int32_t r0, jit_word_t i1)
3182 if (can_zero_extend_int_p(i1))
3185 reg = jit_get_reg(jit_class_gpr);
3191 return (_jit->pc.w);
3195 _bmcr(jit_state_t *_jit, jit_word_t i0, jit_int32_t r0, jit_int32_t r1)
3199 return (_jit->pc.w);
3203 _bmci(jit_state_t *_jit, jit_word_t i0, jit_int32_t r0, jit_word_t i1)
3206 if (can_zero_extend_int_p(i1))
3209 reg = jit_get_reg(jit_class_gpr);
3215 return (_jit->pc.w);
3219 _boaddr(jit_state_t *_jit, jit_word_t i0, jit_int32_t r0, jit_int32_t r1)
3223 return (_jit->pc.w);
3227 _boaddi(jit_state_t *_jit, jit_word_t i0, jit_int32_t r0, jit_word_t i1)
3230 if (can_sign_extend_int_p(i1)) {
3233 return (_jit->pc.w);
3235 reg = jit_get_reg(jit_class_gpr|jit_class_nospill);
3238 return (boaddr(i0, r0, rn(reg)));
3242 _boaddr_u(jit_state_t *_jit, jit_word_t i0, jit_int32_t r0, jit_int32_t r1)
3246 return (_jit->pc.w);
3250 _boaddi_u(jit_state_t *_jit, jit_word_t i0, jit_int32_t r0, jit_word_t i1)
3253 if (can_sign_extend_int_p(i1)) {
3256 return (_jit->pc.w);
3258 reg = jit_get_reg(jit_class_gpr|jit_class_nospill);
3261 return (boaddr_u(i0, r0, rn(reg)));
3265 _bxaddr(jit_state_t *_jit, jit_word_t i0, jit_int32_t r0, jit_int32_t r1)
3269 return (_jit->pc.w);
3273 _bxaddi(jit_state_t *_jit, jit_word_t i0, jit_int32_t r0, jit_word_t i1)
3276 if (can_sign_extend_int_p(i1)) {
3279 return (_jit->pc.w);
3281 reg = jit_get_reg(jit_class_gpr|jit_class_nospill);
3284 return (bxaddr(i0, r0, rn(reg)));
3288 _bxaddr_u(jit_state_t *_jit, jit_word_t i0, jit_int32_t r0, jit_int32_t r1)
3292 return (_jit->pc.w);
3296 _bxaddi_u(jit_state_t *_jit, jit_word_t i0, jit_int32_t r0, jit_word_t i1)
3299 if (can_sign_extend_int_p(i1)) {
3302 return (_jit->pc.w);
3304 reg = jit_get_reg(jit_class_gpr|jit_class_nospill);
3307 return (bxaddr_u(i0, r0, rn(reg)));
3311 _bosubr(jit_state_t *_jit, jit_word_t i0, jit_int32_t r0, jit_int32_t r1)
3315 return (_jit->pc.w);
3319 _bosubi(jit_state_t *_jit, jit_word_t i0, jit_int32_t r0, jit_word_t i1)
3322 if (can_sign_extend_int_p(i1)) {
3325 return (_jit->pc.w);
3327 reg = jit_get_reg(jit_class_gpr|jit_class_nospill);
3330 return (bosubr(i0, r0, rn(reg)));
3334 _bosubr_u(jit_state_t *_jit, jit_word_t i0, jit_int32_t r0, jit_int32_t r1)
3338 return (_jit->pc.w);
3342 _bosubi_u(jit_state_t *_jit, jit_word_t i0, jit_int32_t r0, jit_word_t i1)
3345 if (can_sign_extend_int_p(i1)) {
3348 return (_jit->pc.w);
3350 reg = jit_get_reg(jit_class_gpr|jit_class_nospill);
3353 return (bosubr_u(i0, r0, rn(reg)));
3357 _bxsubr(jit_state_t *_jit, jit_word_t i0, jit_int32_t r0, jit_int32_t r1)
3361 return (_jit->pc.w);
3365 _bxsubi(jit_state_t *_jit, jit_word_t i0, jit_int32_t r0, jit_word_t i1)
3368 if (can_sign_extend_int_p(i1)) {
3371 return (_jit->pc.w);
3373 reg = jit_get_reg(jit_class_gpr|jit_class_nospill);
3376 return (bxsubr(i0, r0, rn(reg)));
3380 _bxsubr_u(jit_state_t *_jit, jit_word_t i0, jit_int32_t r0, jit_int32_t r1)
3384 return (_jit->pc.w);
3388 _bxsubi_u(jit_state_t *_jit, jit_word_t i0, jit_int32_t r0, jit_word_t i1)
3391 if (can_sign_extend_int_p(i1)) {
3394 return (_jit->pc.w);
3396 reg = jit_get_reg(jit_class_gpr|jit_class_nospill);
3399 return (bxsubr_u(i0, r0, rn(reg)));
3403 _callr(jit_state_t *_jit, jit_int32_t r0)
3405 rex(0, 0, _NOREG, _NOREG, r0);
3407 mrm(0x03, 0x02, r7(r0));
3411 _calli(jit_state_t *_jit, jit_word_t i0)
3417 reg = jit_get_reg(jit_class_gpr);
3418 word = movi_p(rn(reg), i0);
3424 w = i0 - (_jit->pc.w + 4);
3432 _jmpr(jit_state_t *_jit, jit_int32_t r0)
3434 rex(0, WIDE, _NOREG, _NOREG, r0);
3436 mrm(0x03, 0x04, r7(r0));
3440 _jmpi(jit_state_t *_jit, jit_word_t i0)
3444 w = i0 - (_jit->pc.w + 4);
3446 return (_jit->pc.w);
3450 _jmpsi(jit_state_t *_jit, jit_uint8_t i0)
3457 _prolog(jit_state_t *_jit, jit_node_t *node)
3460 if (_jitc->function->define_frame || _jitc->function->assume_frame) {
3461 jit_int32_t frame = -_jitc->function->frame;
3462 assert(_jitc->function->self.aoff >= frame);
3463 if (_jitc->function->assume_frame)
3465 _jitc->function->self.aoff = frame;
3467 if (_jitc->function->allocar)
3468 _jitc->function->self.aoff &= -16;
3469 #if __X64 && (__CYGWIN__ || _WIN32)
3470 _jitc->function->stack = (((/* first 32 bytes must be allocated */
3471 (_jitc->function->self.alen > 32 ?
3472 _jitc->function->self.alen : 32) -
3473 /* align stack at 16 bytes */
3474 _jitc->function->self.aoff) + 15) & -16) +
3477 _jitc->function->stack = (((_jitc->function->self.alen -
3478 _jitc->function->self.aoff) + 15) & -16) +
3481 subi(_RSP_REGNO, _RSP_REGNO, stack_framesize - REAL_WORDSIZE);
3482 /* callee save registers */
3484 if (jit_regset_tstbit(&_jitc->function->regset, _RDI))
3485 stxi(12, _RSP_REGNO, _RDI_REGNO);
3486 if (jit_regset_tstbit(&_jitc->function->regset, _RSI))
3487 stxi( 8, _RSP_REGNO, _RSI_REGNO);
3488 if (jit_regset_tstbit(&_jitc->function->regset, _RBX))
3489 stxi( 4, _RSP_REGNO, _RBX_REGNO);
3491 # if __CYGWIN__ || _WIN32
3492 if (jit_regset_tstbit(&_jitc->function->regset, _XMM15))
3493 sse_stxi_d(136, _RSP_REGNO, _XMM15_REGNO);
3494 if (jit_regset_tstbit(&_jitc->function->regset, _XMM14))
3495 sse_stxi_d(128, _RSP_REGNO, _XMM14_REGNO);
3496 if (jit_regset_tstbit(&_jitc->function->regset, _XMM13))
3497 sse_stxi_d(120, _RSP_REGNO, _XMM13_REGNO);
3498 if (jit_regset_tstbit(&_jitc->function->regset, _XMM12))
3499 sse_stxi_d(112, _RSP_REGNO, _XMM12_REGNO);
3500 if (jit_regset_tstbit(&_jitc->function->regset, _XMM11))
3501 sse_stxi_d(104, _RSP_REGNO, _XMM11_REGNO);
3502 if (jit_regset_tstbit(&_jitc->function->regset, _XMM10))
3503 sse_stxi_d(96, _RSP_REGNO, _XMM10_REGNO);
3504 if (jit_regset_tstbit(&_jitc->function->regset, _XMM9))
3505 sse_stxi_d(88, _RSP_REGNO, _XMM9_REGNO);
3506 if (jit_regset_tstbit(&_jitc->function->regset, _XMM8))
3507 sse_stxi_d(80, _RSP_REGNO, _XMM8_REGNO);
3508 if (jit_regset_tstbit(&_jitc->function->regset, _XMM7))
3509 sse_stxi_d(72, _RSP_REGNO, _XMM7_REGNO);
3510 if (jit_regset_tstbit(&_jitc->function->regset, _XMM6))
3511 sse_stxi_d(64, _RSP_REGNO, _XMM6_REGNO);
3512 if (jit_regset_tstbit(&_jitc->function->regset, _R15))
3513 stxi(56, _RSP_REGNO, _R15_REGNO);
3514 if (jit_regset_tstbit(&_jitc->function->regset, _R14))
3515 stxi(48, _RSP_REGNO, _R14_REGNO);
3516 if (jit_regset_tstbit(&_jitc->function->regset, _R13))
3517 stxi(40, _RSP_REGNO, _R13_REGNO);
3518 if (jit_regset_tstbit(&_jitc->function->regset, _R12))
3519 stxi(32, _RSP_REGNO, _R12_REGNO);
3520 if (jit_regset_tstbit(&_jitc->function->regset, _RSI))
3521 stxi(24, _RSP_REGNO, _RSI_REGNO);
3522 if (jit_regset_tstbit(&_jitc->function->regset, _RDI))
3523 stxi(16, _RSP_REGNO, _RDI_REGNO);
3524 if (jit_regset_tstbit(&_jitc->function->regset, _RBX))
3525 stxi( 8, _RSP_REGNO, _RBX_REGNO);
3527 if (jit_regset_tstbit(&_jitc->function->regset, _RBX))
3528 stxi(40, _RSP_REGNO, _RBX_REGNO);
3529 if (jit_regset_tstbit(&_jitc->function->regset, _R12))
3530 stxi(32, _RSP_REGNO, _R12_REGNO);
3531 if (jit_regset_tstbit(&_jitc->function->regset, _R13))
3532 stxi(24, _RSP_REGNO, _R13_REGNO);
3533 if (jit_regset_tstbit(&_jitc->function->regset, _R14))
3534 stxi(16, _RSP_REGNO, _R14_REGNO);
3535 if (jit_regset_tstbit(&_jitc->function->regset, _R15))
3536 stxi( 8, _RSP_REGNO, _R15_REGNO);
3539 stxi(0, _RSP_REGNO, _RBP_REGNO);
3540 movr(_RBP_REGNO, _RSP_REGNO);
3543 subi(_RSP_REGNO, _RSP_REGNO, _jitc->function->stack);
3544 if (_jitc->function->allocar) {
3545 reg = jit_get_reg(jit_class_gpr);
3546 movi(rn(reg), _jitc->function->self.aoff);
3547 stxi_i(_jitc->function->aoffoff, _RBP_REGNO, rn(reg));
3551 #if __X64 && !(__CYGWIN__ || _WIN32)
3552 if (_jitc->function->self.call & jit_call_varargs) {
3553 jit_word_t nofp_code;
3555 /* Save gp registers in the save area, if any is a vararg */
3556 for (reg = first_gp_from_offset(_jitc->function->vagp);
3557 jit_arg_reg_p(reg); ++reg)
3558 stxi(_jitc->function->vaoff + first_gp_offset +
3559 reg * 8, _RBP_REGNO, rn(JIT_RA0 - reg));
3561 reg = first_fp_from_offset(_jitc->function->vafp);
3562 if (jit_arg_f_reg_p(reg)) {
3563 /* Skip over if no float registers were passed as argument */
3568 nofp_code = _jit->pc.w;
3570 /* Save fp registers in the save area, if any is a vararg */
3571 /* Note that the full 16 byte xmm is not saved, because
3572 * lightning only handles float and double, and, while
3573 * attempting to provide a va_list compatible pointer as
3574 * jit_va_start return, does not guarantee it (on all ports). */
3575 for (; jit_arg_f_reg_p(reg); ++reg)
3576 sse_stxi_d(_jitc->function->vaoff + first_fp_offset +
3577 reg * va_fp_increment, _RBP_REGNO, rn(_XMM0 - reg));
3579 patch_rel_char(nofp_code, _jit->pc.w);
3586 _epilog(jit_state_t *_jit, jit_node_t *node)
3588 if (_jitc->function->assume_frame)
3590 /* callee save registers */
3591 movr(_RSP_REGNO, _RBP_REGNO);
3593 if (jit_regset_tstbit(&_jitc->function->regset, _RDI))
3594 ldxi(_RDI_REGNO, _RSP_REGNO, 12);
3595 if (jit_regset_tstbit(&_jitc->function->regset, _RSI))
3596 ldxi(_RSI_REGNO, _RSP_REGNO, 8);
3597 if (jit_regset_tstbit(&_jitc->function->regset, _RBX))
3598 ldxi(_RBX_REGNO, _RSP_REGNO, 4);
3600 # if __CYGWIN__ || _WIN32
3601 if (jit_regset_tstbit(&_jitc->function->regset, _XMM15))
3602 sse_ldxi_d(_XMM15_REGNO, _RSP_REGNO, 136);
3603 if (jit_regset_tstbit(&_jitc->function->regset, _XMM14))
3604 sse_ldxi_d(_XMM14_REGNO, _RSP_REGNO, 128);
3605 if (jit_regset_tstbit(&_jitc->function->regset, _XMM13))
3606 sse_ldxi_d(_XMM13_REGNO, _RSP_REGNO, 120);
3607 if (jit_regset_tstbit(&_jitc->function->regset, _XMM12))
3608 sse_ldxi_d(_XMM12_REGNO, _RSP_REGNO, 112);
3609 if (jit_regset_tstbit(&_jitc->function->regset, _XMM11))
3610 sse_ldxi_d(_XMM11_REGNO, _RSP_REGNO, 104);
3611 if (jit_regset_tstbit(&_jitc->function->regset, _XMM10))
3612 sse_ldxi_d(_XMM10_REGNO, _RSP_REGNO, 96);
3613 if (jit_regset_tstbit(&_jitc->function->regset, _XMM9))
3614 sse_ldxi_d(_XMM9_REGNO, _RSP_REGNO, 88);
3615 if (jit_regset_tstbit(&_jitc->function->regset, _XMM8))
3616 sse_ldxi_d(_XMM8_REGNO, _RSP_REGNO, 80);
3617 if (jit_regset_tstbit(&_jitc->function->regset, _XMM7))
3618 sse_ldxi_d(_XMM7_REGNO, _RSP_REGNO, 72);
3619 if (jit_regset_tstbit(&_jitc->function->regset, _XMM6))
3620 sse_ldxi_d(_XMM6_REGNO, _RSP_REGNO, 64);
3621 if (jit_regset_tstbit(&_jitc->function->regset, _R15))
3622 ldxi(_R15_REGNO, _RSP_REGNO, 56);
3623 if (jit_regset_tstbit(&_jitc->function->regset, _R14))
3624 ldxi(_R14_REGNO, _RSP_REGNO, 48);
3625 if (jit_regset_tstbit(&_jitc->function->regset, _R13))
3626 ldxi(_R13_REGNO, _RSP_REGNO, 40);
3627 if (jit_regset_tstbit(&_jitc->function->regset, _R12))
3628 ldxi(_R12_REGNO, _RSP_REGNO, 32);
3629 if (jit_regset_tstbit(&_jitc->function->regset, _RSI))
3630 ldxi(_RSI_REGNO, _RSP_REGNO, 24);
3631 if (jit_regset_tstbit(&_jitc->function->regset, _RDI))
3632 ldxi(_RDI_REGNO, _RSP_REGNO, 16);
3633 if (jit_regset_tstbit(&_jitc->function->regset, _RBX))
3634 ldxi(_RBX_REGNO, _RSP_REGNO, 8);
3636 if (jit_regset_tstbit(&_jitc->function->regset, _RBX))
3637 ldxi(_RBX_REGNO, _RSP_REGNO, 40);
3638 if (jit_regset_tstbit(&_jitc->function->regset, _R12))
3639 ldxi(_R12_REGNO, _RSP_REGNO, 32);
3640 if (jit_regset_tstbit(&_jitc->function->regset, _R13))
3641 ldxi(_R13_REGNO, _RSP_REGNO, 24);
3642 if (jit_regset_tstbit(&_jitc->function->regset, _R14))
3643 ldxi(_R14_REGNO, _RSP_REGNO, 16);
3644 if (jit_regset_tstbit(&_jitc->function->regset, _R15))
3645 ldxi(_R15_REGNO, _RSP_REGNO, 8);
3648 ldxi(_RBP_REGNO, _RSP_REGNO, 0);
3649 addi(_RSP_REGNO, _RSP_REGNO, stack_framesize - REAL_WORDSIZE);
3655 _vastart(jit_state_t *_jit, jit_int32_t r0)
3657 #if __X32 || __CYGWIN__ || _WIN32
3658 assert(_jitc->function->self.call & jit_call_varargs);
3659 addi(r0, _RBP_REGNO, _jitc->function->self.size);
3663 assert(_jitc->function->self.call & jit_call_varargs);
3665 /* Return jit_va_list_t in the register argument */
3666 addi(r0, _RBP_REGNO, _jitc->function->vaoff);
3667 reg = jit_get_reg(jit_class_gpr);
3669 /* Initialize gp offset in the save area. */
3670 movi(rn(reg), _jitc->function->vagp);
3671 stxi_i(offsetof(jit_va_list_t, gpoff), r0, rn(reg));
3673 /* Initialize fp offset in the save area. */
3674 movi(rn(reg), _jitc->function->vafp);
3675 stxi_i(offsetof(jit_va_list_t, fpoff), r0, rn(reg));
3677 /* Initialize overflow pointer to the first stack argument. */
3678 addi(rn(reg), _RBP_REGNO, _jitc->function->self.size);
3679 stxi(offsetof(jit_va_list_t, over), r0, rn(reg));
3681 /* Initialize register save area pointer. */
3682 addi(rn(reg), r0, first_gp_offset);
3683 stxi(offsetof(jit_va_list_t, save), r0, rn(reg));
3690 _vaarg(jit_state_t *_jit, jit_int32_t r0, jit_int32_t r1)
3692 #if __X32 || __CYGWIN__ || _WIN32
3693 assert(_jitc->function->self.call & jit_call_varargs);
3695 addi(r1, r1, va_gp_increment);
3702 assert(_jitc->function->self.call & jit_call_varargs);
3704 rg0 = jit_get_reg(jit_class_gpr);
3705 rg1 = jit_get_reg(jit_class_gpr);
3707 /* Load the gp offset in save area in the first temporary. */
3708 ldxi_i(rn(rg0), r1, offsetof(jit_va_list_t, gpoff));
3710 /* Jump over if there are no remaining arguments in the save area. */
3711 icmpi(rn(rg0), va_gp_max_offset);
3713 ge_code = _jit->pc.w;
3715 /* Load the save area pointer in the second temporary. */
3716 ldxi(rn(rg1), r1, offsetof(jit_va_list_t, save));
3718 /* Load the vararg argument in the first argument. */
3719 ldxr(r0, rn(rg1), rn(rg0));
3721 /* Update the gp offset. */
3722 addi(rn(rg0), rn(rg0), 8);
3723 stxi_i(offsetof(jit_va_list_t, gpoff), r1, rn(rg0));
3725 /* Will only need one temporary register below. */
3728 /* Jump over overflow code. */
3730 lt_code = _jit->pc.w;
3732 /* Where to land if argument is in overflow area. */
3733 patch_rel_char(ge_code, _jit->pc.w);
3735 /* Load overflow pointer. */
3736 ldxi(rn(rg0), r1, offsetof(jit_va_list_t, over));
3738 /* Load argument. */
3741 /* Update overflow pointer. */
3742 addi(rn(rg0), rn(rg0), va_gp_increment);
3743 stxi(offsetof(jit_va_list_t, over), r1, rn(rg0));
3745 /* Where to land if argument is in save area. */
3746 patch_rel_char(lt_code, _jit->pc.w);
3752 /* The x87 boolean argument tells if will put the result in a x87
3753 * register if non false, in a sse register otherwise. */
3755 _vaarg_d(jit_state_t *_jit, jit_int32_t r0, jit_int32_t r1, jit_bool_t x87)
3757 #if __X32 || __CYGWIN__ || _WIN32
3758 assert(_jitc->function->self.call & jit_call_varargs);
3770 assert(_jitc->function->self.call & jit_call_varargs);
3772 rg0 = jit_get_reg(jit_class_gpr);
3773 rg1 = jit_get_reg(jit_class_gpr);
3775 /* Load the fp offset in save area in the first temporary. */
3776 ldxi_i(rn(rg0), r1, offsetof(jit_va_list_t, fpoff));
3778 /* Jump over if there are no remaining arguments in the save area. */
3779 icmpi(rn(rg0), va_fp_max_offset);
3781 ge_code = _jit->pc.w;
3783 /* Load the save area pointer in the second temporary. */
3784 ldxi(rn(rg1), r1, offsetof(jit_va_list_t, save));
3786 /* Load the vararg argument in the first argument. */
3788 x87_ldxr_d(r0, rn(rg1), rn(rg0));
3790 sse_ldxr_d(r0, rn(rg1), rn(rg0));
3792 /* Update the fp offset. */
3793 addi(rn(rg0), rn(rg0), va_fp_increment);
3794 stxi_i(offsetof(jit_va_list_t, fpoff), r1, rn(rg0));
3796 /* Will only need one temporary register below. */
3799 /* Jump over overflow code. */
3801 lt_code = _jit->pc.w;
3803 /* Where to land if argument is in overflow area. */
3804 patch_rel_char(ge_code, _jit->pc.w);
3806 /* Load overflow pointer. */
3807 ldxi(rn(rg0), r1, offsetof(jit_va_list_t, over));
3809 /* Load argument. */
3811 x87_ldr_d(r0, rn(rg0));
3813 sse_ldr_d(r0, rn(rg0));
3815 /* Update overflow pointer. */
3816 addi(rn(rg0), rn(rg0), 8);
3817 stxi(offsetof(jit_va_list_t, over), r1, rn(rg0));
3819 /* Where to land if argument is in save area. */
3820 patch_rel_char(lt_code, _jit->pc.w);
3827 _patch_at(jit_state_t *_jit, jit_node_t *node,
3828 jit_word_t instr, jit_word_t label)
3830 switch (node->code) {
3832 case jit_code_calli:
3835 patch_abs(instr, label);
3838 patch_rel(instr, label);