notaz.gp2x.de / pcsx_rearmed.git / blame
? search:
summary | shortlog | log | commit | commitdiff | tree
blob | blame (incremental) | history | HEAD
Merge pull request #657 from pcercuei/update-lightrec-20220529
[pcsx_rearmed.git] / deps / lightning / lib / jit_x86-cpu.c
CommitLineData
4a71579b
PC		 1/*
2 * Copyright (C) 2012-2019 Free Software Foundation, Inc.
3 *
4 * This file is part of GNU lightning.
5 *
6 * GNU lightning is free software; you can redistribute it and/or modify it
7 * under the terms of the GNU Lesser General Public License as published
8 * by the Free Software Foundation; either version 3, or (at your option)
9 * any later version.
10 *
11 * GNU lightning is distributed in the hope that it will be useful, but
12 * WITHOUT ANY WARRANTY; without even the implied warranty of MERCHANTABILITY
13 * or FITNESS FOR A PARTICULAR PURPOSE. See the GNU Lesser General Public
14 * License for more details.
15 *
16 * Authors:
17 * Paulo Cesar Pereira de Andrade
18 */
19
20/* avoid using it due to partial stalls */
21#define USE_INC_DEC 0
22
23#if PROTO
24# if __X32 || __X64_32
25# define WIDE 0
26# define ldi(u, v) ldi_i(u, v)
27# define ldr(u, v) ldr_i(u, v)
28# define ldxr(u, v, w) ldxr_i(u, v, w)
29# define ldxi(u, v, w) ldxi_i(u, v, w)
30# define sti(u, v) sti_i(u, v)
31# define stxi(u, v, w) stxi_i(u, v, w)
32# define can_sign_extend_int_p(im) 1
33# define can_zero_extend_int_p(im) 1
34# define fits_uint32_p(im) 1
35# else
36# define WIDE 1
37# define ldi(u, v) ldi_l(u, v)
38# define ldr(u, v) ldr_l(u, v)
39# define ldxr(u, v, w) ldxr_l(u, v, w)
40# define ldxi(u, v, w) ldxi_l(u, v, w)
41# define sti(u, v) sti_l(u, v)
42# define stxi(u, v, w) stxi_l(u, v, w)
43# define can_sign_extend_int_p(im) \
44 (((im) >= 0 && (long long)(im) <= 0x7fffffffLL) || \
45 ((im) < 0 && (long long)(im) > -0x80000000LL))
46# define can_zero_extend_int_p(im) \
47 ((im) >= 0 && (im) < 0x80000000LL)
48# define fits_uint32_p(im) (((im) & 0xffffffff00000000LL) == 0)
49# endif
50# if __X32 || __CYGWIN__ || __X64_32 || _WIN32
51# define reg8_p(rn) \
52 ((rn) >= _RAX_REGNO && (rn) <= _RBX_REGNO)
53# else
54# define reg8_p(rn) 1
55# endif
56# define _RAX_REGNO 0
57# define _RCX_REGNO 1
58# define _RDX_REGNO 2
59# define _RBX_REGNO 3
60# define _RSP_REGNO 4
61# define _RBP_REGNO 5
62# define _RSI_REGNO 6
63# define _RDI_REGNO 7
64# define _R8_REGNO 8
65# define _R9_REGNO 9
66# define _R10_REGNO 10
67# define _R11_REGNO 11
68# define _R12_REGNO 12
69# define _R13_REGNO 13
70# define _R14_REGNO 14
71# define _R15_REGNO 15
72# define r7(reg) ((reg) & 7)
73# define r8(reg) ((reg) & 15)
74# define _SCL1 0x00
75# define _SCL2 0x01
76# define _SCL4 0x02
77# define _SCL8 0x03
78# define X86_ADD 0
79# define X86_OR 1 << 3
80# define X86_ADC 2 << 3
81# define X86_SBB 3 << 3
82# define X86_AND 4 << 3
83# define X86_SUB 5 << 3
84# define X86_XOR 6 << 3
85# define X86_CMP 7 << 3
86# define X86_ROL 0
87# define X86_ROR 1
88# define X86_RCL 2
89# define X86_RCR 3
90# define X86_SHL 4
91# define X86_SHR 5
92# define X86_SAR 7
93# define X86_NOT 2
94# define X86_NEG 3
95# define X86_MUL 4
96# define X86_IMUL 5
97# define X86_DIV 6
98# define X86_IDIV 7
99# define X86_CC_O 0x0
100# define X86_CC_NO 0x1
101# define X86_CC_NAE 0x2
102# define X86_CC_B 0x2
103# define X86_CC_C 0x2
104# define X86_CC_AE 0x3
105# define X86_CC_NB 0x3
106# define X86_CC_NC 0x3
107# define X86_CC_E 0x4
108# define X86_CC_Z 0x4
109# define X86_CC_NE 0x5
110# define X86_CC_NZ 0x5
111# define X86_CC_BE 0x6
112# define X86_CC_NA 0x6
113# define X86_CC_A 0x7
114# define X86_CC_NBE 0x7
115# define X86_CC_S 0x8
116# define X86_CC_NS 0x9
117# define X86_CC_P 0xa
118# define X86_CC_PE 0xa
119# define X86_CC_NP 0xb
120# define X86_CC_PO 0xb
121# define X86_CC_L 0xc
122# define X86_CC_NGE 0xc
123# define X86_CC_GE 0xd
124# define X86_CC_NL 0xd
125# define X86_CC_LE 0xe
126# define X86_CC_NG 0xe
127# define X86_CC_G 0xf
128# define X86_CC_NLE 0xf
129# define mrm(md, r, m) *_jit->pc.uc++ = (md<<6) | (r<<3) | m
130# define sib(sc, i, b) *_jit->pc.uc++ = (sc<<6) | (i<<3) | b
131# define ic(c) *_jit->pc.uc++ = c
132# define is(s) *_jit->pc.us++ = s
133# define ii(i) *_jit->pc.ui++ = i
134# if __X64 && !__X64_32
135# define il(l) *_jit->pc.ul++ = l
136# else
137# define il(l) ii(l)
138# endif
139# define patch_abs(instr, label) \
140 *(jit_word_t *)(instr - sizeof(jit_word_t)) = label
141# define patch_rel(instr, label) \
142 *(jit_int32_t *)(instr - 4) = label - instr
143# define patch_rel_char(instr, label) \
144 *(jit_int8_t *)(instr - 1) = label - instr
145# define rex(l, w, r, x, b) _rex(_jit, l, w, r, x, b)
146static void
147_rex(jit_state_t*,jit_int32_t,jit_int32_t,jit_int32_t,jit_int32_t,jit_int32_t);
148# define rx(rd, md, rb, ri, ms) _rx(_jit, rd, md, rb, ri, ms)
149static void
150_rx(jit_state_t*,jit_int32_t,jit_int32_t,jit_int32_t,jit_int32_t,jit_int32_t);
151# define nop(n) _nop(_jit, n)
152static void _nop(jit_state_t*, jit_int32_t);
153# define emms() is(0x770f)
154# define lea(md, rb, ri, ms, rd) _lea(_jit, md, rb, ri, ms, rd)
155static void
156_lea(jit_state_t*,jit_int32_t,jit_int32_t,jit_int32_t,jit_int32_t,jit_int32_t);
157# define pushr(r0) _pushr(_jit, r0)
158static void _pushr(jit_state_t*, jit_int32_t) maybe_unused;
159# define popr(r0) _popr(_jit, r0)
160static void _popr(jit_state_t*, jit_int32_t) maybe_unused;
161# define xchgr(r0, r1) _xchgr(_jit, r0, r1)
162static void _xchgr(jit_state_t*, jit_int32_t, jit_int32_t);
163# define testr(r0, r1) _testr(_jit, r0, r1)
164static void _testr(jit_state_t*, jit_int32_t, jit_int32_t);
165# define testi(r0, i0) _testi(_jit, r0, i0)
166static void _testi(jit_state_t*, jit_int32_t, jit_word_t);
167# define cc(code, r0) _cc(_jit, code, r0)
168static void _cc(jit_state_t*, jit_int32_t, jit_int32_t);
169# define icmpr(r0, r1) alur(X86_CMP, r0, r1)
170# define alur(code, r0, r1) _alur(_jit, code, r0, r1)
171static void _alur(jit_state_t*, jit_int32_t, jit_int32_t, jit_int32_t);
172# define icmpi(r0, i0) alui(X86_CMP, r0, i0)
173# define alui(code, r0, i0) _alui(_jit, code, r0, i0)
174static void _alui(jit_state_t*, jit_int32_t, jit_int32_t, jit_word_t);
175# define iaddr(r0, r1) alur(X86_ADD, r0, r1)
176# define save(r0) _save(_jit, r0)
177static void _save(jit_state_t*, jit_int32_t);
178# define load(r0) _load(_jit, r0)
179static void _load(jit_state_t*, jit_int32_t);
180# define addr(r0, r1, r2) _addr(_jit, r0, r1, r2)
181static void _addr(jit_state_t*, jit_int32_t, jit_int32_t, jit_int32_t);
182# define iaddi(r0, i0) alui(X86_ADD, r0, i0)
183# define addi(r0, r1, i0) _addi(_jit, r0, r1, i0)
184static void _addi(jit_state_t*, jit_int32_t, jit_int32_t, jit_word_t);
185#define addcr(r0, r1, r2) _addcr(_jit, r0, r1, r2)
186static void _addcr(jit_state_t*, jit_int32_t, jit_int32_t, jit_int32_t);
187#define addci(r0, r1, i0) _addci(_jit, r0, r1, i0)
188static void _addci(jit_state_t*, jit_int32_t, jit_int32_t, jit_word_t);
189# define iaddxr(r0, r1) alur(X86_ADC, r0, r1)
190# define addxr(r0, r1, r2) _addxr(_jit, r0, r1, r2)
191static void _addxr(jit_state_t*, jit_int32_t, jit_int32_t, jit_int32_t);
192# define iaddxi(r0, i0) alui(X86_ADC, r0, i0)
193# define addxi(r0, r1, i0) _addxi(_jit, r0, r1, i0)
194static void _addxi(jit_state_t*, jit_int32_t, jit_int32_t, jit_word_t);
195# define isubr(r0, r1) alur(X86_SUB, r0, r1)
196# define subr(r0, r1, r2) _subr(_jit, r0, r1, r2)
197static void _subr(jit_state_t*, jit_int32_t, jit_int32_t, jit_int32_t);
198# define isubi(r0, i0) alui(X86_SUB, r0, i0)
199# define subi(r0, r1, i0) _subi(_jit, r0, r1, i0)
200static void _subi(jit_state_t*, jit_int32_t, jit_int32_t, jit_word_t);
201# define subcr(r0, r1, r2) _subcr(_jit, r0, r1, r2)
202static void _subcr(jit_state_t*,jit_int32_t,jit_int32_t,jit_int32_t);
203# define subci(r0, r1, i0) _subci(_jit, r0, r1, i0)
204static void _subci(jit_state_t*,jit_int32_t,jit_int32_t,jit_word_t);
205# define isubxr(r0, r1) alur(X86_SBB, r0, r1)
206# define subxr(r0, r1, r2) _subxr(_jit, r0, r1, r2)
207static void _subxr(jit_state_t*,jit_int32_t,jit_int32_t,jit_int32_t);
208# define isubxi(r0, i0) alui(X86_SBB, r0, i0)
209# define subxi(r0, r1, i0) _subxi(_jit, r0, r1, i0)
210static void _subxi(jit_state_t*,jit_int32_t,jit_int32_t,jit_word_t);
211# define rsbi(r0, r1, i0) _rsbi(_jit, r0, r1, i0)
212static void _rsbi(jit_state_t*,jit_int32_t,jit_int32_t,jit_word_t);
213# define imulr(r0, r1) _imulr(_jit, r0, r1)
214static void _imulr(jit_state_t*, jit_int32_t, jit_int32_t);
215# define imuli(r0, r1, i0) _imuli(_jit, r0, r1, i0)
216static void _imuli(jit_state_t*, jit_int32_t, jit_int32_t, jit_word_t);
217# define mulr(r0, r1, r2) _mulr(_jit, r0, r1, r2)
218static void _mulr(jit_state_t*, jit_int32_t, jit_int32_t, jit_int32_t);
219# define muli(r0, r1, i0) _muli(_jit, r0, r1, i0)
220static void _muli(jit_state_t*, jit_int32_t, jit_int32_t, jit_word_t);
221# define umulr(r0) unr(X86_IMUL, r0)
222# define umulr_u(r0) unr(X86_MUL, r0)
223# define qmulr(r0, r1, r2, r3) _iqmulr(_jit, r0, r1, r2, r3, 1)
224# define qmulr_u(r0, r1, r2, r3) _iqmulr(_jit, r0, r1, r2, r3, 0)
225# define iqmulr(r0, r1, r2, r3, sign) _iqmulr(_jit, r0, r1, r2, r3, sign)
226static void _iqmulr(jit_state_t*, jit_int32_t, jit_int32_t,
227 jit_int32_t,jit_int32_t, jit_bool_t);
228# define qmuli(r0, r1, r2, i0) _iqmuli(_jit, r0, r1, r2, i0, 1)
229# define qmuli_u(r0, r1, r2, i0) _iqmuli(_jit, r0, r1, r2, i0, 0)
230# define iqmuli(r0, r1, r2, i0, sign) _iqmuli(_jit, r0, r1, r2, i0, sign)
231static void _iqmuli(jit_state_t*, jit_int32_t, jit_int32_t,
232 jit_int32_t,jit_word_t, jit_bool_t);
233# define sign_extend_rdx_rax() _sign_extend_rdx_rax(_jit)
234static void _sign_extend_rdx_rax(jit_state_t*);
235# define idivr(r0) unr(X86_IDIV, r0)
236# define idivr_u(r0) unr(X86_DIV, r0)
237# define divremr(r0, r1, r2, i0, i1) _divremr(_jit, r0, r1, r2, i0, i1)
238static void
239_divremr(jit_state_t*,jit_int32_t,jit_int32_t,jit_int32_t,
240 jit_bool_t,jit_bool_t);
241# define divremi(r0, r1, i0, i1, i2) _divremi(_jit, r0, r1, i0, i1, i2)
242static void
243_divremi(jit_state_t*,jit_int32_t,jit_int32_t,jit_word_t,jit_bool_t,jit_bool_t);
244# define divr(r0, r1, r2) divremr(r0, r1, r2, 1, 1)
245# define divi(r0, r1, i0) divremi(r0, r1, i0, 1, 1)
246# define divr_u(r0, r1, r2) divremr(r0, r1, r2, 0, 1)
247# define divi_u(r0, r1, i0) divremi(r0, r1, i0, 0, 1)
248# define qdivr(r0, r1, r2, r3) _iqdivr(_jit, r0, r1, r2, r3, 1)
249# define qdivr_u(r0, r1, r2, r3) _iqdivr(_jit, r0, r1, r2, r3, 0)
250# define iqdivr(r0, r1, r2, r3, sign) _iqdivr(_jit, r0, r1, r2, r3, sign)
251static void _iqdivr(jit_state_t*, jit_int32_t, jit_int32_t,
252 jit_int32_t,jit_int32_t, jit_bool_t);
253# define qdivi(r0, r1, r2, i0) _iqdivi(_jit, r0, r1, r2, i0, 1)
254# define qdivi_u(r0, r1, r2, i0) _iqdivi(_jit, r0, r1, r2, i0, 0)
255# define iqdivi(r0, r1, r2, i0, sign) _iqdivi(_jit, r0, r1, r2, i0, sign)
256static void _iqdivi(jit_state_t*, jit_int32_t, jit_int32_t,
257 jit_int32_t,jit_word_t, jit_bool_t);
258# define remr(r0, r1, r2) divremr(r0, r1, r2, 1, 0)
259# define remi(r0, r1, i0) divremi(r0, r1, i0, 1, 0)
260# define remr_u(r0, r1, r2) divremr(r0, r1, r2, 0, 0)
261# define remi_u(r0, r1, i0) divremi(r0, r1, i0, 0, 0)
262# define iandr(r0, r1) alur(X86_AND, r0, r1)
263# define andr(r0, r1, r2) _andr(_jit, r0, r1, r2)
264static void _andr(jit_state_t*,jit_int32_t,jit_int32_t,jit_int32_t);
265# define iandi(r0, i0) alui(X86_AND, r0, i0)
266# define andi(r0, r1, i0) _andi(_jit, r0, r1, i0)
267static void _andi(jit_state_t*, jit_int32_t,jit_int32_t,jit_word_t);
268# define iorr(r0, r1) alur(X86_OR, r0, r1)
269# define orr(r0, r1, r2) _orr(_jit, r0, r1, r2)
270static void _orr(jit_state_t*, jit_int32_t,jit_int32_t,jit_int32_t);
271# define iori(r0, i0) alui(X86_OR, r0, i0)
272# define ori(r0, r1, i0) _ori(_jit, r0, r1, i0)
273static void _ori(jit_state_t*, jit_int32_t,jit_int32_t,jit_word_t);
274# define ixorr(r0, r1) alur(X86_XOR, r0, r1)
275# define xorr(r0, r1, r2) _xorr(_jit, r0, r1, r2)
276static void _xorr(jit_state_t*, jit_int32_t,jit_int32_t,jit_int32_t);
277# define ixori(r0, i0) alui(X86_XOR, r0, i0)
278# define xori(r0, r1, i0) _xori(_jit, r0, r1, i0)
279static void _xori(jit_state_t*, jit_int32_t,jit_int32_t,jit_word_t);
280# define irotshr(code, r0) _irotshr(_jit, code, r0)
281static void _irotshr(jit_state_t*, jit_int32_t, jit_int32_t);
282# define rotshr(code, r0, r1, r2) _rotshr(_jit, code, r0, r1, r2)
283static void
284_rotshr(jit_state_t*,jit_int32_t,jit_int32_t,jit_int32_t,jit_int32_t);
285# define irotshi(code, r0, i0) _irotshi(_jit, code, r0, i0)
286static void _irotshi(jit_state_t*, jit_int32_t, jit_int32_t, jit_word_t);
287# define rotshi(code, r0, r1, i0) _rotshi(_jit, code, r0, r1, i0)
288static void
289_rotshi(jit_state_t*,jit_int32_t,jit_int32_t,jit_int32_t,jit_word_t);
290# define lshr(r0, r1, r2) rotshr(X86_SHL, r0, r1, r2)
291# define lshi(r0, r1, i0) _lshi(_jit, r0, r1, i0)
292static void _lshi(jit_state_t*, jit_int32_t, jit_int32_t, jit_word_t);
293# define rshr(r0, r1, r2) rotshr(X86_SAR, r0, r1, r2)
294# define rshi(r0, r1, i0) rotshi(X86_SAR, r0, r1, i0)
295# define rshr_u(r0, r1, r2) rotshr(X86_SHR, r0, r1, r2)
296# define rshi_u(r0, r1, i0) rotshi(X86_SHR, r0, r1, i0)
297# define unr(code, r0) _unr(_jit, code, r0)
298static void _unr(jit_state_t*, jit_int32_t, jit_int32_t);
299# define inegr(r0) unr(X86_NEG, r0)
300# define negr(r0, r1) _negr(_jit, r0, r1)
301static void _negr(jit_state_t*, jit_int32_t, jit_int32_t);
302# define icomr(r0) unr(X86_NOT, r0)
303# define comr(r0, r1) _comr(_jit, r0, r1)
304static void _comr(jit_state_t*, jit_int32_t, jit_int32_t);
305# if USE_INC_DEC
306# define incr(r0, r1) _incr(_jit, r0, r1)
307static void _incr(jit_state_t*, jit_int32_t, jit_int32_t);
308# define decr(r0, r1) _decr(_jit, r0, r1)
309static void _decr(jit_state_t*, jit_int32_t, jit_int32_t);
310# endif
311# define cr(code, r0, r1, r2) _cr(_jit, code, r0, r1, r2)
312static void
313_cr(jit_state_t*, jit_int32_t, jit_int32_t, jit_int32_t, jit_int32_t);
314# define ci(code, r0, r1, i0) _ci(_jit, code, r0, r1, i0)
315static void
316_ci(jit_state_t *_jit, jit_int32_t, jit_int32_t, jit_int32_t, jit_word_t);
317# define ci0(code, r0, r1) _ci0(_jit, code, r0, r1)
318static void _ci0(jit_state_t*, jit_int32_t, jit_int32_t, jit_int32_t);
319# define ltr(r0, r1, r2) _ltr(_jit, r0, r1, r2)
320static void _ltr(jit_state_t*, jit_int32_t, jit_int32_t, jit_int32_t);
321# define lti(r0, r1, i0) _lti(_jit, r0, r1, i0)
322static void _lti(jit_state_t*, jit_int32_t, jit_int32_t, jit_word_t);
323# define ltr_u(r0, r1, r2) _ltr_u(_jit, r0, r1, r2)
324static void _ltr_u(jit_state_t*, jit_int32_t, jit_int32_t, jit_int32_t);
325# define lti_u(r0, r1, i0) ci(X86_CC_B, r0, r1, i0)
326# define ler(r0, r1, r2) _ler(_jit, r0, r1, r2)
327static void _ler(jit_state_t*, jit_int32_t, jit_int32_t, jit_int32_t);
328# define lei(r0, r1, i0) ci(X86_CC_LE, r0, r1, i0)
329# define ler_u(r0, r1, r2) _ler_u(_jit, r0, r1, r2)
330static void _ler_u(jit_state_t*, jit_int32_t, jit_int32_t, jit_int32_t);
331# define lei_u(r0, r1, i0) _lei_u(_jit, r0, r1, i0)
332static void _lei_u(jit_state_t*, jit_int32_t, jit_int32_t, jit_word_t);
333# define eqr(r0, r1, r2) _eqr(_jit, r0, r1, r2)
334static void _eqr(jit_state_t*, jit_int32_t, jit_int32_t, jit_int32_t);
335# define eqi(r0, r1, i0) _eqi(_jit, r0, r1, i0)
336static void _eqi(jit_state_t*, jit_int32_t, jit_int32_t, jit_word_t);
337# define ger(r0, r1, r2) _ger(_jit, r0, r1, r2)
338static void _ger(jit_state_t*, jit_int32_t, jit_int32_t, jit_int32_t);
339# define gei(r0, r1, i0) _gei(_jit, r0, r1, i0)
340static void _gei(jit_state_t*, jit_int32_t, jit_int32_t, jit_word_t);
341# define ger_u(r0, r1, r2) _ger_u(_jit, r0, r1, r2)
342static void _ger_u(jit_state_t*, jit_int32_t, jit_int32_t, jit_int32_t);
343# define gei_u(r0, r1, i0) _gei_u(_jit, r0, r1, i0)
344static void _gei_u(jit_state_t*, jit_int32_t, jit_int32_t, jit_word_t);
345# define gtr(r0, r1, r2) _gtr(_jit, r0, r1, r2)
346static void _gtr(jit_state_t*, jit_int32_t, jit_int32_t, jit_int32_t);
347# define gti(r0, r1, i0) _ci(_jit, X86_CC_G, r0, r1, i0)
348# define gtr_u(r0, r1, r2) _gtr_u(_jit, r0, r1, r2)
349static void _gtr_u(jit_state_t*, jit_int32_t, jit_int32_t, jit_int32_t);
350# define gti_u(r0, r1, i0) _gti_u(_jit, r0, r1, i0)
351static void _gti_u(jit_state_t*, jit_int32_t, jit_int32_t, jit_word_t);
352# define ner(r0, r1, r2) _ner(_jit, r0, r1, r2)
353static void _ner(jit_state_t*, jit_int32_t, jit_int32_t, jit_int32_t);
354# define nei(r0, r1, i0) _nei(_jit, r0, r1, i0)
355static void _nei(jit_state_t*, jit_int32_t, jit_int32_t, jit_word_t);
356# define movr(r0, r1) _movr(_jit, r0, r1)
357static void _movr(jit_state_t*, jit_int32_t, jit_int32_t);
358# define imovi(r0, i0) _imovi(_jit, r0, i0)
359static void _imovi(jit_state_t*, jit_int32_t, jit_word_t);
360# define movi(r0, i0) _movi(_jit, r0, i0)
361static void _movi(jit_state_t*, jit_int32_t, jit_word_t);
362# define movi_p(r0, i0) _movi_p(_jit, r0, i0)
363static jit_word_t _movi_p(jit_state_t*, jit_int32_t, jit_word_t);
364# define movcr(r0, r1) _movcr(_jit, r0, r1)
365static void _movcr(jit_state_t*,jit_int32_t,jit_int32_t);
366# define movcr_u(r0, r1) _movcr_u(_jit, r0, r1)
367static void _movcr_u(jit_state_t*,jit_int32_t,jit_int32_t);
368# define movsr(r0, r1) _movsr(_jit, r0, r1)
369static void _movsr(jit_state_t*,jit_int32_t,jit_int32_t);
370# define movsr_u(r0, r1) _movsr_u(_jit, r0, r1)
371static void _movsr_u(jit_state_t*,jit_int32_t,jit_int32_t);
1f22b268
PC		 372#define movnr(r0, r1, r2) _movnr(_jit, r0, r1, r2)
373static void _movnr(jit_state_t*, jit_int32_t, jit_int32_t, jit_int32_t);
374#define movzr(r0, r1, r2) _movzr(_jit, r0, r1, r2)
375static void _movzr(jit_state_t*, jit_int32_t, jit_int32_t, jit_int32_t);
4a71579b
PC		 376# if __X64 && !__X64_32
377# define movir(r0, r1) _movir(_jit, r0, r1)
378static void _movir(jit_state_t*,jit_int32_t,jit_int32_t);
379# define movir_u(r0, r1) _movir_u(_jit, r0, r1)
380static void _movir_u(jit_state_t*,jit_int32_t,jit_int32_t);
381# endif
40a44dcb
PC		 382# define bswapr_us(r0, r1) _bswapr_us(_jit, r0, r1)
383static void _bswapr_us(jit_state_t*,jit_int32_t,jit_int32_t);
384# define bswapr_ui(r0, r1) _bswapr_ui(_jit, r0, r1)
385static void _bswapr_ui(jit_state_t*,jit_int32_t,jit_int32_t);
4a71579b 386# if __X64 && !__X64_32
40a44dcb
PC		 387#define bswapr_ul(r0, r1) _bswapr_ul(_jit, r0, r1)
388static void _bswapr_ul(jit_state_t*,jit_int32_t,jit_int32_t);
4a71579b
PC		 389#endif
390# define extr_c(r0, r1) _extr_c(_jit, r0, r1)
391static void _extr_c(jit_state_t*,jit_int32_t,jit_int32_t);
392# define extr_uc(r0, r1) _extr_uc(_jit, r0, r1)
393static void _extr_uc(jit_state_t*,jit_int32_t,jit_int32_t);
394# define extr_s(r0, r1) movsr(r0, r1)
395# define extr_us(r0, r1) movsr_u(r0, r1)
396# if __X64 && !__X64_32
397# define extr_i(r0, r1) movir(r0, r1)
398# define extr_ui(r0, r1) movir_u(r0, r1)
399# endif
400# define ldr_c(r0, r1) _ldr_c(_jit, r0, r1)
401static void _ldr_c(jit_state_t*, jit_int32_t, jit_int32_t);
402# define ldi_c(r0, i0) _ldi_c(_jit, r0, i0)
403static void _ldi_c(jit_state_t*, jit_int32_t, jit_word_t);
404# define ldr_uc(r0, r1) _ldr_uc(_jit, r0, r1)
405static void _ldr_uc(jit_state_t*, jit_int32_t, jit_int32_t);
406# define ldi_uc(r0, i0) _ldi_uc(_jit, r0, i0)
407static void _ldi_uc(jit_state_t*, jit_int32_t, jit_word_t);
408# define ldr_s(r0, r1) _ldr_s(_jit, r0, r1)
409static void _ldr_s(jit_state_t*, jit_int32_t, jit_int32_t);
410# define ldi_s(r0, i0) _ldi_s(_jit, r0, i0)
411static void _ldi_s(jit_state_t*, jit_int32_t, jit_word_t);
412# define ldr_us(r0, r1) _ldr_us(_jit, r0, r1)
413static void _ldr_us(jit_state_t*, jit_int32_t, jit_int32_t);
414# define ldi_us(r0, i0) _ldi_us(_jit, r0, i0)
415static void _ldi_us(jit_state_t*, jit_int32_t, jit_word_t);
416# if __X32 || !__X64_32
417# define ldr_i(r0, r1) _ldr_i(_jit, r0, r1)
418static void _ldr_i(jit_state_t*, jit_int32_t, jit_int32_t);
419# define ldi_i(r0, i0) _ldi_i(_jit, r0, i0)
420static void _ldi_i(jit_state_t*, jit_int32_t, jit_word_t);
421# endif
422# if __X64
423# if __X64_32
424# define ldr_i(r0, r1) _ldr_ui(_jit, r0, r1)
425# define ldi_i(r0, i0) _ldi_ui(_jit, r0, i0)
426# else
427# define ldr_ui(r0, r1) _ldr_ui(_jit, r0, r1)
428# define ldi_ui(r0, i0) _ldi_ui(_jit, r0, i0)
429# endif
430static void _ldr_ui(jit_state_t*, jit_int32_t, jit_int32_t);
431static void _ldi_ui(jit_state_t*, jit_int32_t, jit_word_t);
432# if !__X64_32
433# define ldr_l(r0, r1) _ldr_l(_jit, r0, r1)
434static void _ldr_l(jit_state_t*, jit_int32_t, jit_int32_t);
435# define ldi_l(r0, i0) _ldi_l(_jit, r0, i0)
436static void _ldi_l(jit_state_t*, jit_int32_t, jit_word_t);
437# endif
438# endif
439# define ldxr_c(r0, r1, r2) _ldxr_c(_jit, r0, r1, r2)
440static void _ldxr_c(jit_state_t*, jit_int32_t, jit_int32_t, jit_int32_t);
441# define ldxi_c(r0, r1, i0) _ldxi_c(_jit, r0, r1, i0)
442static void _ldxi_c(jit_state_t*, jit_int32_t, jit_int32_t, jit_word_t);
443# define ldxr_uc(r0, r1, r2) _ldxr_uc(_jit, r0, r1, r2)
444static void _ldxr_uc(jit_state_t*, jit_int32_t, jit_int32_t, jit_int32_t);
445# define ldxi_uc(r0, r1, i0) _ldxi_uc(_jit, r0, r1, i0)
446static void _ldxi_uc(jit_state_t*, jit_int32_t, jit_int32_t, jit_word_t);
447# define ldxr_s(r0, r1, r2) _ldxr_s(_jit, r0, r1, r2)
448static void _ldxr_s(jit_state_t*, jit_int32_t, jit_int32_t, jit_int32_t);
449# define ldxi_s(r0, r1, i0) _ldxi_s(_jit, r0, r1, i0)
450static void _ldxi_s(jit_state_t*, jit_int32_t, jit_int32_t, jit_word_t);
451# define ldxr_us(r0, r1, r2) _ldxr_us(_jit, r0, r1, r2)
452static void _ldxr_us(jit_state_t*, jit_int32_t, jit_int32_t, jit_int32_t);
453# define ldxi_us(r0, r1, i0) _ldxi_us(_jit, r0, r1, i0)
454static void _ldxi_us(jit_state_t*, jit_int32_t, jit_int32_t, jit_word_t);
455# if __X32 || !__X64_32
456# define ldxr_i(r0, r1, r2) _ldxr_i(_jit, r0, r1, r2)
457static void _ldxr_i(jit_state_t*, jit_int32_t, jit_int32_t, jit_int32_t);
458# define ldxi_i(r0, r1, i0) _ldxi_i(_jit, r0, r1, i0)
459static void _ldxi_i(jit_state_t*, jit_int32_t, jit_int32_t, jit_word_t);
460# endif
461# if __X64
462# if __X64_32
463# define ldxr_i(r0, r1, r2) _ldxr_ui(_jit, r0, r1, r2)
464# define ldxi_i(r0, r1, i0) _ldxi_ui(_jit, r0, r1, i0)
465# else
466# define ldxr_ui(r0, r1, r2) _ldxr_ui(_jit, r0, r1, r2)
467# define ldxi_ui(r0, r1, i0) _ldxi_ui(_jit, r0, r1, i0)
468# endif
469static void _ldxr_ui(jit_state_t*, jit_int32_t, jit_int32_t, jit_int32_t);
470static void _ldxi_ui(jit_state_t*, jit_int32_t, jit_int32_t, jit_word_t);
471# if !__X64_32
472# define ldxr_l(r0, r1, r2) _ldxr_l(_jit, r0, r1, r2)
473static void _ldxr_l(jit_state_t*, jit_int32_t, jit_int32_t, jit_int32_t);
474# define ldxi_l(r0, r1, i0) _ldxi_l(_jit, r0, r1, i0)
475static void _ldxi_l(jit_state_t*, jit_int32_t, jit_int32_t, jit_word_t);
476# endif
477# endif
478# define str_c(r0, r1) _str_c(_jit, r0, r1)
479static void _str_c(jit_state_t*, jit_int32_t, jit_int32_t);
480# define sti_c(i0, r0) _sti_c(_jit, i0, r0)
481static void _sti_c(jit_state_t*, jit_word_t, jit_int32_t);
482# define str_s(r0, r1) _str_s(_jit, r0, r1)
483static void _str_s(jit_state_t*, jit_int32_t, jit_int32_t);
484# define sti_s(i0, r0) _sti_s(_jit, i0, r0)
485static void _sti_s(jit_state_t*, jit_word_t, jit_int32_t);
486# define str_i(r0, r1) _str_i(_jit, r0, r1)
487static void _str_i(jit_state_t*, jit_int32_t, jit_int32_t);
488# define sti_i(i0, r0) _sti_i(_jit, i0, r0)
489static void _sti_i(jit_state_t*, jit_word_t, jit_int32_t);
490# if __X64 && !__X64_32
491# define str_l(r0, r1) _str_l(_jit, r0, r1)
492static void _str_l(jit_state_t*, jit_int32_t, jit_int32_t);
493# define sti_l(i0, r0) _sti_l(_jit, i0, r0)
494static void _sti_l(jit_state_t*, jit_word_t, jit_int32_t);
495# endif
496# define stxr_c(r0, r1, r2) _stxr_c(_jit, r0, r1, r2)
497static void _stxr_c(jit_state_t*, jit_int32_t, jit_int32_t, jit_int32_t);
498# define stxi_c(i0, r0, r1) _stxi_c(_jit, i0, r0, r1)
499static void _stxi_c(jit_state_t*, jit_word_t, jit_int32_t, jit_int32_t);
500# define stxr_s(r0, r1, r2) _stxr_s(_jit, r0, r1, r2)
501static void _stxr_s(jit_state_t*, jit_int32_t, jit_int32_t, jit_int32_t);
502# define stxi_s(i0, r0, r1) _stxi_s(_jit, i0, r0, r1)
503static void _stxi_s(jit_state_t*, jit_word_t, jit_int32_t, jit_int32_t);
504# define stxr_i(r0, r1, r2) _stxr_i(_jit, r0, r1, r2)
505static void _stxr_i(jit_state_t*, jit_int32_t, jit_int32_t, jit_int32_t);
506# define stxi_i(i0, r0, r1) _stxi_i(_jit, i0, r0, r1)
507static void _stxi_i(jit_state_t*, jit_word_t, jit_int32_t, jit_int32_t);
508# if __X64 && !__X64_32
509# define stxr_l(r0, r1, r2) _stxr_l(_jit, r0, r1, r2)
510static void _stxr_l(jit_state_t*, jit_int32_t, jit_int32_t, jit_int32_t);
511# define stxi_l(i0, r0, r1) _stxi_l(_jit, i0, r0, r1)
512static void _stxi_l(jit_state_t*, jit_word_t, jit_int32_t, jit_int32_t);
513# endif
514# define jcc(code, i0) _jcc(_jit, code, i0)
515# define jo(i0) jcc(X86_CC_O, i0)
516# define jno(i0) jcc(X86_CC_NO, i0)
517# define jnae(i0) jcc(X86_CC_NAE, i0)
518# define jb(i0) jcc(X86_CC_B, i0)
519# define jc(i0) jcc(X86_CC_C, i0)
520# define jae(i0) jcc(X86_CC_AE, i0)
521# define jnb(i0) jcc(X86_CC_NB, i0)
522# define jnc(i0) jcc(X86_CC_NC, i0)
523# define je(i0) jcc(X86_CC_E, i0)
524# define jz(i0) jcc(X86_CC_Z, i0)
525# define jne(i0) jcc(X86_CC_NE, i0)
526# define jnz(i0) jcc(X86_CC_NZ, i0)
527# define jbe(i0) jcc(X86_CC_BE, i0)
528# define jna(i0) jcc(X86_CC_NA, i0)
529# define ja(i0) jcc(X86_CC_A, i0)
530# define jnbe(i0) jcc(X86_CC_NBE, i0)
531# define js(i0) jcc(X86_CC_S, i0)
532# define jns(i0) jcc(X86_CC_NS, i0)
533# define jp(i0) jcc(X86_CC_P, i0)
534# define jpe(i0) jcc(X86_CC_PE, i0)
535# define jnp(i0) jcc(X86_CC_NP, i0)
536# define jpo(i0) jcc(X86_CC_PO, i0)
537# define jl(i0) jcc(X86_CC_L, i0)
538# define jnge(i0) jcc(X86_CC_NGE, i0)
539# define jge(i0) jcc(X86_CC_GE, i0)
540# define jnl(i0) jcc(X86_CC_NL, i0)
541# define jle(i0) jcc(X86_CC_LE, i0)
542# define jng(i0) jcc(X86_CC_NG, i0)
543# define jg(i0) jcc(X86_CC_G, i0)
544# define jnle(i0) jcc(X86_CC_NLE, i0)
545static void _jcc(jit_state_t*, jit_int32_t, jit_word_t);
546# define jccs(code, i0) _jccs(_jit, code, i0)
547# define jos(i0) jccs(X86_CC_O, i0)
548# define jnos(i0) jccs(X86_CC_NO, i0)
549# define jnaes(i0) jccs(X86_CC_NAE, i0)
550# define jbs(i0) jccs(X86_CC_B, i0)
551# define jcs(i0) jccs(X86_CC_C, i0)
552# define jaes(i0) jccs(X86_CC_AE, i0)
553# define jnbs(i0) jccs(X86_CC_NB, i0)
554# define jncs(i0) jccs(X86_CC_NC, i0)
555# define jes(i0) jccs(X86_CC_E, i0)
556# define jzs(i0) jccs(X86_CC_Z, i0)
557# define jnes(i0) jccs(X86_CC_NE, i0)
558# define jnzs(i0) jccs(X86_CC_NZ, i0)
559# define jbes(i0) jccs(X86_CC_BE, i0)
560# define jnas(i0) jccs(X86_CC_NA, i0)
561# define jas(i0) jccs(X86_CC_A, i0)
562# define jnbes(i0) jccs(X86_CC_NBE, i0)
563# define jss(i0) jccs(X86_CC_S, i0)
564# define jnss(i0) jccs(X86_CC_NS, i0)
565# define jps(i0) jccs(X86_CC_P, i0)
566# define jpes(i0) jccs(X86_CC_PE, i0)
567# define jnps(i0) jccs(X86_CC_NP, i0)
568# define jpos(i0) jccs(X86_CC_PO, i0)
569# define jls(i0) jccs(X86_CC_L, i0)
570# define jnges(i0) jccs(X86_CC_NGE, i0)
571# define jges(i0) jccs(X86_CC_GE, i0)
572# define jnls(i0) jccs(X86_CC_NL, i0)
573# define jles(i0) jccs(X86_CC_LE, i0)
574# define jngs(i0) jccs(X86_CC_NG, i0)
575# define jgs(i0) jccs(X86_CC_G, i0)
576# define jnles(i0) jccs(X86_CC_NLE, i0)
577static void _jccs(jit_state_t*, jit_int32_t, jit_word_t);
578# define jcr(code, i0, r0, r1) _jcr(_jit, code, i0, r0, r1)
579static void _jcr(jit_state_t*,jit_int32_t,jit_word_t,jit_int32_t,jit_int32_t);
580# define jci(code, i0, r0, i1) _jci(_jit, code, i0, r0, i1)
581static void _jci(jit_state_t*,jit_int32_t,jit_word_t,jit_int32_t,jit_word_t);
582# define jci0(code, i0, r0) _jci0(_jit, code, i0, r0)
583static void _jci0(jit_state_t*, jit_int32_t, jit_word_t, jit_int32_t);
584# define bltr(i0, r0, r1) _bltr(_jit, i0, r0, r1)
585static jit_word_t _bltr(jit_state_t*, jit_word_t, jit_int32_t, jit_int32_t);
586# define blti(i0, r0, i1) _blti(_jit, i0, r0, i1)
587static jit_word_t _blti(jit_state_t*, jit_word_t, jit_int32_t, jit_word_t);
588# define bltr_u(i0, r0, r1) _bltr_u(_jit, i0, r0, r1)
589static jit_word_t _bltr_u(jit_state_t*, jit_word_t, jit_int32_t, jit_int32_t);
590# define blti_u(i0, r0, i1) _blti_u(_jit, i0, r0, i1)
591static jit_word_t _blti_u(jit_state_t*, jit_word_t, jit_int32_t, jit_word_t);
592# define bler(i0, r0, r1) _bler(_jit, i0, r0, r1)
593static jit_word_t _bler(jit_state_t*, jit_word_t, jit_int32_t, jit_int32_t);
594# define blei(i0, r0, i1) _blei(_jit, i0, r0, i1)
595static jit_word_t _blei(jit_state_t*, jit_word_t, jit_int32_t, jit_word_t);
596# define bler_u(i0, r0, r1) _bler_u(_jit, i0, r0, r1)
597static jit_word_t _bler_u(jit_state_t*, jit_word_t, jit_int32_t, jit_int32_t);
598# define blei_u(i0, r0, i1) _blei_u(_jit, i0, r0, i1)
599static jit_word_t _blei_u(jit_state_t*, jit_word_t, jit_int32_t, jit_word_t);
600# define beqr(i0, r0, r1) _beqr(_jit, i0, r0, r1)
601static jit_word_t _beqr(jit_state_t*, jit_word_t, jit_int32_t, jit_int32_t);
602# define beqi(i0, r0, i1) _beqi(_jit, i0, r0, i1)
603static jit_word_t _beqi(jit_state_t*, jit_word_t, jit_int32_t, jit_word_t);
604# define bger(i0, r0, r1) _bger(_jit, i0, r0, r1)
605static jit_word_t _bger(jit_state_t*, jit_word_t, jit_int32_t, jit_int32_t);
606# define bgei(i0, r0, i1) _bgei(_jit, i0, r0, i1)
607static jit_word_t _bgei(jit_state_t*, jit_word_t, jit_int32_t, jit_word_t);
608# define bger_u(i0, r0, r1) _bger_u(_jit, i0, r0, r1)
609static jit_word_t _bger_u(jit_state_t*, jit_word_t, jit_int32_t, jit_int32_t);
610# define bgei_u(i0, r0, i1) _bgei_u(_jit, i0, r0, i1)
611static jit_word_t _bgei_u(jit_state_t*, jit_word_t, jit_int32_t, jit_word_t);
612# define bgtr(i0, r0, r1) _bgtr(_jit, i0, r0, r1)
613static jit_word_t _bgtr(jit_state_t*, jit_word_t, jit_int32_t, jit_int32_t);
614# define bgti(i0, r0, i1) _bgti(_jit, i0, r0, i1)
615static jit_word_t _bgti(jit_state_t*, jit_word_t, jit_int32_t, jit_word_t);
616# define bgtr_u(i0, r0, r1) _bgtr_u(_jit, i0, r0, r1)
617static jit_word_t _bgtr_u(jit_state_t*, jit_word_t, jit_int32_t, jit_int32_t);
618# define bgti_u(i0, r0, i1) _bgti_u(_jit, i0, r0, i1)
619static jit_word_t _bgti_u(jit_state_t*, jit_word_t, jit_int32_t, jit_word_t);
620# define bner(i0, r0, r1) _bner(_jit, i0, r0, r1)
621static jit_word_t _bner(jit_state_t*, jit_word_t, jit_int32_t, jit_int32_t);
622# define bnei(i0, r0, i1) _bnei(_jit, i0, r0, i1)
623static jit_word_t _bnei(jit_state_t*, jit_word_t, jit_int32_t, jit_word_t);
624# define bmsr(i0, r0, r1) _bmsr(_jit, i0, r0, r1)
625static jit_word_t _bmsr(jit_state_t*,jit_word_t,jit_int32_t,jit_int32_t);
626# define bmsi(i0, r0, i1) _bmsi(_jit, i0, r0, i1)
627static jit_word_t _bmsi(jit_state_t*,jit_word_t,jit_int32_t,jit_word_t);
628# define bmcr(i0, r0, r1) _bmcr(_jit, i0, r0, r1)
629static jit_word_t _bmcr(jit_state_t*,jit_word_t,jit_int32_t,jit_int32_t);
630# define bmci(i0, r0, i1) _bmci(_jit, i0, r0, i1)
631static jit_word_t _bmci(jit_state_t*,jit_word_t,jit_int32_t,jit_word_t);
632# define boaddr(i0, r0, r1) _boaddr(_jit, i0, r0, r1)
633static jit_word_t _boaddr(jit_state_t*,jit_word_t,jit_int32_t,jit_int32_t);
634# define boaddi(i0, r0, i1) _boaddi(_jit, i0, r0, i1)
635static jit_word_t _boaddi(jit_state_t*,jit_word_t,jit_int32_t,jit_word_t);
636# define boaddr_u(i0, r0, r1) _boaddr_u(_jit, i0, r0, r1)
637static jit_word_t _boaddr_u(jit_state_t*,jit_word_t,jit_int32_t,jit_int32_t);
638# define boaddi_u(i0, r0, i1) _boaddi_u(_jit, i0, r0, i1)
639static jit_word_t _boaddi_u(jit_state_t*,jit_word_t,jit_int32_t,jit_word_t);
640# define bxaddr(i0, r0, r1) _bxaddr(_jit, i0, r0, r1)
641static jit_word_t _bxaddr(jit_state_t*,jit_word_t,jit_int32_t,jit_int32_t);
642# define bxaddi(i0, r0, i1) _bxaddi(_jit, i0, r0, i1)
643static jit_word_t _bxaddi(jit_state_t*,jit_word_t,jit_int32_t,jit_word_t);
644# define bxaddr_u(i0, r0, r1) _bxaddr_u(_jit, i0, r0, r1)
645static jit_word_t _bxaddr_u(jit_state_t*,jit_word_t,jit_int32_t,jit_int32_t);
646# define bxaddi_u(i0, r0, i1) _bxaddi_u(_jit, i0, r0, i1)
647static jit_word_t _bxaddi_u(jit_state_t*,jit_word_t,jit_int32_t,jit_word_t);
648# define bosubr(i0, r0, r1) _bosubr(_jit, i0, r0, r1)
649static jit_word_t _bosubr(jit_state_t*,jit_word_t,jit_int32_t,jit_int32_t);
650# define bosubi(i0, r0, i1) _bosubi(_jit, i0, r0, i1)
651static jit_word_t _bosubi(jit_state_t*,jit_word_t,jit_int32_t,jit_word_t);
652# define bosubr_u(i0, r0, r1) _bosubr_u(_jit, i0, r0, r1)
653static jit_word_t _bosubr_u(jit_state_t*,jit_word_t,jit_int32_t,jit_int32_t);
654# define bosubi_u(i0, r0, i1) _bosubi_u(_jit, i0, r0, i1)
655static jit_word_t _bosubi_u(jit_state_t*,jit_word_t,jit_int32_t,jit_word_t);
656# define bxsubr(i0, r0, r1) _bxsubr(_jit, i0, r0, r1)
657static jit_word_t _bxsubr(jit_state_t*,jit_word_t,jit_int32_t,jit_int32_t);
658# define bxsubi(i0, r0, i1) _bxsubi(_jit, i0, r0, i1)
659static jit_word_t _bxsubi(jit_state_t*,jit_word_t,jit_int32_t,jit_word_t);
660# define bxsubr_u(i0, r0, r1) _bxsubr_u(_jit, i0, r0, r1)
661static jit_word_t _bxsubr_u(jit_state_t*,jit_word_t,jit_int32_t,jit_int32_t);
662# define bxsubi_u(i0, r0, i1) _bxsubi_u(_jit, i0, r0, i1)
663static jit_word_t _bxsubi_u(jit_state_t*,jit_word_t,jit_int32_t,jit_word_t);
664# define callr(r0) _callr(_jit, r0)
665static void _callr(jit_state_t*, jit_int32_t);
666# define calli(i0) _calli(_jit, i0)
667static jit_word_t _calli(jit_state_t*, jit_word_t);
519a9ea1
PC		 668# if __X64
669# define calli_p(i0) _calli_p(_jit, i0)
670static jit_word_t _calli_p(jit_state_t*, jit_word_t);
671# else
672# define calli_p(i0) calli(i0)
673# endif
4a71579b
PC		 674# define jmpr(r0) _jmpr(_jit, r0)
675static void _jmpr(jit_state_t*, jit_int32_t);
676# define jmpi(i0) _jmpi(_jit, i0)
677static jit_word_t _jmpi(jit_state_t*, jit_word_t);
519a9ea1
PC		 678# if __X64
679# define jmpi_p(i0) _jmpi_p(_jit, i0)
680static jit_word_t _jmpi_p(jit_state_t*, jit_word_t);
681# else
682# define jmpi_p(i0) jmpi(i0)
683# endif
4a71579b
PC		 684# define jmpsi(i0) _jmpsi(_jit, i0)
685static void _jmpsi(jit_state_t*, jit_uint8_t);
686# define prolog(node) _prolog(_jit, node)
687static void _prolog(jit_state_t*, jit_node_t*);
688# define epilog(node) _epilog(_jit, node)
689static void _epilog(jit_state_t*, jit_node_t*);
690# define vastart(r0) _vastart(_jit, r0)
691static void _vastart(jit_state_t*, jit_int32_t);
692# define vaarg(r0, r1) _vaarg(_jit, r0, r1)
693static void _vaarg(jit_state_t*, jit_int32_t, jit_int32_t);
694# define vaarg_d(r0, r1, i0) _vaarg_d(_jit, r0, r1, i0)
695static void _vaarg_d(jit_state_t*, jit_int32_t, jit_int32_t, jit_bool_t);
696# define patch_at(node, instr, label) _patch_at(_jit, node, instr, label)
697static void _patch_at(jit_state_t*, jit_node_t*, jit_word_t, jit_word_t);
698# if !defined(HAVE_FFSL)
699# if __X32
700# define ffsl(i) __builtin_ffs(i)
701# else
702# define ffsl(l) __builtin_ffsl(l)
703# endif
704# endif
1f22b268 705# define jit_cmov_p() jit_cpu.cmov
4a71579b
PC		 706#endif
707
708#if CODE
709static void
710_rex(jit_state_t *_jit, jit_int32_t l, jit_int32_t w,
711 jit_int32_t r, jit_int32_t x, jit_int32_t b)
712{
713#if __X64
714 jit_int32_t v = 0x40 | (w << 3);
715
716 if (r != _NOREG)
717 v |= (r & 8) >> 1;
718 if (x != _NOREG)
719 v |= (x & 8) >> 2;
720 if (b != _NOREG)
721 v |= (b & 8) >> 3;
722 if (l || v != 0x40)
723 ic(v);
724#endif
725}
726
727static void
728_rx(jit_state_t *_jit, jit_int32_t rd, jit_int32_t md,
729 jit_int32_t rb, jit_int32_t ri, jit_int32_t ms)
730{
731 if (ri == _NOREG) {
732 if (rb == _NOREG) {
733#if __X32
734 mrm(0x00, r7(rd), 0x05);
735#else
736 mrm(0x00, r7(rd), 0x04);
737 sib(_SCL1, 0x04, 0x05);
738#endif
739 ii(md);
740 }
741 else if (r7(rb) == _RSP_REGNO) {
742 if (md == 0) {
743 mrm(0x00, r7(rd), 0x04);
744 sib(ms, 0x04, 0x04);
745 }
746 else if ((jit_int8_t)md == md) {
747 mrm(0x01, r7(rd), 0x04);
748 sib(ms, 0x04, 0x04);
749 ic(md);
750 }
751 else {
752 mrm(0x02, r7(rd), 0x04);
753 sib(ms, 0x04, 0x04);
754 ii(md);
755 }
756 }
757 else {
758 if (md == 0 && r7(rb) != _RBP_REGNO)
759 mrm(0x00, r7(rd), r7(rb));
760 else if ((jit_int8_t)md == md) {
761 mrm(0x01, r7(rd), r7(rb));
762 ic(md);
763 }
764 else {
765 mrm(0x02, r7(rd), r7(rb));
766 ii(md);
767 }
768 }
769 }
770 else if (rb == _NOREG) {
771 mrm(0x00, r7(rd), 0x04);
772 sib(ms, r7(ri), 0x05);
773 ii(md);
774 }
775 else if (r8(ri) != _RSP_REGNO) {
776 if (md == 0 && r7(rb) != _RBP_REGNO) {
777 mrm(0x00, r7(rd), 0x04);
778 sib(ms, r7(ri), r7(rb));
779 }
780 else if ((jit_int8_t)md == md) {
781 mrm(0x01, r7(rd), 0x04);
782 sib(ms, r7(ri), r7(rb));
783 ic(md);
784 }
785 else {
786 mrm(0x02, r7(rd), 0x04);
787 sib(ms, r7(ri), r7(rb));
788 ic(md);
789 }
790 }
791 else {
792 fprintf(stderr, "illegal index register");
793 abort();
794 }
795}
796
797static void
798_nop(jit_state_t *_jit, jit_int32_t count)
799{
800 switch (count) {
801 case 0:
802 break;
803 case 1: /* NOP */
804 ic(0x90); break;
805 case 2: /* 66 NOP */
806 ic(0x66); ic(0x90);
807 break;
808 case 3: /* NOP DWORD ptr [EAX] */
809 ic(0x0f); ic(0x1f); ic(0x00);
810 break;
811 case 4: /* NOP DWORD ptr [EAX + 00H] */
812 ic(0x0f); ic(0x1f); ic(0x40); ic(0x00);
813 break;
814 case 5: /* NOP DWORD ptr [EAX + EAX*1 + 00H] */
815 ic(0x0f); ic(0x1f); ic(0x44); ic(0x00);
816 ic(0x00);
817 break;
818 case 6: /* 66 NOP DWORD ptr [EAX + EAX*1 + 00H] */
819 ic(0x66); ic(0x0f); ic(0x1f); ic(0x44);
820 ic(0x00); ic(0x00);
821 break;
822 case 7: /* NOP DWORD ptr [EAX + 00000000H] */
823 ic(0x0f); ic(0x1f); ic(0x80); ii(0x0000);
824 break;
825 case 8: /* NOP DWORD ptr [EAX + EAX*1 + 00000000H] */
826 ic(0x0f); ic(0x1f); ic(0x84); ic(0x00);
827 ii(0x0000);
828 break;
829 case 9: /* 66 NOP DWORD ptr [EAX + EAX*1 + 00000000H] */
830 ic(0x66); ic(0x0f); ic(0x1f); ic(0x84);
831 ic(0x00); ii(0x0000);
832 break;
833 default:
834 abort();
835 }
836}
837
838static void
839_lea(jit_state_t *_jit, jit_int32_t md, jit_int32_t rb,
840 jit_int32_t ri, jit_int32_t ms, jit_int32_t rd)
841{
842 rex(0, WIDE, rd, ri, rb);
843 ic(0x8d);
844 rx(rd, md, rb, ri, ms);
845}
846
847static void
848_pushr(jit_state_t *_jit, jit_int32_t r0)
849{
850 rex(0, WIDE, 0, 0, r0);
851 ic(0x50 | r7(r0));
852}
853
854static void
855_popr(jit_state_t *_jit, jit_int32_t r0)
856{
857 rex(0, WIDE, 0, 0, r0);
858 ic(0x58 | r7(r0));
859}
860
861static void
862_xchgr(jit_state_t *_jit, jit_int32_t r0, jit_int32_t r1)
863{
864 rex(0, WIDE, r1, _NOREG, r0);
865 ic(0x87);
866 mrm(0x03, r7(r1), r7(r0));
867}
868
869static void
870_testr(jit_state_t *_jit, jit_int32_t r0, jit_int32_t r1)
871{
872 rex(0, WIDE, r1, _NOREG, r0);
873 ic(0x85);
874 mrm(0x03, r7(r1), r7(r0));
875}
876
877static void
878_testi(jit_state_t *_jit, jit_int32_t r0, jit_word_t i0)
879{
880 rex(0, WIDE, _NOREG, _NOREG, r0);
881 if (r0 == _RAX_REGNO)
882 ic(0xa9);
883 else {
884 ic(0xf7);
885 mrm(0x03, 0x00, r7(r0));
886 }
887 ii(i0);
888}
889
890static void
891_cc(jit_state_t *_jit, jit_int32_t code, jit_int32_t r0)
892{
893 rex(0, 0, _NOREG, _NOREG, r0);
894 ic(0x0f);
895 ic(0x90 | code);
896 mrm(0x03, 0x00, r7(r0));
897}
898
899static void
900_alur(jit_state_t *_jit, jit_int32_t code, jit_int32_t r0, jit_int32_t r1)
901{
902 rex(0, WIDE, r1, _NOREG, r0);
903 ic(code | 0x01);
904 mrm(0x03, r7(r1), r7(r0));
905}
906
907static void
908_alui(jit_state_t *_jit, jit_int32_t code, jit_int32_t r0, jit_word_t i0)
909{
910 jit_int32_t reg;
911 if (can_sign_extend_int_p(i0)) {
912 rex(0, WIDE, _NOREG, _NOREG, r0);
913 if ((jit_int8_t)i0 == i0) {
914 ic(0x83);
915 ic(0xc0 | code | r7(r0));
916 ic(i0);
917 }
918 else {
919 if (r0 == _RAX_REGNO)
920 ic(code | 0x05);
921 else {
922 ic(0x81);
923 ic(0xc0 | code | r7(r0));
924 }
925 ii(i0);
926 }
927 }
928 else {
929 reg = jit_get_reg(jit_class_gpr);
930 movi(rn(reg), i0);
931 alur(code, r0, rn(reg));
932 jit_unget_reg(reg);
933 }
934}
935
936static void
937_save(jit_state_t *_jit, jit_int32_t r0)
938{
939 if (!_jitc->function->regoff[r0]) {
940 _jitc->function->regoff[r0] = jit_allocai(sizeof(jit_word_t));
941 _jitc->again = 1;
942 }
943 assert(!jit_regset_tstbit(&_jitc->regsav, r0));
944 jit_regset_setbit(&_jitc->regsav, r0);
945 stxi(_jitc->function->regoff[r0], _RBP_REGNO, r0);
946}
947
948static void
949_load(jit_state_t *_jit, jit_int32_t r0)
950{
951 assert(_jitc->function->regoff[r0]);
952 assert(jit_regset_tstbit(&_jitc->regsav, r0));
953 jit_regset_clrbit(&_jitc->regsav, r0);
954 ldxi(r0, _RBP_REGNO, _jitc->function->regoff[r0]);
955}
956
957static void
958_addr(jit_state_t *_jit, jit_int32_t r0, jit_int32_t r1, jit_int32_t r2)
959{
960 if (r0 == r1)
961 iaddr(r0, r2);
962 else if (r0 == r2)
963 iaddr(r0, r1);
964 else
965 lea(0, r1, r2, _SCL1, r0);
966}
967
968static void
969_addi(jit_state_t *_jit, jit_int32_t r0, jit_int32_t r1, jit_word_t i0)
970{
971 jit_int32_t reg;
972 if (i0 == 0)
973 movr(r0, r1);
974#if USE_INC_DEC
975 else if (i0 == 1)
976 incr(r0, r1);
977 else if (i0 == -1)
978 decr(r0, r1);
979#endif
980 else if (can_sign_extend_int_p(i0)) {
981 if (r0 == r1)
982 iaddi(r0, i0);
983 else
984 lea(i0, r1, _NOREG, _SCL1, r0);
985 }
986 else if (r0 != r1) {
987 movi(r0, i0);
988 iaddr(r0, r1);
989 }
990 else {
991 reg = jit_get_reg(jit_class_gpr);
992 movi(rn(reg), i0);
993 iaddr(r0, rn(reg));
994 jit_unget_reg(reg);
995 }
996}
997
998static void
999_addcr(jit_state_t *_jit, jit_int32_t r0, jit_int32_t r1, jit_int32_t r2)
1000{
1001 if (r0 == r2)
1002 iaddr(r0, r1);
1003 else {
1004 movr(r0, r1);
1005 iaddr(r0, r2);
1006 }
1007}
1008
1009static void
1010_addci(jit_state_t *_jit, jit_int32_t r0, jit_int32_t r1, jit_word_t i0)
1011{
1012 jit_int32_t reg;
1013 if (can_sign_extend_int_p(i0)) {
1014 movr(r0, r1);
1015 iaddi(r0, i0);
1016 }
1017 else if (r0 == r1) {
1018 reg = jit_get_reg(jit_class_gpr);
1019 movi(rn(reg), i0);
1020 iaddr(r0, rn(reg));
1021 jit_unget_reg(reg);
1022 }
1023 else {
1024 movi(r0, i0);
1025 iaddr(r0, r1);
1026 }
1027}
1028
1029static void
1030_addxr(jit_state_t *_jit, jit_int32_t r0, jit_int32_t r1, jit_int32_t r2)
1031{
1032 if (r0 == r2)
1033 iaddxr(r0, r1);
1034 else {
1035 movr(r0, r1);
1036 iaddxr(r0, r2);
1037 }
1038}
1039
1040static void
1041_addxi(jit_state_t *_jit, jit_int32_t r0, jit_int32_t r1, jit_word_t i0)
1042{
1043 jit_int32_t reg;
1044 if (can_sign_extend_int_p(i0)) {
1045 movr(r0, r1);
1046 iaddxi(r0, i0);
1047 }
1048 else if (r0 == r1) {
1049 reg = jit_get_reg(jit_class_gpr);
1050 movi(rn(reg), i0);
1051 iaddxr(r0, rn(reg));
1052 jit_unget_reg(reg);
1053 }
1054 else {
1055 movi(r0, i0);
1056 iaddxr(r0, r1);
1057 }
1058}
1059
1060static void
1061_subr(jit_state_t *_jit, jit_int32_t r0, jit_int32_t r1, jit_int32_t r2)
1062{
1063 if (r1 == r2)
1064 ixorr(r0, r0);
1065 else if (r0 == r2) {
1066 isubr(r0, r1);
1067 inegr(r0);
1068 }
1069 else {
1070 movr(r0, r1);
1071 isubr(r0, r2);
1072 }
1073}
1074
1075static void
1076_subi(jit_state_t *_jit, jit_int32_t r0, jit_int32_t r1, jit_word_t i0)
1077{
1078 jit_int32_t reg;
1079 if (i0 == 0)
1080 movr(r0, r1);
1081#if USE_INC_DEC
1082 else if (i0 == 1)
1083 decr(r0, r1);
1084 else if (i0 == -1)
1085 incr(r0, r1);
1086#endif
1087 else if (can_sign_extend_int_p(i0)) {
1088 if (r0 == r1)
1089 isubi(r0, i0);
1090 else
1091 lea(-i0, r1, _NOREG, _SCL1, r0);
1092 }
1093 else if (r0 != r1) {
1094 movi(r0, -i0);
1095 iaddr(r0, r1);
1096 }
1097 else {
1098 reg = jit_get_reg(jit_class_gpr);
1099 movi(rn(reg), i0);
1100 isubr(r0, rn(reg));
1101 jit_unget_reg(reg);
1102 }
1103}
1104
1105static void
1106_subcr(jit_state_t *_jit, jit_int32_t r0, jit_int32_t r1, jit_int32_t r2)
1107{
1108 jit_int32_t reg;
1109 if (r0 == r2 && r0 != r1) {
1110 reg = jit_get_reg(jit_class_gpr);
1111 movr(rn(reg), r0);
1112 movr(r0, r1);
1113 isubr(r0, rn(reg));
1114 jit_unget_reg(reg);
1115 }
1116 else {
1117 movr(r0, r1);
1118 isubr(r0, r2);
1119 }
1120}
1121
1122static void
1123_subci(jit_state_t *_jit, jit_int32_t r0, jit_int32_t r1, jit_word_t i0)
1124{
1125 jit_int32_t reg;
1126 movr(r0, r1);
1127 if (can_sign_extend_int_p(i0))
1128 isubi(r0, i0);
1129 else {
1130 reg = jit_get_reg(jit_class_gpr);
1131 movi(rn(reg), i0);
1132 isubr(r0, rn(reg));
1133 jit_unget_reg(reg);
1134 }
1135}
1136
1137static void
1138_subxr(jit_state_t *_jit, jit_int32_t r0, jit_int32_t r1, jit_int32_t r2)
1139{
1140 jit_int32_t reg;
1141 if (r0 == r2 && r0 != r1) {
1142 reg = jit_get_reg(jit_class_gpr);
1143 movr(rn(reg), r0);
1144 movr(r0, r1);
1145 isubxr(r0, rn(reg));
1146 jit_unget_reg(reg);
1147 }
1148 else {
1149 movr(r0, r1);
1150 isubxr(r0, r2);
1151 }
1152}
1153
1154static void
1155_subxi(jit_state_t *_jit, jit_int32_t r0, jit_int32_t r1, jit_word_t i0)
1156{
1157 jit_int32_t reg;
1158 movr(r0, r1);
1159 if (can_sign_extend_int_p(i0))
1160 isubxi(r0, i0);
1161 else {
1162 reg = jit_get_reg(jit_class_gpr);
1163 imovi(rn(reg), i0);
1164 isubxr(r0, rn(reg));
1165 jit_unget_reg(reg);
1166 }
1167}
1168
1169static void
1170_rsbi(jit_state_t *_jit, jit_int32_t r0, jit_int32_t r1, jit_word_t i0)
1171{
1172 subi(r0, r1, i0);
1173 negr(r0, r0);
1174}
1175
1176static void
1177_imulr(jit_state_t *_jit, jit_int32_t r0, jit_int32_t r1)
1178{
1179 rex(0, WIDE, r0, _NOREG, r1);
1180 ic(0x0f);
1181 ic(0xaf);
1182 mrm(0x03, r7(r0), r7(r1));
1183}
1184
1185static void
1186_imuli(jit_state_t *_jit, jit_int32_t r0, jit_int32_t r1, jit_word_t i0)
1187{
1188 jit_int32_t reg;
1189 if (can_sign_extend_int_p(i0)) {
1190 rex(0, WIDE, r0, _NOREG, r1);
1191 if ((jit_int8_t)i0 == i0) {
1192 ic(0x6b);
1193 mrm(0x03, r7(r0), r7(r1));
1194 ic(i0);
1195 }
1196 else {
1197 ic(0x69);
1198 mrm(0x03, r7(r0), r7(r1));
1199 ii(i0);
1200 }
1201 }
1202 else {
1203 reg = jit_get_reg(jit_class_gpr);
1204 movi(rn(reg), i0);
1205 imulr(r0, rn(reg));
1206 jit_unget_reg(reg);
1207 }
1208}
1209
1210static void
1211_mulr(jit_state_t *_jit, jit_int32_t r0, jit_int32_t r1, jit_int32_t r2)
1212{
1213 if (r0 == r1)
1214 imulr(r0, r2);
1215 else if (r0 == r2)
1216 imulr(r0, r1);
1217 else {
1218 movr(r0, r1);
1219 imulr(r0, r2);
1220 }
1221}
1222
1223static void
1224_muli(jit_state_t *_jit, jit_int32_t r0, jit_int32_t r1, jit_word_t i0)
1225{
1226 switch (i0) {
1227 case 0:
1228 ixorr(r0, r0);
1229 break;
1230 case 1:
1231 movr(r0, r1);
1232 break;
1233 case -1:
1234 negr(r0, r1);
1235 break;
1236 case 2:
1237 lea(0, _NOREG, r1, _SCL2, r0);
1238 break;
1239 case 4:
1240 lea(0, _NOREG, r1, _SCL4, r0);
1241 break;
1242 case 8:
1243 lea(0, _NOREG, r1, _SCL8, r0);
1244 break;
1245 default:
1246 if (i0 > 0 && !(i0 & (i0 - 1)))
1247 lshi(r0, r1, ffsl(i0) - 1);
1248 else if (can_sign_extend_int_p(i0))
1249 imuli(r0, r1, i0);
1250 else if (r0 != r1) {
1251 movi(r0, i0);
1252 imulr(r0, r1);
1253 }
1254 else
1255 imuli(r0, r0, i0);
1256 break;
1257 }
1258}
1259
1260#define savset(rn) \
1261 if (r0 != rn) { \
1262 sav |= 1 << rn; \
1263 if (r1 != rn && r2 != rn) \
1264 set |= 1 << rn; \
1265 }
1266#define isavset(rn) \
1267 if (r0 != rn) { \
1268 sav |= 1 << rn; \
1269 if (r1 != rn) \
1270 set |= 1 << rn; \
1271 }
1272#define qsavset(rn) \
1273 if (r0 != rn && r1 != rn) { \
1274 sav |= 1 << rn; \
1275 if (r2 != rn && r3 != rn) \
1276 set |= 1 << rn; \
1277 }
1278#define allocr(rn, rv) \
1279 if (set & (1 << rn)) \
1280 (void)jit_get_reg(rv|jit_class_gpr|jit_class_named); \
1281 if (sav & (1 << rn)) { \
1282 if ( jit_regset_tstbit(&_jitc->regsav, rv) || \
1283 !jit_regset_tstbit(&_jitc->reglive, rv)) \
1284 sav &= ~(1 << rn); \
1285 else \
1286 save(rv); \
1287 }
1288#define clear(rn, rv) \
1289 if (set & (1 << rn)) \
1290 jit_unget_reg(rv); \
1291 if (sav & (1 << rn)) \
1292 load(rv);
1293static void
1294_iqmulr(jit_state_t *_jit, jit_int32_t r0, jit_int32_t r1,
1295 jit_int32_t r2, jit_int32_t r3, jit_bool_t sign)
1296{
1297 jit_int32_t mul;
1298 jit_int32_t sav;
1299 jit_int32_t set;
1300
1301 sav = set = 0;
1302 qsavset(_RDX_REGNO);
1303 qsavset(_RAX_REGNO);
1304 allocr(_RDX_REGNO, _RDX);
1305 allocr(_RAX_REGNO, _RAX);
1306
1307 if (r3 == _RAX_REGNO)
1308 mul = r2;
1309 else {
1310 mul = r3;
1311 movr(_RAX_REGNO, r2);
1312 }
1313 if (sign)
1314 umulr(mul);
1315 else
1316 umulr_u(mul);
1317
1318 if (r0 == _RDX_REGNO && r1 == _RAX_REGNO)
1319 xchgr(_RAX_REGNO, _RDX_REGNO);
1320 else {
1321 if (r0 != _RDX_REGNO)
1322 movr(r0, _RAX_REGNO);
1323 movr(r1, _RDX_REGNO);
1324 if (r0 == _RDX_REGNO)
1325 movr(r0, _RAX_REGNO);
1326 }
1327
1328 clear(_RDX_REGNO, _RDX);
1329 clear(_RAX_REGNO, _RAX);
1330}
1331
1332static void
1333_iqmuli(jit_state_t *_jit, jit_int32_t r0, jit_int32_t r1,
1334 jit_int32_t r2, jit_word_t i0, jit_bool_t sign)
1335{
1336 jit_int32_t reg;
1337
1338 if (i0 == 0) {
1339 ixorr(r0, r0);
1340 ixorr(r1, r1);
1341 }
1342 else {
1343 reg = jit_get_reg(jit_class_gpr);
1344 movi(rn(reg), i0);
1345 if (sign)
1346 qmulr(r0, r1, r2, rn(reg));
1347 else
1348 qmulr_u(r0, r1, r2, rn(reg));
1349 jit_unget_reg(reg);
1350 }
1351}
1352
1353static void
1354_sign_extend_rdx_rax(jit_state_t *_jit)
1355{
1356 rex(0, WIDE, 0, 0, 0);
1357 ic(0x99);
1358}
1359
1360static void
1361_divremr(jit_state_t *_jit, jit_int32_t r0, jit_int32_t r1, jit_int32_t r2,
1362 jit_bool_t sign, jit_bool_t divide)
1363{
1364 jit_int32_t div;
1365 jit_int32_t reg;
1366 jit_int32_t set;
1367 jit_int32_t sav;
1368 jit_int32_t use;
1369
1370 sav = set = use = 0;
1371 savset(_RDX_REGNO);
1372 savset(_RAX_REGNO);
1373 allocr(_RDX_REGNO, _RDX);
1374 allocr(_RAX_REGNO, _RAX);
1375
1376 if (r2 == _RAX_REGNO) {
1377 if (r0 == _RAX_REGNO || r0 == _RDX_REGNO) {
1378 if ((reg = jit_get_reg(jit_class_gpr|jit_class_chk)) == JIT_NOREG)
1379 reg = jit_get_reg((r1 == _RCX_REGNO ? _RBX : _RCX) |
1380 jit_class_gpr|jit_class_named);
1381 use = 1;
1382 div = rn(reg);
1383 movr(div, _RAX_REGNO);
1384 if (r1 != _RAX_REGNO)
1385 movr(_RAX_REGNO, r1);
1386 }
1387 else {
1388 if (r0 == r1)
1389 xchgr(r0, _RAX_REGNO);
1390 else {
1391 if (r0 != _RAX_REGNO)
1392 movr(r0, _RAX_REGNO);
1393 if (r1 != _RAX_REGNO)
1394 movr(_RAX_REGNO, r1);
1395 }
1396 div = r0;
1397 }
1398 }
1399 else if (r2 == _RDX_REGNO) {
1400 if (r0 == _RAX_REGNO || r0 == _RDX_REGNO) {
1401 if ((reg = jit_get_reg(jit_class_gpr|jit_class_chk)) == JIT_NOREG)
1402 reg = jit_get_reg((r1 == _RCX_REGNO ? _RBX : _RCX) |
1403 jit_class_gpr|jit_class_named);
1404 use = 1;
1405 div = rn(reg);
1406 movr(div, _RDX_REGNO);
1407 if (r1 != _RAX_REGNO)
1408 movr(_RAX_REGNO, r1);
1409 }
1410 else {
1411 if (r1 != _RAX_REGNO)
1412 movr(_RAX_REGNO, r1);
1413 movr(r0, _RDX_REGNO);
1414 div = r0;
1415 }
1416 }
1417 else {
1418 if (r1 != _RAX_REGNO)
1419 movr(_RAX_REGNO, r1);
1420 div = r2;
1421 }
1422
1423 if (sign) {
1424 sign_extend_rdx_rax();
1425 idivr(div);
1426 }
1427 else {
1428 ixorr(_RDX_REGNO, _RDX_REGNO);
1429 idivr_u(div);
1430 }
1431
1432 if (use)
1433 jit_unget_reg(reg);
1434
1435 if (divide)
1436 movr(r0, _RAX_REGNO);
1437 else
1438 movr(r0, _RDX_REGNO);
1439
1440 clear(_RDX_REGNO, _RDX);
1441 clear(_RAX_REGNO, _RAX);
1442}
1443
1444static void
1445_divremi(jit_state_t *_jit, jit_int32_t r0, jit_int32_t r1, jit_word_t i0,
1446 jit_bool_t sign, jit_bool_t divide)
1447{
1448 jit_int32_t reg;
1449 jit_int32_t div;
1450 jit_int32_t sav;
1451 jit_int32_t set;
1452 jit_int32_t use;
1453
1454 if (divide) {
1455 switch (i0) {
1456 case 1:
1457 movr(r0, r1);
1458 return;
1459 case -1:
1460 if (sign) {
1461 negr(r0, r1);
1462 return;
1463 }
1464 break;
1465 default:
1466 if (i0 > 0 && !(i0 & (i0 - 1))) {
1467 movr(r0, r1);
1468 if (sign)
1469 rshi(r0, r0, ffsl(i0) - 1);
1470 else
1471 rshi_u(r0, r0, ffsl(i0) - 1);
1472 return;
1473 }
1474 break;
1475 }
1476 }
1477 else if (i0 == 1 || (sign && i0 == -1)) {
1478 ixorr(r0, r0);
1479 return;
1480 }
1481 else if (!sign && i0 > 0 && !(i0 & (i0 - 1))) {
1482 if (can_sign_extend_int_p(i0)) {
1483 movr(r0, r1);
1484 iandi(r0, i0 - 1);
1485 }
1486 else if (r0 != r1) {
1487 movi(r0, i0 - 1);
1488 iandr(r0, r1);
1489 }
1490 else {
1491 reg = jit_get_reg(jit_class_gpr);
1492 movi(rn(reg), i0 - 1);
1493 iandr(r0, rn(reg));
1494 jit_unget_reg(reg);
1495 }
1496 return;
1497 }
1498
1499 sav = set = use = 0;
1500 isavset(_RDX_REGNO);
1501 isavset(_RAX_REGNO);
1502 allocr(_RDX_REGNO, _RDX);
1503 allocr(_RAX_REGNO, _RAX);
1504
1505 if (r0 == _RAX_REGNO || r0 == _RDX_REGNO || r0 == r1) {
1506 if ((reg = jit_get_reg(jit_class_gpr|jit_class_chk)) == JIT_NOREG)
1507 reg = jit_get_reg((r1 == _RCX_REGNO ? _RBX : _RCX) |
1508 jit_class_gpr|jit_class_named);
1509 use = 1;
1510 div = rn(reg);
1511 }
1512 else
1513 div = r0;
1514
1515 movi(div, i0);
1516 movr(_RAX_REGNO, r1);
1517
1518 if (sign) {
1519 sign_extend_rdx_rax();
1520 idivr(div);
1521 }
1522 else {
1523 ixorr(_RDX_REGNO, _RDX_REGNO);
1524 idivr_u(div);
1525 }
1526
1527 if (use)
1528 jit_unget_reg(reg);
1529
1530 if (divide)
1531 movr(r0, _RAX_REGNO);
1532 else
1533 movr(r0, _RDX_REGNO);
1534
1535 clear(_RDX_REGNO, _RDX);
1536 clear(_RAX_REGNO, _RAX);
1537}
1538
1539static void
1540_iqdivr(jit_state_t *_jit, jit_int32_t r0, jit_int32_t r1,
1541 jit_int32_t r2, jit_int32_t r3, jit_bool_t sign)
1542{
1543 jit_int32_t div;
1544 jit_int32_t reg;
1545 jit_int32_t sav;
1546 jit_int32_t set;
1547 jit_int32_t use;
1548
1549 sav = set = use = 0;
1550 qsavset(_RDX_REGNO);
1551 qsavset(_RAX_REGNO);
1552 allocr(_RDX_REGNO, _RDX);
1553 allocr(_RAX_REGNO, _RAX);
1554 if (r3 == _RAX_REGNO) {
1555 if (r0 == _RAX_REGNO || r0 == _RDX_REGNO) {
1556 if ((reg = jit_get_reg(jit_class_gpr|jit_class_chk)) == JIT_NOREG)
1557 reg = jit_get_reg((r1 == _RCX_REGNO ? _RBX : _RCX) |
1558 jit_class_gpr|jit_class_named);
1559 use = 1;
1560 div = rn(reg);
1561 movr(div, _RAX_REGNO);
1562 if (r2 != _RAX_REGNO)
1563 movr(_RAX_REGNO, r2);
1564 }
1565 else {
1566 if (r0 == r2)
1567 xchgr(r0, _RAX_REGNO);
1568 else {
1569 if (r0 != _RAX_REGNO)
1570 movr(r0, _RAX_REGNO);
1571 if (r2 != _RAX_REGNO)
1572 movr(_RAX_REGNO, r2);
1573 }
1574 div = r0;
1575 }
1576 }
1577 else if (r3 == _RDX_REGNO) {
1578 if (r0 == _RAX_REGNO || r0 == _RDX_REGNO) {
1579 if ((reg = jit_get_reg(jit_class_gpr|jit_class_chk)) == JIT_NOREG)
1580 reg = jit_get_reg((r1 == _RCX_REGNO ? _RBX : _RCX) |
1581 jit_class_gpr|jit_class_named);
1582 use = 1;
1583 div = rn(reg);
1584 movr(div, _RDX_REGNO);
1585 if (r2 != _RAX_REGNO)
1586 movr(_RAX_REGNO, r2);
1587 }
1588 else {
1589 if (r2 != _RAX_REGNO)
1590 movr(_RAX_REGNO, r2);
1591 movr(r0, _RDX_REGNO);
1592 div = r0;
1593 }
1594 }
1595 else {
1596 if (r2 != _RAX_REGNO)
1597 movr(_RAX_REGNO, r2);
1598 div = r3;
1599 }
1600 if (sign) {
1601 sign_extend_rdx_rax();
1602 idivr(div);
1603 }
1604 else {
1605 ixorr(_RDX_REGNO, _RDX_REGNO);
1606 idivr_u(div);
1607 }
1608 if (use)
1609 jit_unget_reg(reg);
1610
1611 if (r0 == _RDX_REGNO && r1 == _RAX_REGNO)
1612 xchgr(_RAX_REGNO, _RDX_REGNO);
1613 else {
1614 if (r0 != _RDX_REGNO)
1615 movr(r0, _RAX_REGNO);
1616 movr(r1, _RDX_REGNO);
1617 if (r0 == _RDX_REGNO)
1618 movr(r0, _RAX_REGNO);
1619 }
1620
1621 clear(_RDX_REGNO, _RDX);
1622 clear(_RAX_REGNO, _RAX);
1623}
1624
1625static void
1626_iqdivi(jit_state_t *_jit, jit_int32_t r0, jit_int32_t r1,
1627 jit_int32_t r2, jit_word_t i0, jit_bool_t sign)
1628{
1629 jit_int32_t reg;
1630
1631 reg = jit_get_reg(jit_class_gpr);
1632 movi(rn(reg), i0);
1633 if (sign)
1634 qdivr(r0, r1, r2, rn(reg));
1635 else
1636 qdivr_u(r0, r1, r2, rn(reg));
1637 jit_unget_reg(reg);
1638}
1639#undef clear
1640#undef allocr
1641#undef savset
1642
1643static void
1644_andr(jit_state_t *_jit, jit_int32_t r0, jit_int32_t r1, jit_int32_t r2)
1645{
1646 if (r1 == r2)
1647 movr(r0, r1);
1648 else if (r0 == r1)
1649 iandr(r0, r2);
1650 else if (r0 == r2)
1651 iandr(r0, r1);
1652 else {
1653 movr(r0, r1);
1654 iandr(r0, r2);
1655 }
1656}
1657
1658static void
1659_andi(jit_state_t *_jit, jit_int32_t r0, jit_int32_t r1, jit_word_t i0)
1660{
1661 jit_int32_t reg;
1662
1663 if (i0 == 0)
1664 ixorr(r0, r0);
1665 else if (i0 == -1)
1666 movr(r0, r1);
1667 else if (r0 == r1) {
1668 if (can_sign_extend_int_p(i0))
1669 iandi(r0, i0);
1670 else {
1671 reg = jit_get_reg(jit_class_gpr);
1672 movi(rn(reg), i0);
1673 iandr(r0, rn(reg));
1674 jit_unget_reg(reg);
1675 }
1676 }
1677 else {
1678 movi(r0, i0);
1679 iandr(r0, r1);
1680 }
1681}
1682
1683static void
1684_orr(jit_state_t *_jit, jit_int32_t r0, jit_int32_t r1, jit_int32_t r2)
1685{
1686 if (r1 == r2)
1687 movr(r0, r1);
1688 else if (r0 == r1)
1689 iorr(r0, r2);
1690 else if (r0 == r2)
1691 iorr(r0, r1);
1692 else {
1693 movr(r0, r1);
1694 iorr(r0, r2);
1695 }
1696}
1697
1698static void
1699_ori(jit_state_t *_jit, jit_int32_t r0, jit_int32_t r1, jit_word_t i0)
1700{
1701 jit_int32_t reg;
1702 if (i0 == 0)
1703 movr(r0, r1);
1704 else if (i0 == -1)
1705 movi(r0, -1);
1706 else if (can_sign_extend_int_p(i0)) {
1707 movr(r0, r1);
1708 iori(r0, i0);
1709 }
1710 else if (r0 != r1) {
1711 movi(r0, i0);
1712 iorr(r0, r1);
1713 }
1714 else {
1715 reg = jit_get_reg(jit_class_gpr);
1716 movi(rn(reg), i0);
1717 iorr(r0, rn(reg));
1718 jit_unget_reg(reg);
1719 }
1720}
1721
1722static void
1723_xorr(jit_state_t *_jit, jit_int32_t r0, jit_int32_t r1, jit_int32_t r2)
1724{
1725 if (r1 == r2)
1726 ixorr(r0, r0);
1727 else if (r0 == r1)
1728 ixorr(r0, r2);
1729 else if (r0 == r2)
1730 ixorr(r0, r1);
1731 else {
1732 movr(r0, r1);
1733 ixorr(r0, r2);
1734 }
1735}
1736
1737static void
1738_xori(jit_state_t *_jit, jit_int32_t r0, jit_int32_t r1, jit_word_t i0)
1739{
1740 jit_int32_t reg;
1741 if (i0 == 0)
1742 movr(r0, r1);
1743 else if (i0 == -1)
1744 comr(r0, r1);
1745 else if (can_sign_extend_int_p(i0)) {
1746 movr(r0, r1);
1747 ixori(r0, i0);
1748 }
1749 else if (r0 != r1) {
1750 movi(r0, i0);
1751 ixorr(r0, r1);
1752 }
1753 else {
1754 reg = jit_get_reg(jit_class_gpr);
1755 movi(rn(reg), i0);
1756 ixorr(r0, rn(reg));
1757 jit_unget_reg(reg);
1758 }
1759}
1760
1761static void
1762_irotshr(jit_state_t *_jit, jit_int32_t code, jit_int32_t r0)
1763{
1764 rex(0, WIDE, _RCX_REGNO, _NOREG, r0);
1765 ic(0xd3);
1766 mrm(0x03, code, r7(r0));
1767}
1768
1769static void
1770_rotshr(jit_state_t *_jit, jit_int32_t code,
1771 jit_int32_t r0, jit_int32_t r1, jit_int32_t r2)
1772{
1773 jit_int32_t reg;
1774 jit_int32_t use;
1775
1776 if (r0 == _RCX_REGNO) {
1777 reg = jit_get_reg(jit_class_gpr);
1778 movr(rn(reg), r1);
1779 if (r2 != _RCX_REGNO)
1780 movr(_RCX_REGNO, r2);
1781 irotshr(code, rn(reg));
1782 movr(_RCX_REGNO, rn(reg));
1783 jit_unget_reg(reg);
1784 }
1785 else if (r2 != _RCX_REGNO) {
1786 use = !jit_reg_free_p(_RCX);
1787 if (use) {
1788 reg = jit_get_reg(jit_class_gpr);
1789 movr(rn(reg), _RCX_REGNO);
1790 }
1791 else
1792 reg = 0;
1793 if (r1 == _RCX_REGNO) {
1794 if (r0 == r2)
1795 xchgr(r0, _RCX_REGNO);
1796 else {
1797 movr(r0, r1);
1798 movr(_RCX_REGNO, r2);
1799 }
1800 }
1801 else {
1802 movr(_RCX_REGNO, r2);
1803 movr(r0, r1);
1804 }
1805 irotshr(code, r0);
1806 if (use) {
1807 movr(_RCX_REGNO, rn(reg));
1808 jit_unget_reg(reg);
1809 }
1810 }
1811 else {
1812 movr(r0, r1);
1813 irotshr(code, r0);
1814 }
1815}
1816
1817static void
1818_irotshi(jit_state_t *_jit, jit_int32_t code, jit_int32_t r0, jit_word_t i0)
1819{
1820 rex(0, WIDE, _NOREG, _NOREG, r0);
1821 if (i0 == 1) {
1822 ic(0xd1);
1823 mrm(0x03, code, r7(r0));
1824 }
1825 else {
1826 ic(0xc1);
1827 mrm(0x03, code, r7(r0));
1828 ic(i0);
1829 }
1830}
1831
1832static void
1833_rotshi(jit_state_t *_jit, jit_int32_t code,
1834 jit_int32_t r0, jit_int32_t r1, jit_word_t i0)
1835{
1836 movr(r0, r1);
1837 if (i0)
1838 irotshi(code, r0, i0);
1839}
1840
1841static void
1842_lshi(jit_state_t *_jit, jit_int32_t r0, jit_int32_t r1, jit_word_t i0)
1843{
1844 if (i0 == 0)
1845 movr(r0, r1);
1846 else if (i0 <= 3)
1847 lea(0, _NOREG, r1, i0 == 1 ? _SCL2 : i0 == 2 ? _SCL4 : _SCL8, r0);
1848 else
1849 rotshi(X86_SHL, r0, r1, i0);
1850}
1851
1852static void
1853_unr(jit_state_t *_jit, jit_int32_t code, jit_int32_t r0)
1854{
1855 rex(0, WIDE, _NOREG, _NOREG, r0);
1856 ic(0xf7);
1857 mrm(0x03, code, r7(r0));
1858}
1859
1860static void
1861_negr(jit_state_t *_jit, jit_int32_t r0, jit_int32_t r1)
1862{
1863 if (r0 == r1)
1864 inegr(r0);
1865 else {
1866 ixorr(r0, r0);
1867 isubr(r0, r1);
1868 }
1869}
1870
1871static void
1872_comr(jit_state_t *_jit, jit_int32_t r0, jit_int32_t r1)
1873{
1874 movr(r0, r1);
1875 icomr(r0);
1876}
1877
1878#if USE_INC_DEC
1879static void
1880_incr(jit_state_t *_jit, jit_int32_t r0, jit_int32_t r1)
1881{
1882 movr(r0, r1);
1883# if __X64
1884 rex(0, WIDE, _NOREG, _NOREG, r0);
1885 ic(0xff);
1886 ic(0xc0 | r7(r0));
1887# else
1888 ic(0x40 | r7(r0));
1889# endif
1890}
1891
1892static void
1893_decr(jit_state_t *_jit, jit_int32_t r0, jit_int32_t r1)
1894{
1895 movr(r0, r1);
1896# if __X64
1897 rex(0, WIDE, _NOREG, _NOREG, r0);
1898 ic(0xff);
1899 ic(0xc8 | r7(r0));
1900# else
1901 ic(0x48 | r7(r0));
1902# endif
1903}
1904#endif
1905
1906static void
1907_cr(jit_state_t *_jit,
1908 jit_int32_t code, jit_int32_t r0, jit_int32_t r1, jit_int32_t r2)
1909{
1910 jit_int32_t reg;
1911 jit_bool_t same;
1912 if (reg8_p(r0)) {
1913 same = r0 == r1 || r0 == r2;
1914 if (!same)
1915 ixorr(r0, r0);
1916 icmpr(r1, r2);
1917 if (same)
1918 imovi(r0, 0);
1919 cc(code, r0);
1920 }
1921 else {
1922 reg = jit_get_reg(jit_class_gpr|jit_class_rg8);
1923 ixorr(rn(reg), rn(reg));
1924 icmpr(r1, r2);
1925 cc(code, rn(reg));
1926 movr(r0, rn(reg));
1927 jit_unget_reg(reg);
1928 }
1929}
1930
1931static void
1932_ci(jit_state_t *_jit,
1933 jit_int32_t code, jit_int32_t r0, jit_int32_t r1, jit_word_t i0)
1934{
1935 jit_int32_t reg;
1936 jit_bool_t same;
1937 if (reg8_p(r0)) {
1938 same = r0 == r1;
1939 if (!same)
1940 ixorr(r0, r0);
1941 icmpi(r1, i0);
1942 if (same)
1943 imovi(r0, 0);
1944 cc(code, r0);
1945 }
1946 else {
1947 reg = jit_get_reg(jit_class_gpr|jit_class_rg8);
1948 ixorr(rn(reg), rn(reg));
1949 icmpi(r1, i0);
1950 cc(code, rn(reg));
1951 movr(r0, rn(reg));
1952 jit_unget_reg(reg);
1953 }
1954}
1955
1956static void
1957_ci0(jit_state_t *_jit, jit_int32_t code, jit_int32_t r0, jit_int32_t r1)
1958{
1959 jit_int32_t reg;
1960 jit_bool_t same;
1961 if (reg8_p(r0)) {
1962 same = r0 == r1;
1963 if (!same)
1964 ixorr(r0, r0);
1965 testr(r1, r1);
1966 if (same)
1967 imovi(r0, 0);
1968 cc(code, r0);
1969 }
1970 else {
1971 reg = jit_get_reg(jit_class_gpr|jit_class_rg8);
1972 ixorr(rn(reg), rn(reg));
1973 testr(r1, r1);
1974 cc(code, rn(reg));
1975 movr(r0, rn(reg));
1976 jit_unget_reg(reg);
1977 }
1978}
1979
1980static void
1981_ltr(jit_state_t *_jit, jit_int32_t r0, jit_int32_t r1, jit_int32_t r2)
1982{
1983 if (r1 == r2)
1984 movi(r0, 0);
1985 else
1986 cr(X86_CC_L, r0, r1, r2);
1987}
1988
1989static void
1990_lti(jit_state_t *_jit, jit_int32_t r0, jit_int32_t r1, jit_word_t i0)
1991{
1992 if (i0)
1993 ci(X86_CC_L, r0, r1, i0);
1994 else
1995 ci0(X86_CC_S, r0, r1);
1996}
1997
1998static void
1999_ltr_u(jit_state_t *_jit, jit_int32_t r0, jit_int32_t r1, jit_int32_t r2)
2000{
2001 if (r1 == r2)
2002 movi(r0, 0);
2003 else
2004 cr(X86_CC_B, r0, r1, r2);
2005}
2006
2007static void
2008_ler(jit_state_t *_jit, jit_int32_t r0, jit_int32_t r1, jit_int32_t r2)
2009{
2010 if (r1 == r2)
2011 movi(r0, 1);
2012 else
2013 cr(X86_CC_LE, r0, r1, r2);
2014}
2015
2016static void
2017_ler_u(jit_state_t *_jit, jit_int32_t r0, jit_int32_t r1, jit_int32_t r2)
2018{
2019 if (r1 == r2)
2020 movi(r0, 1);
2021 else
2022 cr(X86_CC_BE, r0, r1, r2);
2023}
2024
2025static void
2026_lei_u(jit_state_t *_jit, jit_int32_t r0, jit_int32_t r1, jit_word_t i0)
2027{
2028 if (i0)
2029 ci(X86_CC_BE, r0, r1, i0);
2030 else
2031 ci0(X86_CC_E, r0, r1);
2032}
2033
2034static void
2035_eqr(jit_state_t *_jit, jit_int32_t r0, jit_int32_t r1, jit_int32_t r2)
2036{
2037 if (r1 == r2)
2038 movi(r0, 1);
2039 else
2040 cr(X86_CC_E, r0, r1, r2);
2041}
2042
2043static void
2044_eqi(jit_state_t *_jit, jit_int32_t r0, jit_int32_t r1, jit_word_t i0)
2045{
2046 if (i0)
2047 ci(X86_CC_E, r0, r1, i0);
2048 else
2049 ci0(X86_CC_E, r0, r1);
2050}
2051
2052static void
2053_ger(jit_state_t *_jit, jit_int32_t r0, jit_int32_t r1, jit_int32_t r2)
2054{
2055 if (r1 == r2)
2056 movi(r0, 1);
2057 else
2058 cr(X86_CC_GE, r0, r1, r2);
2059}
2060
2061static void
2062_gei(jit_state_t *_jit, jit_int32_t r0, jit_int32_t r1, jit_word_t i0)
2063{
2064 if (i0)
2065 ci(X86_CC_GE, r0, r1, i0);
2066 else
2067 ci0(X86_CC_NS, r0, r1);
2068}
2069
2070static void
2071_ger_u(jit_state_t *_jit, jit_int32_t r0, jit_int32_t r1, jit_int32_t r2)
2072{
2073 if (r1 == r2)
2074 movi(r0, 1);
2075 else
2076 cr(X86_CC_AE, r0, r1, r2);
2077}
2078
2079static void
2080_gei_u(jit_state_t *_jit, jit_int32_t r0, jit_int32_t r1, jit_word_t i0)
2081{
2082 if (i0)
2083 ci(X86_CC_AE, r0, r1, i0);
2084 else
2085 ci0(X86_CC_NB, r0, r1);
2086}
2087
2088static void
2089_gtr(jit_state_t *_jit, jit_int32_t r0, jit_int32_t r1, jit_int32_t r2)
2090{
2091 if (r1 == r2)
2092 movi(r0, 0);
2093 else
2094 cr(X86_CC_G, r0, r1, r2);
2095}
2096
2097static void
2098_gtr_u(jit_state_t *_jit, jit_int32_t r0, jit_int32_t r1, jit_int32_t r2)
2099{
2100 if (r1 == r2)
2101 movi(r0, 0);
2102 else
2103 cr(X86_CC_A, r0, r1, r2);
2104}
2105
2106static void
2107_gti_u(jit_state_t *_jit, jit_int32_t r0, jit_int32_t r1, jit_word_t i0)
2108{
2109 if (i0)
2110 ci(X86_CC_A, r0, r1, i0);
2111 else
2112 ci0(X86_CC_NE, r0, r1);
2113}
2114
2115static void
2116_ner(jit_state_t *_jit, jit_int32_t r0, jit_int32_t r1, jit_int32_t r2)
2117{
2118 if (r1 == r2)
2119 movi(r0, 0);
2120 else
2121 cr(X86_CC_NE, r0, r1, r2);
2122}
2123
2124static void
2125_nei(jit_state_t *_jit, jit_int32_t r0, jit_int32_t r1, jit_word_t i0)
2126{
2127 if (i0)
2128 ci(X86_CC_NE, r0, r1, i0);
2129 else
2130 ci0(X86_CC_NE, r0, r1);
2131}
2132
2133static void
2134_movr(jit_state_t *_jit, jit_int32_t r0, jit_int32_t r1)
2135{
2136 if (r0 != r1) {
2137 rex(0, 1, r1, _NOREG, r0);
2138 ic(0x89);
2139 ic(0xc0 | (r1 << 3) | r7(r0));
2140 }
2141}
2142
2143static void
2144_imovi(jit_state_t *_jit, jit_int32_t r0, jit_word_t i0)
2145{
2146#if __X64
2147# if !__X64_32
2148 if (fits_uint32_p(i0)) {
2149# endif
2150 rex(0, 0, _NOREG, _NOREG, r0);
2151 ic(0xb8 | r7(r0));
2152 ii(i0);
2153# if !__X64_32
2154 }
2155 else {
2156 rex(0, 1, _NOREG, _NOREG, r0);
2157 ic(0xb8 | r7(r0));
2158 il(i0);
2159 }
2160# endif
2161#else
2162 ic(0xb8 | r7(r0));
2163 ii(i0);
2164#endif
2165}
2166
2167static void
2168_movi(jit_state_t *_jit, jit_int32_t r0, jit_word_t i0)
2169{
2170 if (i0)
2171 imovi(r0, i0);
2172 else
2173 ixorr(r0, r0);
2174}
2175
2176static jit_word_t
2177_movi_p(jit_state_t *_jit, jit_int32_t r0, jit_word_t i0)
2178{
2179 rex(0, WIDE, _NOREG, _NOREG, r0);
2180 ic(0xb8 | r7(r0));
2181 il(i0);
2182 return (_jit->pc.w);
2183}
2184
2185static void
2186_movcr(jit_state_t *_jit, jit_int32_t r0, jit_int32_t r1)
2187{
2188 rex(0, WIDE, r0, _NOREG, r1);
2189 ic(0x0f);
2190 ic(0xbe);
2191 mrm(0x03, r7(r0), r7(r1));
2192}
2193
2194static void
2195_movcr_u(jit_state_t *_jit, jit_int32_t r0, jit_int32_t r1)
2196{
2197 rex(0, WIDE, r0, _NOREG, r1);
2198 ic(0x0f);
2199 ic(0xb6);
2200 mrm(0x03, r7(r0), r7(r1));
2201}
2202
2203static void
2204_movsr(jit_state_t *_jit, jit_int32_t r0, jit_int32_t r1)
2205{
2206 rex(0, WIDE, r0, _NOREG, r1);
2207 ic(0x0f);
2208 ic(0xbf);
2209 mrm(0x03, r7(r0), r7(r1));
2210}
2211
2212static void
2213_movsr_u(jit_state_t *_jit, jit_int32_t r0, jit_int32_t r1)
2214{
2215 rex(0, WIDE, r0, _NOREG, r1);
2216 ic(0x0f);
2217 ic(0xb7);
2218 mrm(0x03, r7(r0), r7(r1));
2219}
2220
1f22b268
PC		 2221static void
2222_movnr(jit_state_t *_jit, jit_int32_t r0, jit_int32_t r1, jit_int32_t r2)
2223{
2224 assert(jit_cmov_p());
2225
2226 testr(r2, r2);
2227
2228 rex(0, WIDE, r0, _NOREG, r1);
2229 ic(0x0f);
2230 ic(0x45);
2231 mrm(0x03, r7(r0), r7(r1));
2232}
2233
2234static void
2235_movzr(jit_state_t *_jit, jit_int32_t r0, jit_int32_t r1, jit_int32_t r2)
2236{
2237 assert(jit_cmov_p());
2238
2239 testr(r2, r2);
2240
2241 rex(0, WIDE, r0, _NOREG, r1);
2242 ic(0x0f);
2243 ic(0x44);
2244 mrm(0x03, r7(r0), r7(r1));
2245}
2246
4a71579b
PC		 2247#if __X64
2248static void
2249_movir(jit_state_t *_jit, jit_int32_t r0, jit_int32_t r1)
2250{
2251 rex(0, 1, r0, _NOREG, r1);
2252 ic(0x63);
2253 mrm(0x03, r7(r0), r7(r1));
2254}
2255
2256static void
2257_movir_u(jit_state_t *_jit, jit_int32_t r0, jit_int32_t r1)
2258{
2259 rex(0, 0, r1, _NOREG, r0);
2260 ic(0x89);
2261 ic(0xc0 | (r1 << 3) | r7(r0));
2262}
2263#endif
2264
2265static void
40a44dcb 2266_bswapr_us(jit_state_t *_jit, jit_int32_t r0, jit_int32_t r1)
4a71579b
PC		 2267{
2268 extr_us(r0, r1);
2269 ic(0x66);
2270 rex(0, 0, _NOREG, _NOREG, r0);
2271 ic(0xc1);
2272 mrm(0x03, X86_ROR, r7(r0));
2273 ic(8);
2274}
2275
2276static void
40a44dcb 2277_bswapr_ui(jit_state_t *_jit, jit_int32_t r0, jit_int32_t r1)
4a71579b
PC		 2278{
2279 movr(r0, r1);
2280 rex(0, 0, _NOREG, _NOREG, r0);
2281 ic(0x0f);
2282 ic(0xc8 | r7(r0));
2283}
2284
2285#if __X64 && !__X64_32
2286static void
40a44dcb 2287_bswapr_ul(jit_state_t *_jit, jit_int32_t r0, jit_int32_t r1)
4a71579b
PC		 2288{
2289 movr(r0, r1);
2290 rex(0, 1, _NOREG, _NOREG, r0);
2291 ic(0x0f);
2292 ic(0xc8 | r7(r0));
2293}
2294#endif
2295
2296static void
2297_extr_c(jit_state_t *_jit, jit_int32_t r0, jit_int32_t r1)
2298{
2299 jit_int32_t reg;
2300 if (reg8_p(r1))
2301 movcr(r0, r1);
2302 else {
2303 reg = jit_get_reg(jit_class_gpr|jit_class_rg8);
2304 movr(rn(reg), r1);
2305 movcr(r0, rn(reg));
2306 jit_unget_reg(reg);
2307 }
2308}
2309
2310static void
2311_extr_uc(jit_state_t *_jit, jit_int32_t r0, jit_int32_t r1)
2312{
2313 jit_int32_t reg;
2314 if (reg8_p(r1))
2315 movcr_u(r0, r1);
2316 else {
2317 reg = jit_get_reg(jit_class_gpr|jit_class_rg8);
2318 movr(rn(reg), r1);
2319 movcr_u(r0, rn(reg));
2320 jit_unget_reg(reg);
2321 }
2322}
2323
2324static void
2325_ldr_c(jit_state_t *_jit, jit_int32_t r0, jit_int32_t r1)
2326{
2327 rex(0, WIDE, r0, _NOREG, r1);
2328 ic(0x0f);
2329 ic(0xbe);
2330 rx(r0, 0, r1, _NOREG, _SCL1);
2331}
2332
2333static void
2334_ldi_c(jit_state_t *_jit, jit_int32_t r0, jit_word_t i0)
2335{
2336 jit_int32_t reg;
2337 if (can_sign_extend_int_p(i0)) {
2338 rex(0, WIDE, r0, _NOREG, _NOREG);
2339 ic(0x0f);
2340 ic(0xbe);
2341 rx(r0, i0, _NOREG, _NOREG, _SCL1);
2342 }
2343 else {
2344 reg = jit_get_reg(jit_class_gpr);
2345 movi(rn(reg), i0);
2346 ldr_c(r0, rn(reg));
2347 jit_unget_reg(reg);
2348 }
2349}
2350
2351static void
2352_ldr_uc(jit_state_t *_jit, jit_int32_t r0, jit_int32_t r1)
2353{
2354 rex(0, WIDE, r0, _NOREG, r1);
2355 ic(0x0f);
2356 ic(0xb6);
2357 rx(r0, 0, r1, _NOREG, _SCL1);
2358}
2359
2360static void
2361_ldi_uc(jit_state_t *_jit, jit_int32_t r0, jit_word_t i0)
2362{
2363 jit_int32_t reg;
2364 if (can_sign_extend_int_p(i0)) {
2365 rex(0, WIDE, r0, _NOREG, _NOREG);
2366 ic(0x0f);
2367 ic(0xb6);
2368 rx(r0, i0, _NOREG, _NOREG, _SCL1);
2369 }
2370 else {
2371 reg = jit_get_reg(jit_class_gpr);
2372 movi(rn(reg), i0);
2373 ldr_uc(r0, rn(reg));
2374 jit_unget_reg(reg);
2375 }
2376}
2377
2378static void
2379_ldr_s(jit_state_t *_jit, jit_int32_t r0, jit_int32_t r1)
2380{
2381 rex(0, WIDE, r0, _NOREG, r1);
2382 ic(0x0f);
2383 ic(0xbf);
2384 rx(r0, 0, r1, _NOREG, _SCL1);
2385}
2386
2387static void
2388_ldi_s(jit_state_t *_jit, jit_int32_t r0, jit_word_t i0)
2389{
2390 jit_int32_t reg;
2391 if (can_sign_extend_int_p(i0)) {
2392 rex(0, WIDE, r0, _NOREG, _NOREG);
2393 ic(0x0f);
2394 ic(0xbf);
2395 rx(r0, i0, _NOREG, _NOREG, _SCL1);
2396 }
2397 else {
2398 reg = jit_get_reg(jit_class_gpr);
2399 movi(rn(reg), i0);
2400 ldr_s(r0, rn(reg));
2401 jit_unget_reg(reg);
2402 }
2403}
2404
2405static void
2406_ldr_us(jit_state_t *_jit, jit_int32_t r0, jit_int32_t r1)
2407{
2408 rex(0, WIDE, r0, _NOREG, r1);
2409 ic(0x0f);
2410 ic(0xb7);
2411 rx(r0, 0, r1, _NOREG, _SCL1);
2412}
2413
2414static void
2415_ldi_us(jit_state_t *_jit, jit_int32_t r0, jit_word_t i0)
2416{
2417 jit_int32_t reg;
2418 if (can_sign_extend_int_p(i0)) {
2419 rex(0, WIDE, r0, _NOREG, _NOREG);
2420 ic(0x0f);
2421 ic(0xb7);
2422 rx(r0, i0, _NOREG, _NOREG, _SCL1);
2423 }
2424 else {
2425 reg = jit_get_reg(jit_class_gpr);
2426 movi(rn(reg), i0);
2427 ldr_us(r0, rn(reg));
2428 jit_unget_reg(reg);
2429 }
2430}
2431
2432#if __X32 || !__X64_32
2433static void
2434_ldr_i(jit_state_t *_jit, jit_int32_t r0, jit_int32_t r1)
2435{
2436#if __X64
2437 rex(0, WIDE, r0, _NOREG, r1);
2438 ic(0x63);
2439#else
2440 ic(0x8b);
2441#endif
2442 rx(r0, 0, r1, _NOREG, _SCL1);
2443}
2444
2445static void
2446_ldi_i(jit_state_t *_jit, jit_int32_t r0, jit_word_t i0)
2447{
2448 jit_int32_t reg;
2449 if (can_sign_extend_int_p(i0)) {
2450#if __X64
2451 rex(0, WIDE, r0, _NOREG, _NOREG);
2452 ic(0x63);
2453#else
2454 ic(0x8b);
2455#endif
2456 rx(r0, i0, _NOREG, _NOREG, _SCL1);
2457 }
2458 else {
2459 reg = jit_get_reg(jit_class_gpr);
2460 movi(rn(reg), i0);
2461 ldr_i(r0, rn(reg));
2462 jit_unget_reg(reg);
2463 }
2464}
2465#endif
2466
2467#if __X64
2468static void
2469_ldr_ui(jit_state_t *_jit, jit_int32_t r0, jit_int32_t r1)
2470{
2471 rex(0, 0, r0, _NOREG, r1);
2472 ic(0x63);
2473 rx(r0, 0, r1, _NOREG, _SCL1);
2474}
2475
2476static void
2477_ldi_ui(jit_state_t *_jit, jit_int32_t r0, jit_word_t i0)
2478{
2479 jit_int32_t reg;
2480 if (can_sign_extend_int_p(i0)) {
2481 rex(0, 0, r0, _NOREG, _NOREG);
2482 ic(0x63);
2483 rx(r0, i0, _NOREG, _NOREG, _SCL1);
2484 }
2485 else {
2486 reg = jit_get_reg(jit_class_gpr);
2487 movi(rn(reg), i0);
2488 ldr_ui(r0, rn(reg));
2489 jit_unget_reg(reg);
2490 }
2491}
2492
2493# if !__X64_32
2494static void
2495_ldr_l(jit_state_t *_jit, jit_int32_t r0, jit_int32_t r1)
2496{
2497 rex(0, 1, r0, _NOREG, r1);
2498 ic(0x8b);
2499 rx(r0, 0, r1, _NOREG, _SCL1);
2500}
2501
2502static void
2503_ldi_l(jit_state_t *_jit, jit_int32_t r0, jit_word_t i0)
2504{
2505 jit_int32_t reg;
2506 if (can_sign_extend_int_p(i0)) {
2507 rex(0, 1, r0, _NOREG, _NOREG);
2508 ic(0x8b);
2509 rx(r0, i0, _NOREG, _NOREG, _SCL1);
2510 }
2511 else {
2512 reg = jit_get_reg(jit_class_gpr);
2513 movi(rn(reg), i0);
2514 ldr_l(r0, rn(reg));
2515 jit_unget_reg(reg);
2516 }
2517}
2518# endif
2519#endif
2520
2521static void
2522_ldxr_c(jit_state_t *_jit, jit_int32_t r0, jit_int32_t r1, jit_int32_t r2)
2523{
2524#if __X64_32
2525 addr(r0, r1, r2);
2526 ldr_c(r0, r0);
2527#else
2528 rex(0, WIDE, r0, r1, r2);
2529 ic(0x0f);
2530 ic(0xbe);
2531 rx(r0, 0, r2, r1, _SCL1);
2532#endif
2533}
2534
2535static void
2536_ldxi_c(jit_state_t *_jit, jit_int32_t r0, jit_int32_t r1, jit_word_t i0)
2537{
2538 jit_int32_t reg;
2539 if (can_sign_extend_int_p(i0)) {
2540 rex(0, WIDE, r0, _NOREG, r1);
2541 ic(0x0f);
2542 ic(0xbe);
2543 rx(r0, i0, r1, _NOREG, _SCL1);
2544 }
2545 else {
2546 reg = jit_get_reg(jit_class_gpr);
2547 movi(rn(reg), i0);
2548 ldxr_c(r0, r1, rn(reg));
2549 jit_unget_reg(reg);
2550 }
2551}
2552
2553static void
2554_ldxr_uc(jit_state_t *_jit, jit_int32_t r0, jit_int32_t r1, jit_int32_t r2)
2555{
2556#if __X64_32
2557 addr(r0, r1, r2);
2558 ldr_uc(r0, r0);
2559#else
2560 rex(0, WIDE, r0, r1, r2);
2561 ic(0x0f);
2562 ic(0xb6);
2563 rx(r0, 0, r2, r1, _SCL1);
2564#endif
2565}
2566
2567static void
2568_ldxi_uc(jit_state_t *_jit, jit_int32_t r0, jit_int32_t r1, jit_word_t i0)
2569{
2570 jit_int32_t reg;
2571 if (can_sign_extend_int_p(i0)) {
2572 rex(0, WIDE, r0, _NOREG, r1);
2573 ic(0x0f);
2574 ic(0xb6);
2575 rx(r0, i0, r1, _NOREG, _SCL1);
2576 }
2577 else {
2578 reg = jit_get_reg(jit_class_gpr);
2579 movi(rn(reg), i0);
2580 ldxr_uc(r0, r1, rn(reg));
2581 jit_unget_reg(reg);
2582 }
2583}
2584
2585static void
2586_ldxr_s(jit_state_t *_jit, jit_int32_t r0, jit_int32_t r1, jit_int32_t r2)
2587{
2588#if __X64_32
2589 addr(r0, r1, r2);
2590 ldr_s(r0, r0);
2591#else
2592 rex(0, WIDE, r0, r1, r2);
2593 ic(0x0f);
2594 ic(0xbf);
2595 rx(r0, 0, r2, r1, _SCL1);
2596#endif
2597}
2598
2599static void
2600_ldxi_s(jit_state_t *_jit, jit_int32_t r0, jit_int32_t r1, jit_word_t i0)
2601{
2602 jit_int32_t reg;
2603 if (can_sign_extend_int_p(i0)) {
2604 rex(0, WIDE, r0, _NOREG, r1);
2605 ic(0x0f);
2606 ic(0xbf);
2607 rx(r0, i0, r1, _NOREG, _SCL1);
2608 }
2609 else {
2610 reg = jit_get_reg(jit_class_gpr);
2611 movi(rn(reg), i0);
2612 ldxr_s(r0, r1, rn(reg));
2613 jit_unget_reg(reg);
2614 }
2615}
2616
2617static void
2618_ldxr_us(jit_state_t *_jit, jit_int32_t r0, jit_int32_t r1, jit_int32_t r2)
2619{
2620#if __X64_32
2621 addr(r0, r1, r2);
2622 ldr_us(r0, r0);
2623#else
2624 rex(0, WIDE, r0, r1, r2);
2625 ic(0x0f);
2626 ic(0xb7);
2627 rx(r0, 0, r2, r1, _SCL1);
2628#endif
2629}
2630
2631static void
2632_ldxi_us(jit_state_t *_jit, jit_int32_t r0, jit_int32_t r1, jit_word_t i0)
2633{
2634 jit_int32_t reg;
2635 if (can_sign_extend_int_p(i0)) {
2636 rex(0, WIDE, r0, _NOREG, r1);
2637 ic(0x0f);
2638 ic(0xb7);
2639 rx(r0, i0, r1, _NOREG, _SCL1);
2640 }
2641 else {
2642 reg = jit_get_reg(jit_class_gpr);
2643 movi(rn(reg), i0);
2644 ldxr_us(r0, r1, rn(reg));
2645 jit_unget_reg(reg);
2646 }
2647}
2648
2649#if __X64 || !__X64_32
2650static void
2651_ldxr_i(jit_state_t *_jit, jit_int32_t r0, jit_int32_t r1, jit_int32_t r2)
2652{
2653#if __X64
2654 rex(0, WIDE, r0, r1, r2);
2655 ic(0x63);
2656#else
2657 ic(0x8b);
2658#endif
2659 rx(r0, 0, r2, r1, _SCL1);
2660}
2661
2662static void
2663_ldxi_i(jit_state_t *_jit, jit_int32_t r0, jit_int32_t r1, jit_word_t i0)
2664{
2665 jit_int32_t reg;
2666 if (can_sign_extend_int_p(i0)) {
2667#if __X64
2668 rex(0, WIDE, r0, _NOREG, r1);
2669 ic(0x63);
2670#else
2671 ic(0x8b);
2672#endif
2673 rx(r0, i0, r1, _NOREG, _SCL1);
2674 }
2675 else {
2676 reg = jit_get_reg(jit_class_gpr);
2677 movi(rn(reg), i0);
2678 ldxr_i(r0, r1, rn(reg));
2679 jit_unget_reg(reg);
2680 }
2681}
2682#endif
2683
2684#if __X64
2685static void
2686_ldxr_ui(jit_state_t *_jit, jit_int32_t r0, jit_int32_t r1, jit_int32_t r2)
2687{
2688#if __X64_32
2689 addr(r0, r1, r2);
2690 /* to avoid confusion with macro renames */
2691 _ldr_ui(_jit, r0, r0);
2692#else
2693 rex(0, 0, r0, r1, r2);
2694 ic(0x8b);
2695 rx(r0, 0, r2, r1, _SCL1);
2696#endif
2697}
2698
2699static void
2700_ldxi_ui(jit_state_t *_jit, jit_int32_t r0, jit_int32_t r1, jit_word_t i0)
2701{
2702 jit_int32_t reg;
2703 if (can_sign_extend_int_p(i0)) {
2704 rex(0, 0, r0, _NOREG, r1);
2705 ic(0x8b);
2706 rx(r0, i0, r1, _NOREG, _SCL1);
2707 }
2708 else {
2709 reg = jit_get_reg(jit_class_gpr);
2710 movi(rn(reg), i0);
2711 ldxr_ui(r0, r1, rn(reg));
2712 jit_unget_reg(reg);
2713 }
2714}
2715
2716# if !__X64_32
2717static void
2718_ldxr_l(jit_state_t *_jit, jit_int32_t r0, jit_int32_t r1, jit_int32_t r2)
2719{
2720 rex(0, 1, r0, r1, r2);
2721 ic(0x8b);
2722 rx(r0, 0, r2, r1, _SCL1);
2723}
2724
2725static void
2726_ldxi_l(jit_state_t *_jit, jit_int32_t r0, jit_int32_t r1, jit_word_t i0)
2727{
2728 jit_int32_t reg;
2729 if (can_sign_extend_int_p(i0)) {
2730 rex(0, 1, r0, _NOREG, r1);
2731 ic(0x8b);
2732 rx(r0, i0, r1, _NOREG, _SCL1);
2733 }
2734 else {
2735 reg = jit_get_reg(jit_class_gpr);
2736 movi(rn(reg), i0);
2737 ldxr_l(r0, r1, rn(reg));
2738 jit_unget_reg(reg);
2739 }
2740}
2741# endif
2742#endif
2743
2744static void
2745_str_c(jit_state_t *_jit, jit_int32_t r0, jit_int32_t r1)
2746{
2747 jit_int32_t reg;
2748 if (reg8_p(r1)) {
2749 rex(0, 0, r1, _NOREG, r0);
2750 ic(0x88);
2751 rx(r1, 0, r0, _NOREG, _SCL1);
2752 }
2753 else {
2754 reg = jit_get_reg(jit_class_gpr|jit_class_rg8);
2755 movr(rn(reg), r1);
2756 rex(0, 0, rn(reg), _NOREG, r0);
2757 ic(0x88);
2758 rx(rn(reg), 0, r0, _NOREG, _SCL1);
2759 jit_unget_reg(reg);
2760 }
2761}
2762
2763static void
2764_sti_c(jit_state_t *_jit, jit_word_t i0, jit_int32_t r0)
2765{
2766 jit_int32_t reg;
2767 if (can_sign_extend_int_p(i0)) {
2768 if (reg8_p(r0)) {
2769 rex(0, 0, r0, _NOREG, _NOREG);
2770 ic(0x88);
2771 rx(r0, i0, _NOREG, _NOREG, _SCL1);
2772 }
2773 else {
2774 reg = jit_get_reg(jit_class_gpr|jit_class_rg8);
2775 movr(rn(reg), r0);
2776 rex(0, 0, rn(reg), _NOREG, _NOREG);
2777 ic(0x88);
2778 rx(rn(reg), i0, _NOREG, _NOREG, _SCL1);
2779 jit_unget_reg(reg);
2780 }
2781 }
2782 else {
2783 reg = jit_get_reg(jit_class_gpr);
2784 movi(rn(reg), i0);
2785 str_c(rn(reg), r0);
2786 jit_unget_reg(reg);
2787 }
2788}
2789
2790static void
2791_str_s(jit_state_t *_jit, jit_int32_t r0, jit_int32_t r1)
2792{
2793 ic(0x66);
2794 rex(0, 0, r1, _NOREG, r0);
2795 ic(0x89);
2796 rx(r1, 0, r0, _NOREG, _SCL1);
2797}
2798
2799static void
2800_sti_s(jit_state_t *_jit, jit_word_t i0, jit_int32_t r0)
2801{
2802 jit_int32_t reg;
2803 if (can_sign_extend_int_p(i0)) {
2804 ic(0x66);
2805 rex(0, 0, r0, _NOREG, _NOREG);
2806 ic(0x89);
2807 rx(r0, i0, _NOREG, _NOREG, _SCL1);
2808 }
2809 else {
2810 reg = jit_get_reg(jit_class_gpr);
2811 movi(rn(reg), i0);
2812 str_s(rn(reg), r0);
2813 jit_unget_reg(reg);
2814 }
2815}
2816
2817static void
2818_str_i(jit_state_t *_jit, jit_int32_t r0, jit_int32_t r1)
2819{
2820 rex(0, 0, r1, _NOREG, r0);
2821 ic(0x89);
2822 rx(r1, 0, r0, _NOREG, _SCL1);
2823}
2824
2825static void
2826_sti_i(jit_state_t *_jit, jit_word_t i0, jit_int32_t r0)
2827{
2828 jit_int32_t reg;
2829 if (can_sign_extend_int_p(i0)) {
2830 rex(0, 0, r0, _NOREG, _NOREG);
2831 ic(0x89);
2832 rx(r0, i0, _NOREG, _NOREG, _SCL1);
2833 }
2834 else {
2835 reg = jit_get_reg(jit_class_gpr);
2836 movi(rn(reg), i0);
2837 str_i(rn(reg), r0);
2838 jit_unget_reg(reg);
2839 }
2840}
2841
2842#if __X64 && !__X64_32
2843static void
2844_str_l(jit_state_t *_jit, jit_int32_t r0, jit_int32_t r1)
2845{
2846 rex(0, 1, r1, _NOREG, r0);
2847 ic(0x89);
2848 rx(r1, 0, r0, _NOREG, _SCL1);
2849}
2850
2851static void
2852_sti_l(jit_state_t *_jit, jit_word_t i0, jit_int32_t r0)
2853{
2854 jit_int32_t reg;
2855 if (can_sign_extend_int_p(i0)) {
2856 rex(0, 1, r0, _NOREG, _NOREG);
2857 ic(0x89);
2858 rx(r0, i0, _NOREG, _NOREG, _SCL1);
2859 }
2860 else {
2861 reg = jit_get_reg(jit_class_gpr);
2862 movi(rn(reg), i0);
2863 str_l(rn(reg), r0);
2864 jit_unget_reg(reg);
2865 }
2866}
2867#endif
2868
2869static void
2870_stxr_c(jit_state_t *_jit, jit_int32_t r0, jit_int32_t r1, jit_int32_t r2)
2871{
2872 jit_int32_t reg;
2873#if __X64_32
2874 reg = jit_get_reg(jit_class_gpr);
2875 addr(rn(reg), r0, r1);
2876 str_c(rn(reg), r2);
2877 jit_unget_reg(reg);
2878#else
2879 if (reg8_p(r2)) {
2880 rex(0, 0, r2, r1, r0);
2881 ic(0x88);
2882 rx(r2, 0, r0, r1, _SCL1);
2883 }
2884 else {
2885 reg = jit_get_reg(jit_class_gpr|jit_class_rg8);
2886 movr(rn(reg), r2);
2887 rex(0, 0, rn(reg), r1, r0);
2888 ic(0x88);
2889 rx(rn(reg), 0, r0, r1, _SCL1);
2890 jit_unget_reg(reg);
2891 }
2892#endif
2893}
2894
2895static void
2896_stxi_c(jit_state_t *_jit, jit_word_t i0, jit_int32_t r0, jit_int32_t r1)
2897{
2898 jit_int32_t reg;
2899 if (can_sign_extend_int_p(i0)) {
2900 if (reg8_p(r1)) {
2901 rex(0, 0, r1, _NOREG, r0);
2902 ic(0x88);
2903 rx(r1, i0, r0, _NOREG, _SCL1);
2904 }
2905 else {
2906 reg = jit_get_reg(jit_class_gpr|jit_class_rg8);
2907 movr(rn(reg), r1);
2908 rex(0, 0, rn(reg), _NOREG, r0);
2909 ic(0x88);
2910 rx(rn(reg), i0, r0, _NOREG, _SCL1);
2911 jit_unget_reg(reg);
2912 }
2913 }
2914 else {
2915 reg = jit_get_reg(jit_class_gpr);
2916 movi(rn(reg), i0);
2917 stxr_c(rn(reg), r0, r1);
2918 jit_unget_reg(reg);
2919 }
2920}
2921
2922static void
2923_stxr_s(jit_state_t *_jit, jit_int32_t r0, jit_int32_t r1, jit_int32_t r2)
2924{
2925#if __X64_32
2926 jit_int32_t reg;
2927 reg = jit_get_reg(jit_class_gpr);
2928 addr(rn(reg), r0, r1);
2929 str_s(rn(reg), r2);
2930 jit_unget_reg(reg);
2931#else
2932 ic(0x66);
2933 rex(0, 0, r2, r1, r0);
2934 ic(0x89);
2935 rx(r2, 0, r0, r1, _SCL1);
2936#endif
2937}
2938
2939static void
2940_stxi_s(jit_state_t *_jit, jit_word_t i0, jit_int32_t r0, jit_int32_t r1)
2941{
2942 jit_int32_t reg;
2943 if (can_sign_extend_int_p(i0)) {
2944 ic(0x66);
2945 rex(0, 0, r1, _NOREG, r0);
2946 ic(0x89);
2947 rx(r1, i0, r0, _NOREG, _SCL1);
2948 }
2949 else {
2950 reg = jit_get_reg(jit_class_gpr);
2951 movi(rn(reg), i0);
2952 stxr_s(rn(reg), r0, r1);
2953 jit_unget_reg(reg);
2954 }
2955}
2956
2957static void
2958_stxr_i(jit_state_t *_jit, jit_int32_t r0, jit_int32_t r1, jit_int32_t r2)
2959{
2960#if __X64_32
2961 jit_int32_t reg;
2962 reg = jit_get_reg(jit_class_gpr);
2963 addr(rn(reg), r0, r1);
2964 str_i(rn(reg), r2);
2965 jit_unget_reg(reg);
2966#else
2967 rex(0, 0, r2, r1, r0);
2968 ic(0x89);
2969 rx(r2, 0, r0, r1, _SCL1);
2970#endif
2971}
2972
2973static void
2974_stxi_i(jit_state_t *_jit, jit_word_t i0, jit_int32_t r0, jit_int32_t r1)
2975{
2976 jit_int32_t reg;
2977 if (can_sign_extend_int_p(i0)) {
2978 rex(0, 0, r1, _NOREG, r0);
2979 ic(0x89);
2980 rx(r1, i0, r0, _NOREG, _SCL1);
2981 }
2982 else {
2983 reg = jit_get_reg(jit_class_gpr);
2984 movi(rn(reg), i0);
2985 stxr_i(rn(reg), r0, r1);
2986 jit_unget_reg(reg);
2987 }
2988}
2989
2990#if __X64 && !__X64_32
2991static void
2992_stxr_l(jit_state_t *_jit, jit_int32_t r0, jit_int32_t r1, jit_int32_t r2)
2993{
2994 rex(0, 1, r2, r1, r0);
2995 ic(0x89);
2996 rx(r2, 0, r0, r1, _SCL1);
2997}
2998
2999static void
3000_stxi_l(jit_state_t *_jit, jit_word_t i0, jit_int32_t r0, jit_int32_t r1)
3001{
3002 jit_int32_t reg;
3003 if (can_sign_extend_int_p(i0)) {
3004 rex(0, 1, r1, _NOREG, r0);
3005 ic(0x89);
3006 rx(r1, i0, r0, _NOREG, _SCL1);
3007 }
3008 else {
3009 reg = jit_get_reg(jit_class_gpr);
3010 movi(rn(reg), i0);
3011 stxr_l(rn(reg), r0, r1);
3012 jit_unget_reg(reg);
3013 }
3014}
3015#endif
3016
3017static void
3018_jccs(jit_state_t *_jit, jit_int32_t code, jit_word_t i0)
3019{
3020 jit_word_t w;
3021 ic(0x70 | code);
3022 w = i0 - (_jit->pc.w + 1);
3023 ic(w);
3024}
3025
3026static void
3027_jcc(jit_state_t *_jit, jit_int32_t code, jit_word_t i0)
3028{
3029 jit_word_t w;
3030 ic(0x0f);
3031 ic(0x80 | code);
3032 w = i0 - (_jit->pc.w + 4);
3033 ii(w);
3034}
3035
3036static void
3037_jcr(jit_state_t *_jit,
3038 jit_int32_t code, jit_word_t i0, jit_int32_t r0, jit_int32_t r1)
3039{
3040 alur(X86_CMP, r0, r1);
3041 jcc(code, i0);
3042}
3043
3044static void
3045_jci(jit_state_t *_jit,
3046 jit_int32_t code, jit_word_t i0, jit_int32_t r0, jit_word_t i1)
3047{
3048 alui(X86_CMP, r0, i1);
3049 jcc(code, i0);
3050}
3051
3052static void
3053_jci0(jit_state_t *_jit, jit_int32_t code, jit_word_t i0, jit_int32_t r0)
3054{
3055 testr(r0, r0);
3056 jcc(code, i0);
3057}
3058
3059static jit_word_t
3060_bltr(jit_state_t *_jit, jit_word_t i0, jit_int32_t r0, jit_int32_t r1)
3061{
3062 jcr(X86_CC_L, i0, r0, r1);
3063 return (_jit->pc.w);
3064}
3065
3066static jit_word_t
3067_blti(jit_state_t *_jit, jit_word_t i0, jit_int32_t r0, jit_word_t i1)
3068{
3069 if (i1) jci (X86_CC_L, i0, r0, i1);
3070 else jci0(X86_CC_S, i0, r0);
3071 return (_jit->pc.w);
3072}
3073
3074static jit_word_t
3075_bltr_u(jit_state_t *_jit, jit_word_t i0, jit_int32_t r0, jit_int32_t r1)
3076{
3077 jcr(X86_CC_B, i0, r0, r1);
3078 return (_jit->pc.w);
3079}
3080
3081static jit_word_t
3082_blti_u(jit_state_t *_jit, jit_word_t i0, jit_int32_t r0, jit_word_t i1)
3083{
3084 if (i1) jci (X86_CC_B, i0, r0, i1);
3085 else jci0(X86_CC_B, i0, r0);
3086 return (_jit->pc.w);
3087}
3088
3089static jit_word_t
3090_bler(jit_state_t *_jit, jit_word_t i0, jit_int32_t r0, jit_int32_t r1)
3091{
3092 if (r0 == r1) jmpi(i0);
3093 else jcr (X86_CC_LE, i0, r0, r1);
3094 return (_jit->pc.w);
3095}
3096
3097static jit_word_t
3098_blei(jit_state_t *_jit, jit_word_t i0, jit_int32_t r0, jit_word_t i1)
3099{
3100 if (i1) jci (X86_CC_LE, i0, r0, i1);
3101 else jci0(X86_CC_LE, i0, r0);
3102 return (_jit->pc.w);
3103}
3104
3105static jit_word_t
3106_bler_u(jit_state_t *_jit, jit_word_t i0, jit_int32_t r0, jit_int32_t r1)
3107{
3108 if (r0 == r1) jmpi(i0);
3109 else jcr (X86_CC_BE, i0, r0, r1);
3110 return (_jit->pc.w);
3111}
3112
3113static jit_word_t
3114_blei_u(jit_state_t *_jit, jit_word_t i0, jit_int32_t r0, jit_word_t i1)
3115{
3116 if (i1) jci (X86_CC_BE, i0, r0, i1);
3117 else jci0(X86_CC_BE, i0, r0);
3118 return (_jit->pc.w);
3119}
3120
3121static jit_word_t
3122_beqr(jit_state_t *_jit, jit_word_t i0, jit_int32_t r0, jit_int32_t r1)
3123{
3124 if (r0 == r1) jmpi(i0);
3125 else jcr (X86_CC_E, i0, r0, r1);
3126 return (_jit->pc.w);
3127}
3128
3129static jit_word_t
3130_beqi(jit_state_t *_jit, jit_word_t i0, jit_int32_t r0, jit_word_t i1)
3131{
3132 if (i1) jci (X86_CC_E, i0, r0, i1);
3133 else jci0(X86_CC_E, i0, r0);
3134 return (_jit->pc.w);
3135}
3136
3137static jit_word_t
3138_bger(jit_state_t *_jit, jit_word_t i0, jit_int32_t r0, jit_int32_t r1)
3139{
3140 if (r0 == r1) jmpi(i0);
3141 else jcr (X86_CC_GE, i0, r0, r1);
3142 return (_jit->pc.w);
3143}
3144
3145static jit_word_t
3146_bgei(jit_state_t *_jit, jit_word_t i0, jit_int32_t r0, jit_word_t i1)
3147{
3148 if (i1) jci (X86_CC_GE, i0, r0, i1);
3149 else jci0(X86_CC_NS, i0, r0);
3150 return (_jit->pc.w);
3151}
3152
3153static jit_word_t
3154_bger_u(jit_state_t *_jit, jit_word_t i0, jit_int32_t r0, jit_int32_t r1)
3155{
3156 if (r0 == r1) jmpi(i0);
3157 else jcr (X86_CC_AE, i0, r0, r1);
3158 return (_jit->pc.w);
3159}
3160
3161static jit_word_t
3162_bgei_u(jit_state_t *_jit, jit_word_t i0, jit_int32_t r0, jit_word_t i1)
3163{
3164 if (i1) jci (X86_CC_AE, i0, r0, i1);
3165 else jmpi(i0);
3166 return (_jit->pc.w);
3167}
3168
3169static jit_word_t
3170_bgtr(jit_state_t *_jit, jit_word_t i0, jit_int32_t r0, jit_int32_t r1)
3171{
3172 jcr(X86_CC_G, i0, r0, r1);
3173 return (_jit->pc.w);
3174}
3175
3176static jit_word_t
3177_bgti(jit_state_t *_jit, jit_word_t i0, jit_int32_t r0, jit_word_t i1)
3178{
3179 jci(X86_CC_G, i0, r0, i1);
3180 return (_jit->pc.w);
3181}
3182
3183static jit_word_t
3184_bgtr_u(jit_state_t *_jit, jit_word_t i0, jit_int32_t r0, jit_int32_t r1)
3185{
3186 jcr(X86_CC_A, i0, r0, r1);
3187 return (_jit->pc.w);
3188}
3189
3190static jit_word_t
3191_bgti_u(jit_state_t *_jit, jit_word_t i0, jit_int32_t r0, jit_word_t i1)
3192{
3193 if (i1) jci (X86_CC_A, i0, r0, i1);
3194 else jci0(X86_CC_NE, i0, r0);
3195 return (_jit->pc.w);
3196}
3197
3198static jit_word_t
3199_bner(jit_state_t *_jit, jit_word_t i0, jit_int32_t r0, jit_int32_t r1)
3200{
3201 jcr(X86_CC_NE, i0, r0, r1);
3202 return (_jit->pc.w);
3203}
3204
3205static jit_word_t
3206_bnei(jit_state_t *_jit, jit_word_t i0, jit_int32_t r0, jit_word_t i1)
3207{
3208 if (i1) jci (X86_CC_NE, i0, r0, i1);
3209 else jci0(X86_CC_NE, i0, r0);
3210 return (_jit->pc.w);
3211}
3212
3213static jit_word_t
3214_bmsr(jit_state_t *_jit, jit_word_t i0, jit_int32_t r0, jit_int32_t r1)
3215{
3216 testr(r0, r1);
3217 jnz(i0);
3218 return (_jit->pc.w);
3219}
3220
3221static jit_word_t
3222_bmsi(jit_state_t *_jit, jit_word_t i0, jit_int32_t r0, jit_word_t i1)
3223{
3224 jit_int32_t reg;
3225 if (can_zero_extend_int_p(i1))
3226 testi(r0, i1);
3227 else {
3228 reg = jit_get_reg(jit_class_gpr);
3229 movi(rn(reg), i1);
3230 testr(r0, rn(reg));
3231 jit_unget_reg(reg);
3232 }
3233 jnz(i0);
3234 return (_jit->pc.w);
3235}
3236
3237static jit_word_t
3238_bmcr(jit_state_t *_jit, jit_word_t i0, jit_int32_t r0, jit_int32_t r1)
3239{
3240 testr(r0, r1);
3241 jz(i0);
3242 return (_jit->pc.w);
3243}
3244
3245static jit_word_t
3246_bmci(jit_state_t *_jit, jit_word_t i0, jit_int32_t r0, jit_word_t i1)
3247{
3248 jit_int32_t reg;
3249 if (can_zero_extend_int_p(i1))
3250 testi(r0, i1);
3251 else {
3252 reg = jit_get_reg(jit_class_gpr);
3253 movi(rn(reg), i1);
3254 testr(r0, rn(reg));
3255 jit_unget_reg(reg);
3256 }
3257 jz(i0);
3258 return (_jit->pc.w);
3259}
3260
3261static jit_word_t
3262_boaddr(jit_state_t *_jit, jit_word_t i0, jit_int32_t r0, jit_int32_t r1)
3263{
3264 iaddr(r0, r1);
3265 jo(i0);
3266 return (_jit->pc.w);
3267}
3268
3269static jit_word_t
3270_boaddi(jit_state_t *_jit, jit_word_t i0, jit_int32_t r0, jit_word_t i1)
3271{
3272 jit_int32_t reg;
3273 if (can_sign_extend_int_p(i1)) {
3274 iaddi(r0, i1);
3275 jo(i0);
3276 return (_jit->pc.w);
3277 }
3278 reg = jit_get_reg(jit_class_gpr|jit_class_nospill);
3279 movi(rn(reg), i1);
3280 jit_unget_reg(reg);
3281 return (boaddr(i0, r0, rn(reg)));
3282}
3283
3284static jit_word_t
3285_boaddr_u(jit_state_t *_jit, jit_word_t i0, jit_int32_t r0, jit_int32_t r1)
3286{
3287 iaddr(r0, r1);
3288 jc(i0);
3289 return (_jit->pc.w);
3290}
3291
3292static jit_word_t
3293_boaddi_u(jit_state_t *_jit, jit_word_t i0, jit_int32_t r0, jit_word_t i1)
3294{
3295 jit_int32_t reg;
3296 if (can_sign_extend_int_p(i1)) {
3297 iaddi(r0, i1);
3298 jc(i0);
3299 return (_jit->pc.w);
3300 }
3301 reg = jit_get_reg(jit_class_gpr|jit_class_nospill);
3302 movi(rn(reg), i1);
3303 jit_unget_reg(reg);
3304 return (boaddr_u(i0, r0, rn(reg)));
3305}
3306
3307static jit_word_t
3308_bxaddr(jit_state_t *_jit, jit_word_t i0, jit_int32_t r0, jit_int32_t r1)
3309{
3310 iaddr(r0, r1);
3311 jno(i0);
3312 return (_jit->pc.w);
3313}
3314
3315static jit_word_t
3316_bxaddi(jit_state_t *_jit, jit_word_t i0, jit_int32_t r0, jit_word_t i1)
3317{
3318 jit_int32_t reg;
3319 if (can_sign_extend_int_p(i1)) {
3320 iaddi(r0, i1);
3321 jno(i0);
3322 return (_jit->pc.w);
3323 }
3324 reg = jit_get_reg(jit_class_gpr|jit_class_nospill);
3325 movi(rn(reg), i1);
3326 jit_unget_reg(reg);
3327 return (bxaddr(i0, r0, rn(reg)));
3328}
3329
3330static jit_word_t
3331_bxaddr_u(jit_state_t *_jit, jit_word_t i0, jit_int32_t r0, jit_int32_t r1)
3332{
3333 iaddr(r0, r1);
3334 jnc(i0);
3335 return (_jit->pc.w);
3336}
3337
3338static jit_word_t
3339_bxaddi_u(jit_state_t *_jit, jit_word_t i0, jit_int32_t r0, jit_word_t i1)
3340{
3341 jit_int32_t reg;
3342 if (can_sign_extend_int_p(i1)) {
3343 iaddi(r0, i1);
3344 jnc(i0);
3345 return (_jit->pc.w);
3346 }
3347 reg = jit_get_reg(jit_class_gpr|jit_class_nospill);
3348 movi(rn(reg), i1);
3349 jit_unget_reg(reg);
3350 return (bxaddr_u(i0, r0, rn(reg)));
3351}
3352
3353static jit_word_t
3354_bosubr(jit_state_t *_jit, jit_word_t i0, jit_int32_t r0, jit_int32_t r1)
3355{
3356 isubr(r0, r1);
3357 jo(i0);
3358 return (_jit->pc.w);
3359}
3360
3361static jit_word_t
3362_bosubi(jit_state_t *_jit, jit_word_t i0, jit_int32_t r0, jit_word_t i1)
3363{
3364 jit_int32_t reg;
3365 if (can_sign_extend_int_p(i1)) {
3366 isubi(r0, i1);
3367 jo(i0);
3368 return (_jit->pc.w);
3369 }
3370 reg = jit_get_reg(jit_class_gpr|jit_class_nospill);
3371 movi(rn(reg), i1);
3372 jit_unget_reg(reg);
3373 return (bosubr(i0, r0, rn(reg)));
3374}
3375
3376static jit_word_t
3377_bosubr_u(jit_state_t *_jit, jit_word_t i0, jit_int32_t r0, jit_int32_t r1)
3378{
3379 isubr(r0, r1);
3380 jc(i0);
3381 return (_jit->pc.w);
3382}
3383
3384static jit_word_t
3385_bosubi_u(jit_state_t *_jit, jit_word_t i0, jit_int32_t r0, jit_word_t i1)
3386{
3387 jit_int32_t reg;
3388 if (can_sign_extend_int_p(i1)) {
3389 isubi(r0, i1);
3390 jc(i0);
3391 return (_jit->pc.w);
3392 }
3393 reg = jit_get_reg(jit_class_gpr|jit_class_nospill);
3394 movi(rn(reg), i1);
3395 jit_unget_reg(reg);
3396 return (bosubr_u(i0, r0, rn(reg)));
3397}
3398
3399static jit_word_t
3400_bxsubr(jit_state_t *_jit, jit_word_t i0, jit_int32_t r0, jit_int32_t r1)
3401{
3402 isubr(r0, r1);
3403 jno(i0);
3404 return (_jit->pc.w);
3405}
3406
3407static jit_word_t
3408_bxsubi(jit_state_t *_jit, jit_word_t i0, jit_int32_t r0, jit_word_t i1)
3409{
3410 jit_int32_t reg;
3411 if (can_sign_extend_int_p(i1)) {
3412 isubi(r0, i1);
3413 jno(i0);
3414 return (_jit->pc.w);
3415 }
3416 reg = jit_get_reg(jit_class_gpr|jit_class_nospill);
3417 movi(rn(reg), i1);
3418 jit_unget_reg(reg);
3419 return (bxsubr(i0, r0, rn(reg)));
3420}
3421
3422static jit_word_t
3423_bxsubr_u(jit_state_t *_jit, jit_word_t i0, jit_int32_t r0, jit_int32_t r1)
3424{
3425 isubr(r0, r1);
3426 jnc(i0);
3427 return (_jit->pc.w);
3428}
3429
3430static jit_word_t
3431_bxsubi_u(jit_state_t *_jit, jit_word_t i0, jit_int32_t r0, jit_word_t i1)
3432{
3433 jit_int32_t reg;
3434 if (can_sign_extend_int_p(i1)) {
3435 isubi(r0, i1);
3436 jnc(i0);
3437 return (_jit->pc.w);
3438 }
3439 reg = jit_get_reg(jit_class_gpr|jit_class_nospill);
3440 movi(rn(reg), i1);
3441 jit_unget_reg(reg);
3442 return (bxsubr_u(i0, r0, rn(reg)));
3443}
3444
3445static void
3446_callr(jit_state_t *_jit, jit_int32_t r0)
3447{
3448 rex(0, 0, _NOREG, _NOREG, r0);
3449 ic(0xff);
3450 mrm(0x03, 0x02, r7(r0));
3451}
3452
3453static jit_word_t
3454_calli(jit_state_t *_jit, jit_word_t i0)
3455{
3456 jit_word_t word;
519a9ea1 3457 jit_word_t w;
4a71579b 3458#if __X64
519a9ea1
PC		 3459 w = i0 - (_jit->pc.w + 5);
3460 if ((jit_int32_t)w == w) {
3461#endif
3462 ic(0xe8);
3463 w = i0 - (_jit->pc.w + 4);
3464 ii(w);
3465 word = _jit->pc.w;
3466#if __X64
3467 }
3468 else
3469 word = calli_p(i0);
3470#endif
3471 return (word);
3472}
4a71579b 3473
519a9ea1
PC		 3474#if __X64
3475static jit_word_t
3476_calli_p(jit_state_t *_jit, jit_word_t i0)
3477{
3478 jit_word_t word;
3479 jit_int32_t reg;
4a71579b
PC		 3480 reg = jit_get_reg(jit_class_gpr);
3481 word = movi_p(rn(reg), i0);
3482 callr(rn(reg));
3483 jit_unget_reg(reg);
4a71579b
PC		 3484 return (word);
3485}
519a9ea1 3486#endif
4a71579b
PC		 3487
3488static void
3489_jmpr(jit_state_t *_jit, jit_int32_t r0)
3490{
519a9ea1 3491 rex(0, 0, _NOREG, _NOREG, r0);
4a71579b
PC		 3492 ic(0xff);
3493 mrm(0x03, 0x04, r7(r0));
3494}
3495
3496static jit_word_t
3497_jmpi(jit_state_t *_jit, jit_word_t i0)
3498{
519a9ea1 3499 jit_word_t word;
4a71579b 3500 jit_word_t w;
519a9ea1
PC		 3501#if __X64
3502 w = i0 - (_jit->pc.w + 5);
3503 if ((jit_int32_t)w == w) {
3504#endif
3505 ic(0xe9);
3506 w = i0 - (_jit->pc.w + 4);
3507 ii(w);
3508 word = _jit->pc.w;
3509#if __X64
3510 }
3511 else
3512 word = jmpi_p(i0);
3513#endif
3514 return (word);
4a71579b
PC		 3515}
3516
519a9ea1
PC		 3517#if __X64
3518static jit_word_t
3519_jmpi_p(jit_state_t *_jit, jit_word_t i0)
3520{
3521 jit_word_t word;
3522 jit_int32_t reg;
3523 reg = jit_get_reg(jit_class_gpr|jit_class_nospill);
3524 word = movi_p(rn(reg), i0);
3525 jmpr(rn(reg));
3526 jit_unget_reg(reg);
3527 return (word);
3528}
3529#endif
3530
4a71579b
PC		 3531static void
3532_jmpsi(jit_state_t *_jit, jit_uint8_t i0)
3533{
3534 ic(0xeb);
3535 ic(i0);
3536}
3537
3538static void
3539_prolog(jit_state_t *_jit, jit_node_t *node)
3540{
3541 jit_int32_t reg;
3542 if (_jitc->function->define_frame || _jitc->function->assume_frame) {
3543 jit_int32_t frame = -_jitc->function->frame;
3544 assert(_jitc->function->self.aoff >= frame);
3545 if (_jitc->function->assume_frame)
3546 return;
3547 _jitc->function->self.aoff = frame;
3548 }
3549 if (_jitc->function->allocar)
3550 _jitc->function->self.aoff &= -16;
3551#if __X64 && (__CYGWIN__ || _WIN32)
3552 _jitc->function->stack = (((/* first 32 bytes must be allocated */
3553 (_jitc->function->self.alen > 32 ?
3554 _jitc->function->self.alen : 32) -
3555 /* align stack at 16 bytes */
3556 _jitc->function->self.aoff) + 15) & -16) +
3557 stack_adjust;
3558#else
3559 _jitc->function->stack = (((_jitc->function->self.alen -
3560 _jitc->function->self.aoff) + 15) & -16) +
3561 stack_adjust;
3562#endif
3563 subi(_RSP_REGNO, _RSP_REGNO, stack_framesize - REAL_WORDSIZE);
3564 /* callee save registers */
3565#if __X32
3566 if (jit_regset_tstbit(&_jitc->function->regset, _RDI))
3567 stxi(12, _RSP_REGNO, _RDI_REGNO);
3568 if (jit_regset_tstbit(&_jitc->function->regset, _RSI))
3569 stxi( 8, _RSP_REGNO, _RSI_REGNO);
3570 if (jit_regset_tstbit(&_jitc->function->regset, _RBX))
3571 stxi( 4, _RSP_REGNO, _RBX_REGNO);
3572#else
3573# if __CYGWIN__ || _WIN32
3574 if (jit_regset_tstbit(&_jitc->function->regset, _XMM15))
3575 sse_stxi_d(136, _RSP_REGNO, _XMM15_REGNO);
3576 if (jit_regset_tstbit(&_jitc->function->regset, _XMM14))
3577 sse_stxi_d(128, _RSP_REGNO, _XMM14_REGNO);
3578 if (jit_regset_tstbit(&_jitc->function->regset, _XMM13))
3579 sse_stxi_d(120, _RSP_REGNO, _XMM13_REGNO);
3580 if (jit_regset_tstbit(&_jitc->function->regset, _XMM12))
3581 sse_stxi_d(112, _RSP_REGNO, _XMM12_REGNO);
3582 if (jit_regset_tstbit(&_jitc->function->regset, _XMM11))
3583 sse_stxi_d(104, _RSP_REGNO, _XMM11_REGNO);
3584 if (jit_regset_tstbit(&_jitc->function->regset, _XMM10))
3585 sse_stxi_d(96, _RSP_REGNO, _XMM10_REGNO);
3586 if (jit_regset_tstbit(&_jitc->function->regset, _XMM9))
3587 sse_stxi_d(88, _RSP_REGNO, _XMM9_REGNO);
3588 if (jit_regset_tstbit(&_jitc->function->regset, _XMM8))
3589 sse_stxi_d(80, _RSP_REGNO, _XMM8_REGNO);
3590 if (jit_regset_tstbit(&_jitc->function->regset, _XMM7))
3591 sse_stxi_d(72, _RSP_REGNO, _XMM7_REGNO);
3592 if (jit_regset_tstbit(&_jitc->function->regset, _XMM6))
3593 sse_stxi_d(64, _RSP_REGNO, _XMM6_REGNO);
3594 if (jit_regset_tstbit(&_jitc->function->regset, _R15))
3595 stxi(56, _RSP_REGNO, _R15_REGNO);
3596 if (jit_regset_tstbit(&_jitc->function->regset, _R14))
3597 stxi(48, _RSP_REGNO, _R14_REGNO);
3598 if (jit_regset_tstbit(&_jitc->function->regset, _R13))
3599 stxi(40, _RSP_REGNO, _R13_REGNO);
3600 if (jit_regset_tstbit(&_jitc->function->regset, _R12))
3601 stxi(32, _RSP_REGNO, _R12_REGNO);
3602 if (jit_regset_tstbit(&_jitc->function->regset, _RSI))
3603 stxi(24, _RSP_REGNO, _RSI_REGNO);
3604 if (jit_regset_tstbit(&_jitc->function->regset, _RDI))
3605 stxi(16, _RSP_REGNO, _RDI_REGNO);
3606 if (jit_regset_tstbit(&_jitc->function->regset, _RBX))
3607 stxi( 8, _RSP_REGNO, _RBX_REGNO);
3608# else
3609 if (jit_regset_tstbit(&_jitc->function->regset, _RBX))
3610 stxi(40, _RSP_REGNO, _RBX_REGNO);
3611 if (jit_regset_tstbit(&_jitc->function->regset, _R12))
3612 stxi(32, _RSP_REGNO, _R12_REGNO);
3613 if (jit_regset_tstbit(&_jitc->function->regset, _R13))
3614 stxi(24, _RSP_REGNO, _R13_REGNO);
3615 if (jit_regset_tstbit(&_jitc->function->regset, _R14))
3616 stxi(16, _RSP_REGNO, _R14_REGNO);
3617 if (jit_regset_tstbit(&_jitc->function->regset, _R15))
3618 stxi( 8, _RSP_REGNO, _R15_REGNO);
3619# endif
3620#endif
3621 stxi(0, _RSP_REGNO, _RBP_REGNO);
3622 movr(_RBP_REGNO, _RSP_REGNO);
3623
3624 /* alloca */
3625 subi(_RSP_REGNO, _RSP_REGNO, _jitc->function->stack);
3626 if (_jitc->function->allocar) {
3627 reg = jit_get_reg(jit_class_gpr);
3628 movi(rn(reg), _jitc->function->self.aoff);
3629 stxi_i(_jitc->function->aoffoff, _RBP_REGNO, rn(reg));
3630 jit_unget_reg(reg);
3631 }
3632
3633#if __X64 && !(__CYGWIN__ || _WIN32)
3634 if (_jitc->function->self.call & jit_call_varargs) {
3635 jit_word_t nofp_code;
3636
3637 /* Save gp registers in the save area, if any is a vararg */
3638 for (reg = first_gp_from_offset(_jitc->function->vagp);
3639 jit_arg_reg_p(reg); ++reg)
3640 stxi(_jitc->function->vaoff + first_gp_offset +
3641 reg * 8, _RBP_REGNO, rn(JIT_RA0 - reg));
3642
3643 reg = first_fp_from_offset(_jitc->function->vafp);
3644 if (jit_arg_f_reg_p(reg)) {
3645 /* Skip over if no float registers were passed as argument */
3646 /* test %al, %al */
3647 ic(0x84);
3648 ic(0xc0);
3649 jes(0);
3650 nofp_code = _jit->pc.w;
3651
3652 /* Save fp registers in the save area, if any is a vararg */
3653 /* Note that the full 16 byte xmm is not saved, because
3654 * lightning only handles float and double, and, while
3655 * attempting to provide a va_list compatible pointer as
3656 * jit_va_start return, does not guarantee it (on all ports). */
3657 for (; jit_arg_f_reg_p(reg); ++reg)
3658 sse_stxi_d(_jitc->function->vaoff + first_fp_offset +
3659 reg * va_fp_increment, _RBP_REGNO, rn(_XMM0 - reg));
3660
3661 patch_rel_char(nofp_code, _jit->pc.w);
3662 }
3663 }
3664#endif
3665}
3666
3667static void
3668_epilog(jit_state_t *_jit, jit_node_t *node)
3669{
3670 if (_jitc->function->assume_frame)
3671 return;
3672 /* callee save registers */
3673 movr(_RSP_REGNO, _RBP_REGNO);
3674#if __X32
3675 if (jit_regset_tstbit(&_jitc->function->regset, _RDI))
3676 ldxi(_RDI_REGNO, _RSP_REGNO, 12);
3677 if (jit_regset_tstbit(&_jitc->function->regset, _RSI))
3678 ldxi(_RSI_REGNO, _RSP_REGNO, 8);
3679 if (jit_regset_tstbit(&_jitc->function->regset, _RBX))
3680 ldxi(_RBX_REGNO, _RSP_REGNO, 4);
3681#else
3682# if __CYGWIN__ || _WIN32
3683 if (jit_regset_tstbit(&_jitc->function->regset, _XMM15))
3684 sse_ldxi_d(_XMM15_REGNO, _RSP_REGNO, 136);
3685 if (jit_regset_tstbit(&_jitc->function->regset, _XMM14))
3686 sse_ldxi_d(_XMM14_REGNO, _RSP_REGNO, 128);
3687 if (jit_regset_tstbit(&_jitc->function->regset, _XMM13))
3688 sse_ldxi_d(_XMM13_REGNO, _RSP_REGNO, 120);
3689 if (jit_regset_tstbit(&_jitc->function->regset, _XMM12))
3690 sse_ldxi_d(_XMM12_REGNO, _RSP_REGNO, 112);
3691 if (jit_regset_tstbit(&_jitc->function->regset, _XMM11))
3692 sse_ldxi_d(_XMM11_REGNO, _RSP_REGNO, 104);
3693 if (jit_regset_tstbit(&_jitc->function->regset, _XMM10))
3694 sse_ldxi_d(_XMM10_REGNO, _RSP_REGNO, 96);
3695 if (jit_regset_tstbit(&_jitc->function->regset, _XMM9))
3696 sse_ldxi_d(_XMM9_REGNO, _RSP_REGNO, 88);
3697 if (jit_regset_tstbit(&_jitc->function->regset, _XMM8))
3698 sse_ldxi_d(_XMM8_REGNO, _RSP_REGNO, 80);
3699 if (jit_regset_tstbit(&_jitc->function->regset, _XMM7))
3700 sse_ldxi_d(_XMM7_REGNO, _RSP_REGNO, 72);
3701 if (jit_regset_tstbit(&_jitc->function->regset, _XMM6))
3702 sse_ldxi_d(_XMM6_REGNO, _RSP_REGNO, 64);
3703 if (jit_regset_tstbit(&_jitc->function->regset, _R15))
3704 ldxi(_R15_REGNO, _RSP_REGNO, 56);
3705 if (jit_regset_tstbit(&_jitc->function->regset, _R14))
3706 ldxi(_R14_REGNO, _RSP_REGNO, 48);
3707 if (jit_regset_tstbit(&_jitc->function->regset, _R13))
3708 ldxi(_R13_REGNO, _RSP_REGNO, 40);
3709 if (jit_regset_tstbit(&_jitc->function->regset, _R12))
3710 ldxi(_R12_REGNO, _RSP_REGNO, 32);
3711 if (jit_regset_tstbit(&_jitc->function->regset, _RSI))
3712 ldxi(_RSI_REGNO, _RSP_REGNO, 24);
3713 if (jit_regset_tstbit(&_jitc->function->regset, _RDI))
3714 ldxi(_RDI_REGNO, _RSP_REGNO, 16);
3715 if (jit_regset_tstbit(&_jitc->function->regset, _RBX))
3716 ldxi(_RBX_REGNO, _RSP_REGNO, 8);
3717# else
3718 if (jit_regset_tstbit(&_jitc->function->regset, _RBX))
3719 ldxi(_RBX_REGNO, _RSP_REGNO, 40);
3720 if (jit_regset_tstbit(&_jitc->function->regset, _R12))
3721 ldxi(_R12_REGNO, _RSP_REGNO, 32);
3722 if (jit_regset_tstbit(&_jitc->function->regset, _R13))
3723 ldxi(_R13_REGNO, _RSP_REGNO, 24);
3724 if (jit_regset_tstbit(&_jitc->function->regset, _R14))
3725 ldxi(_R14_REGNO, _RSP_REGNO, 16);
3726 if (jit_regset_tstbit(&_jitc->function->regset, _R15))
3727 ldxi(_R15_REGNO, _RSP_REGNO, 8);
3728# endif
3729#endif
3730 ldxi(_RBP_REGNO, _RSP_REGNO, 0);
3731 addi(_RSP_REGNO, _RSP_REGNO, stack_framesize - REAL_WORDSIZE);
3732
3733 ic(0xc3);
3734}
3735
3736static void
3737_vastart(jit_state_t *_jit, jit_int32_t r0)
3738{
3739#if __X32 || __CYGWIN__ || _WIN32
3740 assert(_jitc->function->self.call & jit_call_varargs);
3741 addi(r0, _RBP_REGNO, _jitc->function->self.size);
3742#else
3743 jit_int32_t reg;
3744
3745 assert(_jitc->function->self.call & jit_call_varargs);
3746
3747 /* Return jit_va_list_t in the register argument */
3748 addi(r0, _RBP_REGNO, _jitc->function->vaoff);
3749 reg = jit_get_reg(jit_class_gpr);
3750
3751 /* Initialize gp offset in the save area. */
3752 movi(rn(reg), _jitc->function->vagp);
3753 stxi_i(offsetof(jit_va_list_t, gpoff), r0, rn(reg));
3754
3755 /* Initialize fp offset in the save area. */
3756 movi(rn(reg), _jitc->function->vafp);
3757 stxi_i(offsetof(jit_va_list_t, fpoff), r0, rn(reg));
3758
3759 /* Initialize overflow pointer to the first stack argument. */
3760 addi(rn(reg), _RBP_REGNO, _jitc->function->self.size);
3761 stxi(offsetof(jit_va_list_t, over), r0, rn(reg));
3762
3763 /* Initialize register save area pointer. */
3764 addi(rn(reg), r0, first_gp_offset);
3765 stxi(offsetof(jit_va_list_t, save), r0, rn(reg));
3766
3767 jit_unget_reg(reg);
3768#endif
3769}
3770
3771static void
3772_vaarg(jit_state_t *_jit, jit_int32_t r0, jit_int32_t r1)
3773{
3774#if __X32 || __CYGWIN__ || _WIN32
3775 assert(_jitc->function->self.call & jit_call_varargs);
3776 ldr(r0, r1);
3777 addi(r1, r1, va_gp_increment);
3778#else
3779 jit_int32_t rg0;
3780 jit_int32_t rg1;
3781 jit_word_t ge_code;
3782 jit_word_t lt_code;
3783
3784 assert(_jitc->function->self.call & jit_call_varargs);
3785
3786 rg0 = jit_get_reg(jit_class_gpr);
3787 rg1 = jit_get_reg(jit_class_gpr);
3788
3789 /* Load the gp offset in save area in the first temporary. */
3790 ldxi_i(rn(rg0), r1, offsetof(jit_va_list_t, gpoff));
3791
3792 /* Jump over if there are no remaining arguments in the save area. */
3793 icmpi(rn(rg0), va_gp_max_offset);
3794 jaes(0);
3795 ge_code = _jit->pc.w;
3796
3797 /* Load the save area pointer in the second temporary. */
3798 ldxi(rn(rg1), r1, offsetof(jit_va_list_t, save));
3799
3800 /* Load the vararg argument in the first argument. */
3801 ldxr(r0, rn(rg1), rn(rg0));
3802
3803 /* Update the gp offset. */
3804 addi(rn(rg0), rn(rg0), 8);
3805 stxi_i(offsetof(jit_va_list_t, gpoff), r1, rn(rg0));
3806
3807 /* Will only need one temporary register below. */
3808 jit_unget_reg(rg1);
3809
3810 /* Jump over overflow code. */
3811 jmpsi(0);
3812 lt_code = _jit->pc.w;
3813
3814 /* Where to land if argument is in overflow area. */
3815 patch_rel_char(ge_code, _jit->pc.w);
3816
3817 /* Load overflow pointer. */
3818 ldxi(rn(rg0), r1, offsetof(jit_va_list_t, over));
3819
3820 /* Load argument. */
3821 ldr(r0, rn(rg0));
3822
3823 /* Update overflow pointer. */
3824 addi(rn(rg0), rn(rg0), va_gp_increment);
3825 stxi(offsetof(jit_va_list_t, over), r1, rn(rg0));
3826
3827 /* Where to land if argument is in save area. */
3828 patch_rel_char(lt_code, _jit->pc.w);
3829
3830 jit_unget_reg(rg0);
3831#endif
3832}
3833
3834/* The x87 boolean argument tells if will put the result in a x87
3835 * register if non false, in a sse register otherwise. */
3836static void
3837_vaarg_d(jit_state_t *_jit, jit_int32_t r0, jit_int32_t r1, jit_bool_t x87)
3838{
3839#if __X32 || __CYGWIN__ || _WIN32
3840 assert(_jitc->function->self.call & jit_call_varargs);
3841 if (x87)
3842 x87_ldr_d(r0, r1);
3843 else
3844 sse_ldr_d(r0, r1);
3845 addi(r1, r1, 8);
3846#else
3847 jit_int32_t rg0;
3848 jit_int32_t rg1;
3849 jit_word_t ge_code;
3850 jit_word_t lt_code;
3851
3852 assert(_jitc->function->self.call & jit_call_varargs);
3853
3854 rg0 = jit_get_reg(jit_class_gpr);
3855 rg1 = jit_get_reg(jit_class_gpr);
3856
3857 /* Load the fp offset in save area in the first temporary. */
3858 ldxi_i(rn(rg0), r1, offsetof(jit_va_list_t, fpoff));
3859
3860 /* Jump over if there are no remaining arguments in the save area. */
3861 icmpi(rn(rg0), va_fp_max_offset);
3862 jaes(0);
3863 ge_code = _jit->pc.w;
3864
3865 /* Load the save area pointer in the second temporary. */
3866 ldxi(rn(rg1), r1, offsetof(jit_va_list_t, save));
3867
3868 /* Load the vararg argument in the first argument. */
3869 if (x87)
3870 x87_ldxr_d(r0, rn(rg1), rn(rg0));
3871 else
3872 sse_ldxr_d(r0, rn(rg1), rn(rg0));
3873
3874 /* Update the fp offset. */
3875 addi(rn(rg0), rn(rg0), va_fp_increment);
3876 stxi_i(offsetof(jit_va_list_t, fpoff), r1, rn(rg0));
3877
3878 /* Will only need one temporary register below. */
3879 jit_unget_reg(rg1);
3880
3881 /* Jump over overflow code. */
3882 jmpsi(0);
3883 lt_code = _jit->pc.w;
3884
3885 /* Where to land if argument is in overflow area. */
3886 patch_rel_char(ge_code, _jit->pc.w);
3887
3888 /* Load overflow pointer. */
3889 ldxi(rn(rg0), r1, offsetof(jit_va_list_t, over));
3890
3891 /* Load argument. */
3892 if (x87)
3893 x87_ldr_d(r0, rn(rg0));
3894 else
3895 sse_ldr_d(r0, rn(rg0));
3896
3897 /* Update overflow pointer. */
3898 addi(rn(rg0), rn(rg0), 8);
3899 stxi(offsetof(jit_va_list_t, over), r1, rn(rg0));
3900
3901 /* Where to land if argument is in save area. */
3902 patch_rel_char(lt_code, _jit->pc.w);
3903
3904 jit_unget_reg(rg0);
3905#endif
3906}
3907
3908static void
3909_patch_at(jit_state_t *_jit, jit_node_t *node,
3910 jit_word_t instr, jit_word_t label)
3911{
3912 switch (node->code) {
3913# if __X64
3914 case jit_code_calli:
519a9ea1 3915 case jit_code_jmpi:
4a71579b
PC		 3916# endif
3917 case jit_code_movi:
3918 patch_abs(instr, label);
3919 break;
3920 default:
3921 patch_rel(instr, label);
3922 break;
3923 }
3924}
3925#endif
ARM optimized PCSX fork