notaz.gp2x.de / pcsx_rearmed.git / blame
? search:
summary | shortlog | log | commit | commitdiff | tree
blob | blame (incremental) | history | HEAD
spu: adjust irqs again
[pcsx_rearmed.git] / deps / lightning / lib / jit_x86-cpu.c
CommitLineData
4a71579b 1/*
79bfeef6 2 * Copyright (C) 2012-2023 Free Software Foundation, Inc.
4a71579b
PC		 3 *
4 * This file is part of GNU lightning.
5 *
6 * GNU lightning is free software; you can redistribute it and/or modify it
7 * under the terms of the GNU Lesser General Public License as published
8 * by the Free Software Foundation; either version 3, or (at your option)
9 * any later version.
10 *
11 * GNU lightning is distributed in the hope that it will be useful, but
12 * WITHOUT ANY WARRANTY; without even the implied warranty of MERCHANTABILITY
13 * or FITNESS FOR A PARTICULAR PURPOSE. See the GNU Lesser General Public
14 * License for more details.
15 *
16 * Authors:
17 * Paulo Cesar Pereira de Andrade
18 */
19
20/* avoid using it due to partial stalls */
21#define USE_INC_DEC 0
22
23#if PROTO
24# if __X32 || __X64_32
25# define WIDE 0
26# define ldi(u, v) ldi_i(u, v)
27# define ldr(u, v) ldr_i(u, v)
28# define ldxr(u, v, w) ldxr_i(u, v, w)
29# define ldxi(u, v, w) ldxi_i(u, v, w)
30# define sti(u, v) sti_i(u, v)
31# define stxi(u, v, w) stxi_i(u, v, w)
32# define can_sign_extend_int_p(im) 1
33# define can_zero_extend_int_p(im) 1
34# define fits_uint32_p(im) 1
35# else
36# define WIDE 1
37# define ldi(u, v) ldi_l(u, v)
38# define ldr(u, v) ldr_l(u, v)
39# define ldxr(u, v, w) ldxr_l(u, v, w)
40# define ldxi(u, v, w) ldxi_l(u, v, w)
41# define sti(u, v) sti_l(u, v)
42# define stxi(u, v, w) stxi_l(u, v, w)
43# define can_sign_extend_int_p(im) \
44 (((im) >= 0 && (long long)(im) <= 0x7fffffffLL) || \
45 ((im) < 0 && (long long)(im) > -0x80000000LL))
46# define can_zero_extend_int_p(im) \
47 ((im) >= 0 && (im) < 0x80000000LL)
48# define fits_uint32_p(im) (((im) & 0xffffffff00000000LL) == 0)
49# endif
50# if __X32 || __CYGWIN__ || __X64_32 || _WIN32
51# define reg8_p(rn) \
52 ((rn) >= _RAX_REGNO && (rn) <= _RBX_REGNO)
53# else
54# define reg8_p(rn) 1
55# endif
56# define _RAX_REGNO 0
57# define _RCX_REGNO 1
58# define _RDX_REGNO 2
59# define _RBX_REGNO 3
60# define _RSP_REGNO 4
61# define _RBP_REGNO 5
62# define _RSI_REGNO 6
63# define _RDI_REGNO 7
64# define _R8_REGNO 8
65# define _R9_REGNO 9
66# define _R10_REGNO 10
67# define _R11_REGNO 11
68# define _R12_REGNO 12
69# define _R13_REGNO 13
70# define _R14_REGNO 14
71# define _R15_REGNO 15
72# define r7(reg) ((reg) & 7)
73# define r8(reg) ((reg) & 15)
74# define _SCL1 0x00
75# define _SCL2 0x01
76# define _SCL4 0x02
77# define _SCL8 0x03
78# define X86_ADD 0
79# define X86_OR 1 << 3
80# define X86_ADC 2 << 3
81# define X86_SBB 3 << 3
82# define X86_AND 4 << 3
83# define X86_SUB 5 << 3
84# define X86_XOR 6 << 3
85# define X86_CMP 7 << 3
86# define X86_ROL 0
87# define X86_ROR 1
88# define X86_RCL 2
89# define X86_RCR 3
90# define X86_SHL 4
91# define X86_SHR 5
92# define X86_SAR 7
93# define X86_NOT 2
94# define X86_NEG 3
95# define X86_MUL 4
96# define X86_IMUL 5
97# define X86_DIV 6
98# define X86_IDIV 7
99# define X86_CC_O 0x0
100# define X86_CC_NO 0x1
101# define X86_CC_NAE 0x2
102# define X86_CC_B 0x2
103# define X86_CC_C 0x2
104# define X86_CC_AE 0x3
105# define X86_CC_NB 0x3
106# define X86_CC_NC 0x3
107# define X86_CC_E 0x4
108# define X86_CC_Z 0x4
109# define X86_CC_NE 0x5
110# define X86_CC_NZ 0x5
111# define X86_CC_BE 0x6
112# define X86_CC_NA 0x6
113# define X86_CC_A 0x7
114# define X86_CC_NBE 0x7
115# define X86_CC_S 0x8
116# define X86_CC_NS 0x9
117# define X86_CC_P 0xa
118# define X86_CC_PE 0xa
119# define X86_CC_NP 0xb
120# define X86_CC_PO 0xb
121# define X86_CC_L 0xc
122# define X86_CC_NGE 0xc
123# define X86_CC_GE 0xd
124# define X86_CC_NL 0xd
125# define X86_CC_LE 0xe
126# define X86_CC_NG 0xe
127# define X86_CC_G 0xf
128# define X86_CC_NLE 0xf
129# define mrm(md, r, m) *_jit->pc.uc++ = (md<<6) | (r<<3) | m
130# define sib(sc, i, b) *_jit->pc.uc++ = (sc<<6) | (i<<3) | b
131# define ic(c) *_jit->pc.uc++ = c
132# define is(s) *_jit->pc.us++ = s
133# define ii(i) *_jit->pc.ui++ = i
134# if __X64 && !__X64_32
135# define il(l) *_jit->pc.ul++ = l
136# else
137# define il(l) ii(l)
138# endif
4a71579b
PC		 139# define rex(l, w, r, x, b) _rex(_jit, l, w, r, x, b)
140static void
141_rex(jit_state_t*,jit_int32_t,jit_int32_t,jit_int32_t,jit_int32_t,jit_int32_t);
142# define rx(rd, md, rb, ri, ms) _rx(_jit, rd, md, rb, ri, ms)
143static void
144_rx(jit_state_t*,jit_int32_t,jit_int32_t,jit_int32_t,jit_int32_t,jit_int32_t);
145# define nop(n) _nop(_jit, n)
146static void _nop(jit_state_t*, jit_int32_t);
147# define emms() is(0x770f)
148# define lea(md, rb, ri, ms, rd) _lea(_jit, md, rb, ri, ms, rd)
149static void
150_lea(jit_state_t*,jit_int32_t,jit_int32_t,jit_int32_t,jit_int32_t,jit_int32_t);
151# define pushr(r0) _pushr(_jit, r0)
152static void _pushr(jit_state_t*, jit_int32_t) maybe_unused;
153# define popr(r0) _popr(_jit, r0)
154static void _popr(jit_state_t*, jit_int32_t) maybe_unused;
155# define xchgr(r0, r1) _xchgr(_jit, r0, r1)
156static void _xchgr(jit_state_t*, jit_int32_t, jit_int32_t);
157# define testr(r0, r1) _testr(_jit, r0, r1)
158static void _testr(jit_state_t*, jit_int32_t, jit_int32_t);
159# define testi(r0, i0) _testi(_jit, r0, i0)
160static void _testi(jit_state_t*, jit_int32_t, jit_word_t);
161# define cc(code, r0) _cc(_jit, code, r0)
162static void _cc(jit_state_t*, jit_int32_t, jit_int32_t);
163# define icmpr(r0, r1) alur(X86_CMP, r0, r1)
164# define alur(code, r0, r1) _alur(_jit, code, r0, r1)
165static void _alur(jit_state_t*, jit_int32_t, jit_int32_t, jit_int32_t);
166# define icmpi(r0, i0) alui(X86_CMP, r0, i0)
167# define alui(code, r0, i0) _alui(_jit, code, r0, i0)
168static void _alui(jit_state_t*, jit_int32_t, jit_int32_t, jit_word_t);
169# define iaddr(r0, r1) alur(X86_ADD, r0, r1)
170# define save(r0) _save(_jit, r0)
171static void _save(jit_state_t*, jit_int32_t);
172# define load(r0) _load(_jit, r0)
173static void _load(jit_state_t*, jit_int32_t);
174# define addr(r0, r1, r2) _addr(_jit, r0, r1, r2)
175static void _addr(jit_state_t*, jit_int32_t, jit_int32_t, jit_int32_t);
176# define iaddi(r0, i0) alui(X86_ADD, r0, i0)
177# define addi(r0, r1, i0) _addi(_jit, r0, r1, i0)
178static void _addi(jit_state_t*, jit_int32_t, jit_int32_t, jit_word_t);
179#define addcr(r0, r1, r2) _addcr(_jit, r0, r1, r2)
180static void _addcr(jit_state_t*, jit_int32_t, jit_int32_t, jit_int32_t);
181#define addci(r0, r1, i0) _addci(_jit, r0, r1, i0)
182static void _addci(jit_state_t*, jit_int32_t, jit_int32_t, jit_word_t);
79bfeef6
PC		 183# define iaddxr(r0, r1) _iaddxr(_jit, r0, r1)
184static void _iaddxr(jit_state_t*, jit_int32_t, jit_int32_t);
4a71579b
PC		 185# define addxr(r0, r1, r2) _addxr(_jit, r0, r1, r2)
186static void _addxr(jit_state_t*, jit_int32_t, jit_int32_t, jit_int32_t);
187# define iaddxi(r0, i0) alui(X86_ADC, r0, i0)
188# define addxi(r0, r1, i0) _addxi(_jit, r0, r1, i0)
189static void _addxi(jit_state_t*, jit_int32_t, jit_int32_t, jit_word_t);
190# define isubr(r0, r1) alur(X86_SUB, r0, r1)
191# define subr(r0, r1, r2) _subr(_jit, r0, r1, r2)
192static void _subr(jit_state_t*, jit_int32_t, jit_int32_t, jit_int32_t);
193# define isubi(r0, i0) alui(X86_SUB, r0, i0)
194# define subi(r0, r1, i0) _subi(_jit, r0, r1, i0)
195static void _subi(jit_state_t*, jit_int32_t, jit_int32_t, jit_word_t);
196# define subcr(r0, r1, r2) _subcr(_jit, r0, r1, r2)
197static void _subcr(jit_state_t*,jit_int32_t,jit_int32_t,jit_int32_t);
198# define subci(r0, r1, i0) _subci(_jit, r0, r1, i0)
199static void _subci(jit_state_t*,jit_int32_t,jit_int32_t,jit_word_t);
200# define isubxr(r0, r1) alur(X86_SBB, r0, r1)
201# define subxr(r0, r1, r2) _subxr(_jit, r0, r1, r2)
202static void _subxr(jit_state_t*,jit_int32_t,jit_int32_t,jit_int32_t);
203# define isubxi(r0, i0) alui(X86_SBB, r0, i0)
204# define subxi(r0, r1, i0) _subxi(_jit, r0, r1, i0)
205static void _subxi(jit_state_t*,jit_int32_t,jit_int32_t,jit_word_t);
206# define rsbi(r0, r1, i0) _rsbi(_jit, r0, r1, i0)
207static void _rsbi(jit_state_t*,jit_int32_t,jit_int32_t,jit_word_t);
208# define imulr(r0, r1) _imulr(_jit, r0, r1)
209static void _imulr(jit_state_t*, jit_int32_t, jit_int32_t);
210# define imuli(r0, r1, i0) _imuli(_jit, r0, r1, i0)
211static void _imuli(jit_state_t*, jit_int32_t, jit_int32_t, jit_word_t);
212# define mulr(r0, r1, r2) _mulr(_jit, r0, r1, r2)
213static void _mulr(jit_state_t*, jit_int32_t, jit_int32_t, jit_int32_t);
214# define muli(r0, r1, i0) _muli(_jit, r0, r1, i0)
215static void _muli(jit_state_t*, jit_int32_t, jit_int32_t, jit_word_t);
216# define umulr(r0) unr(X86_IMUL, r0)
217# define umulr_u(r0) unr(X86_MUL, r0)
218# define qmulr(r0, r1, r2, r3) _iqmulr(_jit, r0, r1, r2, r3, 1)
219# define qmulr_u(r0, r1, r2, r3) _iqmulr(_jit, r0, r1, r2, r3, 0)
220# define iqmulr(r0, r1, r2, r3, sign) _iqmulr(_jit, r0, r1, r2, r3, sign)
221static void _iqmulr(jit_state_t*, jit_int32_t, jit_int32_t,
222 jit_int32_t,jit_int32_t, jit_bool_t);
223# define qmuli(r0, r1, r2, i0) _iqmuli(_jit, r0, r1, r2, i0, 1)
224# define qmuli_u(r0, r1, r2, i0) _iqmuli(_jit, r0, r1, r2, i0, 0)
225# define iqmuli(r0, r1, r2, i0, sign) _iqmuli(_jit, r0, r1, r2, i0, sign)
226static void _iqmuli(jit_state_t*, jit_int32_t, jit_int32_t,
227 jit_int32_t,jit_word_t, jit_bool_t);
228# define sign_extend_rdx_rax() _sign_extend_rdx_rax(_jit)
229static void _sign_extend_rdx_rax(jit_state_t*);
230# define idivr(r0) unr(X86_IDIV, r0)
231# define idivr_u(r0) unr(X86_DIV, r0)
232# define divremr(r0, r1, r2, i0, i1) _divremr(_jit, r0, r1, r2, i0, i1)
233static void
234_divremr(jit_state_t*,jit_int32_t,jit_int32_t,jit_int32_t,
235 jit_bool_t,jit_bool_t);
236# define divremi(r0, r1, i0, i1, i2) _divremi(_jit, r0, r1, i0, i1, i2)
237static void
238_divremi(jit_state_t*,jit_int32_t,jit_int32_t,jit_word_t,jit_bool_t,jit_bool_t);
239# define divr(r0, r1, r2) divremr(r0, r1, r2, 1, 1)
240# define divi(r0, r1, i0) divremi(r0, r1, i0, 1, 1)
241# define divr_u(r0, r1, r2) divremr(r0, r1, r2, 0, 1)
242# define divi_u(r0, r1, i0) divremi(r0, r1, i0, 0, 1)
243# define qdivr(r0, r1, r2, r3) _iqdivr(_jit, r0, r1, r2, r3, 1)
244# define qdivr_u(r0, r1, r2, r3) _iqdivr(_jit, r0, r1, r2, r3, 0)
245# define iqdivr(r0, r1, r2, r3, sign) _iqdivr(_jit, r0, r1, r2, r3, sign)
246static void _iqdivr(jit_state_t*, jit_int32_t, jit_int32_t,
247 jit_int32_t,jit_int32_t, jit_bool_t);
248# define qdivi(r0, r1, r2, i0) _iqdivi(_jit, r0, r1, r2, i0, 1)
249# define qdivi_u(r0, r1, r2, i0) _iqdivi(_jit, r0, r1, r2, i0, 0)
250# define iqdivi(r0, r1, r2, i0, sign) _iqdivi(_jit, r0, r1, r2, i0, sign)
251static void _iqdivi(jit_state_t*, jit_int32_t, jit_int32_t,
252 jit_int32_t,jit_word_t, jit_bool_t);
253# define remr(r0, r1, r2) divremr(r0, r1, r2, 1, 0)
254# define remi(r0, r1, i0) divremi(r0, r1, i0, 1, 0)
255# define remr_u(r0, r1, r2) divremr(r0, r1, r2, 0, 0)
256# define remi_u(r0, r1, i0) divremi(r0, r1, i0, 0, 0)
257# define iandr(r0, r1) alur(X86_AND, r0, r1)
258# define andr(r0, r1, r2) _andr(_jit, r0, r1, r2)
259static void _andr(jit_state_t*,jit_int32_t,jit_int32_t,jit_int32_t);
260# define iandi(r0, i0) alui(X86_AND, r0, i0)
261# define andi(r0, r1, i0) _andi(_jit, r0, r1, i0)
262static void _andi(jit_state_t*, jit_int32_t,jit_int32_t,jit_word_t);
263# define iorr(r0, r1) alur(X86_OR, r0, r1)
264# define orr(r0, r1, r2) _orr(_jit, r0, r1, r2)
265static void _orr(jit_state_t*, jit_int32_t,jit_int32_t,jit_int32_t);
266# define iori(r0, i0) alui(X86_OR, r0, i0)
267# define ori(r0, r1, i0) _ori(_jit, r0, r1, i0)
268static void _ori(jit_state_t*, jit_int32_t,jit_int32_t,jit_word_t);
269# define ixorr(r0, r1) alur(X86_XOR, r0, r1)
270# define xorr(r0, r1, r2) _xorr(_jit, r0, r1, r2)
271static void _xorr(jit_state_t*, jit_int32_t,jit_int32_t,jit_int32_t);
272# define ixori(r0, i0) alui(X86_XOR, r0, i0)
273# define xori(r0, r1, i0) _xori(_jit, r0, r1, i0)
274static void _xori(jit_state_t*, jit_int32_t,jit_int32_t,jit_word_t);
275# define irotshr(code, r0) _irotshr(_jit, code, r0)
276static void _irotshr(jit_state_t*, jit_int32_t, jit_int32_t);
277# define rotshr(code, r0, r1, r2) _rotshr(_jit, code, r0, r1, r2)
278static void
279_rotshr(jit_state_t*,jit_int32_t,jit_int32_t,jit_int32_t,jit_int32_t);
280# define irotshi(code, r0, i0) _irotshi(_jit, code, r0, i0)
281static void _irotshi(jit_state_t*, jit_int32_t, jit_int32_t, jit_word_t);
282# define rotshi(code, r0, r1, i0) _rotshi(_jit, code, r0, r1, i0)
283static void
284_rotshi(jit_state_t*,jit_int32_t,jit_int32_t,jit_int32_t,jit_word_t);
285# define lshr(r0, r1, r2) rotshr(X86_SHL, r0, r1, r2)
286# define lshi(r0, r1, i0) _lshi(_jit, r0, r1, i0)
287static void _lshi(jit_state_t*, jit_int32_t, jit_int32_t, jit_word_t);
288# define rshr(r0, r1, r2) rotshr(X86_SAR, r0, r1, r2)
289# define rshi(r0, r1, i0) rotshi(X86_SAR, r0, r1, i0)
290# define rshr_u(r0, r1, r2) rotshr(X86_SHR, r0, r1, r2)
291# define rshi_u(r0, r1, i0) rotshi(X86_SHR, r0, r1, i0)
292# define unr(code, r0) _unr(_jit, code, r0)
293static void _unr(jit_state_t*, jit_int32_t, jit_int32_t);
294# define inegr(r0) unr(X86_NEG, r0)
295# define negr(r0, r1) _negr(_jit, r0, r1)
296static void _negr(jit_state_t*, jit_int32_t, jit_int32_t);
297# define icomr(r0) unr(X86_NOT, r0)
298# define comr(r0, r1) _comr(_jit, r0, r1)
299static void _comr(jit_state_t*, jit_int32_t, jit_int32_t);
300# if USE_INC_DEC
301# define incr(r0, r1) _incr(_jit, r0, r1)
302static void _incr(jit_state_t*, jit_int32_t, jit_int32_t);
303# define decr(r0, r1) _decr(_jit, r0, r1)
304static void _decr(jit_state_t*, jit_int32_t, jit_int32_t);
305# endif
79bfeef6
PC		 306# define clor(r0, r1) _clor(_jit, r0, r1)
307static void _clor(jit_state_t*, jit_int32_t, jit_int32_t);
308# define clzr(r0, r1) _clzr(_jit, r0, r1)
309static void _clzr(jit_state_t*, jit_int32_t, jit_int32_t);
310# define ctor(r0, r1) _ctor(_jit, r0, r1)
311static void _ctor(jit_state_t*, jit_int32_t, jit_int32_t);
312# define ctzr(r0, r1) _ctzr(_jit, r0, r1)
313static void _ctzr(jit_state_t*, jit_int32_t, jit_int32_t);
4a71579b
PC		 314# define cr(code, r0, r1, r2) _cr(_jit, code, r0, r1, r2)
315static void
316_cr(jit_state_t*, jit_int32_t, jit_int32_t, jit_int32_t, jit_int32_t);
317# define ci(code, r0, r1, i0) _ci(_jit, code, r0, r1, i0)
318static void
319_ci(jit_state_t *_jit, jit_int32_t, jit_int32_t, jit_int32_t, jit_word_t);
320# define ci0(code, r0, r1) _ci0(_jit, code, r0, r1)
321static void _ci0(jit_state_t*, jit_int32_t, jit_int32_t, jit_int32_t);
322# define ltr(r0, r1, r2) _ltr(_jit, r0, r1, r2)
323static void _ltr(jit_state_t*, jit_int32_t, jit_int32_t, jit_int32_t);
324# define lti(r0, r1, i0) _lti(_jit, r0, r1, i0)
325static void _lti(jit_state_t*, jit_int32_t, jit_int32_t, jit_word_t);
326# define ltr_u(r0, r1, r2) _ltr_u(_jit, r0, r1, r2)
327static void _ltr_u(jit_state_t*, jit_int32_t, jit_int32_t, jit_int32_t);
328# define lti_u(r0, r1, i0) ci(X86_CC_B, r0, r1, i0)
329# define ler(r0, r1, r2) _ler(_jit, r0, r1, r2)
330static void _ler(jit_state_t*, jit_int32_t, jit_int32_t, jit_int32_t);
331# define lei(r0, r1, i0) ci(X86_CC_LE, r0, r1, i0)
332# define ler_u(r0, r1, r2) _ler_u(_jit, r0, r1, r2)
333static void _ler_u(jit_state_t*, jit_int32_t, jit_int32_t, jit_int32_t);
334# define lei_u(r0, r1, i0) _lei_u(_jit, r0, r1, i0)
335static void _lei_u(jit_state_t*, jit_int32_t, jit_int32_t, jit_word_t);
336# define eqr(r0, r1, r2) _eqr(_jit, r0, r1, r2)
337static void _eqr(jit_state_t*, jit_int32_t, jit_int32_t, jit_int32_t);
338# define eqi(r0, r1, i0) _eqi(_jit, r0, r1, i0)
339static void _eqi(jit_state_t*, jit_int32_t, jit_int32_t, jit_word_t);
340# define ger(r0, r1, r2) _ger(_jit, r0, r1, r2)
341static void _ger(jit_state_t*, jit_int32_t, jit_int32_t, jit_int32_t);
342# define gei(r0, r1, i0) _gei(_jit, r0, r1, i0)
343static void _gei(jit_state_t*, jit_int32_t, jit_int32_t, jit_word_t);
344# define ger_u(r0, r1, r2) _ger_u(_jit, r0, r1, r2)
345static void _ger_u(jit_state_t*, jit_int32_t, jit_int32_t, jit_int32_t);
346# define gei_u(r0, r1, i0) _gei_u(_jit, r0, r1, i0)
347static void _gei_u(jit_state_t*, jit_int32_t, jit_int32_t, jit_word_t);
348# define gtr(r0, r1, r2) _gtr(_jit, r0, r1, r2)
349static void _gtr(jit_state_t*, jit_int32_t, jit_int32_t, jit_int32_t);
350# define gti(r0, r1, i0) _ci(_jit, X86_CC_G, r0, r1, i0)
351# define gtr_u(r0, r1, r2) _gtr_u(_jit, r0, r1, r2)
352static void _gtr_u(jit_state_t*, jit_int32_t, jit_int32_t, jit_int32_t);
353# define gti_u(r0, r1, i0) _gti_u(_jit, r0, r1, i0)
354static void _gti_u(jit_state_t*, jit_int32_t, jit_int32_t, jit_word_t);
355# define ner(r0, r1, r2) _ner(_jit, r0, r1, r2)
356static void _ner(jit_state_t*, jit_int32_t, jit_int32_t, jit_int32_t);
357# define nei(r0, r1, i0) _nei(_jit, r0, r1, i0)
358static void _nei(jit_state_t*, jit_int32_t, jit_int32_t, jit_word_t);
359# define movr(r0, r1) _movr(_jit, r0, r1)
360static void _movr(jit_state_t*, jit_int32_t, jit_int32_t);
361# define imovi(r0, i0) _imovi(_jit, r0, i0)
362static void _imovi(jit_state_t*, jit_int32_t, jit_word_t);
363# define movi(r0, i0) _movi(_jit, r0, i0)
79bfeef6
PC		 364static
365# if CAN_RIP_ADDRESS
366jit_word_t
367# else
368void
369# endif
370_movi(jit_state_t*, jit_int32_t, jit_word_t);
4a71579b
PC		 371# define movi_p(r0, i0) _movi_p(_jit, r0, i0)
372static jit_word_t _movi_p(jit_state_t*, jit_int32_t, jit_word_t);
373# define movcr(r0, r1) _movcr(_jit, r0, r1)
374static void _movcr(jit_state_t*,jit_int32_t,jit_int32_t);
375# define movcr_u(r0, r1) _movcr_u(_jit, r0, r1)
376static void _movcr_u(jit_state_t*,jit_int32_t,jit_int32_t);
377# define movsr(r0, r1) _movsr(_jit, r0, r1)
378static void _movsr(jit_state_t*,jit_int32_t,jit_int32_t);
379# define movsr_u(r0, r1) _movsr_u(_jit, r0, r1)
380static void _movsr_u(jit_state_t*,jit_int32_t,jit_int32_t);
ba3814c1
PC		 381# define casx(r0, r1, r2, r3, i0) _casx(_jit, r0, r1, r2, r3, i0)
382static void _casx(jit_state_t *_jit,jit_int32_t,jit_int32_t,
383 jit_int32_t,jit_int32_t,jit_word_t);
384#define casr(r0, r1, r2, r3) casx(r0, r1, r2, r3, 0)
385#define casi(r0, i0, r1, r2) casx(r0, _NOREG, r1, r2, i0)
1f22b268
PC		 386#define movnr(r0, r1, r2) _movnr(_jit, r0, r1, r2)
387static void _movnr(jit_state_t*, jit_int32_t, jit_int32_t, jit_int32_t);
388#define movzr(r0, r1, r2) _movzr(_jit, r0, r1, r2)
389static void _movzr(jit_state_t*, jit_int32_t, jit_int32_t, jit_int32_t);
4a71579b
PC		 390# if __X64 && !__X64_32
391# define movir(r0, r1) _movir(_jit, r0, r1)
392static void _movir(jit_state_t*,jit_int32_t,jit_int32_t);
393# define movir_u(r0, r1) _movir_u(_jit, r0, r1)
394static void _movir_u(jit_state_t*,jit_int32_t,jit_int32_t);
395# endif
40a44dcb
PC		 396# define bswapr_us(r0, r1) _bswapr_us(_jit, r0, r1)
397static void _bswapr_us(jit_state_t*,jit_int32_t,jit_int32_t);
398# define bswapr_ui(r0, r1) _bswapr_ui(_jit, r0, r1)
399static void _bswapr_ui(jit_state_t*,jit_int32_t,jit_int32_t);
4a71579b 400# if __X64 && !__X64_32
40a44dcb
PC		 401#define bswapr_ul(r0, r1) _bswapr_ul(_jit, r0, r1)
402static void _bswapr_ul(jit_state_t*,jit_int32_t,jit_int32_t);
4a71579b
PC		 403#endif
404# define extr_c(r0, r1) _extr_c(_jit, r0, r1)
405static void _extr_c(jit_state_t*,jit_int32_t,jit_int32_t);
406# define extr_uc(r0, r1) _extr_uc(_jit, r0, r1)
407static void _extr_uc(jit_state_t*,jit_int32_t,jit_int32_t);
408# define extr_s(r0, r1) movsr(r0, r1)
409# define extr_us(r0, r1) movsr_u(r0, r1)
410# if __X64 && !__X64_32
411# define extr_i(r0, r1) movir(r0, r1)
412# define extr_ui(r0, r1) movir_u(r0, r1)
413# endif
414# define ldr_c(r0, r1) _ldr_c(_jit, r0, r1)
415static void _ldr_c(jit_state_t*, jit_int32_t, jit_int32_t);
416# define ldi_c(r0, i0) _ldi_c(_jit, r0, i0)
417static void _ldi_c(jit_state_t*, jit_int32_t, jit_word_t);
418# define ldr_uc(r0, r1) _ldr_uc(_jit, r0, r1)
419static void _ldr_uc(jit_state_t*, jit_int32_t, jit_int32_t);
420# define ldi_uc(r0, i0) _ldi_uc(_jit, r0, i0)
421static void _ldi_uc(jit_state_t*, jit_int32_t, jit_word_t);
422# define ldr_s(r0, r1) _ldr_s(_jit, r0, r1)
423static void _ldr_s(jit_state_t*, jit_int32_t, jit_int32_t);
424# define ldi_s(r0, i0) _ldi_s(_jit, r0, i0)
425static void _ldi_s(jit_state_t*, jit_int32_t, jit_word_t);
426# define ldr_us(r0, r1) _ldr_us(_jit, r0, r1)
427static void _ldr_us(jit_state_t*, jit_int32_t, jit_int32_t);
428# define ldi_us(r0, i0) _ldi_us(_jit, r0, i0)
429static void _ldi_us(jit_state_t*, jit_int32_t, jit_word_t);
430# if __X32 || !__X64_32
431# define ldr_i(r0, r1) _ldr_i(_jit, r0, r1)
432static void _ldr_i(jit_state_t*, jit_int32_t, jit_int32_t);
433# define ldi_i(r0, i0) _ldi_i(_jit, r0, i0)
434static void _ldi_i(jit_state_t*, jit_int32_t, jit_word_t);
435# endif
436# if __X64
437# if __X64_32
438# define ldr_i(r0, r1) _ldr_ui(_jit, r0, r1)
439# define ldi_i(r0, i0) _ldi_ui(_jit, r0, i0)
440# else
441# define ldr_ui(r0, r1) _ldr_ui(_jit, r0, r1)
442# define ldi_ui(r0, i0) _ldi_ui(_jit, r0, i0)
443# endif
444static void _ldr_ui(jit_state_t*, jit_int32_t, jit_int32_t);
445static void _ldi_ui(jit_state_t*, jit_int32_t, jit_word_t);
446# if !__X64_32
447# define ldr_l(r0, r1) _ldr_l(_jit, r0, r1)
448static void _ldr_l(jit_state_t*, jit_int32_t, jit_int32_t);
449# define ldi_l(r0, i0) _ldi_l(_jit, r0, i0)
450static void _ldi_l(jit_state_t*, jit_int32_t, jit_word_t);
451# endif
452# endif
453# define ldxr_c(r0, r1, r2) _ldxr_c(_jit, r0, r1, r2)
454static void _ldxr_c(jit_state_t*, jit_int32_t, jit_int32_t, jit_int32_t);
455# define ldxi_c(r0, r1, i0) _ldxi_c(_jit, r0, r1, i0)
456static void _ldxi_c(jit_state_t*, jit_int32_t, jit_int32_t, jit_word_t);
457# define ldxr_uc(r0, r1, r2) _ldxr_uc(_jit, r0, r1, r2)
458static void _ldxr_uc(jit_state_t*, jit_int32_t, jit_int32_t, jit_int32_t);
459# define ldxi_uc(r0, r1, i0) _ldxi_uc(_jit, r0, r1, i0)
460static void _ldxi_uc(jit_state_t*, jit_int32_t, jit_int32_t, jit_word_t);
461# define ldxr_s(r0, r1, r2) _ldxr_s(_jit, r0, r1, r2)
462static void _ldxr_s(jit_state_t*, jit_int32_t, jit_int32_t, jit_int32_t);
463# define ldxi_s(r0, r1, i0) _ldxi_s(_jit, r0, r1, i0)
464static void _ldxi_s(jit_state_t*, jit_int32_t, jit_int32_t, jit_word_t);
465# define ldxr_us(r0, r1, r2) _ldxr_us(_jit, r0, r1, r2)
466static void _ldxr_us(jit_state_t*, jit_int32_t, jit_int32_t, jit_int32_t);
467# define ldxi_us(r0, r1, i0) _ldxi_us(_jit, r0, r1, i0)
468static void _ldxi_us(jit_state_t*, jit_int32_t, jit_int32_t, jit_word_t);
469# if __X32 || !__X64_32
470# define ldxr_i(r0, r1, r2) _ldxr_i(_jit, r0, r1, r2)
471static void _ldxr_i(jit_state_t*, jit_int32_t, jit_int32_t, jit_int32_t);
472# define ldxi_i(r0, r1, i0) _ldxi_i(_jit, r0, r1, i0)
473static void _ldxi_i(jit_state_t*, jit_int32_t, jit_int32_t, jit_word_t);
474# endif
475# if __X64
476# if __X64_32
477# define ldxr_i(r0, r1, r2) _ldxr_ui(_jit, r0, r1, r2)
478# define ldxi_i(r0, r1, i0) _ldxi_ui(_jit, r0, r1, i0)
479# else
480# define ldxr_ui(r0, r1, r2) _ldxr_ui(_jit, r0, r1, r2)
481# define ldxi_ui(r0, r1, i0) _ldxi_ui(_jit, r0, r1, i0)
482# endif
483static void _ldxr_ui(jit_state_t*, jit_int32_t, jit_int32_t, jit_int32_t);
484static void _ldxi_ui(jit_state_t*, jit_int32_t, jit_int32_t, jit_word_t);
485# if !__X64_32
486# define ldxr_l(r0, r1, r2) _ldxr_l(_jit, r0, r1, r2)
487static void _ldxr_l(jit_state_t*, jit_int32_t, jit_int32_t, jit_int32_t);
488# define ldxi_l(r0, r1, i0) _ldxi_l(_jit, r0, r1, i0)
489static void _ldxi_l(jit_state_t*, jit_int32_t, jit_int32_t, jit_word_t);
490# endif
491# endif
492# define str_c(r0, r1) _str_c(_jit, r0, r1)
493static void _str_c(jit_state_t*, jit_int32_t, jit_int32_t);
494# define sti_c(i0, r0) _sti_c(_jit, i0, r0)
495static void _sti_c(jit_state_t*, jit_word_t, jit_int32_t);
496# define str_s(r0, r1) _str_s(_jit, r0, r1)
497static void _str_s(jit_state_t*, jit_int32_t, jit_int32_t);
498# define sti_s(i0, r0) _sti_s(_jit, i0, r0)
499static void _sti_s(jit_state_t*, jit_word_t, jit_int32_t);
500# define str_i(r0, r1) _str_i(_jit, r0, r1)
501static void _str_i(jit_state_t*, jit_int32_t, jit_int32_t);
502# define sti_i(i0, r0) _sti_i(_jit, i0, r0)
503static void _sti_i(jit_state_t*, jit_word_t, jit_int32_t);
504# if __X64 && !__X64_32
505# define str_l(r0, r1) _str_l(_jit, r0, r1)
506static void _str_l(jit_state_t*, jit_int32_t, jit_int32_t);
507# define sti_l(i0, r0) _sti_l(_jit, i0, r0)
508static void _sti_l(jit_state_t*, jit_word_t, jit_int32_t);
509# endif
510# define stxr_c(r0, r1, r2) _stxr_c(_jit, r0, r1, r2)
511static void _stxr_c(jit_state_t*, jit_int32_t, jit_int32_t, jit_int32_t);
512# define stxi_c(i0, r0, r1) _stxi_c(_jit, i0, r0, r1)
513static void _stxi_c(jit_state_t*, jit_word_t, jit_int32_t, jit_int32_t);
514# define stxr_s(r0, r1, r2) _stxr_s(_jit, r0, r1, r2)
515static void _stxr_s(jit_state_t*, jit_int32_t, jit_int32_t, jit_int32_t);
516# define stxi_s(i0, r0, r1) _stxi_s(_jit, i0, r0, r1)
517static void _stxi_s(jit_state_t*, jit_word_t, jit_int32_t, jit_int32_t);
518# define stxr_i(r0, r1, r2) _stxr_i(_jit, r0, r1, r2)
519static void _stxr_i(jit_state_t*, jit_int32_t, jit_int32_t, jit_int32_t);
520# define stxi_i(i0, r0, r1) _stxi_i(_jit, i0, r0, r1)
521static void _stxi_i(jit_state_t*, jit_word_t, jit_int32_t, jit_int32_t);
522# if __X64 && !__X64_32
523# define stxr_l(r0, r1, r2) _stxr_l(_jit, r0, r1, r2)
524static void _stxr_l(jit_state_t*, jit_int32_t, jit_int32_t, jit_int32_t);
525# define stxi_l(i0, r0, r1) _stxi_l(_jit, i0, r0, r1)
526static void _stxi_l(jit_state_t*, jit_word_t, jit_int32_t, jit_int32_t);
527# endif
528# define jcc(code, i0) _jcc(_jit, code, i0)
529# define jo(i0) jcc(X86_CC_O, i0)
530# define jno(i0) jcc(X86_CC_NO, i0)
531# define jnae(i0) jcc(X86_CC_NAE, i0)
532# define jb(i0) jcc(X86_CC_B, i0)
533# define jc(i0) jcc(X86_CC_C, i0)
534# define jae(i0) jcc(X86_CC_AE, i0)
535# define jnb(i0) jcc(X86_CC_NB, i0)
536# define jnc(i0) jcc(X86_CC_NC, i0)
537# define je(i0) jcc(X86_CC_E, i0)
538# define jz(i0) jcc(X86_CC_Z, i0)
539# define jne(i0) jcc(X86_CC_NE, i0)
540# define jnz(i0) jcc(X86_CC_NZ, i0)
541# define jbe(i0) jcc(X86_CC_BE, i0)
542# define jna(i0) jcc(X86_CC_NA, i0)
543# define ja(i0) jcc(X86_CC_A, i0)
544# define jnbe(i0) jcc(X86_CC_NBE, i0)
545# define js(i0) jcc(X86_CC_S, i0)
546# define jns(i0) jcc(X86_CC_NS, i0)
547# define jp(i0) jcc(X86_CC_P, i0)
548# define jpe(i0) jcc(X86_CC_PE, i0)
549# define jnp(i0) jcc(X86_CC_NP, i0)
550# define jpo(i0) jcc(X86_CC_PO, i0)
551# define jl(i0) jcc(X86_CC_L, i0)
552# define jnge(i0) jcc(X86_CC_NGE, i0)
553# define jge(i0) jcc(X86_CC_GE, i0)
554# define jnl(i0) jcc(X86_CC_NL, i0)
555# define jle(i0) jcc(X86_CC_LE, i0)
556# define jng(i0) jcc(X86_CC_NG, i0)
557# define jg(i0) jcc(X86_CC_G, i0)
558# define jnle(i0) jcc(X86_CC_NLE, i0)
79bfeef6 559static jit_word_t _jcc(jit_state_t*, jit_int32_t, jit_word_t);
4a71579b
PC		 560# define jccs(code, i0) _jccs(_jit, code, i0)
561# define jos(i0) jccs(X86_CC_O, i0)
562# define jnos(i0) jccs(X86_CC_NO, i0)
563# define jnaes(i0) jccs(X86_CC_NAE, i0)
564# define jbs(i0) jccs(X86_CC_B, i0)
565# define jcs(i0) jccs(X86_CC_C, i0)
566# define jaes(i0) jccs(X86_CC_AE, i0)
567# define jnbs(i0) jccs(X86_CC_NB, i0)
568# define jncs(i0) jccs(X86_CC_NC, i0)
569# define jes(i0) jccs(X86_CC_E, i0)
570# define jzs(i0) jccs(X86_CC_Z, i0)
571# define jnes(i0) jccs(X86_CC_NE, i0)
572# define jnzs(i0) jccs(X86_CC_NZ, i0)
573# define jbes(i0) jccs(X86_CC_BE, i0)
574# define jnas(i0) jccs(X86_CC_NA, i0)
575# define jas(i0) jccs(X86_CC_A, i0)
576# define jnbes(i0) jccs(X86_CC_NBE, i0)
577# define jss(i0) jccs(X86_CC_S, i0)
578# define jnss(i0) jccs(X86_CC_NS, i0)
579# define jps(i0) jccs(X86_CC_P, i0)
580# define jpes(i0) jccs(X86_CC_PE, i0)
581# define jnps(i0) jccs(X86_CC_NP, i0)
582# define jpos(i0) jccs(X86_CC_PO, i0)
583# define jls(i0) jccs(X86_CC_L, i0)
584# define jnges(i0) jccs(X86_CC_NGE, i0)
585# define jges(i0) jccs(X86_CC_GE, i0)
586# define jnls(i0) jccs(X86_CC_NL, i0)
587# define jles(i0) jccs(X86_CC_LE, i0)
588# define jngs(i0) jccs(X86_CC_NG, i0)
589# define jgs(i0) jccs(X86_CC_G, i0)
590# define jnles(i0) jccs(X86_CC_NLE, i0)
79bfeef6 591static jit_word_t _jccs(jit_state_t*, jit_int32_t, jit_word_t);
4a71579b 592# define jcr(code, i0, r0, r1) _jcr(_jit, code, i0, r0, r1)
79bfeef6
PC		 593static jit_word_t _jcr(jit_state_t*,
594 jit_int32_t,jit_word_t,jit_int32_t,jit_int32_t);
4a71579b 595# define jci(code, i0, r0, i1) _jci(_jit, code, i0, r0, i1)
79bfeef6
PC		 596static jit_word_t _jci(jit_state_t*,
597 jit_int32_t,jit_word_t,jit_int32_t,jit_word_t);
4a71579b 598# define jci0(code, i0, r0) _jci0(_jit, code, i0, r0)
79bfeef6 599static jit_word_t _jci0(jit_state_t*, jit_int32_t, jit_word_t, jit_int32_t);
4a71579b
PC		 600# define bltr(i0, r0, r1) _bltr(_jit, i0, r0, r1)
601static jit_word_t _bltr(jit_state_t*, jit_word_t, jit_int32_t, jit_int32_t);
602# define blti(i0, r0, i1) _blti(_jit, i0, r0, i1)
603static jit_word_t _blti(jit_state_t*, jit_word_t, jit_int32_t, jit_word_t);
604# define bltr_u(i0, r0, r1) _bltr_u(_jit, i0, r0, r1)
605static jit_word_t _bltr_u(jit_state_t*, jit_word_t, jit_int32_t, jit_int32_t);
606# define blti_u(i0, r0, i1) _blti_u(_jit, i0, r0, i1)
607static jit_word_t _blti_u(jit_state_t*, jit_word_t, jit_int32_t, jit_word_t);
608# define bler(i0, r0, r1) _bler(_jit, i0, r0, r1)
609static jit_word_t _bler(jit_state_t*, jit_word_t, jit_int32_t, jit_int32_t);
610# define blei(i0, r0, i1) _blei(_jit, i0, r0, i1)
611static jit_word_t _blei(jit_state_t*, jit_word_t, jit_int32_t, jit_word_t);
612# define bler_u(i0, r0, r1) _bler_u(_jit, i0, r0, r1)
613static jit_word_t _bler_u(jit_state_t*, jit_word_t, jit_int32_t, jit_int32_t);
614# define blei_u(i0, r0, i1) _blei_u(_jit, i0, r0, i1)
615static jit_word_t _blei_u(jit_state_t*, jit_word_t, jit_int32_t, jit_word_t);
616# define beqr(i0, r0, r1) _beqr(_jit, i0, r0, r1)
617static jit_word_t _beqr(jit_state_t*, jit_word_t, jit_int32_t, jit_int32_t);
618# define beqi(i0, r0, i1) _beqi(_jit, i0, r0, i1)
619static jit_word_t _beqi(jit_state_t*, jit_word_t, jit_int32_t, jit_word_t);
620# define bger(i0, r0, r1) _bger(_jit, i0, r0, r1)
621static jit_word_t _bger(jit_state_t*, jit_word_t, jit_int32_t, jit_int32_t);
622# define bgei(i0, r0, i1) _bgei(_jit, i0, r0, i1)
623static jit_word_t _bgei(jit_state_t*, jit_word_t, jit_int32_t, jit_word_t);
624# define bger_u(i0, r0, r1) _bger_u(_jit, i0, r0, r1)
625static jit_word_t _bger_u(jit_state_t*, jit_word_t, jit_int32_t, jit_int32_t);
626# define bgei_u(i0, r0, i1) _bgei_u(_jit, i0, r0, i1)
627static jit_word_t _bgei_u(jit_state_t*, jit_word_t, jit_int32_t, jit_word_t);
628# define bgtr(i0, r0, r1) _bgtr(_jit, i0, r0, r1)
629static jit_word_t _bgtr(jit_state_t*, jit_word_t, jit_int32_t, jit_int32_t);
630# define bgti(i0, r0, i1) _bgti(_jit, i0, r0, i1)
631static jit_word_t _bgti(jit_state_t*, jit_word_t, jit_int32_t, jit_word_t);
632# define bgtr_u(i0, r0, r1) _bgtr_u(_jit, i0, r0, r1)
633static jit_word_t _bgtr_u(jit_state_t*, jit_word_t, jit_int32_t, jit_int32_t);
634# define bgti_u(i0, r0, i1) _bgti_u(_jit, i0, r0, i1)
635static jit_word_t _bgti_u(jit_state_t*, jit_word_t, jit_int32_t, jit_word_t);
636# define bner(i0, r0, r1) _bner(_jit, i0, r0, r1)
637static jit_word_t _bner(jit_state_t*, jit_word_t, jit_int32_t, jit_int32_t);
638# define bnei(i0, r0, i1) _bnei(_jit, i0, r0, i1)
639static jit_word_t _bnei(jit_state_t*, jit_word_t, jit_int32_t, jit_word_t);
640# define bmsr(i0, r0, r1) _bmsr(_jit, i0, r0, r1)
641static jit_word_t _bmsr(jit_state_t*,jit_word_t,jit_int32_t,jit_int32_t);
642# define bmsi(i0, r0, i1) _bmsi(_jit, i0, r0, i1)
643static jit_word_t _bmsi(jit_state_t*,jit_word_t,jit_int32_t,jit_word_t);
644# define bmcr(i0, r0, r1) _bmcr(_jit, i0, r0, r1)
645static jit_word_t _bmcr(jit_state_t*,jit_word_t,jit_int32_t,jit_int32_t);
646# define bmci(i0, r0, i1) _bmci(_jit, i0, r0, i1)
647static jit_word_t _bmci(jit_state_t*,jit_word_t,jit_int32_t,jit_word_t);
648# define boaddr(i0, r0, r1) _boaddr(_jit, i0, r0, r1)
649static jit_word_t _boaddr(jit_state_t*,jit_word_t,jit_int32_t,jit_int32_t);
650# define boaddi(i0, r0, i1) _boaddi(_jit, i0, r0, i1)
651static jit_word_t _boaddi(jit_state_t*,jit_word_t,jit_int32_t,jit_word_t);
652# define boaddr_u(i0, r0, r1) _boaddr_u(_jit, i0, r0, r1)
653static jit_word_t _boaddr_u(jit_state_t*,jit_word_t,jit_int32_t,jit_int32_t);
654# define boaddi_u(i0, r0, i1) _boaddi_u(_jit, i0, r0, i1)
655static jit_word_t _boaddi_u(jit_state_t*,jit_word_t,jit_int32_t,jit_word_t);
656# define bxaddr(i0, r0, r1) _bxaddr(_jit, i0, r0, r1)
657static jit_word_t _bxaddr(jit_state_t*,jit_word_t,jit_int32_t,jit_int32_t);
658# define bxaddi(i0, r0, i1) _bxaddi(_jit, i0, r0, i1)
659static jit_word_t _bxaddi(jit_state_t*,jit_word_t,jit_int32_t,jit_word_t);
660# define bxaddr_u(i0, r0, r1) _bxaddr_u(_jit, i0, r0, r1)
661static jit_word_t _bxaddr_u(jit_state_t*,jit_word_t,jit_int32_t,jit_int32_t);
662# define bxaddi_u(i0, r0, i1) _bxaddi_u(_jit, i0, r0, i1)
663static jit_word_t _bxaddi_u(jit_state_t*,jit_word_t,jit_int32_t,jit_word_t);
664# define bosubr(i0, r0, r1) _bosubr(_jit, i0, r0, r1)
665static jit_word_t _bosubr(jit_state_t*,jit_word_t,jit_int32_t,jit_int32_t);
666# define bosubi(i0, r0, i1) _bosubi(_jit, i0, r0, i1)
667static jit_word_t _bosubi(jit_state_t*,jit_word_t,jit_int32_t,jit_word_t);
668# define bosubr_u(i0, r0, r1) _bosubr_u(_jit, i0, r0, r1)
669static jit_word_t _bosubr_u(jit_state_t*,jit_word_t,jit_int32_t,jit_int32_t);
670# define bosubi_u(i0, r0, i1) _bosubi_u(_jit, i0, r0, i1)
671static jit_word_t _bosubi_u(jit_state_t*,jit_word_t,jit_int32_t,jit_word_t);
672# define bxsubr(i0, r0, r1) _bxsubr(_jit, i0, r0, r1)
673static jit_word_t _bxsubr(jit_state_t*,jit_word_t,jit_int32_t,jit_int32_t);
674# define bxsubi(i0, r0, i1) _bxsubi(_jit, i0, r0, i1)
675static jit_word_t _bxsubi(jit_state_t*,jit_word_t,jit_int32_t,jit_word_t);
676# define bxsubr_u(i0, r0, r1) _bxsubr_u(_jit, i0, r0, r1)
677static jit_word_t _bxsubr_u(jit_state_t*,jit_word_t,jit_int32_t,jit_int32_t);
678# define bxsubi_u(i0, r0, i1) _bxsubi_u(_jit, i0, r0, i1)
679static jit_word_t _bxsubi_u(jit_state_t*,jit_word_t,jit_int32_t,jit_word_t);
680# define callr(r0) _callr(_jit, r0)
681static void _callr(jit_state_t*, jit_int32_t);
682# define calli(i0) _calli(_jit, i0)
683static jit_word_t _calli(jit_state_t*, jit_word_t);
519a9ea1
PC		 684# if __X64
685# define calli_p(i0) _calli_p(_jit, i0)
686static jit_word_t _calli_p(jit_state_t*, jit_word_t);
687# else
688# define calli_p(i0) calli(i0)
689# endif
4a71579b
PC		 690# define jmpr(r0) _jmpr(_jit, r0)
691static void _jmpr(jit_state_t*, jit_int32_t);
692# define jmpi(i0) _jmpi(_jit, i0)
693static jit_word_t _jmpi(jit_state_t*, jit_word_t);
519a9ea1
PC		 694# if __X64
695# define jmpi_p(i0) _jmpi_p(_jit, i0)
696static jit_word_t _jmpi_p(jit_state_t*, jit_word_t);
697# else
698# define jmpi_p(i0) jmpi(i0)
699# endif
4a71579b 700# define jmpsi(i0) _jmpsi(_jit, i0)
79bfeef6 701static jit_word_t _jmpsi(jit_state_t*, jit_uint8_t);
4a71579b
PC		 702# define prolog(node) _prolog(_jit, node)
703static void _prolog(jit_state_t*, jit_node_t*);
704# define epilog(node) _epilog(_jit, node)
705static void _epilog(jit_state_t*, jit_node_t*);
706# define vastart(r0) _vastart(_jit, r0)
707static void _vastart(jit_state_t*, jit_int32_t);
708# define vaarg(r0, r1) _vaarg(_jit, r0, r1)
709static void _vaarg(jit_state_t*, jit_int32_t, jit_int32_t);
710# define vaarg_d(r0, r1, i0) _vaarg_d(_jit, r0, r1, i0)
711static void _vaarg_d(jit_state_t*, jit_int32_t, jit_int32_t, jit_bool_t);
79bfeef6
PC		 712# define patch_at(instr, label) _patch_at(_jit, instr, label)
713static void _patch_at(jit_state_t*, jit_word_t, jit_word_t);
4a71579b
PC		 714# if !defined(HAVE_FFSL)
715# if __X32
716# define ffsl(i) __builtin_ffs(i)
717# else
718# define ffsl(l) __builtin_ffsl(l)
719# endif
720# endif
1f22b268 721# define jit_cmov_p() jit_cpu.cmov
4a71579b
PC		 722#endif
723
724#if CODE
725static void
726_rex(jit_state_t *_jit, jit_int32_t l, jit_int32_t w,
727 jit_int32_t r, jit_int32_t x, jit_int32_t b)
728{
729#if __X64
730 jit_int32_t v = 0x40 | (w << 3);
731
732 if (r != _NOREG)
733 v |= (r & 8) >> 1;
734 if (x != _NOREG)
735 v |= (x & 8) >> 2;
736 if (b != _NOREG)
737 v |= (b & 8) >> 3;
738 if (l || v != 0x40)
739 ic(v);
740#endif
741}
742
743static void
744_rx(jit_state_t *_jit, jit_int32_t rd, jit_int32_t md,
745 jit_int32_t rb, jit_int32_t ri, jit_int32_t ms)
746{
747 if (ri == _NOREG) {
748 if (rb == _NOREG) {
79bfeef6
PC		 749 /* Use ms == _SCL8 to tell it is a %rip relative displacement */
750#if __X64
751 if (ms == _SCL8)
752#endif
753 mrm(0x00, r7(rd), 0x05);
754#if __X64
755 else {
756 mrm(0x00, r7(rd), 0x04);
757 sib(_SCL1, 0x04, 0x05);
758 }
4a71579b
PC		 759#endif
760 ii(md);
761 }
762 else if (r7(rb) == _RSP_REGNO) {
763 if (md == 0) {
764 mrm(0x00, r7(rd), 0x04);
765 sib(ms, 0x04, 0x04);
766 }
767 else if ((jit_int8_t)md == md) {
768 mrm(0x01, r7(rd), 0x04);
769 sib(ms, 0x04, 0x04);
770 ic(md);
771 }
772 else {
773 mrm(0x02, r7(rd), 0x04);
774 sib(ms, 0x04, 0x04);
775 ii(md);
776 }
777 }
778 else {
779 if (md == 0 && r7(rb) != _RBP_REGNO)
780 mrm(0x00, r7(rd), r7(rb));
781 else if ((jit_int8_t)md == md) {
782 mrm(0x01, r7(rd), r7(rb));
783 ic(md);
784 }
785 else {
786 mrm(0x02, r7(rd), r7(rb));
787 ii(md);
788 }
789 }
790 }
791 else if (rb == _NOREG) {
792 mrm(0x00, r7(rd), 0x04);
793 sib(ms, r7(ri), 0x05);
794 ii(md);
795 }
796 else if (r8(ri) != _RSP_REGNO) {
797 if (md == 0 && r7(rb) != _RBP_REGNO) {
798 mrm(0x00, r7(rd), 0x04);
799 sib(ms, r7(ri), r7(rb));
800 }
801 else if ((jit_int8_t)md == md) {
802 mrm(0x01, r7(rd), 0x04);
803 sib(ms, r7(ri), r7(rb));
804 ic(md);
805 }
806 else {
807 mrm(0x02, r7(rd), 0x04);
808 sib(ms, r7(ri), r7(rb));
809 ic(md);
810 }
811 }
812 else {
813 fprintf(stderr, "illegal index register");
814 abort();
815 }
816}
817
818static void
819_nop(jit_state_t *_jit, jit_int32_t count)
820{
c0c16242
PC		 821 jit_int32_t i;
822 while (count) {
823 if (count > 9)
824 i = 9;
825 else
826 i = count;
827 switch (i) {
828 case 0:
829 break;
830 case 1: /* NOP */
831 ic(0x90); break;
832 case 2: /* 66 NOP */
833 ic(0x66); ic(0x90);
834 break;
835 case 3: /* NOP DWORD ptr [EAX] */
836 ic(0x0f); ic(0x1f); ic(0x00);
837 break;
838 case 4: /* NOP DWORD ptr [EAX + 00H] */
839 ic(0x0f); ic(0x1f); ic(0x40); ic(0x00);
840 break;
841 case 5: /* NOP DWORD ptr [EAX + EAX*1 + 00H] */
842 ic(0x0f); ic(0x1f); ic(0x44); ic(0x00);
843 ic(0x00);
844 break;
845 case 6: /* 66 NOP DWORD ptr [EAX + EAX*1 + 00H] */
846 ic(0x66); ic(0x0f); ic(0x1f); ic(0x44);
847 ic(0x00); ic(0x00);
848 break;
849 case 7: /* NOP DWORD ptr [EAX + 00000000H] */
850 ic(0x0f); ic(0x1f); ic(0x80); ii(0x0000);
851 break;
852 case 8: /* NOP DWORD ptr [EAX + EAX*1 + 00000000H] */
853 ic(0x0f); ic(0x1f); ic(0x84); ic(0x00);
854 ii(0x0000);
855 break;
856 case 9: /* 66 NOP DWORD ptr [EAX + EAX*1 + 00000000H] */
857 ic(0x66); ic(0x0f); ic(0x1f); ic(0x84);
858 ic(0x00); ii(0x0000);
859 break;
860 }
861 count -= i;
4a71579b
PC		 862 }
863}
4a71579b
PC		 864static void
865_lea(jit_state_t *_jit, jit_int32_t md, jit_int32_t rb,
866 jit_int32_t ri, jit_int32_t ms, jit_int32_t rd)
867{
868 rex(0, WIDE, rd, ri, rb);
869 ic(0x8d);
870 rx(rd, md, rb, ri, ms);
871}
872
873static void
874_pushr(jit_state_t *_jit, jit_int32_t r0)
875{
876 rex(0, WIDE, 0, 0, r0);
877 ic(0x50 | r7(r0));
878}
879
880static void
881_popr(jit_state_t *_jit, jit_int32_t r0)
882{
883 rex(0, WIDE, 0, 0, r0);
884 ic(0x58 | r7(r0));
885}
886
887static void
888_xchgr(jit_state_t *_jit, jit_int32_t r0, jit_int32_t r1)
889{
890 rex(0, WIDE, r1, _NOREG, r0);
891 ic(0x87);
892 mrm(0x03, r7(r1), r7(r0));
893}
894
895static void
896_testr(jit_state_t *_jit, jit_int32_t r0, jit_int32_t r1)
897{
898 rex(0, WIDE, r1, _NOREG, r0);
899 ic(0x85);
900 mrm(0x03, r7(r1), r7(r0));
901}
902
903static void
904_testi(jit_state_t *_jit, jit_int32_t r0, jit_word_t i0)
905{
906 rex(0, WIDE, _NOREG, _NOREG, r0);
907 if (r0 == _RAX_REGNO)
908 ic(0xa9);
909 else {
910 ic(0xf7);
911 mrm(0x03, 0x00, r7(r0));
912 }
913 ii(i0);
914}
915
916static void
917_cc(jit_state_t *_jit, jit_int32_t code, jit_int32_t r0)
918{
919 rex(0, 0, _NOREG, _NOREG, r0);
920 ic(0x0f);
921 ic(0x90 | code);
922 mrm(0x03, 0x00, r7(r0));
923}
924
925static void
926_alur(jit_state_t *_jit, jit_int32_t code, jit_int32_t r0, jit_int32_t r1)
927{
928 rex(0, WIDE, r1, _NOREG, r0);
929 ic(code | 0x01);
930 mrm(0x03, r7(r1), r7(r0));
931}
932
933static void
934_alui(jit_state_t *_jit, jit_int32_t code, jit_int32_t r0, jit_word_t i0)
935{
936 jit_int32_t reg;
937 if (can_sign_extend_int_p(i0)) {
938 rex(0, WIDE, _NOREG, _NOREG, r0);
939 if ((jit_int8_t)i0 == i0) {
940 ic(0x83);
941 ic(0xc0 | code | r7(r0));
942 ic(i0);
943 }
944 else {
945 if (r0 == _RAX_REGNO)
946 ic(code | 0x05);
947 else {
948 ic(0x81);
949 ic(0xc0 | code | r7(r0));
950 }
951 ii(i0);
952 }
953 }
954 else {
955 reg = jit_get_reg(jit_class_gpr);
956 movi(rn(reg), i0);
957 alur(code, r0, rn(reg));
958 jit_unget_reg(reg);
959 }
960}
961
962static void
963_save(jit_state_t *_jit, jit_int32_t r0)
964{
965 if (!_jitc->function->regoff[r0]) {
966 _jitc->function->regoff[r0] = jit_allocai(sizeof(jit_word_t));
967 _jitc->again = 1;
968 }
969 assert(!jit_regset_tstbit(&_jitc->regsav, r0));
970 jit_regset_setbit(&_jitc->regsav, r0);
971 stxi(_jitc->function->regoff[r0], _RBP_REGNO, r0);
972}
973
974static void
975_load(jit_state_t *_jit, jit_int32_t r0)
976{
977 assert(_jitc->function->regoff[r0]);
978 assert(jit_regset_tstbit(&_jitc->regsav, r0));
979 jit_regset_clrbit(&_jitc->regsav, r0);
980 ldxi(r0, _RBP_REGNO, _jitc->function->regoff[r0]);
981}
982
983static void
984_addr(jit_state_t *_jit, jit_int32_t r0, jit_int32_t r1, jit_int32_t r2)
985{
986 if (r0 == r1)
987 iaddr(r0, r2);
988 else if (r0 == r2)
989 iaddr(r0, r1);
990 else
991 lea(0, r1, r2, _SCL1, r0);
992}
993
994static void
995_addi(jit_state_t *_jit, jit_int32_t r0, jit_int32_t r1, jit_word_t i0)
996{
997 jit_int32_t reg;
998 if (i0 == 0)
999 movr(r0, r1);
1000#if USE_INC_DEC
1001 else if (i0 == 1)
1002 incr(r0, r1);
1003 else if (i0 == -1)
1004 decr(r0, r1);
1005#endif
1006 else if (can_sign_extend_int_p(i0)) {
1007 if (r0 == r1)
1008 iaddi(r0, i0);
1009 else
1010 lea(i0, r1, _NOREG, _SCL1, r0);
1011 }
1012 else if (r0 != r1) {
1013 movi(r0, i0);
1014 iaddr(r0, r1);
1015 }
1016 else {
1017 reg = jit_get_reg(jit_class_gpr);
1018 movi(rn(reg), i0);
1019 iaddr(r0, rn(reg));
1020 jit_unget_reg(reg);
1021 }
1022}
1023
1024static void
1025_addcr(jit_state_t *_jit, jit_int32_t r0, jit_int32_t r1, jit_int32_t r2)
1026{
1027 if (r0 == r2)
1028 iaddr(r0, r1);
1029 else {
1030 movr(r0, r1);
1031 iaddr(r0, r2);
1032 }
1033}
1034
1035static void
1036_addci(jit_state_t *_jit, jit_int32_t r0, jit_int32_t r1, jit_word_t i0)
1037{
1038 jit_int32_t reg;
1039 if (can_sign_extend_int_p(i0)) {
1040 movr(r0, r1);
1041 iaddi(r0, i0);
1042 }
1043 else if (r0 == r1) {
1044 reg = jit_get_reg(jit_class_gpr);
1045 movi(rn(reg), i0);
1046 iaddr(r0, rn(reg));
1047 jit_unget_reg(reg);
1048 }
1049 else {
1050 movi(r0, i0);
1051 iaddr(r0, r1);
1052 }
1053}
1054
79bfeef6
PC		 1055static void
1056_iaddxr(jit_state_t *_jit, jit_int32_t r0, jit_int32_t r1)
1057{
1058 /* FIXME: this is not doing what I did expect for the simple test case:
1059 * mov $0xffffffffffffffff, %rax -- rax = 0xffffffffffffffff (-1)
1060 * mov $0xffffffffffffffff, %r10 -- r10 = 0xffffffffffffffff (-1)
1061 * mov $0x1, %r11d -- r11 = 1
1062 * xor %rbx, %rbx -- rbx = 0
1063 * (gdb) p $eflags
1064 * $1 = [ PF ZF IF ]
1065 * add %r11, %rax -- r11 = 0x10000000000000000 (0)
1066 * does not fit in 64 bit ^
1067 * (gdb) p $eflags
1068 * $2 = [ CF PF AF ZF IF ]
1069 * adcx %r10, %rbx -- r10 = 0xffffffffffffffff (-1)
1070 * (gdb) p $eflags
1071 * $3 = [ CF PF AF ZF IF ]
1072 * (gdb) p/x $r10
1073 * $4 = 0xffffffffffffffff
1074 * but, r10 should be zero, as it is:
1075 * -1 (%r10) + 0 (%rbx) + carry (!!eflags.CF)
1076 * FIXME: maybe should only use ADCX in the third operation onward, that
1077 * is, after the first ADC? In either case, the add -1+0+carry should
1078 * have used and consumed the carry? At least this is what is expected
1079 * in Lightning...
1080 */
1081#if 0
1082 /* Significantly longer instruction, but avoid cpu stalls as only
1083 * the carry flag is used in a sequence. */
1084 if (jit_cpu.adx) {
1085 /* ADCX */
1086 ic(0x66);
1087 rex(0, WIDE, r1, _NOREG, r0);
1088 ic(0x0f);
1089 ic(0x38);
1090 ic(0xf6);
1091 mrm(0x03, r7(r1), r7(r0));
1092 }
1093 else
1094#endif
1095 alur(X86_ADC, r0, r1);
1096}
1097
4a71579b
PC		 1098static void
1099_addxr(jit_state_t *_jit, jit_int32_t r0, jit_int32_t r1, jit_int32_t r2)
1100{
1101 if (r0 == r2)
1102 iaddxr(r0, r1);
1103 else {
1104 movr(r0, r1);
1105 iaddxr(r0, r2);
1106 }
1107}
1108
1109static void
1110_addxi(jit_state_t *_jit, jit_int32_t r0, jit_int32_t r1, jit_word_t i0)
1111{
1112 jit_int32_t reg;
79bfeef6
PC		 1113 if (
1114#if 0
1115 /* Do not mix ADC and ADCX */
1116 !jit_cpu.adx &&
1117#endif
1118 can_sign_extend_int_p(i0)) {
4a71579b
PC		 1119 movr(r0, r1);
1120 iaddxi(r0, i0);
1121 }
1122 else if (r0 == r1) {
1123 reg = jit_get_reg(jit_class_gpr);
1124 movi(rn(reg), i0);
1125 iaddxr(r0, rn(reg));
1126 jit_unget_reg(reg);
1127 }
1128 else {
1129 movi(r0, i0);
1130 iaddxr(r0, r1);
1131 }
1132}
1133
1134static void
1135_subr(jit_state_t *_jit, jit_int32_t r0, jit_int32_t r1, jit_int32_t r2)
1136{
1137 if (r1 == r2)
1138 ixorr(r0, r0);
1139 else if (r0 == r2) {
1140 isubr(r0, r1);
1141 inegr(r0);
1142 }
1143 else {
1144 movr(r0, r1);
1145 isubr(r0, r2);
1146 }
1147}
1148
1149static void
1150_subi(jit_state_t *_jit, jit_int32_t r0, jit_int32_t r1, jit_word_t i0)
1151{
1152 jit_int32_t reg;
1153 if (i0 == 0)
1154 movr(r0, r1);
1155#if USE_INC_DEC
1156 else if (i0 == 1)
1157 decr(r0, r1);
1158 else if (i0 == -1)
1159 incr(r0, r1);
1160#endif
1161 else if (can_sign_extend_int_p(i0)) {
1162 if (r0 == r1)
1163 isubi(r0, i0);
1164 else
1165 lea(-i0, r1, _NOREG, _SCL1, r0);
1166 }
1167 else if (r0 != r1) {
1168 movi(r0, -i0);
1169 iaddr(r0, r1);
1170 }
1171 else {
1172 reg = jit_get_reg(jit_class_gpr);
1173 movi(rn(reg), i0);
1174 isubr(r0, rn(reg));
1175 jit_unget_reg(reg);
1176 }
1177}
1178
1179static void
1180_subcr(jit_state_t *_jit, jit_int32_t r0, jit_int32_t r1, jit_int32_t r2)
1181{
1182 jit_int32_t reg;
1183 if (r0 == r2 && r0 != r1) {
1184 reg = jit_get_reg(jit_class_gpr);
1185 movr(rn(reg), r0);
1186 movr(r0, r1);
1187 isubr(r0, rn(reg));
1188 jit_unget_reg(reg);
1189 }
1190 else {
1191 movr(r0, r1);
1192 isubr(r0, r2);
1193 }
1194}
1195
1196static void
1197_subci(jit_state_t *_jit, jit_int32_t r0, jit_int32_t r1, jit_word_t i0)
1198{
1199 jit_int32_t reg;
1200 movr(r0, r1);
1201 if (can_sign_extend_int_p(i0))
1202 isubi(r0, i0);
1203 else {
1204 reg = jit_get_reg(jit_class_gpr);
1205 movi(rn(reg), i0);
1206 isubr(r0, rn(reg));
1207 jit_unget_reg(reg);
1208 }
1209}
1210
1211static void
1212_subxr(jit_state_t *_jit, jit_int32_t r0, jit_int32_t r1, jit_int32_t r2)
1213{
1214 jit_int32_t reg;
1215 if (r0 == r2 && r0 != r1) {
1216 reg = jit_get_reg(jit_class_gpr);
1217 movr(rn(reg), r0);
1218 movr(r0, r1);
1219 isubxr(r0, rn(reg));
1220 jit_unget_reg(reg);
1221 }
1222 else {
1223 movr(r0, r1);
1224 isubxr(r0, r2);
1225 }
1226}
1227
1228static void
1229_subxi(jit_state_t *_jit, jit_int32_t r0, jit_int32_t r1, jit_word_t i0)
1230{
1231 jit_int32_t reg;
1232 movr(r0, r1);
1233 if (can_sign_extend_int_p(i0))
1234 isubxi(r0, i0);
1235 else {
1236 reg = jit_get_reg(jit_class_gpr);
1237 imovi(rn(reg), i0);
1238 isubxr(r0, rn(reg));
1239 jit_unget_reg(reg);
1240 }
1241}
1242
1243static void
1244_rsbi(jit_state_t *_jit, jit_int32_t r0, jit_int32_t r1, jit_word_t i0)
1245{
1246 subi(r0, r1, i0);
1247 negr(r0, r0);
1248}
1249
1250static void
1251_imulr(jit_state_t *_jit, jit_int32_t r0, jit_int32_t r1)
1252{
1253 rex(0, WIDE, r0, _NOREG, r1);
1254 ic(0x0f);
1255 ic(0xaf);
1256 mrm(0x03, r7(r0), r7(r1));
1257}
1258
1259static void
1260_imuli(jit_state_t *_jit, jit_int32_t r0, jit_int32_t r1, jit_word_t i0)
1261{
1262 jit_int32_t reg;
1263 if (can_sign_extend_int_p(i0)) {
1264 rex(0, WIDE, r0, _NOREG, r1);
1265 if ((jit_int8_t)i0 == i0) {
1266 ic(0x6b);
1267 mrm(0x03, r7(r0), r7(r1));
1268 ic(i0);
1269 }
1270 else {
1271 ic(0x69);
1272 mrm(0x03, r7(r0), r7(r1));
1273 ii(i0);
1274 }
1275 }
1276 else {
1277 reg = jit_get_reg(jit_class_gpr);
1278 movi(rn(reg), i0);
1279 imulr(r0, rn(reg));
1280 jit_unget_reg(reg);
1281 }
1282}
1283
1284static void
1285_mulr(jit_state_t *_jit, jit_int32_t r0, jit_int32_t r1, jit_int32_t r2)
1286{
1287 if (r0 == r1)
1288 imulr(r0, r2);
1289 else if (r0 == r2)
1290 imulr(r0, r1);
1291 else {
1292 movr(r0, r1);
1293 imulr(r0, r2);
1294 }
1295}
1296
1297static void
1298_muli(jit_state_t *_jit, jit_int32_t r0, jit_int32_t r1, jit_word_t i0)
1299{
1300 switch (i0) {
1301 case 0:
1302 ixorr(r0, r0);
1303 break;
1304 case 1:
1305 movr(r0, r1);
1306 break;
1307 case -1:
1308 negr(r0, r1);
1309 break;
1310 case 2:
1311 lea(0, _NOREG, r1, _SCL2, r0);
1312 break;
1313 case 4:
1314 lea(0, _NOREG, r1, _SCL4, r0);
1315 break;
1316 case 8:
1317 lea(0, _NOREG, r1, _SCL8, r0);
1318 break;
1319 default:
1320 if (i0 > 0 && !(i0 & (i0 - 1)))
1321 lshi(r0, r1, ffsl(i0) - 1);
1322 else if (can_sign_extend_int_p(i0))
1323 imuli(r0, r1, i0);
1324 else if (r0 != r1) {
1325 movi(r0, i0);
1326 imulr(r0, r1);
1327 }
1328 else
1329 imuli(r0, r0, i0);
1330 break;
1331 }
1332}
1333
1334#define savset(rn) \
1335 if (r0 != rn) { \
1336 sav |= 1 << rn; \
1337 if (r1 != rn && r2 != rn) \
1338 set |= 1 << rn; \
1339 }
1340#define isavset(rn) \
1341 if (r0 != rn) { \
1342 sav |= 1 << rn; \
1343 if (r1 != rn) \
1344 set |= 1 << rn; \
1345 }
1346#define qsavset(rn) \
1347 if (r0 != rn && r1 != rn) { \
1348 sav |= 1 << rn; \
1349 if (r2 != rn && r3 != rn) \
1350 set |= 1 << rn; \
1351 }
1352#define allocr(rn, rv) \
1353 if (set & (1 << rn)) \
1354 (void)jit_get_reg(rv|jit_class_gpr|jit_class_named); \
1355 if (sav & (1 << rn)) { \
1356 if ( jit_regset_tstbit(&_jitc->regsav, rv) || \
1357 !jit_regset_tstbit(&_jitc->reglive, rv)) \
1358 sav &= ~(1 << rn); \
1359 else \
1360 save(rv); \
1361 }
1362#define clear(rn, rv) \
1363 if (set & (1 << rn)) \
1364 jit_unget_reg(rv); \
1365 if (sav & (1 << rn)) \
1366 load(rv);
1367static void
1368_iqmulr(jit_state_t *_jit, jit_int32_t r0, jit_int32_t r1,
1369 jit_int32_t r2, jit_int32_t r3, jit_bool_t sign)
1370{
1371 jit_int32_t mul;
1372 jit_int32_t sav;
1373 jit_int32_t set;
1374
1375 sav = set = 0;
1376 qsavset(_RDX_REGNO);
1377 qsavset(_RAX_REGNO);
1378 allocr(_RDX_REGNO, _RDX);
1379 allocr(_RAX_REGNO, _RAX);
1380
1381 if (r3 == _RAX_REGNO)
1382 mul = r2;
1383 else {
1384 mul = r3;
1385 movr(_RAX_REGNO, r2);
1386 }
1387 if (sign)
1388 umulr(mul);
1389 else
1390 umulr_u(mul);
1391
1392 if (r0 == _RDX_REGNO && r1 == _RAX_REGNO)
1393 xchgr(_RAX_REGNO, _RDX_REGNO);
1394 else {
1395 if (r0 != _RDX_REGNO)
1396 movr(r0, _RAX_REGNO);
1397 movr(r1, _RDX_REGNO);
1398 if (r0 == _RDX_REGNO)
1399 movr(r0, _RAX_REGNO);
1400 }
1401
1402 clear(_RDX_REGNO, _RDX);
1403 clear(_RAX_REGNO, _RAX);
1404}
1405
1406static void
1407_iqmuli(jit_state_t *_jit, jit_int32_t r0, jit_int32_t r1,
1408 jit_int32_t r2, jit_word_t i0, jit_bool_t sign)
1409{
1410 jit_int32_t reg;
1411
1412 if (i0 == 0) {
1413 ixorr(r0, r0);
1414 ixorr(r1, r1);
1415 }
1416 else {
1417 reg = jit_get_reg(jit_class_gpr);
1418 movi(rn(reg), i0);
1419 if (sign)
1420 qmulr(r0, r1, r2, rn(reg));
1421 else
1422 qmulr_u(r0, r1, r2, rn(reg));
1423 jit_unget_reg(reg);
1424 }
1425}
1426
1427static void
1428_sign_extend_rdx_rax(jit_state_t *_jit)
1429{
1430 rex(0, WIDE, 0, 0, 0);
1431 ic(0x99);
1432}
1433
1434static void
1435_divremr(jit_state_t *_jit, jit_int32_t r0, jit_int32_t r1, jit_int32_t r2,
1436 jit_bool_t sign, jit_bool_t divide)
1437{
1438 jit_int32_t div;
1439 jit_int32_t reg;
1440 jit_int32_t set;
1441 jit_int32_t sav;
1442 jit_int32_t use;
1443
1444 sav = set = use = 0;
1445 savset(_RDX_REGNO);
1446 savset(_RAX_REGNO);
1447 allocr(_RDX_REGNO, _RDX);
1448 allocr(_RAX_REGNO, _RAX);
1449
1450 if (r2 == _RAX_REGNO) {
1451 if (r0 == _RAX_REGNO || r0 == _RDX_REGNO) {
1452 if ((reg = jit_get_reg(jit_class_gpr|jit_class_chk)) == JIT_NOREG)
1453 reg = jit_get_reg((r1 == _RCX_REGNO ? _RBX : _RCX) |
1454 jit_class_gpr|jit_class_named);
1455 use = 1;
1456 div = rn(reg);
1457 movr(div, _RAX_REGNO);
1458 if (r1 != _RAX_REGNO)
1459 movr(_RAX_REGNO, r1);
1460 }
1461 else {
1462 if (r0 == r1)
1463 xchgr(r0, _RAX_REGNO);
1464 else {
1465 if (r0 != _RAX_REGNO)
1466 movr(r0, _RAX_REGNO);
1467 if (r1 != _RAX_REGNO)
1468 movr(_RAX_REGNO, r1);
1469 }
1470 div = r0;
1471 }
1472 }
1473 else if (r2 == _RDX_REGNO) {
1474 if (r0 == _RAX_REGNO || r0 == _RDX_REGNO) {
1475 if ((reg = jit_get_reg(jit_class_gpr|jit_class_chk)) == JIT_NOREG)
1476 reg = jit_get_reg((r1 == _RCX_REGNO ? _RBX : _RCX) |
1477 jit_class_gpr|jit_class_named);
1478 use = 1;
1479 div = rn(reg);
1480 movr(div, _RDX_REGNO);
1481 if (r1 != _RAX_REGNO)
1482 movr(_RAX_REGNO, r1);
1483 }
1484 else {
1485 if (r1 != _RAX_REGNO)
1486 movr(_RAX_REGNO, r1);
1487 movr(r0, _RDX_REGNO);
1488 div = r0;
1489 }
1490 }
1491 else {
1492 if (r1 != _RAX_REGNO)
1493 movr(_RAX_REGNO, r1);
1494 div = r2;
1495 }
1496
1497 if (sign) {
1498 sign_extend_rdx_rax();
1499 idivr(div);
1500 }
1501 else {
1502 ixorr(_RDX_REGNO, _RDX_REGNO);
1503 idivr_u(div);
1504 }
1505
1506 if (use)
1507 jit_unget_reg(reg);
1508
1509 if (divide)
1510 movr(r0, _RAX_REGNO);
1511 else
1512 movr(r0, _RDX_REGNO);
1513
1514 clear(_RDX_REGNO, _RDX);
1515 clear(_RAX_REGNO, _RAX);
1516}
1517
1518static void
1519_divremi(jit_state_t *_jit, jit_int32_t r0, jit_int32_t r1, jit_word_t i0,
1520 jit_bool_t sign, jit_bool_t divide)
1521{
1522 jit_int32_t reg;
1523 jit_int32_t div;
1524 jit_int32_t sav;
1525 jit_int32_t set;
1526 jit_int32_t use;
1527
1528 if (divide) {
1529 switch (i0) {
1530 case 1:
1531 movr(r0, r1);
1532 return;
1533 case -1:
1534 if (sign) {
1535 negr(r0, r1);
1536 return;
1537 }
1538 break;
1539 default:
1540 if (i0 > 0 && !(i0 & (i0 - 1))) {
1541 movr(r0, r1);
1542 if (sign)
1543 rshi(r0, r0, ffsl(i0) - 1);
1544 else
1545 rshi_u(r0, r0, ffsl(i0) - 1);
1546 return;
1547 }
1548 break;
1549 }
1550 }
1551 else if (i0 == 1 || (sign && i0 == -1)) {
1552 ixorr(r0, r0);
1553 return;
1554 }
1555 else if (!sign && i0 > 0 && !(i0 & (i0 - 1))) {
1556 if (can_sign_extend_int_p(i0)) {
1557 movr(r0, r1);
1558 iandi(r0, i0 - 1);
1559 }
1560 else if (r0 != r1) {
1561 movi(r0, i0 - 1);
1562 iandr(r0, r1);
1563 }
1564 else {
1565 reg = jit_get_reg(jit_class_gpr);
1566 movi(rn(reg), i0 - 1);
1567 iandr(r0, rn(reg));
1568 jit_unget_reg(reg);
1569 }
1570 return;
1571 }
1572
1573 sav = set = use = 0;
1574 isavset(_RDX_REGNO);
1575 isavset(_RAX_REGNO);
1576 allocr(_RDX_REGNO, _RDX);
1577 allocr(_RAX_REGNO, _RAX);
1578
1579 if (r0 == _RAX_REGNO || r0 == _RDX_REGNO || r0 == r1) {
1580 if ((reg = jit_get_reg(jit_class_gpr|jit_class_chk)) == JIT_NOREG)
1581 reg = jit_get_reg((r1 == _RCX_REGNO ? _RBX : _RCX) |
1582 jit_class_gpr|jit_class_named);
1583 use = 1;
1584 div = rn(reg);
1585 }
1586 else
1587 div = r0;
1588
1589 movi(div, i0);
1590 movr(_RAX_REGNO, r1);
1591
1592 if (sign) {
1593 sign_extend_rdx_rax();
1594 idivr(div);
1595 }
1596 else {
1597 ixorr(_RDX_REGNO, _RDX_REGNO);
1598 idivr_u(div);
1599 }
1600
1601 if (use)
1602 jit_unget_reg(reg);
1603
1604 if (divide)
1605 movr(r0, _RAX_REGNO);
1606 else
1607 movr(r0, _RDX_REGNO);
1608
1609 clear(_RDX_REGNO, _RDX);
1610 clear(_RAX_REGNO, _RAX);
1611}
1612
1613static void
1614_iqdivr(jit_state_t *_jit, jit_int32_t r0, jit_int32_t r1,
1615 jit_int32_t r2, jit_int32_t r3, jit_bool_t sign)
1616{
1617 jit_int32_t div;
1618 jit_int32_t reg;
1619 jit_int32_t sav;
1620 jit_int32_t set;
1621 jit_int32_t use;
1622
1623 sav = set = use = 0;
1624 qsavset(_RDX_REGNO);
1625 qsavset(_RAX_REGNO);
1626 allocr(_RDX_REGNO, _RDX);
1627 allocr(_RAX_REGNO, _RAX);
1628 if (r3 == _RAX_REGNO) {
1629 if (r0 == _RAX_REGNO || r0 == _RDX_REGNO) {
1630 if ((reg = jit_get_reg(jit_class_gpr|jit_class_chk)) == JIT_NOREG)
1631 reg = jit_get_reg((r1 == _RCX_REGNO ? _RBX : _RCX) |
1632 jit_class_gpr|jit_class_named);
1633 use = 1;
1634 div = rn(reg);
1635 movr(div, _RAX_REGNO);
1636 if (r2 != _RAX_REGNO)
1637 movr(_RAX_REGNO, r2);
1638 }
1639 else {
1640 if (r0 == r2)
1641 xchgr(r0, _RAX_REGNO);
1642 else {
1643 if (r0 != _RAX_REGNO)
1644 movr(r0, _RAX_REGNO);
1645 if (r2 != _RAX_REGNO)
1646 movr(_RAX_REGNO, r2);
1647 }
1648 div = r0;
1649 }
1650 }
1651 else if (r3 == _RDX_REGNO) {
1652 if (r0 == _RAX_REGNO || r0 == _RDX_REGNO) {
1653 if ((reg = jit_get_reg(jit_class_gpr|jit_class_chk)) == JIT_NOREG)
1654 reg = jit_get_reg((r1 == _RCX_REGNO ? _RBX : _RCX) |
1655 jit_class_gpr|jit_class_named);
1656 use = 1;
1657 div = rn(reg);
1658 movr(div, _RDX_REGNO);
1659 if (r2 != _RAX_REGNO)
1660 movr(_RAX_REGNO, r2);
1661 }
1662 else {
1663 if (r2 != _RAX_REGNO)
1664 movr(_RAX_REGNO, r2);
1665 movr(r0, _RDX_REGNO);
1666 div = r0;
1667 }
1668 }
1669 else {
1670 if (r2 != _RAX_REGNO)
1671 movr(_RAX_REGNO, r2);
1672 div = r3;
1673 }
1674 if (sign) {
1675 sign_extend_rdx_rax();
1676 idivr(div);
1677 }
1678 else {
1679 ixorr(_RDX_REGNO, _RDX_REGNO);
1680 idivr_u(div);
1681 }
1682 if (use)
1683 jit_unget_reg(reg);
1684
1685 if (r0 == _RDX_REGNO && r1 == _RAX_REGNO)
1686 xchgr(_RAX_REGNO, _RDX_REGNO);
1687 else {
1688 if (r0 != _RDX_REGNO)
1689 movr(r0, _RAX_REGNO);
1690 movr(r1, _RDX_REGNO);
1691 if (r0 == _RDX_REGNO)
1692 movr(r0, _RAX_REGNO);
1693 }
1694
1695 clear(_RDX_REGNO, _RDX);
1696 clear(_RAX_REGNO, _RAX);
1697}
1698
1699static void
1700_iqdivi(jit_state_t *_jit, jit_int32_t r0, jit_int32_t r1,
1701 jit_int32_t r2, jit_word_t i0, jit_bool_t sign)
1702{
1703 jit_int32_t reg;
1704
1705 reg = jit_get_reg(jit_class_gpr);
1706 movi(rn(reg), i0);
1707 if (sign)
1708 qdivr(r0, r1, r2, rn(reg));
1709 else
1710 qdivr_u(r0, r1, r2, rn(reg));
1711 jit_unget_reg(reg);
1712}
1713#undef clear
1714#undef allocr
1715#undef savset
1716
1717static void
1718_andr(jit_state_t *_jit, jit_int32_t r0, jit_int32_t r1, jit_int32_t r2)
1719{
1720 if (r1 == r2)
1721 movr(r0, r1);
1722 else if (r0 == r1)
1723 iandr(r0, r2);
1724 else if (r0 == r2)
1725 iandr(r0, r1);
1726 else {
1727 movr(r0, r1);
1728 iandr(r0, r2);
1729 }
1730}
1731
1732static void
1733_andi(jit_state_t *_jit, jit_int32_t r0, jit_int32_t r1, jit_word_t i0)
1734{
1735 jit_int32_t reg;
1736
1737 if (i0 == 0)
1738 ixorr(r0, r0);
1739 else if (i0 == -1)
1740 movr(r0, r1);
1741 else if (r0 == r1) {
1742 if (can_sign_extend_int_p(i0))
1743 iandi(r0, i0);
1744 else {
1745 reg = jit_get_reg(jit_class_gpr);
1746 movi(rn(reg), i0);
1747 iandr(r0, rn(reg));
1748 jit_unget_reg(reg);
1749 }
1750 }
1751 else {
1752 movi(r0, i0);
1753 iandr(r0, r1);
1754 }
1755}
1756
1757static void
1758_orr(jit_state_t *_jit, jit_int32_t r0, jit_int32_t r1, jit_int32_t r2)
1759{
1760 if (r1 == r2)
1761 movr(r0, r1);
1762 else if (r0 == r1)
1763 iorr(r0, r2);
1764 else if (r0 == r2)
1765 iorr(r0, r1);
1766 else {
1767 movr(r0, r1);
1768 iorr(r0, r2);
1769 }
1770}
1771
1772static void
1773_ori(jit_state_t *_jit, jit_int32_t r0, jit_int32_t r1, jit_word_t i0)
1774{
1775 jit_int32_t reg;
1776 if (i0 == 0)
1777 movr(r0, r1);
1778 else if (i0 == -1)
1779 movi(r0, -1);
1780 else if (can_sign_extend_int_p(i0)) {
1781 movr(r0, r1);
1782 iori(r0, i0);
1783 }
1784 else if (r0 != r1) {
1785 movi(r0, i0);
1786 iorr(r0, r1);
1787 }
1788 else {
1789 reg = jit_get_reg(jit_class_gpr);
1790 movi(rn(reg), i0);
1791 iorr(r0, rn(reg));
1792 jit_unget_reg(reg);
1793 }
1794}
1795
1796static void
1797_xorr(jit_state_t *_jit, jit_int32_t r0, jit_int32_t r1, jit_int32_t r2)
1798{
1799 if (r1 == r2)
1800 ixorr(r0, r0);
1801 else if (r0 == r1)
1802 ixorr(r0, r2);
1803 else if (r0 == r2)
1804 ixorr(r0, r1);
1805 else {
1806 movr(r0, r1);
1807 ixorr(r0, r2);
1808 }
1809}
1810
1811static void
1812_xori(jit_state_t *_jit, jit_int32_t r0, jit_int32_t r1, jit_word_t i0)
1813{
1814 jit_int32_t reg;
1815 if (i0 == 0)
1816 movr(r0, r1);
1817 else if (i0 == -1)
1818 comr(r0, r1);
1819 else if (can_sign_extend_int_p(i0)) {
1820 movr(r0, r1);
1821 ixori(r0, i0);
1822 }
1823 else if (r0 != r1) {
1824 movi(r0, i0);
1825 ixorr(r0, r1);
1826 }
1827 else {
1828 reg = jit_get_reg(jit_class_gpr);
1829 movi(rn(reg), i0);
1830 ixorr(r0, rn(reg));
1831 jit_unget_reg(reg);
1832 }
1833}
1834
1835static void
1836_irotshr(jit_state_t *_jit, jit_int32_t code, jit_int32_t r0)
1837{
1838 rex(0, WIDE, _RCX_REGNO, _NOREG, r0);
1839 ic(0xd3);
1840 mrm(0x03, code, r7(r0));
1841}
1842
1843static void
1844_rotshr(jit_state_t *_jit, jit_int32_t code,
1845 jit_int32_t r0, jit_int32_t r1, jit_int32_t r2)
1846{
1847 jit_int32_t reg;
1848 jit_int32_t use;
1849
1850 if (r0 == _RCX_REGNO) {
1851 reg = jit_get_reg(jit_class_gpr);
1852 movr(rn(reg), r1);
1853 if (r2 != _RCX_REGNO)
1854 movr(_RCX_REGNO, r2);
1855 irotshr(code, rn(reg));
1856 movr(_RCX_REGNO, rn(reg));
1857 jit_unget_reg(reg);
1858 }
1859 else if (r2 != _RCX_REGNO) {
1860 use = !jit_reg_free_p(_RCX);
1861 if (use) {
1862 reg = jit_get_reg(jit_class_gpr);
1863 movr(rn(reg), _RCX_REGNO);
1864 }
1865 else
1866 reg = 0;
1867 if (r1 == _RCX_REGNO) {
1868 if (r0 == r2)
1869 xchgr(r0, _RCX_REGNO);
1870 else {
1871 movr(r0, r1);
1872 movr(_RCX_REGNO, r2);
1873 }
1874 }
1875 else {
1876 movr(_RCX_REGNO, r2);
1877 movr(r0, r1);
1878 }
1879 irotshr(code, r0);
1880 if (use) {
1881 movr(_RCX_REGNO, rn(reg));
1882 jit_unget_reg(reg);
1883 }
1884 }
1885 else {
1886 movr(r0, r1);
1887 irotshr(code, r0);
1888 }
1889}
1890
1891static void
1892_irotshi(jit_state_t *_jit, jit_int32_t code, jit_int32_t r0, jit_word_t i0)
1893{
1894 rex(0, WIDE, _NOREG, _NOREG, r0);
1895 if (i0 == 1) {
1896 ic(0xd1);
1897 mrm(0x03, code, r7(r0));
1898 }
1899 else {
1900 ic(0xc1);
1901 mrm(0x03, code, r7(r0));
1902 ic(i0);
1903 }
1904}
1905
1906static void
1907_rotshi(jit_state_t *_jit, jit_int32_t code,
1908 jit_int32_t r0, jit_int32_t r1, jit_word_t i0)
1909{
1910 movr(r0, r1);
1911 if (i0)
1912 irotshi(code, r0, i0);
1913}
1914
1915static void
1916_lshi(jit_state_t *_jit, jit_int32_t r0, jit_int32_t r1, jit_word_t i0)
1917{
1918 if (i0 == 0)
1919 movr(r0, r1);
1920 else if (i0 <= 3)
1921 lea(0, _NOREG, r1, i0 == 1 ? _SCL2 : i0 == 2 ? _SCL4 : _SCL8, r0);
1922 else
1923 rotshi(X86_SHL, r0, r1, i0);
1924}
1925
1926static void
1927_unr(jit_state_t *_jit, jit_int32_t code, jit_int32_t r0)
1928{
1929 rex(0, WIDE, _NOREG, _NOREG, r0);
1930 ic(0xf7);
1931 mrm(0x03, code, r7(r0));
1932}
1933
1934static void
1935_negr(jit_state_t *_jit, jit_int32_t r0, jit_int32_t r1)
1936{
1937 if (r0 == r1)
1938 inegr(r0);
1939 else {
1940 ixorr(r0, r0);
1941 isubr(r0, r1);
1942 }
1943}
1944
1945static void
1946_comr(jit_state_t *_jit, jit_int32_t r0, jit_int32_t r1)
1947{
1948 movr(r0, r1);
1949 icomr(r0);
1950}
1951
1952#if USE_INC_DEC
1953static void
1954_incr(jit_state_t *_jit, jit_int32_t r0, jit_int32_t r1)
1955{
1956 movr(r0, r1);
1957# if __X64
1958 rex(0, WIDE, _NOREG, _NOREG, r0);
1959 ic(0xff);
1960 ic(0xc0 | r7(r0));
1961# else
1962 ic(0x40 | r7(r0));
1963# endif
1964}
1965
1966static void
1967_decr(jit_state_t *_jit, jit_int32_t r0, jit_int32_t r1)
1968{
1969 movr(r0, r1);
1970# if __X64
1971 rex(0, WIDE, _NOREG, _NOREG, r0);
1972 ic(0xff);
1973 ic(0xc8 | r7(r0));
1974# else
1975 ic(0x48 | r7(r0));
1976# endif
1977}
1978#endif
1979
79bfeef6
PC		 1980static void
1981_clor(jit_state_t *_jit, jit_int32_t r0, jit_int32_t r1)
1982{
1983 comr(r0, r1);
1984 clzr(r0, r0);
1985}
1986
1987static void
1988_clzr(jit_state_t *_jit, jit_int32_t r0, jit_int32_t r1)
1989{
1990 jit_word_t w, x;
1991 /* LZCNT */
1992 if (jit_cpu.abm)
1993 ic(0xf3);
1994 /* else BSR */
1995 rex(0, WIDE, r0, _NOREG, r1);
1996 ic(0x0f);
1997 ic(0xbd);
1998 mrm(0x3, r7(r0), r7(r1));
1999 if (!jit_cpu.abm) {
2000 /* jump if undefined: r1 == 0 */
2001 w = jccs(X86_CC_E, _jit->pc.w);
2002 /* count leading zeros */
2003 rsbi(r0, r0, __WORDSIZE - 1);
2004 /* done */
2005 x = jmpsi(_jit->pc.w);
2006 /* if r1 == 0 */
2007 patch_at(w, _jit->pc.w);
2008 movi(r0, __WORDSIZE);
2009 /* not undefined */
2010 patch_at(x, _jit->pc.w);
2011 }
2012 /* LZCNT has defined behavior for value zero and count leading zeros */
2013}
2014
2015static void
2016_ctor(jit_state_t *_jit, jit_int32_t r0, jit_int32_t r1)
2017{
2018 comr(r0, r1);
2019 ctzr(r0, r0);
2020}
2021
2022static void
2023_ctzr(jit_state_t *_jit, jit_int32_t r0, jit_int32_t r1)
2024{
2025 jit_word_t w;
2026 jit_int32_t t0;
2027 if (!jit_cpu.abm) {
2028 if (jit_cmov_p())
2029 t0 = jit_get_reg(jit_class_gpr|jit_class_nospill|jit_class_chk);
2030 else
2031 t0 = _NOREG;
2032 if (t0 != _NOREG)
2033 movi(rn(t0), __WORDSIZE);
2034 }
2035 /* TZCNT */
2036 if (jit_cpu.abm)
2037 ic(0xf3);
2038 /* else BSF */
2039 rex(0, WIDE, r0, _NOREG, r1);
2040 ic(0x0f);
2041 ic(0xbc);
2042 mrm(0x3, r7(r0), r7(r1));
2043 if (!jit_cpu.abm) {
2044 /* No conditional move or need spill/reload a temporary */
2045 if (t0 == _NOREG) {
2046 w = jccs(X86_CC_E, _jit->pc.w);
2047 movi(r0, __WORDSIZE);
2048 patch_at(w, _jit->pc.w);
2049 }
2050 else {
2051 /* CMOVE */
2052 rex(0, WIDE, r0, _NOREG, rn(t0));
2053 ic(0x0f);
2054 ic(0x44);
2055 mrm(0x3, r7(r0), r7(rn(t0)));
2056 jit_unget_reg(t0);
2057 }
2058 }
2059 /* TZCNT has defined behavior for value zero */
2060}
2061
4a71579b
PC		 2062static void
2063_cr(jit_state_t *_jit,
2064 jit_int32_t code, jit_int32_t r0, jit_int32_t r1, jit_int32_t r2)
2065{
2066 jit_int32_t reg;
2067 jit_bool_t same;
2068 if (reg8_p(r0)) {
2069 same = r0 == r1 || r0 == r2;
2070 if (!same)
2071 ixorr(r0, r0);
2072 icmpr(r1, r2);
2073 if (same)
2074 imovi(r0, 0);
2075 cc(code, r0);
2076 }
2077 else {
2078 reg = jit_get_reg(jit_class_gpr|jit_class_rg8);
2079 ixorr(rn(reg), rn(reg));
2080 icmpr(r1, r2);
2081 cc(code, rn(reg));
2082 movr(r0, rn(reg));
2083 jit_unget_reg(reg);
2084 }
2085}
2086
2087static void
2088_ci(jit_state_t *_jit,
2089 jit_int32_t code, jit_int32_t r0, jit_int32_t r1, jit_word_t i0)
2090{
2091 jit_int32_t reg;
2092 jit_bool_t same;
2093 if (reg8_p(r0)) {
2094 same = r0 == r1;
2095 if (!same)
2096 ixorr(r0, r0);
2097 icmpi(r1, i0);
2098 if (same)
2099 imovi(r0, 0);
2100 cc(code, r0);
2101 }
2102 else {
2103 reg = jit_get_reg(jit_class_gpr|jit_class_rg8);
2104 ixorr(rn(reg), rn(reg));
2105 icmpi(r1, i0);
2106 cc(code, rn(reg));
2107 movr(r0, rn(reg));
2108 jit_unget_reg(reg);
2109 }
2110}
2111
2112static void
2113_ci0(jit_state_t *_jit, jit_int32_t code, jit_int32_t r0, jit_int32_t r1)
2114{
2115 jit_int32_t reg;
2116 jit_bool_t same;
2117 if (reg8_p(r0)) {
2118 same = r0 == r1;
2119 if (!same)
2120 ixorr(r0, r0);
2121 testr(r1, r1);
2122 if (same)
2123 imovi(r0, 0);
2124 cc(code, r0);
2125 }
2126 else {
2127 reg = jit_get_reg(jit_class_gpr|jit_class_rg8);
2128 ixorr(rn(reg), rn(reg));
2129 testr(r1, r1);
2130 cc(code, rn(reg));
2131 movr(r0, rn(reg));
2132 jit_unget_reg(reg);
2133 }
2134}
2135
2136static void
2137_ltr(jit_state_t *_jit, jit_int32_t r0, jit_int32_t r1, jit_int32_t r2)
2138{
2139 if (r1 == r2)
2140 movi(r0, 0);
2141 else
2142 cr(X86_CC_L, r0, r1, r2);
2143}
2144
2145static void
2146_lti(jit_state_t *_jit, jit_int32_t r0, jit_int32_t r1, jit_word_t i0)
2147{
2148 if (i0)
2149 ci(X86_CC_L, r0, r1, i0);
2150 else
2151 ci0(X86_CC_S, r0, r1);
2152}
2153
2154static void
2155_ltr_u(jit_state_t *_jit, jit_int32_t r0, jit_int32_t r1, jit_int32_t r2)
2156{
2157 if (r1 == r2)
2158 movi(r0, 0);
2159 else
2160 cr(X86_CC_B, r0, r1, r2);
2161}
2162
2163static void
2164_ler(jit_state_t *_jit, jit_int32_t r0, jit_int32_t r1, jit_int32_t r2)
2165{
2166 if (r1 == r2)
2167 movi(r0, 1);
2168 else
2169 cr(X86_CC_LE, r0, r1, r2);
2170}
2171
2172static void
2173_ler_u(jit_state_t *_jit, jit_int32_t r0, jit_int32_t r1, jit_int32_t r2)
2174{
2175 if (r1 == r2)
2176 movi(r0, 1);
2177 else
2178 cr(X86_CC_BE, r0, r1, r2);
2179}
2180
2181static void
2182_lei_u(jit_state_t *_jit, jit_int32_t r0, jit_int32_t r1, jit_word_t i0)
2183{
2184 if (i0)
2185 ci(X86_CC_BE, r0, r1, i0);
2186 else
2187 ci0(X86_CC_E, r0, r1);
2188}
2189
2190static void
2191_eqr(jit_state_t *_jit, jit_int32_t r0, jit_int32_t r1, jit_int32_t r2)
2192{
2193 if (r1 == r2)
2194 movi(r0, 1);
2195 else
2196 cr(X86_CC_E, r0, r1, r2);
2197}
2198
2199static void
2200_eqi(jit_state_t *_jit, jit_int32_t r0, jit_int32_t r1, jit_word_t i0)
2201{
2202 if (i0)
2203 ci(X86_CC_E, r0, r1, i0);
2204 else
2205 ci0(X86_CC_E, r0, r1);
2206}
2207
2208static void
2209_ger(jit_state_t *_jit, jit_int32_t r0, jit_int32_t r1, jit_int32_t r2)
2210{
2211 if (r1 == r2)
2212 movi(r0, 1);
2213 else
2214 cr(X86_CC_GE, r0, r1, r2);
2215}
2216
2217static void
2218_gei(jit_state_t *_jit, jit_int32_t r0, jit_int32_t r1, jit_word_t i0)
2219{
2220 if (i0)
2221 ci(X86_CC_GE, r0, r1, i0);
2222 else
2223 ci0(X86_CC_NS, r0, r1);
2224}
2225
2226static void
2227_ger_u(jit_state_t *_jit, jit_int32_t r0, jit_int32_t r1, jit_int32_t r2)
2228{
2229 if (r1 == r2)
2230 movi(r0, 1);
2231 else
2232 cr(X86_CC_AE, r0, r1, r2);
2233}
2234
2235static void
2236_gei_u(jit_state_t *_jit, jit_int32_t r0, jit_int32_t r1, jit_word_t i0)
2237{
2238 if (i0)
2239 ci(X86_CC_AE, r0, r1, i0);
2240 else
2241 ci0(X86_CC_NB, r0, r1);
2242}
2243
2244static void
2245_gtr(jit_state_t *_jit, jit_int32_t r0, jit_int32_t r1, jit_int32_t r2)
2246{
2247 if (r1 == r2)
2248 movi(r0, 0);
2249 else
2250 cr(X86_CC_G, r0, r1, r2);
2251}
2252
2253static void
2254_gtr_u(jit_state_t *_jit, jit_int32_t r0, jit_int32_t r1, jit_int32_t r2)
2255{
2256 if (r1 == r2)
2257 movi(r0, 0);
2258 else
2259 cr(X86_CC_A, r0, r1, r2);
2260}
2261
2262static void
2263_gti_u(jit_state_t *_jit, jit_int32_t r0, jit_int32_t r1, jit_word_t i0)
2264{
2265 if (i0)
2266 ci(X86_CC_A, r0, r1, i0);
2267 else
2268 ci0(X86_CC_NE, r0, r1);
2269}
2270
2271static void
2272_ner(jit_state_t *_jit, jit_int32_t r0, jit_int32_t r1, jit_int32_t r2)
2273{
2274 if (r1 == r2)
2275 movi(r0, 0);
2276 else
2277 cr(X86_CC_NE, r0, r1, r2);
2278}
2279
2280static void
2281_nei(jit_state_t *_jit, jit_int32_t r0, jit_int32_t r1, jit_word_t i0)
2282{
2283 if (i0)
2284 ci(X86_CC_NE, r0, r1, i0);
2285 else
2286 ci0(X86_CC_NE, r0, r1);
2287}
2288
2289static void
2290_movr(jit_state_t *_jit, jit_int32_t r0, jit_int32_t r1)
2291{
2292 if (r0 != r1) {
2293 rex(0, 1, r1, _NOREG, r0);
2294 ic(0x89);
2295 ic(0xc0 | (r1 << 3) | r7(r0));
2296 }
2297}
2298
2299static void
2300_imovi(jit_state_t *_jit, jit_int32_t r0, jit_word_t i0)
2301{
2302#if __X64
2303# if !__X64_32
2304 if (fits_uint32_p(i0)) {
2305# endif
2306 rex(0, 0, _NOREG, _NOREG, r0);
2307 ic(0xb8 | r7(r0));
2308 ii(i0);
2309# if !__X64_32
2310 }
79bfeef6
PC		 2311 else if (can_sign_extend_int_p(i0)) {
2312 rex(0, 1, _NOREG, _NOREG, r0);
2313 ic(0xc7);
2314 ic(0xc0 | r7(r0));
2315 ii(i0);
2316 }
4a71579b
PC		 2317 else {
2318 rex(0, 1, _NOREG, _NOREG, r0);
2319 ic(0xb8 | r7(r0));
2320 il(i0);
2321 }
2322# endif
2323#else
2324 ic(0xb8 | r7(r0));
2325 ii(i0);
2326#endif
2327}
2328
79bfeef6
PC		 2329#if CAN_RIP_ADDRESS
2330static jit_word_t
2331#else
4a71579b 2332static void
79bfeef6 2333#endif
4a71579b
PC		 2334_movi(jit_state_t *_jit, jit_int32_t r0, jit_word_t i0)
2335{
79bfeef6
PC		 2336#if CAN_RIP_ADDRESS
2337 jit_word_t w, rel;
2338 w = _jit->pc.w;
2339 rel = i0 - (w + 8);
2340 rel = rel < 0 ? rel - 8 : rel + 8;
2341 if (can_sign_extend_int_p(rel)) {
2342 /* lea rel(%rip), %r0 */
2343 rex(0, WIDE, r0, _NOREG, _NOREG);
2344 w = _jit->pc.w;
2345 ic(0x8d);
2346 rx(r0, i0 - (_jit->pc.w + 5), _NOREG, _NOREG, _SCL8);
2347 }
2348 else
2349#endif
4a71579b
PC		 2350 if (i0)
2351 imovi(r0, i0);
2352 else
2353 ixorr(r0, r0);
79bfeef6
PC		 2354#if CAN_RIP_ADDRESS
2355 return (w);
2356#endif
4a71579b
PC		 2357}
2358
2359static jit_word_t
2360_movi_p(jit_state_t *_jit, jit_int32_t r0, jit_word_t i0)
2361{
79bfeef6 2362 jit_word_t w;
4a71579b 2363 rex(0, WIDE, _NOREG, _NOREG, r0);
79bfeef6 2364 w = _jit->pc.w;
4a71579b
PC		 2365 ic(0xb8 | r7(r0));
2366 il(i0);
79bfeef6 2367 return (w);
4a71579b
PC		 2368}
2369
2370static void
2371_movcr(jit_state_t *_jit, jit_int32_t r0, jit_int32_t r1)
2372{
2373 rex(0, WIDE, r0, _NOREG, r1);
2374 ic(0x0f);
2375 ic(0xbe);
2376 mrm(0x03, r7(r0), r7(r1));
2377}
2378
2379static void
2380_movcr_u(jit_state_t *_jit, jit_int32_t r0, jit_int32_t r1)
2381{
2382 rex(0, WIDE, r0, _NOREG, r1);
2383 ic(0x0f);
2384 ic(0xb6);
2385 mrm(0x03, r7(r0), r7(r1));
2386}
2387
2388static void
2389_movsr(jit_state_t *_jit, jit_int32_t r0, jit_int32_t r1)
2390{
2391 rex(0, WIDE, r0, _NOREG, r1);
2392 ic(0x0f);
2393 ic(0xbf);
2394 mrm(0x03, r7(r0), r7(r1));
2395}
2396
2397static void
2398_movsr_u(jit_state_t *_jit, jit_int32_t r0, jit_int32_t r1)
2399{
2400 rex(0, WIDE, r0, _NOREG, r1);
2401 ic(0x0f);
2402 ic(0xb7);
2403 mrm(0x03, r7(r0), r7(r1));
2404}
2405
ba3814c1
PC		 2406static void
2407_casx(jit_state_t *_jit, jit_int32_t r0, jit_int32_t r1,
2408 jit_int32_t r2, jit_int32_t r3, jit_word_t i0)
2409{
2410 jit_int32_t save_rax, restore_rax;
2411 jit_int32_t ascasr_reg, ascasr_use;
2412 if (r0 != _RAX_REGNO) { /* result not in %rax */
2413 if (r2 != _RAX_REGNO) { /* old value not in %rax */
2414 save_rax = jit_get_reg(jit_class_gpr);
2415 movr(rn(save_rax), _RAX_REGNO);
2416 restore_rax = 1;
2417 }
2418 else
2419 restore_rax = 0;
2420 }
2421 else
2422 restore_rax = 0;
2423 if (r2 != _RAX_REGNO)
2424 movr(_RAX_REGNO, r2);
2425 if (r1 == _NOREG) { /* using immediate address */
2426 if (!can_sign_extend_int_p(i0)) {
2427 ascasr_reg = jit_get_reg(jit_class_gpr);
2428 if (ascasr_reg == _RAX) {
2429 ascasr_reg = jit_get_reg(jit_class_gpr);
2430 jit_unget_reg(_RAX);
2431 }
2432 ascasr_use = 1;
2433 movi(rn(ascasr_reg), i0);
2434 }
2435 else
2436 ascasr_use = 0;
2437 }
2438 else
2439 ascasr_use = 0;
2440 ic(0xf0); /* lock */
2441 if (ascasr_use)
2442 rex(0, WIDE, r3, _NOREG, rn(ascasr_reg));
2443 else
2444 rex(0, WIDE, r3, _NOREG, r1);
2445 ic(0x0f);
2446 ic(0xb1);
2447 if (r1 != _NOREG) /* casr */
2448 rx(r3, 0, r1, _NOREG, _SCL1);
2449 else { /* casi */
2450 if (ascasr_use)
2451 rx(r3, 0, rn(ascasr_reg), _NOREG, _SCL1); /* address in reg */
2452 else
2453 rx(r3, i0, _NOREG, _NOREG, _SCL1); /* address in offset */
2454 }
2455 cc(X86_CC_E, r0);
2456 if (r0 != _RAX_REGNO)
2457 movr(r0, _RAX_REGNO);
2458 if (restore_rax) {
2459 movr(_RAX_REGNO, rn(save_rax));
2460 jit_unget_reg(save_rax);
2461 }
2462 if (ascasr_use)
2463 jit_unget_reg(ascasr_reg);
2464}
2465
1f22b268
PC		 2466static void
2467_movnr(jit_state_t *_jit, jit_int32_t r0, jit_int32_t r1, jit_int32_t r2)
2468{
2469 assert(jit_cmov_p());
2470
2471 testr(r2, r2);
2472
2473 rex(0, WIDE, r0, _NOREG, r1);
2474 ic(0x0f);
2475 ic(0x45);
2476 mrm(0x03, r7(r0), r7(r1));
2477}
2478
2479static void
2480_movzr(jit_state_t *_jit, jit_int32_t r0, jit_int32_t r1, jit_int32_t r2)
2481{
2482 assert(jit_cmov_p());
2483
2484 testr(r2, r2);
2485
2486 rex(0, WIDE, r0, _NOREG, r1);
2487 ic(0x0f);
2488 ic(0x44);
2489 mrm(0x03, r7(r0), r7(r1));
2490}
2491
4a71579b
PC		 2492#if __X64
2493static void
2494_movir(jit_state_t *_jit, jit_int32_t r0, jit_int32_t r1)
2495{
2496 rex(0, 1, r0, _NOREG, r1);
2497 ic(0x63);
2498 mrm(0x03, r7(r0), r7(r1));
2499}
2500
2501static void
2502_movir_u(jit_state_t *_jit, jit_int32_t r0, jit_int32_t r1)
2503{
2504 rex(0, 0, r1, _NOREG, r0);
2505 ic(0x89);
2506 ic(0xc0 | (r1 << 3) | r7(r0));
2507}
2508#endif
2509
2510static void
40a44dcb 2511_bswapr_us(jit_state_t *_jit, jit_int32_t r0, jit_int32_t r1)
4a71579b
PC		 2512{
2513 extr_us(r0, r1);
2514 ic(0x66);
2515 rex(0, 0, _NOREG, _NOREG, r0);
2516 ic(0xc1);
2517 mrm(0x03, X86_ROR, r7(r0));
2518 ic(8);
2519}
2520
2521static void
40a44dcb 2522_bswapr_ui(jit_state_t *_jit, jit_int32_t r0, jit_int32_t r1)
4a71579b
PC		 2523{
2524 movr(r0, r1);
2525 rex(0, 0, _NOREG, _NOREG, r0);
2526 ic(0x0f);
2527 ic(0xc8 | r7(r0));
2528}
2529
2530#if __X64 && !__X64_32
2531static void
40a44dcb 2532_bswapr_ul(jit_state_t *_jit, jit_int32_t r0, jit_int32_t r1)
4a71579b
PC		 2533{
2534 movr(r0, r1);
2535 rex(0, 1, _NOREG, _NOREG, r0);
2536 ic(0x0f);
2537 ic(0xc8 | r7(r0));
2538}
2539#endif
2540
2541static void
2542_extr_c(jit_state_t *_jit, jit_int32_t r0, jit_int32_t r1)
2543{
2544 jit_int32_t reg;
2545 if (reg8_p(r1))
2546 movcr(r0, r1);
2547 else {
2548 reg = jit_get_reg(jit_class_gpr|jit_class_rg8);
2549 movr(rn(reg), r1);
2550 movcr(r0, rn(reg));
2551 jit_unget_reg(reg);
2552 }
2553}
2554
2555static void
2556_extr_uc(jit_state_t *_jit, jit_int32_t r0, jit_int32_t r1)
2557{
2558 jit_int32_t reg;
2559 if (reg8_p(r1))
2560 movcr_u(r0, r1);
2561 else {
2562 reg = jit_get_reg(jit_class_gpr|jit_class_rg8);
2563 movr(rn(reg), r1);
2564 movcr_u(r0, rn(reg));
2565 jit_unget_reg(reg);
2566 }
2567}
2568
2569static void
2570_ldr_c(jit_state_t *_jit, jit_int32_t r0, jit_int32_t r1)
2571{
2572 rex(0, WIDE, r0, _NOREG, r1);
2573 ic(0x0f);
2574 ic(0xbe);
2575 rx(r0, 0, r1, _NOREG, _SCL1);
2576}
2577
2578static void
2579_ldi_c(jit_state_t *_jit, jit_int32_t r0, jit_word_t i0)
2580{
2581 jit_int32_t reg;
79bfeef6
PC		 2582#if CAN_RIP_ADDRESS
2583 jit_word_t rel = i0 - _jit->pc.w;
2584 rel = rel < 0 ? rel - 8 : rel + 8;
2585 if (can_sign_extend_int_p(rel)) {
2586 rex(0, WIDE, r0, _NOREG, _NOREG);
2587 ic(0x0f);
2588 ic(0xbe);
2589 rx(r0, i0 - (_jit->pc.w + 5), _NOREG, _NOREG, _SCL8);
2590 }
2591 else
2592#endif
2593 if (address_p(i0)) {
4a71579b
PC		 2594 rex(0, WIDE, r0, _NOREG, _NOREG);
2595 ic(0x0f);
2596 ic(0xbe);
2597 rx(r0, i0, _NOREG, _NOREG, _SCL1);
2598 }
2599 else {
2600 reg = jit_get_reg(jit_class_gpr);
2601 movi(rn(reg), i0);
2602 ldr_c(r0, rn(reg));
2603 jit_unget_reg(reg);
2604 }
2605}
2606
2607static void
2608_ldr_uc(jit_state_t *_jit, jit_int32_t r0, jit_int32_t r1)
2609{
2610 rex(0, WIDE, r0, _NOREG, r1);
2611 ic(0x0f);
2612 ic(0xb6);
2613 rx(r0, 0, r1, _NOREG, _SCL1);
2614}
2615
2616static void
2617_ldi_uc(jit_state_t *_jit, jit_int32_t r0, jit_word_t i0)
2618{
2619 jit_int32_t reg;
79bfeef6
PC		 2620#if CAN_RIP_ADDRESS
2621 jit_word_t rel = i0 - _jit->pc.w;
2622 rel = rel < 0 ? rel - 8 : rel + 8;
2623 if (can_sign_extend_int_p(rel)) {
2624 rex(0, WIDE, r0, _NOREG, _NOREG);
2625 ic(0x0f);
2626 ic(0xb6);
2627 rx(r0, i0 - (_jit->pc.w + 5), _NOREG, _NOREG, _SCL8);
2628 }
2629 else
2630#endif
2631 if (address_p(i0)) {
4a71579b
PC		 2632 rex(0, WIDE, r0, _NOREG, _NOREG);
2633 ic(0x0f);
2634 ic(0xb6);
2635 rx(r0, i0, _NOREG, _NOREG, _SCL1);
2636 }
2637 else {
2638 reg = jit_get_reg(jit_class_gpr);
2639 movi(rn(reg), i0);
2640 ldr_uc(r0, rn(reg));
2641 jit_unget_reg(reg);
2642 }
2643}
2644
2645static void
2646_ldr_s(jit_state_t *_jit, jit_int32_t r0, jit_int32_t r1)
2647{
2648 rex(0, WIDE, r0, _NOREG, r1);
2649 ic(0x0f);
2650 ic(0xbf);
2651 rx(r0, 0, r1, _NOREG, _SCL1);
2652}
2653
2654static void
2655_ldi_s(jit_state_t *_jit, jit_int32_t r0, jit_word_t i0)
2656{
2657 jit_int32_t reg;
79bfeef6
PC		 2658#if CAN_RIP_ADDRESS
2659 jit_word_t rel = i0 - _jit->pc.w;
2660 rel = rel < 0 ? rel - 8 : rel + 8;
2661 if (can_sign_extend_int_p(rel)) {
2662 rex(0, WIDE, r0, _NOREG, _NOREG);
2663 ic(0x0f);
2664 ic(0xbf);
2665 rx(r0, i0 - (_jit->pc.w + 5), _NOREG, _NOREG, _SCL8);
2666 }
2667 else
2668#endif
2669 if (address_p(i0)) {
4a71579b
PC		 2670 rex(0, WIDE, r0, _NOREG, _NOREG);
2671 ic(0x0f);
2672 ic(0xbf);
2673 rx(r0, i0, _NOREG, _NOREG, _SCL1);
2674 }
2675 else {
2676 reg = jit_get_reg(jit_class_gpr);
2677 movi(rn(reg), i0);
2678 ldr_s(r0, rn(reg));
2679 jit_unget_reg(reg);
2680 }
2681}
2682
2683static void
2684_ldr_us(jit_state_t *_jit, jit_int32_t r0, jit_int32_t r1)
2685{
2686 rex(0, WIDE, r0, _NOREG, r1);
2687 ic(0x0f);
2688 ic(0xb7);
2689 rx(r0, 0, r1, _NOREG, _SCL1);
2690}
2691
2692static void
2693_ldi_us(jit_state_t *_jit, jit_int32_t r0, jit_word_t i0)
2694{
2695 jit_int32_t reg;
79bfeef6
PC		 2696#if CAN_RIP_ADDRESS
2697 jit_word_t rel = i0 - _jit->pc.w;
2698 rel = rel < 0 ? rel - 8 : rel + 8;
2699 if (can_sign_extend_int_p(rel)) {
2700 rex(0, WIDE, r0, _NOREG, _NOREG);
2701 ic(0x0f);
2702 ic(0xb7);
2703 rx(r0, i0 - (_jit->pc.w + 5), _NOREG, _NOREG, _SCL8);
2704 }
2705 else
2706#endif
2707 if (address_p(i0)) {
4a71579b
PC		 2708 rex(0, WIDE, r0, _NOREG, _NOREG);
2709 ic(0x0f);
2710 ic(0xb7);
2711 rx(r0, i0, _NOREG, _NOREG, _SCL1);
2712 }
2713 else {
2714 reg = jit_get_reg(jit_class_gpr);
2715 movi(rn(reg), i0);
2716 ldr_us(r0, rn(reg));
2717 jit_unget_reg(reg);
2718 }
2719}
2720
2721#if __X32 || !__X64_32
2722static void
2723_ldr_i(jit_state_t *_jit, jit_int32_t r0, jit_int32_t r1)
2724{
2725#if __X64
2726 rex(0, WIDE, r0, _NOREG, r1);
2727 ic(0x63);
2728#else
2729 ic(0x8b);
2730#endif
2731 rx(r0, 0, r1, _NOREG, _SCL1);
2732}
2733
2734static void
2735_ldi_i(jit_state_t *_jit, jit_int32_t r0, jit_word_t i0)
2736{
2737 jit_int32_t reg;
79bfeef6
PC		 2738#if CAN_RIP_ADDRESS
2739 jit_word_t rel = i0 - _jit->pc.w;
2740 rel = rel < 0 ? rel - 8 : rel + 8;
2741 if (can_sign_extend_int_p(rel)) {
2742 rex(0, WIDE, r0, _NOREG, _NOREG);
2743 ic(0x63);
2744 rx(r0, i0 - (_jit->pc.w + 5), _NOREG, _NOREG, _SCL8);
2745 }
2746 else
2747#endif
2748 if (address_p(i0)) {
4a71579b
PC		 2749#if __X64
2750 rex(0, WIDE, r0, _NOREG, _NOREG);
2751 ic(0x63);
2752#else
2753 ic(0x8b);
2754#endif
2755 rx(r0, i0, _NOREG, _NOREG, _SCL1);
2756 }
2757 else {
2758 reg = jit_get_reg(jit_class_gpr);
2759 movi(rn(reg), i0);
2760 ldr_i(r0, rn(reg));
2761 jit_unget_reg(reg);
2762 }
2763}
2764#endif
2765
2766#if __X64
2767static void
2768_ldr_ui(jit_state_t *_jit, jit_int32_t r0, jit_int32_t r1)
2769{
2770 rex(0, 0, r0, _NOREG, r1);
2771 ic(0x63);
2772 rx(r0, 0, r1, _NOREG, _SCL1);
2773}
2774
2775static void
2776_ldi_ui(jit_state_t *_jit, jit_int32_t r0, jit_word_t i0)
2777{
2778 jit_int32_t reg;
79bfeef6
PC		 2779# if !__X64_32
2780 jit_word_t rel = i0 - _jit->pc.w;
2781 rel = rel < 0 ? rel - 8 : rel + 8;
2782 if (can_sign_extend_int_p(rel)) {
2783 rex(0, 0, r0, _NOREG, _NOREG);
2784 ic(0x63);
2785 rx(r0, i0 - (_jit->pc.w + 5), _NOREG, _NOREG, _SCL8);
2786 }
2787 else
2788#endif
2789 if (address_p(i0)) {
4a71579b
PC		 2790 rex(0, 0, r0, _NOREG, _NOREG);
2791 ic(0x63);
2792 rx(r0, i0, _NOREG, _NOREG, _SCL1);
2793 }
2794 else {
2795 reg = jit_get_reg(jit_class_gpr);
2796 movi(rn(reg), i0);
79bfeef6
PC		 2797# if __X64_32
2798 ldr_i(r0, rn(reg));
2799# else
4a71579b 2800 ldr_ui(r0, rn(reg));
79bfeef6 2801# endif
4a71579b
PC		 2802 jit_unget_reg(reg);
2803 }
2804}
2805
2806# if !__X64_32
2807static void
2808_ldr_l(jit_state_t *_jit, jit_int32_t r0, jit_int32_t r1)
2809{
2810 rex(0, 1, r0, _NOREG, r1);
2811 ic(0x8b);
2812 rx(r0, 0, r1, _NOREG, _SCL1);
2813}
2814
2815static void
2816_ldi_l(jit_state_t *_jit, jit_int32_t r0, jit_word_t i0)
2817{
2818 jit_int32_t reg;
79bfeef6
PC		 2819 jit_word_t rel = i0 - _jit->pc.w;
2820 rel = rel < 0 ? rel - 8 : rel + 8;
2821 if (can_sign_extend_int_p(rel)) {
2822 rex(0, WIDE, r0, _NOREG, _NOREG);
2823 ic(0x8b);
2824 rx(r0, i0 - (_jit->pc.w + 5), _NOREG, _NOREG, _SCL8);
2825 }
2826 else if (can_sign_extend_int_p(i0)) {
2827 rex(0, WIDE, r0, _NOREG, _NOREG);
4a71579b
PC		 2828 ic(0x8b);
2829 rx(r0, i0, _NOREG, _NOREG, _SCL1);
2830 }
2831 else {
2832 reg = jit_get_reg(jit_class_gpr);
2833 movi(rn(reg), i0);
2834 ldr_l(r0, rn(reg));
2835 jit_unget_reg(reg);
2836 }
2837}
2838# endif
2839#endif
2840
2841static void
2842_ldxr_c(jit_state_t *_jit, jit_int32_t r0, jit_int32_t r1, jit_int32_t r2)
2843{
2844#if __X64_32
2845 addr(r0, r1, r2);
2846 ldr_c(r0, r0);
2847#else
2848 rex(0, WIDE, r0, r1, r2);
2849 ic(0x0f);
2850 ic(0xbe);
2851 rx(r0, 0, r2, r1, _SCL1);
2852#endif
2853}
2854
2855static void
2856_ldxi_c(jit_state_t *_jit, jit_int32_t r0, jit_int32_t r1, jit_word_t i0)
2857{
2858 jit_int32_t reg;
2859 if (can_sign_extend_int_p(i0)) {
2860 rex(0, WIDE, r0, _NOREG, r1);
2861 ic(0x0f);
2862 ic(0xbe);
2863 rx(r0, i0, r1, _NOREG, _SCL1);
2864 }
2865 else {
2866 reg = jit_get_reg(jit_class_gpr);
2867 movi(rn(reg), i0);
2868 ldxr_c(r0, r1, rn(reg));
2869 jit_unget_reg(reg);
2870 }
2871}
2872
2873static void
2874_ldxr_uc(jit_state_t *_jit, jit_int32_t r0, jit_int32_t r1, jit_int32_t r2)
2875{
2876#if __X64_32
2877 addr(r0, r1, r2);
2878 ldr_uc(r0, r0);
2879#else
2880 rex(0, WIDE, r0, r1, r2);
2881 ic(0x0f);
2882 ic(0xb6);
2883 rx(r0, 0, r2, r1, _SCL1);
2884#endif
2885}
2886
2887static void
2888_ldxi_uc(jit_state_t *_jit, jit_int32_t r0, jit_int32_t r1, jit_word_t i0)
2889{
2890 jit_int32_t reg;
2891 if (can_sign_extend_int_p(i0)) {
2892 rex(0, WIDE, r0, _NOREG, r1);
2893 ic(0x0f);
2894 ic(0xb6);
2895 rx(r0, i0, r1, _NOREG, _SCL1);
2896 }
2897 else {
2898 reg = jit_get_reg(jit_class_gpr);
2899 movi(rn(reg), i0);
2900 ldxr_uc(r0, r1, rn(reg));
2901 jit_unget_reg(reg);
2902 }
2903}
2904
2905static void
2906_ldxr_s(jit_state_t *_jit, jit_int32_t r0, jit_int32_t r1, jit_int32_t r2)
2907{
2908#if __X64_32
2909 addr(r0, r1, r2);
2910 ldr_s(r0, r0);
2911#else
2912 rex(0, WIDE, r0, r1, r2);
2913 ic(0x0f);
2914 ic(0xbf);
2915 rx(r0, 0, r2, r1, _SCL1);
2916#endif
2917}
2918
2919static void
2920_ldxi_s(jit_state_t *_jit, jit_int32_t r0, jit_int32_t r1, jit_word_t i0)
2921{
2922 jit_int32_t reg;
2923 if (can_sign_extend_int_p(i0)) {
2924 rex(0, WIDE, r0, _NOREG, r1);
2925 ic(0x0f);
2926 ic(0xbf);
2927 rx(r0, i0, r1, _NOREG, _SCL1);
2928 }
2929 else {
2930 reg = jit_get_reg(jit_class_gpr);
2931 movi(rn(reg), i0);
2932 ldxr_s(r0, r1, rn(reg));
2933 jit_unget_reg(reg);
2934 }
2935}
2936
2937static void
2938_ldxr_us(jit_state_t *_jit, jit_int32_t r0, jit_int32_t r1, jit_int32_t r2)
2939{
2940#if __X64_32
2941 addr(r0, r1, r2);
2942 ldr_us(r0, r0);
2943#else
2944 rex(0, WIDE, r0, r1, r2);
2945 ic(0x0f);
2946 ic(0xb7);
2947 rx(r0, 0, r2, r1, _SCL1);
2948#endif
2949}
2950
2951static void
2952_ldxi_us(jit_state_t *_jit, jit_int32_t r0, jit_int32_t r1, jit_word_t i0)
2953{
2954 jit_int32_t reg;
2955 if (can_sign_extend_int_p(i0)) {
2956 rex(0, WIDE, r0, _NOREG, r1);
2957 ic(0x0f);
2958 ic(0xb7);
2959 rx(r0, i0, r1, _NOREG, _SCL1);
2960 }
2961 else {
2962 reg = jit_get_reg(jit_class_gpr);
2963 movi(rn(reg), i0);
2964 ldxr_us(r0, r1, rn(reg));
2965 jit_unget_reg(reg);
2966 }
2967}
2968
2969#if __X64 || !__X64_32
2970static void
2971_ldxr_i(jit_state_t *_jit, jit_int32_t r0, jit_int32_t r1, jit_int32_t r2)
2972{
2973#if __X64
2974 rex(0, WIDE, r0, r1, r2);
2975 ic(0x63);
2976#else
2977 ic(0x8b);
2978#endif
2979 rx(r0, 0, r2, r1, _SCL1);
2980}
2981
2982static void
2983_ldxi_i(jit_state_t *_jit, jit_int32_t r0, jit_int32_t r1, jit_word_t i0)
2984{
2985 jit_int32_t reg;
2986 if (can_sign_extend_int_p(i0)) {
2987#if __X64
2988 rex(0, WIDE, r0, _NOREG, r1);
2989 ic(0x63);
2990#else
2991 ic(0x8b);
2992#endif
2993 rx(r0, i0, r1, _NOREG, _SCL1);
2994 }
2995 else {
2996 reg = jit_get_reg(jit_class_gpr);
2997 movi(rn(reg), i0);
2998 ldxr_i(r0, r1, rn(reg));
2999 jit_unget_reg(reg);
3000 }
3001}
3002#endif
3003
3004#if __X64
3005static void
3006_ldxr_ui(jit_state_t *_jit, jit_int32_t r0, jit_int32_t r1, jit_int32_t r2)
3007{
3008#if __X64_32
3009 addr(r0, r1, r2);
3010 /* to avoid confusion with macro renames */
3011 _ldr_ui(_jit, r0, r0);
3012#else
3013 rex(0, 0, r0, r1, r2);
3014 ic(0x8b);
3015 rx(r0, 0, r2, r1, _SCL1);
3016#endif
3017}
3018
3019static void
3020_ldxi_ui(jit_state_t *_jit, jit_int32_t r0, jit_int32_t r1, jit_word_t i0)
3021{
3022 jit_int32_t reg;
3023 if (can_sign_extend_int_p(i0)) {
3024 rex(0, 0, r0, _NOREG, r1);
3025 ic(0x8b);
3026 rx(r0, i0, r1, _NOREG, _SCL1);
3027 }
3028 else {
3029 reg = jit_get_reg(jit_class_gpr);
3030 movi(rn(reg), i0);
79bfeef6
PC		 3031# if __X64_32
3032 ldxr_i(r0, r1, rn(reg));
3033# else
4a71579b 3034 ldxr_ui(r0, r1, rn(reg));
79bfeef6 3035# endif
4a71579b
PC		 3036 jit_unget_reg(reg);
3037 }
3038}
3039
3040# if !__X64_32
3041static void
3042_ldxr_l(jit_state_t *_jit, jit_int32_t r0, jit_int32_t r1, jit_int32_t r2)
3043{
3044 rex(0, 1, r0, r1, r2);
3045 ic(0x8b);
3046 rx(r0, 0, r2, r1, _SCL1);
3047}
3048
3049static void
3050_ldxi_l(jit_state_t *_jit, jit_int32_t r0, jit_int32_t r1, jit_word_t i0)
3051{
3052 jit_int32_t reg;
3053 if (can_sign_extend_int_p(i0)) {
3054 rex(0, 1, r0, _NOREG, r1);
3055 ic(0x8b);
3056 rx(r0, i0, r1, _NOREG, _SCL1);
3057 }
3058 else {
3059 reg = jit_get_reg(jit_class_gpr);
3060 movi(rn(reg), i0);
3061 ldxr_l(r0, r1, rn(reg));
3062 jit_unget_reg(reg);
3063 }
3064}
3065# endif
3066#endif
3067
3068static void
3069_str_c(jit_state_t *_jit, jit_int32_t r0, jit_int32_t r1)
3070{
3071 jit_int32_t reg;
3072 if (reg8_p(r1)) {
3073 rex(0, 0, r1, _NOREG, r0);
3074 ic(0x88);
3075 rx(r1, 0, r0, _NOREG, _SCL1);
3076 }
3077 else {
3078 reg = jit_get_reg(jit_class_gpr|jit_class_rg8);
3079 movr(rn(reg), r1);
3080 rex(0, 0, rn(reg), _NOREG, r0);
3081 ic(0x88);
3082 rx(rn(reg), 0, r0, _NOREG, _SCL1);
3083 jit_unget_reg(reg);
3084 }
3085}
3086
3087static void
3088_sti_c(jit_state_t *_jit, jit_word_t i0, jit_int32_t r0)
3089{
3090 jit_int32_t reg;
79bfeef6
PC		 3091#if CAN_RIP_ADDRESS
3092 jit_word_t rel = i0 - _jit->pc.w;
3093 rel = rel < 0 ? rel - 16 : rel + 16;
3094 if (can_sign_extend_int_p(rel)) {
3095 if (reg8_p(r0)) {
3096 rex(0, 0, r0, _NOREG, _NOREG);
3097 ic(0x88);
3098 rx(r0, i0 - (_jit->pc.w + 5), _NOREG, _NOREG, _SCL8);
3099 }
3100 else {
3101 reg = jit_get_reg(jit_class_gpr|jit_class_rg8);
3102 movr(rn(reg), r0);
3103 rex(0, 0, rn(reg), _NOREG, _NOREG);
3104 ic(0x88);
3105 rx(rn(reg), i0 - (_jit->pc.w + 5), _NOREG, _NOREG, _SCL8);
3106 jit_unget_reg(reg);
3107 }
3108 }
3109 else
3110#endif
3111 if (address_p(i0)) {
4a71579b
PC		 3112 if (reg8_p(r0)) {
3113 rex(0, 0, r0, _NOREG, _NOREG);
3114 ic(0x88);
3115 rx(r0, i0, _NOREG, _NOREG, _SCL1);
3116 }
3117 else {
3118 reg = jit_get_reg(jit_class_gpr|jit_class_rg8);
3119 movr(rn(reg), r0);
3120 rex(0, 0, rn(reg), _NOREG, _NOREG);
3121 ic(0x88);
3122 rx(rn(reg), i0, _NOREG, _NOREG, _SCL1);
3123 jit_unget_reg(reg);
3124 }
3125 }
3126 else {
3127 reg = jit_get_reg(jit_class_gpr);
3128 movi(rn(reg), i0);
3129 str_c(rn(reg), r0);
3130 jit_unget_reg(reg);
3131 }
3132}
3133
3134static void
3135_str_s(jit_state_t *_jit, jit_int32_t r0, jit_int32_t r1)
3136{
3137 ic(0x66);
3138 rex(0, 0, r1, _NOREG, r0);
3139 ic(0x89);
3140 rx(r1, 0, r0, _NOREG, _SCL1);
3141}
3142
3143static void
3144_sti_s(jit_state_t *_jit, jit_word_t i0, jit_int32_t r0)
3145{
3146 jit_int32_t reg;
79bfeef6
PC		 3147#if CAN_RIP_ADDRESS
3148 jit_word_t rel = i0 - _jit->pc.w;
3149 rel = rel < 0 ? rel - 8 : rel + 8;
3150 if (can_sign_extend_int_p(rel)) {
3151 ic(0x66);
3152 rex(0, 0, r0, _NOREG, _NOREG);
3153 ic(0x89);
3154 rx(r0, i0 - (_jit->pc.w + 5), _NOREG, _NOREG, _SCL8);
3155 }
3156 else
3157#endif
3158 if (address_p(i0)) {
4a71579b
PC		 3159 ic(0x66);
3160 rex(0, 0, r0, _NOREG, _NOREG);
3161 ic(0x89);
3162 rx(r0, i0, _NOREG, _NOREG, _SCL1);
3163 }
3164 else {
3165 reg = jit_get_reg(jit_class_gpr);
3166 movi(rn(reg), i0);
3167 str_s(rn(reg), r0);
3168 jit_unget_reg(reg);
3169 }
3170}
3171
3172static void
3173_str_i(jit_state_t *_jit, jit_int32_t r0, jit_int32_t r1)
3174{
3175 rex(0, 0, r1, _NOREG, r0);
3176 ic(0x89);
3177 rx(r1, 0, r0, _NOREG, _SCL1);
3178}
3179
3180static void
3181_sti_i(jit_state_t *_jit, jit_word_t i0, jit_int32_t r0)
3182{
3183 jit_int32_t reg;
79bfeef6
PC		 3184#if CAN_RIP_ADDRESS
3185 jit_word_t rel = i0 - _jit->pc.w;
3186 rel = rel < 0 ? rel - 8 : rel + 8;
3187 if (can_sign_extend_int_p(rel)) {
3188 rex(0, 0, r0, _NOREG, _NOREG);
3189 ic(0x89);
3190 rx(r0, i0 - (_jit->pc.w + 5), _NOREG, _NOREG, _SCL8);
3191 }
3192 else
3193#endif
3194 if (address_p(i0)) {
4a71579b
PC		 3195 rex(0, 0, r0, _NOREG, _NOREG);
3196 ic(0x89);
3197 rx(r0, i0, _NOREG, _NOREG, _SCL1);
3198 }
3199 else {
3200 reg = jit_get_reg(jit_class_gpr);
3201 movi(rn(reg), i0);
3202 str_i(rn(reg), r0);
3203 jit_unget_reg(reg);
3204 }
3205}
3206
3207#if __X64 && !__X64_32
3208static void
3209_str_l(jit_state_t *_jit, jit_int32_t r0, jit_int32_t r1)
3210{
3211 rex(0, 1, r1, _NOREG, r0);
3212 ic(0x89);
3213 rx(r1, 0, r0, _NOREG, _SCL1);
3214}
3215
3216static void
3217_sti_l(jit_state_t *_jit, jit_word_t i0, jit_int32_t r0)
3218{
3219 jit_int32_t reg;
79bfeef6
PC		 3220#if CAN_RIP_ADDRESS
3221 jit_word_t rel = i0 - _jit->pc.w;
3222 rel = rel < 0 ? rel - 8 : rel + 8;
3223 if (can_sign_extend_int_p(rel)) {
3224 rex(0, WIDE, r0, _NOREG, _NOREG);
3225 ic(0x89);
3226 rx(r0, i0 - (_jit->pc.w + 5), _NOREG, _NOREG, _SCL8);
3227 }
3228 else
3229#endif
4a71579b 3230 if (can_sign_extend_int_p(i0)) {
79bfeef6 3231 rex(0, WIDE, r0, _NOREG, _NOREG);
4a71579b
PC		 3232 ic(0x89);
3233 rx(r0, i0, _NOREG, _NOREG, _SCL1);
3234 }
3235 else {
3236 reg = jit_get_reg(jit_class_gpr);
3237 movi(rn(reg), i0);
3238 str_l(rn(reg), r0);
3239 jit_unget_reg(reg);
3240 }
3241}
3242#endif
3243
3244static void
3245_stxr_c(jit_state_t *_jit, jit_int32_t r0, jit_int32_t r1, jit_int32_t r2)
3246{
3247 jit_int32_t reg;
3248#if __X64_32
3249 reg = jit_get_reg(jit_class_gpr);
3250 addr(rn(reg), r0, r1);
3251 str_c(rn(reg), r2);
3252 jit_unget_reg(reg);
3253#else
3254 if (reg8_p(r2)) {
3255 rex(0, 0, r2, r1, r0);
3256 ic(0x88);
3257 rx(r2, 0, r0, r1, _SCL1);
3258 }
3259 else {
3260 reg = jit_get_reg(jit_class_gpr|jit_class_rg8);
3261 movr(rn(reg), r2);
3262 rex(0, 0, rn(reg), r1, r0);
3263 ic(0x88);
3264 rx(rn(reg), 0, r0, r1, _SCL1);
3265 jit_unget_reg(reg);
3266 }
3267#endif
3268}
3269
3270static void
3271_stxi_c(jit_state_t *_jit, jit_word_t i0, jit_int32_t r0, jit_int32_t r1)
3272{
3273 jit_int32_t reg;
3274 if (can_sign_extend_int_p(i0)) {
3275 if (reg8_p(r1)) {
3276 rex(0, 0, r1, _NOREG, r0);
3277 ic(0x88);
3278 rx(r1, i0, r0, _NOREG, _SCL1);
3279 }
3280 else {
3281 reg = jit_get_reg(jit_class_gpr|jit_class_rg8);
3282 movr(rn(reg), r1);
3283 rex(0, 0, rn(reg), _NOREG, r0);
3284 ic(0x88);
3285 rx(rn(reg), i0, r0, _NOREG, _SCL1);
3286 jit_unget_reg(reg);
3287 }
3288 }
3289 else {
3290 reg = jit_get_reg(jit_class_gpr);
3291 movi(rn(reg), i0);
3292 stxr_c(rn(reg), r0, r1);
3293 jit_unget_reg(reg);
3294 }
3295}
3296
3297static void
3298_stxr_s(jit_state_t *_jit, jit_int32_t r0, jit_int32_t r1, jit_int32_t r2)
3299{
3300#if __X64_32
3301 jit_int32_t reg;
3302 reg = jit_get_reg(jit_class_gpr);
3303 addr(rn(reg), r0, r1);
3304 str_s(rn(reg), r2);
3305 jit_unget_reg(reg);
3306#else
3307 ic(0x66);
3308 rex(0, 0, r2, r1, r0);
3309 ic(0x89);
3310 rx(r2, 0, r0, r1, _SCL1);
3311#endif
3312}
3313
3314static void
3315_stxi_s(jit_state_t *_jit, jit_word_t i0, jit_int32_t r0, jit_int32_t r1)
3316{
3317 jit_int32_t reg;
3318 if (can_sign_extend_int_p(i0)) {
3319 ic(0x66);
3320 rex(0, 0, r1, _NOREG, r0);
3321 ic(0x89);
3322 rx(r1, i0, r0, _NOREG, _SCL1);
3323 }
3324 else {
3325 reg = jit_get_reg(jit_class_gpr);
3326 movi(rn(reg), i0);
3327 stxr_s(rn(reg), r0, r1);
3328 jit_unget_reg(reg);
3329 }
3330}
3331
3332static void
3333_stxr_i(jit_state_t *_jit, jit_int32_t r0, jit_int32_t r1, jit_int32_t r2)
3334{
3335#if __X64_32
3336 jit_int32_t reg;
3337 reg = jit_get_reg(jit_class_gpr);
3338 addr(rn(reg), r0, r1);
3339 str_i(rn(reg), r2);
3340 jit_unget_reg(reg);
3341#else
3342 rex(0, 0, r2, r1, r0);
3343 ic(0x89);
3344 rx(r2, 0, r0, r1, _SCL1);
3345#endif
3346}
3347
3348static void
3349_stxi_i(jit_state_t *_jit, jit_word_t i0, jit_int32_t r0, jit_int32_t r1)
3350{
3351 jit_int32_t reg;
3352 if (can_sign_extend_int_p(i0)) {
3353 rex(0, 0, r1, _NOREG, r0);
3354 ic(0x89);
3355 rx(r1, i0, r0, _NOREG, _SCL1);
3356 }
3357 else {
3358 reg = jit_get_reg(jit_class_gpr);
3359 movi(rn(reg), i0);
3360 stxr_i(rn(reg), r0, r1);
3361 jit_unget_reg(reg);
3362 }
3363}
3364
3365#if __X64 && !__X64_32
3366static void
3367_stxr_l(jit_state_t *_jit, jit_int32_t r0, jit_int32_t r1, jit_int32_t r2)
3368{
3369 rex(0, 1, r2, r1, r0);
3370 ic(0x89);
3371 rx(r2, 0, r0, r1, _SCL1);
3372}
3373
3374static void
3375_stxi_l(jit_state_t *_jit, jit_word_t i0, jit_int32_t r0, jit_int32_t r1)
3376{
3377 jit_int32_t reg;
3378 if (can_sign_extend_int_p(i0)) {
3379 rex(0, 1, r1, _NOREG, r0);
3380 ic(0x89);
3381 rx(r1, i0, r0, _NOREG, _SCL1);
3382 }
3383 else {
3384 reg = jit_get_reg(jit_class_gpr);
3385 movi(rn(reg), i0);
3386 stxr_l(rn(reg), r0, r1);
3387 jit_unget_reg(reg);
3388 }
3389}
3390#endif
3391
79bfeef6 3392static jit_word_t
4a71579b
PC		 3393_jccs(jit_state_t *_jit, jit_int32_t code, jit_word_t i0)
3394{
79bfeef6 3395 jit_word_t d;
4a71579b 3396 jit_word_t w;
79bfeef6
PC		 3397 w = _jit->pc.w;
3398 d = i0 - (w + 1);
4a71579b 3399 ic(0x70 | code);
79bfeef6
PC		 3400 ic(d);
3401 return (w);
4a71579b
PC		 3402}
3403
79bfeef6 3404static jit_word_t
4a71579b
PC		 3405_jcc(jit_state_t *_jit, jit_int32_t code, jit_word_t i0)
3406{
79bfeef6 3407 jit_word_t d;
4a71579b 3408 jit_word_t w;
79bfeef6 3409 w = _jit->pc.w;
4a71579b 3410 ic(0x0f);
79bfeef6 3411 d = i0 - (w + 6);
4a71579b 3412 ic(0x80 | code);
79bfeef6
PC		 3413 ii(d);
3414 return (w);
4a71579b
PC		 3415}
3416
79bfeef6 3417static jit_word_t
4a71579b
PC		 3418_jcr(jit_state_t *_jit,
3419 jit_int32_t code, jit_word_t i0, jit_int32_t r0, jit_int32_t r1)
3420{
3421 alur(X86_CMP, r0, r1);
79bfeef6 3422 return (jcc(code, i0));
4a71579b
PC		 3423}
3424
79bfeef6 3425static jit_word_t
4a71579b
PC		 3426_jci(jit_state_t *_jit,
3427 jit_int32_t code, jit_word_t i0, jit_int32_t r0, jit_word_t i1)
3428{
3429 alui(X86_CMP, r0, i1);
79bfeef6 3430 return (jcc(code, i0));
4a71579b
PC		 3431}
3432
79bfeef6 3433static jit_word_t
4a71579b
PC		 3434_jci0(jit_state_t *_jit, jit_int32_t code, jit_word_t i0, jit_int32_t r0)
3435{
3436 testr(r0, r0);
79bfeef6 3437 return (jcc(code, i0));
4a71579b
PC		 3438}
3439
3440static jit_word_t
3441_bltr(jit_state_t *_jit, jit_word_t i0, jit_int32_t r0, jit_int32_t r1)
3442{
79bfeef6 3443 return (jcr(X86_CC_L, i0, r0, r1));
4a71579b
PC		 3444}
3445
3446static jit_word_t
3447_blti(jit_state_t *_jit, jit_word_t i0, jit_int32_t r0, jit_word_t i1)
3448{
79bfeef6
PC		 3449 jit_word_t w;
3450 if (i1) w = jci (X86_CC_L, i0, r0, i1);
3451 else w = jci0(X86_CC_S, i0, r0);
3452 return (w);
4a71579b
PC		 3453}
3454
3455static jit_word_t
3456_bltr_u(jit_state_t *_jit, jit_word_t i0, jit_int32_t r0, jit_int32_t r1)
3457{
79bfeef6 3458 return (jcr(X86_CC_B, i0, r0, r1));
4a71579b
PC		 3459}
3460
3461static jit_word_t
3462_blti_u(jit_state_t *_jit, jit_word_t i0, jit_int32_t r0, jit_word_t i1)
3463{
79bfeef6
PC		 3464 jit_word_t w;
3465 if (i1) w = jci (X86_CC_B, i0, r0, i1);
3466 else w = jci0(X86_CC_B, i0, r0);
3467 return (w);
4a71579b
PC		 3468}
3469
3470static jit_word_t
3471_bler(jit_state_t *_jit, jit_word_t i0, jit_int32_t r0, jit_int32_t r1)
3472{
79bfeef6
PC		 3473 jit_word_t w;
3474 if (r0 == r1) w = jmpi(i0);
3475 else w = jcr (X86_CC_LE, i0, r0, r1);
3476 return (w);
4a71579b
PC		 3477}
3478
3479static jit_word_t
3480_blei(jit_state_t *_jit, jit_word_t i0, jit_int32_t r0, jit_word_t i1)
3481{
79bfeef6
PC		 3482 jit_word_t w;
3483 if (i1) w = jci (X86_CC_LE, i0, r0, i1);
3484 else w = jci0(X86_CC_LE, i0, r0);
3485 return (w);
4a71579b
PC		 3486}
3487
3488static jit_word_t
3489_bler_u(jit_state_t *_jit, jit_word_t i0, jit_int32_t r0, jit_int32_t r1)
3490{
79bfeef6
PC		 3491 jit_word_t w;
3492 if (r0 == r1) w = jmpi(i0);
3493 else w = jcr (X86_CC_BE, i0, r0, r1);
3494 return (w);
4a71579b
PC		 3495}
3496
3497static jit_word_t
3498_blei_u(jit_state_t *_jit, jit_word_t i0, jit_int32_t r0, jit_word_t i1)
3499{
79bfeef6
PC		 3500 jit_word_t w;
3501 if (i1) w = jci (X86_CC_BE, i0, r0, i1);
3502 else w = jci0(X86_CC_BE, i0, r0);
3503 return (w);
4a71579b
PC		 3504}
3505
3506static jit_word_t
3507_beqr(jit_state_t *_jit, jit_word_t i0, jit_int32_t r0, jit_int32_t r1)
3508{
79bfeef6
PC		 3509 jit_word_t w;
3510 if (r0 == r1) w = jmpi(i0);
3511 else w = jcr (X86_CC_E, i0, r0, r1);
3512 return (w);
4a71579b
PC		 3513}
3514
3515static jit_word_t
3516_beqi(jit_state_t *_jit, jit_word_t i0, jit_int32_t r0, jit_word_t i1)
3517{
79bfeef6
PC		 3518 jit_word_t w;
3519 if (i1) w = jci (X86_CC_E, i0, r0, i1);
3520 else w = jci0(X86_CC_E, i0, r0);
3521 return (w);
4a71579b
PC		 3522}
3523
3524static jit_word_t
3525_bger(jit_state_t *_jit, jit_word_t i0, jit_int32_t r0, jit_int32_t r1)
3526{
79bfeef6
PC		 3527 jit_word_t w;
3528 if (r0 == r1) w = jmpi(i0);
3529 else w = jcr (X86_CC_GE, i0, r0, r1);
3530 return (w);
4a71579b
PC		 3531}
3532
3533static jit_word_t
3534_bgei(jit_state_t *_jit, jit_word_t i0, jit_int32_t r0, jit_word_t i1)
3535{
79bfeef6
PC		 3536 jit_word_t w;
3537 if (i1) w = jci (X86_CC_GE, i0, r0, i1);
3538 else w = jci0(X86_CC_NS, i0, r0);
3539 return (w);
4a71579b
PC		 3540}
3541
3542static jit_word_t
3543_bger_u(jit_state_t *_jit, jit_word_t i0, jit_int32_t r0, jit_int32_t r1)
3544{
79bfeef6
PC		 3545 jit_word_t w;
3546 if (r0 == r1) w = jmpi(i0);
3547 else w = jcr (X86_CC_AE, i0, r0, r1);
3548 return (w);
4a71579b
PC		 3549}
3550
3551static jit_word_t
3552_bgei_u(jit_state_t *_jit, jit_word_t i0, jit_int32_t r0, jit_word_t i1)
3553{
79bfeef6
PC		 3554 jit_word_t w;
3555 if (i1) w = jci (X86_CC_AE, i0, r0, i1);
3556 else w = jmpi(i0);
3557 return (w);
4a71579b
PC		 3558}
3559
3560static jit_word_t
3561_bgtr(jit_state_t *_jit, jit_word_t i0, jit_int32_t r0, jit_int32_t r1)
3562{
79bfeef6 3563 return (jcr(X86_CC_G, i0, r0, r1));
4a71579b
PC		 3564}
3565
3566static jit_word_t
3567_bgti(jit_state_t *_jit, jit_word_t i0, jit_int32_t r0, jit_word_t i1)
3568{
79bfeef6 3569 return (jci(X86_CC_G, i0, r0, i1));
4a71579b
PC		 3570}
3571
3572static jit_word_t
3573_bgtr_u(jit_state_t *_jit, jit_word_t i0, jit_int32_t r0, jit_int32_t r1)
3574{
79bfeef6 3575 return (jcr(X86_CC_A, i0, r0, r1));
4a71579b
PC		 3576}
3577
3578static jit_word_t
3579_bgti_u(jit_state_t *_jit, jit_word_t i0, jit_int32_t r0, jit_word_t i1)
3580{
79bfeef6
PC		 3581 jit_word_t w;
3582 if (i1) w = jci (X86_CC_A, i0, r0, i1);
3583 else w = jci0(X86_CC_NE, i0, r0);
3584 return (w);
4a71579b
PC		 3585}
3586
3587static jit_word_t
3588_bner(jit_state_t *_jit, jit_word_t i0, jit_int32_t r0, jit_int32_t r1)
3589{
79bfeef6 3590 return (jcr(X86_CC_NE, i0, r0, r1));
4a71579b
PC		 3591}
3592
3593static jit_word_t
3594_bnei(jit_state_t *_jit, jit_word_t i0, jit_int32_t r0, jit_word_t i1)
3595{
79bfeef6
PC		 3596 jit_word_t w;
3597 if (i1) w = jci (X86_CC_NE, i0, r0, i1);
3598 else w = jci0(X86_CC_NE, i0, r0);
3599 return (w);
4a71579b
PC		 3600}
3601
3602static jit_word_t
3603_bmsr(jit_state_t *_jit, jit_word_t i0, jit_int32_t r0, jit_int32_t r1)
3604{
3605 testr(r0, r1);
79bfeef6 3606 return (jnz(i0));
4a71579b
PC		 3607}
3608
3609static jit_word_t
3610_bmsi(jit_state_t *_jit, jit_word_t i0, jit_int32_t r0, jit_word_t i1)
3611{
3612 jit_int32_t reg;
3613 if (can_zero_extend_int_p(i1))
3614 testi(r0, i1);
3615 else {
3616 reg = jit_get_reg(jit_class_gpr);
3617 movi(rn(reg), i1);
3618 testr(r0, rn(reg));
3619 jit_unget_reg(reg);
3620 }
79bfeef6 3621 return (jnz(i0));
4a71579b
PC		 3622}
3623
3624static jit_word_t
3625_bmcr(jit_state_t *_jit, jit_word_t i0, jit_int32_t r0, jit_int32_t r1)
3626{
3627 testr(r0, r1);
79bfeef6 3628 return (jz(i0));
4a71579b
PC		 3629}
3630
3631static jit_word_t
3632_bmci(jit_state_t *_jit, jit_word_t i0, jit_int32_t r0, jit_word_t i1)
3633{
3634 jit_int32_t reg;
3635 if (can_zero_extend_int_p(i1))
3636 testi(r0, i1);
3637 else {
3638 reg = jit_get_reg(jit_class_gpr);
3639 movi(rn(reg), i1);
3640 testr(r0, rn(reg));
3641 jit_unget_reg(reg);
3642 }
79bfeef6 3643 return (jz(i0));
4a71579b
PC		 3644}
3645
3646static jit_word_t
3647_boaddr(jit_state_t *_jit, jit_word_t i0, jit_int32_t r0, jit_int32_t r1)
3648{
3649 iaddr(r0, r1);
79bfeef6 3650 return (jo(i0));
4a71579b
PC		 3651}
3652
3653static jit_word_t
3654_boaddi(jit_state_t *_jit, jit_word_t i0, jit_int32_t r0, jit_word_t i1)
3655{
3656 jit_int32_t reg;
3657 if (can_sign_extend_int_p(i1)) {
3658 iaddi(r0, i1);
79bfeef6 3659 return (jo(i0));
4a71579b
PC		 3660 }
3661 reg = jit_get_reg(jit_class_gpr|jit_class_nospill);
3662 movi(rn(reg), i1);
3663 jit_unget_reg(reg);
3664 return (boaddr(i0, r0, rn(reg)));
3665}
3666
3667static jit_word_t
3668_boaddr_u(jit_state_t *_jit, jit_word_t i0, jit_int32_t r0, jit_int32_t r1)
3669{
3670 iaddr(r0, r1);
79bfeef6 3671 return (jc(i0));
4a71579b
PC		 3672}
3673
3674static jit_word_t
3675_boaddi_u(jit_state_t *_jit, jit_word_t i0, jit_int32_t r0, jit_word_t i1)
3676{
3677 jit_int32_t reg;
3678 if (can_sign_extend_int_p(i1)) {
3679 iaddi(r0, i1);
79bfeef6 3680 return (jc(i0));
4a71579b
PC		 3681 }
3682 reg = jit_get_reg(jit_class_gpr|jit_class_nospill);
3683 movi(rn(reg), i1);
3684 jit_unget_reg(reg);
3685 return (boaddr_u(i0, r0, rn(reg)));
3686}
3687
3688static jit_word_t
3689_bxaddr(jit_state_t *_jit, jit_word_t i0, jit_int32_t r0, jit_int32_t r1)
3690{
3691 iaddr(r0, r1);
79bfeef6 3692 return (jno(i0));
4a71579b
PC		 3693}
3694
3695static jit_word_t
3696_bxaddi(jit_state_t *_jit, jit_word_t i0, jit_int32_t r0, jit_word_t i1)
3697{
3698 jit_int32_t reg;
3699 if (can_sign_extend_int_p(i1)) {
3700 iaddi(r0, i1);
79bfeef6 3701 return (jno(i0));
4a71579b
PC		 3702 }
3703 reg = jit_get_reg(jit_class_gpr|jit_class_nospill);
3704 movi(rn(reg), i1);
3705 jit_unget_reg(reg);
3706 return (bxaddr(i0, r0, rn(reg)));
3707}
3708
3709static jit_word_t
3710_bxaddr_u(jit_state_t *_jit, jit_word_t i0, jit_int32_t r0, jit_int32_t r1)
3711{
3712 iaddr(r0, r1);
79bfeef6 3713 return (jnc(i0));
4a71579b
PC		 3714}
3715
3716static jit_word_t
3717_bxaddi_u(jit_state_t *_jit, jit_word_t i0, jit_int32_t r0, jit_word_t i1)
3718{
3719 jit_int32_t reg;
3720 if (can_sign_extend_int_p(i1)) {
3721 iaddi(r0, i1);
79bfeef6 3722 return (jnc(i0));
4a71579b
PC		 3723 }
3724 reg = jit_get_reg(jit_class_gpr|jit_class_nospill);
3725 movi(rn(reg), i1);
3726 jit_unget_reg(reg);
3727 return (bxaddr_u(i0, r0, rn(reg)));
3728}
3729
3730static jit_word_t
3731_bosubr(jit_state_t *_jit, jit_word_t i0, jit_int32_t r0, jit_int32_t r1)
3732{
3733 isubr(r0, r1);
79bfeef6 3734 return (jo(i0));
4a71579b
PC		 3735}
3736
3737static jit_word_t
3738_bosubi(jit_state_t *_jit, jit_word_t i0, jit_int32_t r0, jit_word_t i1)
3739{
3740 jit_int32_t reg;
3741 if (can_sign_extend_int_p(i1)) {
3742 isubi(r0, i1);
79bfeef6 3743 return (jo(i0));
4a71579b
PC		 3744 }
3745 reg = jit_get_reg(jit_class_gpr|jit_class_nospill);
3746 movi(rn(reg), i1);
3747 jit_unget_reg(reg);
3748 return (bosubr(i0, r0, rn(reg)));
3749}
3750
3751static jit_word_t
3752_bosubr_u(jit_state_t *_jit, jit_word_t i0, jit_int32_t r0, jit_int32_t r1)
3753{
3754 isubr(r0, r1);
79bfeef6 3755 return (jc(i0));
4a71579b
PC		 3756}
3757
3758static jit_word_t
3759_bosubi_u(jit_state_t *_jit, jit_word_t i0, jit_int32_t r0, jit_word_t i1)
3760{
3761 jit_int32_t reg;
3762 if (can_sign_extend_int_p(i1)) {
3763 isubi(r0, i1);
79bfeef6 3764 return (jc(i0));
4a71579b
PC		 3765 }
3766 reg = jit_get_reg(jit_class_gpr|jit_class_nospill);
3767 movi(rn(reg), i1);
3768 jit_unget_reg(reg);
3769 return (bosubr_u(i0, r0, rn(reg)));
3770}
3771
3772static jit_word_t
3773_bxsubr(jit_state_t *_jit, jit_word_t i0, jit_int32_t r0, jit_int32_t r1)
3774{
3775 isubr(r0, r1);
79bfeef6 3776 return (jno(i0));
4a71579b
PC		 3777}
3778
3779static jit_word_t
3780_bxsubi(jit_state_t *_jit, jit_word_t i0, jit_int32_t r0, jit_word_t i1)
3781{
3782 jit_int32_t reg;
3783 if (can_sign_extend_int_p(i1)) {
3784 isubi(r0, i1);
79bfeef6 3785 return (jno(i0));
4a71579b
PC		 3786 }
3787 reg = jit_get_reg(jit_class_gpr|jit_class_nospill);
3788 movi(rn(reg), i1);
3789 jit_unget_reg(reg);
3790 return (bxsubr(i0, r0, rn(reg)));
3791}
3792
3793static jit_word_t
3794_bxsubr_u(jit_state_t *_jit, jit_word_t i0, jit_int32_t r0, jit_int32_t r1)
3795{
3796 isubr(r0, r1);
79bfeef6 3797 return (jnc(i0));
4a71579b
PC		 3798}
3799
3800static jit_word_t
3801_bxsubi_u(jit_state_t *_jit, jit_word_t i0, jit_int32_t r0, jit_word_t i1)
3802{
3803 jit_int32_t reg;
3804 if (can_sign_extend_int_p(i1)) {
3805 isubi(r0, i1);
79bfeef6 3806 return (jnc(i0));
4a71579b
PC		 3807 }
3808 reg = jit_get_reg(jit_class_gpr|jit_class_nospill);
3809 movi(rn(reg), i1);
3810 jit_unget_reg(reg);
3811 return (bxsubr_u(i0, r0, rn(reg)));
3812}
3813
3814static void
3815_callr(jit_state_t *_jit, jit_int32_t r0)
3816{
3817 rex(0, 0, _NOREG, _NOREG, r0);
3818 ic(0xff);
3819 mrm(0x03, 0x02, r7(r0));
3820}
3821
3822static jit_word_t
3823_calli(jit_state_t *_jit, jit_word_t i0)
3824{
519a9ea1 3825 jit_word_t w;
79bfeef6
PC		 3826 jit_word_t d;
3827 jit_word_t l = _jit->pc.w + 5;
3828 d = i0 - l;
4a71579b 3829#if __X64
79bfeef6
PC		 3830 if (
3831# if __X64_32
3832 !((d < 0) ^ (l < 0)) &&
3833# endif
3834 (jit_int32_t)d == d) {
519a9ea1 3835#endif
79bfeef6 3836 w = _jit->pc.w;
519a9ea1 3837 ic(0xe8);
79bfeef6 3838 ii(d);
519a9ea1
PC		 3839#if __X64
3840 }
3841 else
79bfeef6 3842 w = calli_p(i0);
519a9ea1 3843#endif
79bfeef6 3844 return (w);
519a9ea1 3845}
4a71579b 3846
519a9ea1
PC		 3847#if __X64
3848static jit_word_t
3849_calli_p(jit_state_t *_jit, jit_word_t i0)
3850{
79bfeef6 3851 jit_word_t w;
519a9ea1 3852 jit_int32_t reg;
4a71579b 3853 reg = jit_get_reg(jit_class_gpr);
79bfeef6 3854 w = movi_p(rn(reg), i0);
4a71579b
PC		 3855 callr(rn(reg));
3856 jit_unget_reg(reg);
79bfeef6 3857 return (w);
4a71579b 3858}
519a9ea1 3859#endif
4a71579b
PC		 3860
3861static void
3862_jmpr(jit_state_t *_jit, jit_int32_t r0)
3863{
519a9ea1 3864 rex(0, 0, _NOREG, _NOREG, r0);
4a71579b
PC		 3865 ic(0xff);
3866 mrm(0x03, 0x04, r7(r0));
3867}
3868
3869static jit_word_t
3870_jmpi(jit_state_t *_jit, jit_word_t i0)
3871{
3872 jit_word_t w;
79bfeef6
PC		 3873 jit_word_t d;
3874 jit_word_t l = _jit->pc.w + 5;
3875 d = i0 - l;
519a9ea1 3876#if __X64
79bfeef6
PC		 3877 if (
3878# if __X64_32
3879 !((d < 0) ^ (l < 0)) &&
3880# endif
3881 (jit_int32_t)d == d) {
519a9ea1 3882#endif
79bfeef6 3883 w = _jit->pc.w;
519a9ea1 3884 ic(0xe9);
79bfeef6 3885 ii(d);
519a9ea1
PC		 3886#if __X64
3887 }
3888 else
79bfeef6 3889 w = jmpi_p(i0);
519a9ea1 3890#endif
79bfeef6 3891 return (w);
4a71579b
PC		 3892}
3893
519a9ea1
PC		 3894#if __X64
3895static jit_word_t
3896_jmpi_p(jit_state_t *_jit, jit_word_t i0)
3897{
79bfeef6 3898 jit_word_t w;
519a9ea1
PC		 3899 jit_int32_t reg;
3900 reg = jit_get_reg(jit_class_gpr|jit_class_nospill);
79bfeef6 3901 w = movi_p(rn(reg), i0);
519a9ea1
PC		 3902 jmpr(rn(reg));
3903 jit_unget_reg(reg);
79bfeef6 3904 return (w);
519a9ea1
PC		 3905}
3906#endif
3907
79bfeef6 3908static jit_word_t
4a71579b
PC		 3909_jmpsi(jit_state_t *_jit, jit_uint8_t i0)
3910{
79bfeef6 3911 jit_word_t w = _jit->pc.w;
4a71579b
PC		 3912 ic(0xeb);
3913 ic(i0);
79bfeef6 3914 return (w);
4a71579b
PC		 3915}
3916
3917static void
3918_prolog(jit_state_t *_jit, jit_node_t *node)
3919{
79bfeef6 3920 jit_int32_t reg, offs;
4a71579b
PC		 3921 if (_jitc->function->define_frame || _jitc->function->assume_frame) {
3922 jit_int32_t frame = -_jitc->function->frame;
79bfeef6 3923 jit_check_frame();
4a71579b
PC		 3924 assert(_jitc->function->self.aoff >= frame);
3925 if (_jitc->function->assume_frame)
3926 return;
3927 _jitc->function->self.aoff = frame;
3928 }
3929 if (_jitc->function->allocar)
3930 _jitc->function->self.aoff &= -16;
3931#if __X64 && (__CYGWIN__ || _WIN32)
3932 _jitc->function->stack = (((/* first 32 bytes must be allocated */
3933 (_jitc->function->self.alen > 32 ?
3934 _jitc->function->self.alen : 32) -
3935 /* align stack at 16 bytes */
79bfeef6 3936 _jitc->function->self.aoff) + 15) & -16);
4a71579b
PC		 3937#else
3938 _jitc->function->stack = (((_jitc->function->self.alen -
79bfeef6 3939 _jitc->function->self.aoff) + 15) & -16);
4a71579b 3940#endif
79bfeef6
PC		 3941
3942 if (_jitc->function->stack)
3943 _jitc->function->need_stack = 1;
3944
3945 if (!_jitc->function->need_frame && !_jitc->function->need_stack) {
3946 /* check if any callee save register needs to be saved */
3947 for (reg = 0; reg < _jitc->reglen; ++reg)
3948 if (jit_regset_tstbit(&_jitc->function->regset, reg) &&
3949 (_rvs[reg].spec & jit_class_sav)) {
3950 _jitc->function->need_stack = 1;
3951 break;
3952 }
3953 }
3954
3955 if (_jitc->function->need_frame || _jitc->function->need_stack)
3956 subi(_RSP_REGNO, _RSP_REGNO, jit_framesize());
4a71579b 3957 /* callee save registers */
79bfeef6
PC		 3958 for (reg = 0, offs = REAL_WORDSIZE; reg < jit_size(iregs); reg++) {
3959 if (jit_regset_tstbit(&_jitc->function->regset, iregs[reg])) {
3960 stxi(offs, _RSP_REGNO, rn(iregs[reg]));
3961 offs += REAL_WORDSIZE;
3962 }
3963 }
3964#if __X64 && (__CYGWIN__ || _WIN32)
3965 for (reg = 0; reg < jit_size(fregs); reg++) {
3966 if (jit_regset_tstbit(&_jitc->function->regset, fregs[reg])) {
3967 sse_stxi_d(offs, _RSP_REGNO, rn(fregs[reg]));
3968 offs += sizeof(jit_float64_t);
3969 }
3970 }
4a71579b 3971#endif
79bfeef6
PC		 3972
3973 if (_jitc->function->need_frame) {
3974 stxi(0, _RSP_REGNO, _RBP_REGNO);
3975 movr(_RBP_REGNO, _RSP_REGNO);
3976 }
4a71579b
PC		 3977
3978 /* alloca */
79bfeef6
PC		 3979 if (_jitc->function->stack)
3980 subi(_RSP_REGNO, _RSP_REGNO, _jitc->function->stack);
4a71579b
PC		 3981 if (_jitc->function->allocar) {
3982 reg = jit_get_reg(jit_class_gpr);
3983 movi(rn(reg), _jitc->function->self.aoff);
3984 stxi_i(_jitc->function->aoffoff, _RBP_REGNO, rn(reg));
3985 jit_unget_reg(reg);
3986 }
3987
3988#if __X64 && !(__CYGWIN__ || _WIN32)
3989 if (_jitc->function->self.call & jit_call_varargs) {
3990 jit_word_t nofp_code;
3991
3992 /* Save gp registers in the save area, if any is a vararg */
3993 for (reg = first_gp_from_offset(_jitc->function->vagp);
3994 jit_arg_reg_p(reg); ++reg)
3995 stxi(_jitc->function->vaoff + first_gp_offset +
3996 reg * 8, _RBP_REGNO, rn(JIT_RA0 - reg));
3997
3998 reg = first_fp_from_offset(_jitc->function->vafp);
3999 if (jit_arg_f_reg_p(reg)) {
4000 /* Skip over if no float registers were passed as argument */
4001 /* test %al, %al */
4002 ic(0x84);
4003 ic(0xc0);
79bfeef6 4004 nofp_code = jes(0);
4a71579b
PC		 4005
4006 /* Save fp registers in the save area, if any is a vararg */
4007 /* Note that the full 16 byte xmm is not saved, because
4008 * lightning only handles float and double, and, while
4009 * attempting to provide a va_list compatible pointer as
4010 * jit_va_start return, does not guarantee it (on all ports). */
4011 for (; jit_arg_f_reg_p(reg); ++reg)
4012 sse_stxi_d(_jitc->function->vaoff + first_fp_offset +
4013 reg * va_fp_increment, _RBP_REGNO, rn(_XMM0 - reg));
4014
79bfeef6 4015 patch_at(nofp_code, _jit->pc.w);
4a71579b
PC		 4016 }
4017 }
4018#endif
4019}
4020
4021static void
4022_epilog(jit_state_t *_jit, jit_node_t *node)
4023{
79bfeef6 4024 jit_int32_t reg, offs;
4a71579b
PC		 4025 if (_jitc->function->assume_frame)
4026 return;
79bfeef6
PC		 4027 if (_jitc->function->need_frame)
4028 movr(_RSP_REGNO, _RBP_REGNO);
4029
4a71579b 4030 /* callee save registers */
79bfeef6
PC		 4031 for (reg = 0, offs = REAL_WORDSIZE; reg < jit_size(iregs); reg++) {
4032 if (jit_regset_tstbit(&_jitc->function->regset, iregs[reg])) {
4033 ldxi(rn(iregs[reg]), _RSP_REGNO, offs);
4034 offs += REAL_WORDSIZE;
4035 }
4036 }
4037#if __X64 && (__CYGWIN__ || _WIN32)
4038 for (reg = 0; reg < jit_size(fregs); reg++) {
4039 if (jit_regset_tstbit(&_jitc->function->regset, fregs[reg])) {
4040 sse_ldxi_d(rn(fregs[reg]), _RSP_REGNO, offs);
4041 offs += sizeof(jit_float64_t);
4042 }
4043 }
4a71579b 4044#endif
79bfeef6
PC		 4045
4046 if (_jitc->function->need_frame) {
4047 ldxi(_RBP_REGNO, _RSP_REGNO, 0);
4048 addi(_RSP_REGNO, _RSP_REGNO, jit_framesize());
4049 }
4050 /* This condition does not happen as much as expected because
4051 * it is not safe to not create a frame pointer if any function
4052 * is called, even jit functions, as those might call external
4053 * functions. */
4054 else if (_jitc->function->need_stack)
4055 addi(_RSP_REGNO, _RSP_REGNO, jit_framesize());
4a71579b
PC		 4056
4057 ic(0xc3);
4058}
4059
4060static void
4061_vastart(jit_state_t *_jit, jit_int32_t r0)
4062{
4063#if __X32 || __CYGWIN__ || _WIN32
4064 assert(_jitc->function->self.call & jit_call_varargs);
79bfeef6 4065 addi(r0, _RBP_REGNO, jit_selfsize());
4a71579b
PC		 4066#else
4067 jit_int32_t reg;
4068
4069 assert(_jitc->function->self.call & jit_call_varargs);
4070
4071 /* Return jit_va_list_t in the register argument */
4072 addi(r0, _RBP_REGNO, _jitc->function->vaoff);
4073 reg = jit_get_reg(jit_class_gpr);
4074
4075 /* Initialize gp offset in the save area. */
4076 movi(rn(reg), _jitc->function->vagp);
4077 stxi_i(offsetof(jit_va_list_t, gpoff), r0, rn(reg));
4078
4079 /* Initialize fp offset in the save area. */
4080 movi(rn(reg), _jitc->function->vafp);
4081 stxi_i(offsetof(jit_va_list_t, fpoff), r0, rn(reg));
4082
4083 /* Initialize overflow pointer to the first stack argument. */
79bfeef6 4084 addi(rn(reg), _RBP_REGNO, jit_selfsize());
4a71579b
PC		 4085 stxi(offsetof(jit_va_list_t, over), r0, rn(reg));
4086
4087 /* Initialize register save area pointer. */
4088 addi(rn(reg), r0, first_gp_offset);
4089 stxi(offsetof(jit_va_list_t, save), r0, rn(reg));
4090
4091 jit_unget_reg(reg);
4092#endif
4093}
4094
4095static void
4096_vaarg(jit_state_t *_jit, jit_int32_t r0, jit_int32_t r1)
4097{
4098#if __X32 || __CYGWIN__ || _WIN32
4099 assert(_jitc->function->self.call & jit_call_varargs);
4100 ldr(r0, r1);
4101 addi(r1, r1, va_gp_increment);
4102#else
4103 jit_int32_t rg0;
4104 jit_int32_t rg1;
4105 jit_word_t ge_code;
4106 jit_word_t lt_code;
4107
4108 assert(_jitc->function->self.call & jit_call_varargs);
4109
4110 rg0 = jit_get_reg(jit_class_gpr);
4111 rg1 = jit_get_reg(jit_class_gpr);
4112
4113 /* Load the gp offset in save area in the first temporary. */
4114 ldxi_i(rn(rg0), r1, offsetof(jit_va_list_t, gpoff));
4115
4116 /* Jump over if there are no remaining arguments in the save area. */
4117 icmpi(rn(rg0), va_gp_max_offset);
79bfeef6 4118 ge_code = jaes(0);
4a71579b
PC		 4119
4120 /* Load the save area pointer in the second temporary. */
4121 ldxi(rn(rg1), r1, offsetof(jit_va_list_t, save));
4122
4123 /* Load the vararg argument in the first argument. */
4124 ldxr(r0, rn(rg1), rn(rg0));
4125
4126 /* Update the gp offset. */
4127 addi(rn(rg0), rn(rg0), 8);
4128 stxi_i(offsetof(jit_va_list_t, gpoff), r1, rn(rg0));
4129
4130 /* Will only need one temporary register below. */
4131 jit_unget_reg(rg1);
4132
4133 /* Jump over overflow code. */
79bfeef6 4134 lt_code = jmpsi(0);
4a71579b
PC		 4135
4136 /* Where to land if argument is in overflow area. */
79bfeef6 4137 patch_at(ge_code, _jit->pc.w);
4a71579b
PC		 4138
4139 /* Load overflow pointer. */
4140 ldxi(rn(rg0), r1, offsetof(jit_va_list_t, over));
4141
4142 /* Load argument. */
4143 ldr(r0, rn(rg0));
4144
4145 /* Update overflow pointer. */
4146 addi(rn(rg0), rn(rg0), va_gp_increment);
4147 stxi(offsetof(jit_va_list_t, over), r1, rn(rg0));
4148
4149 /* Where to land if argument is in save area. */
79bfeef6 4150 patch_at(lt_code, _jit->pc.w);
4a71579b
PC		 4151
4152 jit_unget_reg(rg0);
4153#endif
4154}
4155
4156/* The x87 boolean argument tells if will put the result in a x87
4157 * register if non false, in a sse register otherwise. */
4158static void
4159_vaarg_d(jit_state_t *_jit, jit_int32_t r0, jit_int32_t r1, jit_bool_t x87)
4160{
4161#if __X32 || __CYGWIN__ || _WIN32
4162 assert(_jitc->function->self.call & jit_call_varargs);
4163 if (x87)
4164 x87_ldr_d(r0, r1);
4165 else
4166 sse_ldr_d(r0, r1);
4167 addi(r1, r1, 8);
4168#else
4169 jit_int32_t rg0;
4170 jit_int32_t rg1;
4171 jit_word_t ge_code;
4172 jit_word_t lt_code;
4173
4174 assert(_jitc->function->self.call & jit_call_varargs);
4175
4176 rg0 = jit_get_reg(jit_class_gpr);
4177 rg1 = jit_get_reg(jit_class_gpr);
4178
4179 /* Load the fp offset in save area in the first temporary. */
4180 ldxi_i(rn(rg0), r1, offsetof(jit_va_list_t, fpoff));
4181
4182 /* Jump over if there are no remaining arguments in the save area. */
4183 icmpi(rn(rg0), va_fp_max_offset);
79bfeef6 4184 ge_code = jaes(0);
4a71579b
PC		 4185
4186 /* Load the save area pointer in the second temporary. */
4187 ldxi(rn(rg1), r1, offsetof(jit_va_list_t, save));
4188
4189 /* Load the vararg argument in the first argument. */
4190 if (x87)
4191 x87_ldxr_d(r0, rn(rg1), rn(rg0));
4192 else
4193 sse_ldxr_d(r0, rn(rg1), rn(rg0));
4194
4195 /* Update the fp offset. */
4196 addi(rn(rg0), rn(rg0), va_fp_increment);
4197 stxi_i(offsetof(jit_va_list_t, fpoff), r1, rn(rg0));
4198
4199 /* Will only need one temporary register below. */
4200 jit_unget_reg(rg1);
4201
4202 /* Jump over overflow code. */
79bfeef6 4203 lt_code = jmpsi(0);
4a71579b
PC		 4204
4205 /* Where to land if argument is in overflow area. */
79bfeef6 4206 patch_at(ge_code, _jit->pc.w);
4a71579b
PC		 4207
4208 /* Load overflow pointer. */
4209 ldxi(rn(rg0), r1, offsetof(jit_va_list_t, over));
4210
4211 /* Load argument. */
4212 if (x87)
4213 x87_ldr_d(r0, rn(rg0));
4214 else
4215 sse_ldr_d(r0, rn(rg0));
4216
4217 /* Update overflow pointer. */
4218 addi(rn(rg0), rn(rg0), 8);
4219 stxi(offsetof(jit_va_list_t, over), r1, rn(rg0));
4220
4221 /* Where to land if argument is in save area. */
79bfeef6 4222 patch_at(lt_code, _jit->pc.w);
4a71579b
PC		 4223
4224 jit_unget_reg(rg0);
4225#endif
4226}
4227
4228static void
79bfeef6 4229_patch_at(jit_state_t *_jit, jit_word_t instr, jit_word_t label)
4a71579b 4230{
79bfeef6
PC		 4231 jit_word_t disp;
4232 jit_uint8_t *code = (jit_uint8_t *)instr;
4233 ++instr;
4234 switch (code[0]) {
4235 /* movi_p */
4236 case 0xb8 ... 0xbf:
4237 *(jit_word_t *)instr = label;
4a71579b 4238 break;
79bfeef6
PC		 4239 /* forward pc relative address known to be in range */
4240#if CAN_RIP_ADDRESS
4241 /* movi */
4242 case 0x8d:
4243 ++instr;
4244 goto apply;
4245#endif
4246 /* jcc */
4247 case 0x0f:
4248 ++instr;
4249 if (code[1] < 0x80 || code[1] > 0x8f)
4250 goto fail;
4251 /* calli */
4252 case 0xe8:
4253 /* jmpi */
4254 case 0xe9:
4255#if CAN_RIP_ADDRESS
4256 apply:
4257#endif
4258 disp = label - (instr + 4);
4259 assert((jit_int32_t)disp == disp);
4260 *(jit_int32_t *)instr = disp;
4261 break;
4262 /* jccs */
4263 case 0x70 ... 0x7f:
4264 /* jmpsi */
4265 case 0xeb:
4266 disp = label - (instr + 1);
4267 assert((jit_int8_t)disp == disp);
4268 *(jit_int8_t *)instr = disp;
4a71579b 4269 break;
79bfeef6
PC		 4270 default:
4271 fail:
4272 abort();
4a71579b
PC		 4273 }
4274}
4275#endif
ARM optimized PCSX fork