notaz.gp2x.de / pcsx_rearmed.git / blame
? search:
summary | shortlog | log | commit | commitdiff | tree
blob | blame (incremental) | history | HEAD
Update lightrec 20220910 (#686)
[pcsx_rearmed.git] / deps / lightning / lib / jit_x86-cpu.c
CommitLineData
4a71579b
PC		 1/*
2 * Copyright (C) 2012-2019 Free Software Foundation, Inc.
3 *
4 * This file is part of GNU lightning.
5 *
6 * GNU lightning is free software; you can redistribute it and/or modify it
7 * under the terms of the GNU Lesser General Public License as published
8 * by the Free Software Foundation; either version 3, or (at your option)
9 * any later version.
10 *
11 * GNU lightning is distributed in the hope that it will be useful, but
12 * WITHOUT ANY WARRANTY; without even the implied warranty of MERCHANTABILITY
13 * or FITNESS FOR A PARTICULAR PURPOSE. See the GNU Lesser General Public
14 * License for more details.
15 *
16 * Authors:
17 * Paulo Cesar Pereira de Andrade
18 */
19
20/* avoid using it due to partial stalls */
21#define USE_INC_DEC 0
22
23#if PROTO
24# if __X32 || __X64_32
25# define WIDE 0
26# define ldi(u, v) ldi_i(u, v)
27# define ldr(u, v) ldr_i(u, v)
28# define ldxr(u, v, w) ldxr_i(u, v, w)
29# define ldxi(u, v, w) ldxi_i(u, v, w)
30# define sti(u, v) sti_i(u, v)
31# define stxi(u, v, w) stxi_i(u, v, w)
32# define can_sign_extend_int_p(im) 1
33# define can_zero_extend_int_p(im) 1
34# define fits_uint32_p(im) 1
35# else
36# define WIDE 1
37# define ldi(u, v) ldi_l(u, v)
38# define ldr(u, v) ldr_l(u, v)
39# define ldxr(u, v, w) ldxr_l(u, v, w)
40# define ldxi(u, v, w) ldxi_l(u, v, w)
41# define sti(u, v) sti_l(u, v)
42# define stxi(u, v, w) stxi_l(u, v, w)
43# define can_sign_extend_int_p(im) \
44 (((im) >= 0 && (long long)(im) <= 0x7fffffffLL) || \
45 ((im) < 0 && (long long)(im) > -0x80000000LL))
46# define can_zero_extend_int_p(im) \
47 ((im) >= 0 && (im) < 0x80000000LL)
48# define fits_uint32_p(im) (((im) & 0xffffffff00000000LL) == 0)
49# endif
50# if __X32 || __CYGWIN__ || __X64_32 || _WIN32
51# define reg8_p(rn) \
52 ((rn) >= _RAX_REGNO && (rn) <= _RBX_REGNO)
53# else
54# define reg8_p(rn) 1
55# endif
56# define _RAX_REGNO 0
57# define _RCX_REGNO 1
58# define _RDX_REGNO 2
59# define _RBX_REGNO 3
60# define _RSP_REGNO 4
61# define _RBP_REGNO 5
62# define _RSI_REGNO 6
63# define _RDI_REGNO 7
64# define _R8_REGNO 8
65# define _R9_REGNO 9
66# define _R10_REGNO 10
67# define _R11_REGNO 11
68# define _R12_REGNO 12
69# define _R13_REGNO 13
70# define _R14_REGNO 14
71# define _R15_REGNO 15
72# define r7(reg) ((reg) & 7)
73# define r8(reg) ((reg) & 15)
74# define _SCL1 0x00
75# define _SCL2 0x01
76# define _SCL4 0x02
77# define _SCL8 0x03
78# define X86_ADD 0
79# define X86_OR 1 << 3
80# define X86_ADC 2 << 3
81# define X86_SBB 3 << 3
82# define X86_AND 4 << 3
83# define X86_SUB 5 << 3
84# define X86_XOR 6 << 3
85# define X86_CMP 7 << 3
86# define X86_ROL 0
87# define X86_ROR 1
88# define X86_RCL 2
89# define X86_RCR 3
90# define X86_SHL 4
91# define X86_SHR 5
92# define X86_SAR 7
93# define X86_NOT 2
94# define X86_NEG 3
95# define X86_MUL 4
96# define X86_IMUL 5
97# define X86_DIV 6
98# define X86_IDIV 7
99# define X86_CC_O 0x0
100# define X86_CC_NO 0x1
101# define X86_CC_NAE 0x2
102# define X86_CC_B 0x2
103# define X86_CC_C 0x2
104# define X86_CC_AE 0x3
105# define X86_CC_NB 0x3
106# define X86_CC_NC 0x3
107# define X86_CC_E 0x4
108# define X86_CC_Z 0x4
109# define X86_CC_NE 0x5
110# define X86_CC_NZ 0x5
111# define X86_CC_BE 0x6
112# define X86_CC_NA 0x6
113# define X86_CC_A 0x7
114# define X86_CC_NBE 0x7
115# define X86_CC_S 0x8
116# define X86_CC_NS 0x9
117# define X86_CC_P 0xa
118# define X86_CC_PE 0xa
119# define X86_CC_NP 0xb
120# define X86_CC_PO 0xb
121# define X86_CC_L 0xc
122# define X86_CC_NGE 0xc
123# define X86_CC_GE 0xd
124# define X86_CC_NL 0xd
125# define X86_CC_LE 0xe
126# define X86_CC_NG 0xe
127# define X86_CC_G 0xf
128# define X86_CC_NLE 0xf
129# define mrm(md, r, m) *_jit->pc.uc++ = (md<<6) | (r<<3) | m
130# define sib(sc, i, b) *_jit->pc.uc++ = (sc<<6) | (i<<3) | b
131# define ic(c) *_jit->pc.uc++ = c
132# define is(s) *_jit->pc.us++ = s
133# define ii(i) *_jit->pc.ui++ = i
134# if __X64 && !__X64_32
135# define il(l) *_jit->pc.ul++ = l
136# else
137# define il(l) ii(l)
138# endif
139# define patch_abs(instr, label) \
140 *(jit_word_t *)(instr - sizeof(jit_word_t)) = label
141# define patch_rel(instr, label) \
142 *(jit_int32_t *)(instr - 4) = label - instr
143# define patch_rel_char(instr, label) \
144 *(jit_int8_t *)(instr - 1) = label - instr
145# define rex(l, w, r, x, b) _rex(_jit, l, w, r, x, b)
146static void
147_rex(jit_state_t*,jit_int32_t,jit_int32_t,jit_int32_t,jit_int32_t,jit_int32_t);
148# define rx(rd, md, rb, ri, ms) _rx(_jit, rd, md, rb, ri, ms)
149static void
150_rx(jit_state_t*,jit_int32_t,jit_int32_t,jit_int32_t,jit_int32_t,jit_int32_t);
151# define nop(n) _nop(_jit, n)
152static void _nop(jit_state_t*, jit_int32_t);
153# define emms() is(0x770f)
154# define lea(md, rb, ri, ms, rd) _lea(_jit, md, rb, ri, ms, rd)
155static void
156_lea(jit_state_t*,jit_int32_t,jit_int32_t,jit_int32_t,jit_int32_t,jit_int32_t);
157# define pushr(r0) _pushr(_jit, r0)
158static void _pushr(jit_state_t*, jit_int32_t) maybe_unused;
159# define popr(r0) _popr(_jit, r0)
160static void _popr(jit_state_t*, jit_int32_t) maybe_unused;
161# define xchgr(r0, r1) _xchgr(_jit, r0, r1)
162static void _xchgr(jit_state_t*, jit_int32_t, jit_int32_t);
163# define testr(r0, r1) _testr(_jit, r0, r1)
164static void _testr(jit_state_t*, jit_int32_t, jit_int32_t);
165# define testi(r0, i0) _testi(_jit, r0, i0)
166static void _testi(jit_state_t*, jit_int32_t, jit_word_t);
167# define cc(code, r0) _cc(_jit, code, r0)
168static void _cc(jit_state_t*, jit_int32_t, jit_int32_t);
169# define icmpr(r0, r1) alur(X86_CMP, r0, r1)
170# define alur(code, r0, r1) _alur(_jit, code, r0, r1)
171static void _alur(jit_state_t*, jit_int32_t, jit_int32_t, jit_int32_t);
172# define icmpi(r0, i0) alui(X86_CMP, r0, i0)
173# define alui(code, r0, i0) _alui(_jit, code, r0, i0)
174static void _alui(jit_state_t*, jit_int32_t, jit_int32_t, jit_word_t);
175# define iaddr(r0, r1) alur(X86_ADD, r0, r1)
176# define save(r0) _save(_jit, r0)
177static void _save(jit_state_t*, jit_int32_t);
178# define load(r0) _load(_jit, r0)
179static void _load(jit_state_t*, jit_int32_t);
180# define addr(r0, r1, r2) _addr(_jit, r0, r1, r2)
181static void _addr(jit_state_t*, jit_int32_t, jit_int32_t, jit_int32_t);
182# define iaddi(r0, i0) alui(X86_ADD, r0, i0)
183# define addi(r0, r1, i0) _addi(_jit, r0, r1, i0)
184static void _addi(jit_state_t*, jit_int32_t, jit_int32_t, jit_word_t);
185#define addcr(r0, r1, r2) _addcr(_jit, r0, r1, r2)
186static void _addcr(jit_state_t*, jit_int32_t, jit_int32_t, jit_int32_t);
187#define addci(r0, r1, i0) _addci(_jit, r0, r1, i0)
188static void _addci(jit_state_t*, jit_int32_t, jit_int32_t, jit_word_t);
189# define iaddxr(r0, r1) alur(X86_ADC, r0, r1)
190# define addxr(r0, r1, r2) _addxr(_jit, r0, r1, r2)
191static void _addxr(jit_state_t*, jit_int32_t, jit_int32_t, jit_int32_t);
192# define iaddxi(r0, i0) alui(X86_ADC, r0, i0)
193# define addxi(r0, r1, i0) _addxi(_jit, r0, r1, i0)
194static void _addxi(jit_state_t*, jit_int32_t, jit_int32_t, jit_word_t);
195# define isubr(r0, r1) alur(X86_SUB, r0, r1)
196# define subr(r0, r1, r2) _subr(_jit, r0, r1, r2)
197static void _subr(jit_state_t*, jit_int32_t, jit_int32_t, jit_int32_t);
198# define isubi(r0, i0) alui(X86_SUB, r0, i0)
199# define subi(r0, r1, i0) _subi(_jit, r0, r1, i0)
200static void _subi(jit_state_t*, jit_int32_t, jit_int32_t, jit_word_t);
201# define subcr(r0, r1, r2) _subcr(_jit, r0, r1, r2)
202static void _subcr(jit_state_t*,jit_int32_t,jit_int32_t,jit_int32_t);
203# define subci(r0, r1, i0) _subci(_jit, r0, r1, i0)
204static void _subci(jit_state_t*,jit_int32_t,jit_int32_t,jit_word_t);
205# define isubxr(r0, r1) alur(X86_SBB, r0, r1)
206# define subxr(r0, r1, r2) _subxr(_jit, r0, r1, r2)
207static void _subxr(jit_state_t*,jit_int32_t,jit_int32_t,jit_int32_t);
208# define isubxi(r0, i0) alui(X86_SBB, r0, i0)
209# define subxi(r0, r1, i0) _subxi(_jit, r0, r1, i0)
210static void _subxi(jit_state_t*,jit_int32_t,jit_int32_t,jit_word_t);
211# define rsbi(r0, r1, i0) _rsbi(_jit, r0, r1, i0)
212static void _rsbi(jit_state_t*,jit_int32_t,jit_int32_t,jit_word_t);
213# define imulr(r0, r1) _imulr(_jit, r0, r1)
214static void _imulr(jit_state_t*, jit_int32_t, jit_int32_t);
215# define imuli(r0, r1, i0) _imuli(_jit, r0, r1, i0)
216static void _imuli(jit_state_t*, jit_int32_t, jit_int32_t, jit_word_t);
217# define mulr(r0, r1, r2) _mulr(_jit, r0, r1, r2)
218static void _mulr(jit_state_t*, jit_int32_t, jit_int32_t, jit_int32_t);
219# define muli(r0, r1, i0) _muli(_jit, r0, r1, i0)
220static void _muli(jit_state_t*, jit_int32_t, jit_int32_t, jit_word_t);
221# define umulr(r0) unr(X86_IMUL, r0)
222# define umulr_u(r0) unr(X86_MUL, r0)
223# define qmulr(r0, r1, r2, r3) _iqmulr(_jit, r0, r1, r2, r3, 1)
224# define qmulr_u(r0, r1, r2, r3) _iqmulr(_jit, r0, r1, r2, r3, 0)
225# define iqmulr(r0, r1, r2, r3, sign) _iqmulr(_jit, r0, r1, r2, r3, sign)
226static void _iqmulr(jit_state_t*, jit_int32_t, jit_int32_t,
227 jit_int32_t,jit_int32_t, jit_bool_t);
228# define qmuli(r0, r1, r2, i0) _iqmuli(_jit, r0, r1, r2, i0, 1)
229# define qmuli_u(r0, r1, r2, i0) _iqmuli(_jit, r0, r1, r2, i0, 0)
230# define iqmuli(r0, r1, r2, i0, sign) _iqmuli(_jit, r0, r1, r2, i0, sign)
231static void _iqmuli(jit_state_t*, jit_int32_t, jit_int32_t,
232 jit_int32_t,jit_word_t, jit_bool_t);
233# define sign_extend_rdx_rax() _sign_extend_rdx_rax(_jit)
234static void _sign_extend_rdx_rax(jit_state_t*);
235# define idivr(r0) unr(X86_IDIV, r0)
236# define idivr_u(r0) unr(X86_DIV, r0)
237# define divremr(r0, r1, r2, i0, i1) _divremr(_jit, r0, r1, r2, i0, i1)
238static void
239_divremr(jit_state_t*,jit_int32_t,jit_int32_t,jit_int32_t,
240 jit_bool_t,jit_bool_t);
241# define divremi(r0, r1, i0, i1, i2) _divremi(_jit, r0, r1, i0, i1, i2)
242static void
243_divremi(jit_state_t*,jit_int32_t,jit_int32_t,jit_word_t,jit_bool_t,jit_bool_t);
244# define divr(r0, r1, r2) divremr(r0, r1, r2, 1, 1)
245# define divi(r0, r1, i0) divremi(r0, r1, i0, 1, 1)
246# define divr_u(r0, r1, r2) divremr(r0, r1, r2, 0, 1)
247# define divi_u(r0, r1, i0) divremi(r0, r1, i0, 0, 1)
248# define qdivr(r0, r1, r2, r3) _iqdivr(_jit, r0, r1, r2, r3, 1)
249# define qdivr_u(r0, r1, r2, r3) _iqdivr(_jit, r0, r1, r2, r3, 0)
250# define iqdivr(r0, r1, r2, r3, sign) _iqdivr(_jit, r0, r1, r2, r3, sign)
251static void _iqdivr(jit_state_t*, jit_int32_t, jit_int32_t,
252 jit_int32_t,jit_int32_t, jit_bool_t);
253# define qdivi(r0, r1, r2, i0) _iqdivi(_jit, r0, r1, r2, i0, 1)
254# define qdivi_u(r0, r1, r2, i0) _iqdivi(_jit, r0, r1, r2, i0, 0)
255# define iqdivi(r0, r1, r2, i0, sign) _iqdivi(_jit, r0, r1, r2, i0, sign)
256static void _iqdivi(jit_state_t*, jit_int32_t, jit_int32_t,
257 jit_int32_t,jit_word_t, jit_bool_t);
258# define remr(r0, r1, r2) divremr(r0, r1, r2, 1, 0)
259# define remi(r0, r1, i0) divremi(r0, r1, i0, 1, 0)
260# define remr_u(r0, r1, r2) divremr(r0, r1, r2, 0, 0)
261# define remi_u(r0, r1, i0) divremi(r0, r1, i0, 0, 0)
262# define iandr(r0, r1) alur(X86_AND, r0, r1)
263# define andr(r0, r1, r2) _andr(_jit, r0, r1, r2)
264static void _andr(jit_state_t*,jit_int32_t,jit_int32_t,jit_int32_t);
265# define iandi(r0, i0) alui(X86_AND, r0, i0)
266# define andi(r0, r1, i0) _andi(_jit, r0, r1, i0)
267static void _andi(jit_state_t*, jit_int32_t,jit_int32_t,jit_word_t);
268# define iorr(r0, r1) alur(X86_OR, r0, r1)
269# define orr(r0, r1, r2) _orr(_jit, r0, r1, r2)
270static void _orr(jit_state_t*, jit_int32_t,jit_int32_t,jit_int32_t);
271# define iori(r0, i0) alui(X86_OR, r0, i0)
272# define ori(r0, r1, i0) _ori(_jit, r0, r1, i0)
273static void _ori(jit_state_t*, jit_int32_t,jit_int32_t,jit_word_t);
274# define ixorr(r0, r1) alur(X86_XOR, r0, r1)
275# define xorr(r0, r1, r2) _xorr(_jit, r0, r1, r2)
276static void _xorr(jit_state_t*, jit_int32_t,jit_int32_t,jit_int32_t);
277# define ixori(r0, i0) alui(X86_XOR, r0, i0)
278# define xori(r0, r1, i0) _xori(_jit, r0, r1, i0)
279static void _xori(jit_state_t*, jit_int32_t,jit_int32_t,jit_word_t);
280# define irotshr(code, r0) _irotshr(_jit, code, r0)
281static void _irotshr(jit_state_t*, jit_int32_t, jit_int32_t);
282# define rotshr(code, r0, r1, r2) _rotshr(_jit, code, r0, r1, r2)
283static void
284_rotshr(jit_state_t*,jit_int32_t,jit_int32_t,jit_int32_t,jit_int32_t);
285# define irotshi(code, r0, i0) _irotshi(_jit, code, r0, i0)
286static void _irotshi(jit_state_t*, jit_int32_t, jit_int32_t, jit_word_t);
287# define rotshi(code, r0, r1, i0) _rotshi(_jit, code, r0, r1, i0)
288static void
289_rotshi(jit_state_t*,jit_int32_t,jit_int32_t,jit_int32_t,jit_word_t);
290# define lshr(r0, r1, r2) rotshr(X86_SHL, r0, r1, r2)
291# define lshi(r0, r1, i0) _lshi(_jit, r0, r1, i0)
292static void _lshi(jit_state_t*, jit_int32_t, jit_int32_t, jit_word_t);
293# define rshr(r0, r1, r2) rotshr(X86_SAR, r0, r1, r2)
294# define rshi(r0, r1, i0) rotshi(X86_SAR, r0, r1, i0)
295# define rshr_u(r0, r1, r2) rotshr(X86_SHR, r0, r1, r2)
296# define rshi_u(r0, r1, i0) rotshi(X86_SHR, r0, r1, i0)
297# define unr(code, r0) _unr(_jit, code, r0)
298static void _unr(jit_state_t*, jit_int32_t, jit_int32_t);
299# define inegr(r0) unr(X86_NEG, r0)
300# define negr(r0, r1) _negr(_jit, r0, r1)
301static void _negr(jit_state_t*, jit_int32_t, jit_int32_t);
302# define icomr(r0) unr(X86_NOT, r0)
303# define comr(r0, r1) _comr(_jit, r0, r1)
304static void _comr(jit_state_t*, jit_int32_t, jit_int32_t);
305# if USE_INC_DEC
306# define incr(r0, r1) _incr(_jit, r0, r1)
307static void _incr(jit_state_t*, jit_int32_t, jit_int32_t);
308# define decr(r0, r1) _decr(_jit, r0, r1)
309static void _decr(jit_state_t*, jit_int32_t, jit_int32_t);
310# endif
311# define cr(code, r0, r1, r2) _cr(_jit, code, r0, r1, r2)
312static void
313_cr(jit_state_t*, jit_int32_t, jit_int32_t, jit_int32_t, jit_int32_t);
314# define ci(code, r0, r1, i0) _ci(_jit, code, r0, r1, i0)
315static void
316_ci(jit_state_t *_jit, jit_int32_t, jit_int32_t, jit_int32_t, jit_word_t);
317# define ci0(code, r0, r1) _ci0(_jit, code, r0, r1)
318static void _ci0(jit_state_t*, jit_int32_t, jit_int32_t, jit_int32_t);
319# define ltr(r0, r1, r2) _ltr(_jit, r0, r1, r2)
320static void _ltr(jit_state_t*, jit_int32_t, jit_int32_t, jit_int32_t);
321# define lti(r0, r1, i0) _lti(_jit, r0, r1, i0)
322static void _lti(jit_state_t*, jit_int32_t, jit_int32_t, jit_word_t);
323# define ltr_u(r0, r1, r2) _ltr_u(_jit, r0, r1, r2)
324static void _ltr_u(jit_state_t*, jit_int32_t, jit_int32_t, jit_int32_t);
325# define lti_u(r0, r1, i0) ci(X86_CC_B, r0, r1, i0)
326# define ler(r0, r1, r2) _ler(_jit, r0, r1, r2)
327static void _ler(jit_state_t*, jit_int32_t, jit_int32_t, jit_int32_t);
328# define lei(r0, r1, i0) ci(X86_CC_LE, r0, r1, i0)
329# define ler_u(r0, r1, r2) _ler_u(_jit, r0, r1, r2)
330static void _ler_u(jit_state_t*, jit_int32_t, jit_int32_t, jit_int32_t);
331# define lei_u(r0, r1, i0) _lei_u(_jit, r0, r1, i0)
332static void _lei_u(jit_state_t*, jit_int32_t, jit_int32_t, jit_word_t);
333# define eqr(r0, r1, r2) _eqr(_jit, r0, r1, r2)
334static void _eqr(jit_state_t*, jit_int32_t, jit_int32_t, jit_int32_t);
335# define eqi(r0, r1, i0) _eqi(_jit, r0, r1, i0)
336static void _eqi(jit_state_t*, jit_int32_t, jit_int32_t, jit_word_t);
337# define ger(r0, r1, r2) _ger(_jit, r0, r1, r2)
338static void _ger(jit_state_t*, jit_int32_t, jit_int32_t, jit_int32_t);
339# define gei(r0, r1, i0) _gei(_jit, r0, r1, i0)
340static void _gei(jit_state_t*, jit_int32_t, jit_int32_t, jit_word_t);
341# define ger_u(r0, r1, r2) _ger_u(_jit, r0, r1, r2)
342static void _ger_u(jit_state_t*, jit_int32_t, jit_int32_t, jit_int32_t);
343# define gei_u(r0, r1, i0) _gei_u(_jit, r0, r1, i0)
344static void _gei_u(jit_state_t*, jit_int32_t, jit_int32_t, jit_word_t);
345# define gtr(r0, r1, r2) _gtr(_jit, r0, r1, r2)
346static void _gtr(jit_state_t*, jit_int32_t, jit_int32_t, jit_int32_t);
347# define gti(r0, r1, i0) _ci(_jit, X86_CC_G, r0, r1, i0)
348# define gtr_u(r0, r1, r2) _gtr_u(_jit, r0, r1, r2)
349static void _gtr_u(jit_state_t*, jit_int32_t, jit_int32_t, jit_int32_t);
350# define gti_u(r0, r1, i0) _gti_u(_jit, r0, r1, i0)
351static void _gti_u(jit_state_t*, jit_int32_t, jit_int32_t, jit_word_t);
352# define ner(r0, r1, r2) _ner(_jit, r0, r1, r2)
353static void _ner(jit_state_t*, jit_int32_t, jit_int32_t, jit_int32_t);
354# define nei(r0, r1, i0) _nei(_jit, r0, r1, i0)
355static void _nei(jit_state_t*, jit_int32_t, jit_int32_t, jit_word_t);
356# define movr(r0, r1) _movr(_jit, r0, r1)
357static void _movr(jit_state_t*, jit_int32_t, jit_int32_t);
358# define imovi(r0, i0) _imovi(_jit, r0, i0)
359static void _imovi(jit_state_t*, jit_int32_t, jit_word_t);
360# define movi(r0, i0) _movi(_jit, r0, i0)
361static void _movi(jit_state_t*, jit_int32_t, jit_word_t);
362# define movi_p(r0, i0) _movi_p(_jit, r0, i0)
363static jit_word_t _movi_p(jit_state_t*, jit_int32_t, jit_word_t);
364# define movcr(r0, r1) _movcr(_jit, r0, r1)
365static void _movcr(jit_state_t*,jit_int32_t,jit_int32_t);
366# define movcr_u(r0, r1) _movcr_u(_jit, r0, r1)
367static void _movcr_u(jit_state_t*,jit_int32_t,jit_int32_t);
368# define movsr(r0, r1) _movsr(_jit, r0, r1)
369static void _movsr(jit_state_t*,jit_int32_t,jit_int32_t);
370# define movsr_u(r0, r1) _movsr_u(_jit, r0, r1)
371static void _movsr_u(jit_state_t*,jit_int32_t,jit_int32_t);
ba3814c1
PC		 372# define casx(r0, r1, r2, r3, i0) _casx(_jit, r0, r1, r2, r3, i0)
373static void _casx(jit_state_t *_jit,jit_int32_t,jit_int32_t,
374 jit_int32_t,jit_int32_t,jit_word_t);
375#define casr(r0, r1, r2, r3) casx(r0, r1, r2, r3, 0)
376#define casi(r0, i0, r1, r2) casx(r0, _NOREG, r1, r2, i0)
1f22b268
PC		 377#define movnr(r0, r1, r2) _movnr(_jit, r0, r1, r2)
378static void _movnr(jit_state_t*, jit_int32_t, jit_int32_t, jit_int32_t);
379#define movzr(r0, r1, r2) _movzr(_jit, r0, r1, r2)
380static void _movzr(jit_state_t*, jit_int32_t, jit_int32_t, jit_int32_t);
4a71579b
PC		 381# if __X64 && !__X64_32
382# define movir(r0, r1) _movir(_jit, r0, r1)
383static void _movir(jit_state_t*,jit_int32_t,jit_int32_t);
384# define movir_u(r0, r1) _movir_u(_jit, r0, r1)
385static void _movir_u(jit_state_t*,jit_int32_t,jit_int32_t);
386# endif
40a44dcb
PC		 387# define bswapr_us(r0, r1) _bswapr_us(_jit, r0, r1)
388static void _bswapr_us(jit_state_t*,jit_int32_t,jit_int32_t);
389# define bswapr_ui(r0, r1) _bswapr_ui(_jit, r0, r1)
390static void _bswapr_ui(jit_state_t*,jit_int32_t,jit_int32_t);
4a71579b 391# if __X64 && !__X64_32
40a44dcb
PC		 392#define bswapr_ul(r0, r1) _bswapr_ul(_jit, r0, r1)
393static void _bswapr_ul(jit_state_t*,jit_int32_t,jit_int32_t);
4a71579b
PC		 394#endif
395# define extr_c(r0, r1) _extr_c(_jit, r0, r1)
396static void _extr_c(jit_state_t*,jit_int32_t,jit_int32_t);
397# define extr_uc(r0, r1) _extr_uc(_jit, r0, r1)
398static void _extr_uc(jit_state_t*,jit_int32_t,jit_int32_t);
399# define extr_s(r0, r1) movsr(r0, r1)
400# define extr_us(r0, r1) movsr_u(r0, r1)
401# if __X64 && !__X64_32
402# define extr_i(r0, r1) movir(r0, r1)
403# define extr_ui(r0, r1) movir_u(r0, r1)
404# endif
405# define ldr_c(r0, r1) _ldr_c(_jit, r0, r1)
406static void _ldr_c(jit_state_t*, jit_int32_t, jit_int32_t);
407# define ldi_c(r0, i0) _ldi_c(_jit, r0, i0)
408static void _ldi_c(jit_state_t*, jit_int32_t, jit_word_t);
409# define ldr_uc(r0, r1) _ldr_uc(_jit, r0, r1)
410static void _ldr_uc(jit_state_t*, jit_int32_t, jit_int32_t);
411# define ldi_uc(r0, i0) _ldi_uc(_jit, r0, i0)
412static void _ldi_uc(jit_state_t*, jit_int32_t, jit_word_t);
413# define ldr_s(r0, r1) _ldr_s(_jit, r0, r1)
414static void _ldr_s(jit_state_t*, jit_int32_t, jit_int32_t);
415# define ldi_s(r0, i0) _ldi_s(_jit, r0, i0)
416static void _ldi_s(jit_state_t*, jit_int32_t, jit_word_t);
417# define ldr_us(r0, r1) _ldr_us(_jit, r0, r1)
418static void _ldr_us(jit_state_t*, jit_int32_t, jit_int32_t);
419# define ldi_us(r0, i0) _ldi_us(_jit, r0, i0)
420static void _ldi_us(jit_state_t*, jit_int32_t, jit_word_t);
421# if __X32 || !__X64_32
422# define ldr_i(r0, r1) _ldr_i(_jit, r0, r1)
423static void _ldr_i(jit_state_t*, jit_int32_t, jit_int32_t);
424# define ldi_i(r0, i0) _ldi_i(_jit, r0, i0)
425static void _ldi_i(jit_state_t*, jit_int32_t, jit_word_t);
426# endif
427# if __X64
428# if __X64_32
429# define ldr_i(r0, r1) _ldr_ui(_jit, r0, r1)
430# define ldi_i(r0, i0) _ldi_ui(_jit, r0, i0)
431# else
432# define ldr_ui(r0, r1) _ldr_ui(_jit, r0, r1)
433# define ldi_ui(r0, i0) _ldi_ui(_jit, r0, i0)
434# endif
435static void _ldr_ui(jit_state_t*, jit_int32_t, jit_int32_t);
436static void _ldi_ui(jit_state_t*, jit_int32_t, jit_word_t);
437# if !__X64_32
438# define ldr_l(r0, r1) _ldr_l(_jit, r0, r1)
439static void _ldr_l(jit_state_t*, jit_int32_t, jit_int32_t);
440# define ldi_l(r0, i0) _ldi_l(_jit, r0, i0)
441static void _ldi_l(jit_state_t*, jit_int32_t, jit_word_t);
442# endif
443# endif
444# define ldxr_c(r0, r1, r2) _ldxr_c(_jit, r0, r1, r2)
445static void _ldxr_c(jit_state_t*, jit_int32_t, jit_int32_t, jit_int32_t);
446# define ldxi_c(r0, r1, i0) _ldxi_c(_jit, r0, r1, i0)
447static void _ldxi_c(jit_state_t*, jit_int32_t, jit_int32_t, jit_word_t);
448# define ldxr_uc(r0, r1, r2) _ldxr_uc(_jit, r0, r1, r2)
449static void _ldxr_uc(jit_state_t*, jit_int32_t, jit_int32_t, jit_int32_t);
450# define ldxi_uc(r0, r1, i0) _ldxi_uc(_jit, r0, r1, i0)
451static void _ldxi_uc(jit_state_t*, jit_int32_t, jit_int32_t, jit_word_t);
452# define ldxr_s(r0, r1, r2) _ldxr_s(_jit, r0, r1, r2)
453static void _ldxr_s(jit_state_t*, jit_int32_t, jit_int32_t, jit_int32_t);
454# define ldxi_s(r0, r1, i0) _ldxi_s(_jit, r0, r1, i0)
455static void _ldxi_s(jit_state_t*, jit_int32_t, jit_int32_t, jit_word_t);
456# define ldxr_us(r0, r1, r2) _ldxr_us(_jit, r0, r1, r2)
457static void _ldxr_us(jit_state_t*, jit_int32_t, jit_int32_t, jit_int32_t);
458# define ldxi_us(r0, r1, i0) _ldxi_us(_jit, r0, r1, i0)
459static void _ldxi_us(jit_state_t*, jit_int32_t, jit_int32_t, jit_word_t);
460# if __X32 || !__X64_32
461# define ldxr_i(r0, r1, r2) _ldxr_i(_jit, r0, r1, r2)
462static void _ldxr_i(jit_state_t*, jit_int32_t, jit_int32_t, jit_int32_t);
463# define ldxi_i(r0, r1, i0) _ldxi_i(_jit, r0, r1, i0)
464static void _ldxi_i(jit_state_t*, jit_int32_t, jit_int32_t, jit_word_t);
465# endif
466# if __X64
467# if __X64_32
468# define ldxr_i(r0, r1, r2) _ldxr_ui(_jit, r0, r1, r2)
469# define ldxi_i(r0, r1, i0) _ldxi_ui(_jit, r0, r1, i0)
470# else
471# define ldxr_ui(r0, r1, r2) _ldxr_ui(_jit, r0, r1, r2)
472# define ldxi_ui(r0, r1, i0) _ldxi_ui(_jit, r0, r1, i0)
473# endif
474static void _ldxr_ui(jit_state_t*, jit_int32_t, jit_int32_t, jit_int32_t);
475static void _ldxi_ui(jit_state_t*, jit_int32_t, jit_int32_t, jit_word_t);
476# if !__X64_32
477# define ldxr_l(r0, r1, r2) _ldxr_l(_jit, r0, r1, r2)
478static void _ldxr_l(jit_state_t*, jit_int32_t, jit_int32_t, jit_int32_t);
479# define ldxi_l(r0, r1, i0) _ldxi_l(_jit, r0, r1, i0)
480static void _ldxi_l(jit_state_t*, jit_int32_t, jit_int32_t, jit_word_t);
481# endif
482# endif
483# define str_c(r0, r1) _str_c(_jit, r0, r1)
484static void _str_c(jit_state_t*, jit_int32_t, jit_int32_t);
485# define sti_c(i0, r0) _sti_c(_jit, i0, r0)
486static void _sti_c(jit_state_t*, jit_word_t, jit_int32_t);
487# define str_s(r0, r1) _str_s(_jit, r0, r1)
488static void _str_s(jit_state_t*, jit_int32_t, jit_int32_t);
489# define sti_s(i0, r0) _sti_s(_jit, i0, r0)
490static void _sti_s(jit_state_t*, jit_word_t, jit_int32_t);
491# define str_i(r0, r1) _str_i(_jit, r0, r1)
492static void _str_i(jit_state_t*, jit_int32_t, jit_int32_t);
493# define sti_i(i0, r0) _sti_i(_jit, i0, r0)
494static void _sti_i(jit_state_t*, jit_word_t, jit_int32_t);
495# if __X64 && !__X64_32
496# define str_l(r0, r1) _str_l(_jit, r0, r1)
497static void _str_l(jit_state_t*, jit_int32_t, jit_int32_t);
498# define sti_l(i0, r0) _sti_l(_jit, i0, r0)
499static void _sti_l(jit_state_t*, jit_word_t, jit_int32_t);
500# endif
501# define stxr_c(r0, r1, r2) _stxr_c(_jit, r0, r1, r2)
502static void _stxr_c(jit_state_t*, jit_int32_t, jit_int32_t, jit_int32_t);
503# define stxi_c(i0, r0, r1) _stxi_c(_jit, i0, r0, r1)
504static void _stxi_c(jit_state_t*, jit_word_t, jit_int32_t, jit_int32_t);
505# define stxr_s(r0, r1, r2) _stxr_s(_jit, r0, r1, r2)
506static void _stxr_s(jit_state_t*, jit_int32_t, jit_int32_t, jit_int32_t);
507# define stxi_s(i0, r0, r1) _stxi_s(_jit, i0, r0, r1)
508static void _stxi_s(jit_state_t*, jit_word_t, jit_int32_t, jit_int32_t);
509# define stxr_i(r0, r1, r2) _stxr_i(_jit, r0, r1, r2)
510static void _stxr_i(jit_state_t*, jit_int32_t, jit_int32_t, jit_int32_t);
511# define stxi_i(i0, r0, r1) _stxi_i(_jit, i0, r0, r1)
512static void _stxi_i(jit_state_t*, jit_word_t, jit_int32_t, jit_int32_t);
513# if __X64 && !__X64_32
514# define stxr_l(r0, r1, r2) _stxr_l(_jit, r0, r1, r2)
515static void _stxr_l(jit_state_t*, jit_int32_t, jit_int32_t, jit_int32_t);
516# define stxi_l(i0, r0, r1) _stxi_l(_jit, i0, r0, r1)
517static void _stxi_l(jit_state_t*, jit_word_t, jit_int32_t, jit_int32_t);
518# endif
519# define jcc(code, i0) _jcc(_jit, code, i0)
520# define jo(i0) jcc(X86_CC_O, i0)
521# define jno(i0) jcc(X86_CC_NO, i0)
522# define jnae(i0) jcc(X86_CC_NAE, i0)
523# define jb(i0) jcc(X86_CC_B, i0)
524# define jc(i0) jcc(X86_CC_C, i0)
525# define jae(i0) jcc(X86_CC_AE, i0)
526# define jnb(i0) jcc(X86_CC_NB, i0)
527# define jnc(i0) jcc(X86_CC_NC, i0)
528# define je(i0) jcc(X86_CC_E, i0)
529# define jz(i0) jcc(X86_CC_Z, i0)
530# define jne(i0) jcc(X86_CC_NE, i0)
531# define jnz(i0) jcc(X86_CC_NZ, i0)
532# define jbe(i0) jcc(X86_CC_BE, i0)
533# define jna(i0) jcc(X86_CC_NA, i0)
534# define ja(i0) jcc(X86_CC_A, i0)
535# define jnbe(i0) jcc(X86_CC_NBE, i0)
536# define js(i0) jcc(X86_CC_S, i0)
537# define jns(i0) jcc(X86_CC_NS, i0)
538# define jp(i0) jcc(X86_CC_P, i0)
539# define jpe(i0) jcc(X86_CC_PE, i0)
540# define jnp(i0) jcc(X86_CC_NP, i0)
541# define jpo(i0) jcc(X86_CC_PO, i0)
542# define jl(i0) jcc(X86_CC_L, i0)
543# define jnge(i0) jcc(X86_CC_NGE, i0)
544# define jge(i0) jcc(X86_CC_GE, i0)
545# define jnl(i0) jcc(X86_CC_NL, i0)
546# define jle(i0) jcc(X86_CC_LE, i0)
547# define jng(i0) jcc(X86_CC_NG, i0)
548# define jg(i0) jcc(X86_CC_G, i0)
549# define jnle(i0) jcc(X86_CC_NLE, i0)
550static void _jcc(jit_state_t*, jit_int32_t, jit_word_t);
551# define jccs(code, i0) _jccs(_jit, code, i0)
552# define jos(i0) jccs(X86_CC_O, i0)
553# define jnos(i0) jccs(X86_CC_NO, i0)
554# define jnaes(i0) jccs(X86_CC_NAE, i0)
555# define jbs(i0) jccs(X86_CC_B, i0)
556# define jcs(i0) jccs(X86_CC_C, i0)
557# define jaes(i0) jccs(X86_CC_AE, i0)
558# define jnbs(i0) jccs(X86_CC_NB, i0)
559# define jncs(i0) jccs(X86_CC_NC, i0)
560# define jes(i0) jccs(X86_CC_E, i0)
561# define jzs(i0) jccs(X86_CC_Z, i0)
562# define jnes(i0) jccs(X86_CC_NE, i0)
563# define jnzs(i0) jccs(X86_CC_NZ, i0)
564# define jbes(i0) jccs(X86_CC_BE, i0)
565# define jnas(i0) jccs(X86_CC_NA, i0)
566# define jas(i0) jccs(X86_CC_A, i0)
567# define jnbes(i0) jccs(X86_CC_NBE, i0)
568# define jss(i0) jccs(X86_CC_S, i0)
569# define jnss(i0) jccs(X86_CC_NS, i0)
570# define jps(i0) jccs(X86_CC_P, i0)
571# define jpes(i0) jccs(X86_CC_PE, i0)
572# define jnps(i0) jccs(X86_CC_NP, i0)
573# define jpos(i0) jccs(X86_CC_PO, i0)
574# define jls(i0) jccs(X86_CC_L, i0)
575# define jnges(i0) jccs(X86_CC_NGE, i0)
576# define jges(i0) jccs(X86_CC_GE, i0)
577# define jnls(i0) jccs(X86_CC_NL, i0)
578# define jles(i0) jccs(X86_CC_LE, i0)
579# define jngs(i0) jccs(X86_CC_NG, i0)
580# define jgs(i0) jccs(X86_CC_G, i0)
581# define jnles(i0) jccs(X86_CC_NLE, i0)
582static void _jccs(jit_state_t*, jit_int32_t, jit_word_t);
583# define jcr(code, i0, r0, r1) _jcr(_jit, code, i0, r0, r1)
584static void _jcr(jit_state_t*,jit_int32_t,jit_word_t,jit_int32_t,jit_int32_t);
585# define jci(code, i0, r0, i1) _jci(_jit, code, i0, r0, i1)
586static void _jci(jit_state_t*,jit_int32_t,jit_word_t,jit_int32_t,jit_word_t);
587# define jci0(code, i0, r0) _jci0(_jit, code, i0, r0)
588static void _jci0(jit_state_t*, jit_int32_t, jit_word_t, jit_int32_t);
589# define bltr(i0, r0, r1) _bltr(_jit, i0, r0, r1)
590static jit_word_t _bltr(jit_state_t*, jit_word_t, jit_int32_t, jit_int32_t);
591# define blti(i0, r0, i1) _blti(_jit, i0, r0, i1)
592static jit_word_t _blti(jit_state_t*, jit_word_t, jit_int32_t, jit_word_t);
593# define bltr_u(i0, r0, r1) _bltr_u(_jit, i0, r0, r1)
594static jit_word_t _bltr_u(jit_state_t*, jit_word_t, jit_int32_t, jit_int32_t);
595# define blti_u(i0, r0, i1) _blti_u(_jit, i0, r0, i1)
596static jit_word_t _blti_u(jit_state_t*, jit_word_t, jit_int32_t, jit_word_t);
597# define bler(i0, r0, r1) _bler(_jit, i0, r0, r1)
598static jit_word_t _bler(jit_state_t*, jit_word_t, jit_int32_t, jit_int32_t);
599# define blei(i0, r0, i1) _blei(_jit, i0, r0, i1)
600static jit_word_t _blei(jit_state_t*, jit_word_t, jit_int32_t, jit_word_t);
601# define bler_u(i0, r0, r1) _bler_u(_jit, i0, r0, r1)
602static jit_word_t _bler_u(jit_state_t*, jit_word_t, jit_int32_t, jit_int32_t);
603# define blei_u(i0, r0, i1) _blei_u(_jit, i0, r0, i1)
604static jit_word_t _blei_u(jit_state_t*, jit_word_t, jit_int32_t, jit_word_t);
605# define beqr(i0, r0, r1) _beqr(_jit, i0, r0, r1)
606static jit_word_t _beqr(jit_state_t*, jit_word_t, jit_int32_t, jit_int32_t);
607# define beqi(i0, r0, i1) _beqi(_jit, i0, r0, i1)
608static jit_word_t _beqi(jit_state_t*, jit_word_t, jit_int32_t, jit_word_t);
609# define bger(i0, r0, r1) _bger(_jit, i0, r0, r1)
610static jit_word_t _bger(jit_state_t*, jit_word_t, jit_int32_t, jit_int32_t);
611# define bgei(i0, r0, i1) _bgei(_jit, i0, r0, i1)
612static jit_word_t _bgei(jit_state_t*, jit_word_t, jit_int32_t, jit_word_t);
613# define bger_u(i0, r0, r1) _bger_u(_jit, i0, r0, r1)
614static jit_word_t _bger_u(jit_state_t*, jit_word_t, jit_int32_t, jit_int32_t);
615# define bgei_u(i0, r0, i1) _bgei_u(_jit, i0, r0, i1)
616static jit_word_t _bgei_u(jit_state_t*, jit_word_t, jit_int32_t, jit_word_t);
617# define bgtr(i0, r0, r1) _bgtr(_jit, i0, r0, r1)
618static jit_word_t _bgtr(jit_state_t*, jit_word_t, jit_int32_t, jit_int32_t);
619# define bgti(i0, r0, i1) _bgti(_jit, i0, r0, i1)
620static jit_word_t _bgti(jit_state_t*, jit_word_t, jit_int32_t, jit_word_t);
621# define bgtr_u(i0, r0, r1) _bgtr_u(_jit, i0, r0, r1)
622static jit_word_t _bgtr_u(jit_state_t*, jit_word_t, jit_int32_t, jit_int32_t);
623# define bgti_u(i0, r0, i1) _bgti_u(_jit, i0, r0, i1)
624static jit_word_t _bgti_u(jit_state_t*, jit_word_t, jit_int32_t, jit_word_t);
625# define bner(i0, r0, r1) _bner(_jit, i0, r0, r1)
626static jit_word_t _bner(jit_state_t*, jit_word_t, jit_int32_t, jit_int32_t);
627# define bnei(i0, r0, i1) _bnei(_jit, i0, r0, i1)
628static jit_word_t _bnei(jit_state_t*, jit_word_t, jit_int32_t, jit_word_t);
629# define bmsr(i0, r0, r1) _bmsr(_jit, i0, r0, r1)
630static jit_word_t _bmsr(jit_state_t*,jit_word_t,jit_int32_t,jit_int32_t);
631# define bmsi(i0, r0, i1) _bmsi(_jit, i0, r0, i1)
632static jit_word_t _bmsi(jit_state_t*,jit_word_t,jit_int32_t,jit_word_t);
633# define bmcr(i0, r0, r1) _bmcr(_jit, i0, r0, r1)
634static jit_word_t _bmcr(jit_state_t*,jit_word_t,jit_int32_t,jit_int32_t);
635# define bmci(i0, r0, i1) _bmci(_jit, i0, r0, i1)
636static jit_word_t _bmci(jit_state_t*,jit_word_t,jit_int32_t,jit_word_t);
637# define boaddr(i0, r0, r1) _boaddr(_jit, i0, r0, r1)
638static jit_word_t _boaddr(jit_state_t*,jit_word_t,jit_int32_t,jit_int32_t);
639# define boaddi(i0, r0, i1) _boaddi(_jit, i0, r0, i1)
640static jit_word_t _boaddi(jit_state_t*,jit_word_t,jit_int32_t,jit_word_t);
641# define boaddr_u(i0, r0, r1) _boaddr_u(_jit, i0, r0, r1)
642static jit_word_t _boaddr_u(jit_state_t*,jit_word_t,jit_int32_t,jit_int32_t);
643# define boaddi_u(i0, r0, i1) _boaddi_u(_jit, i0, r0, i1)
644static jit_word_t _boaddi_u(jit_state_t*,jit_word_t,jit_int32_t,jit_word_t);
645# define bxaddr(i0, r0, r1) _bxaddr(_jit, i0, r0, r1)
646static jit_word_t _bxaddr(jit_state_t*,jit_word_t,jit_int32_t,jit_int32_t);
647# define bxaddi(i0, r0, i1) _bxaddi(_jit, i0, r0, i1)
648static jit_word_t _bxaddi(jit_state_t*,jit_word_t,jit_int32_t,jit_word_t);
649# define bxaddr_u(i0, r0, r1) _bxaddr_u(_jit, i0, r0, r1)
650static jit_word_t _bxaddr_u(jit_state_t*,jit_word_t,jit_int32_t,jit_int32_t);
651# define bxaddi_u(i0, r0, i1) _bxaddi_u(_jit, i0, r0, i1)
652static jit_word_t _bxaddi_u(jit_state_t*,jit_word_t,jit_int32_t,jit_word_t);
653# define bosubr(i0, r0, r1) _bosubr(_jit, i0, r0, r1)
654static jit_word_t _bosubr(jit_state_t*,jit_word_t,jit_int32_t,jit_int32_t);
655# define bosubi(i0, r0, i1) _bosubi(_jit, i0, r0, i1)
656static jit_word_t _bosubi(jit_state_t*,jit_word_t,jit_int32_t,jit_word_t);
657# define bosubr_u(i0, r0, r1) _bosubr_u(_jit, i0, r0, r1)
658static jit_word_t _bosubr_u(jit_state_t*,jit_word_t,jit_int32_t,jit_int32_t);
659# define bosubi_u(i0, r0, i1) _bosubi_u(_jit, i0, r0, i1)
660static jit_word_t _bosubi_u(jit_state_t*,jit_word_t,jit_int32_t,jit_word_t);
661# define bxsubr(i0, r0, r1) _bxsubr(_jit, i0, r0, r1)
662static jit_word_t _bxsubr(jit_state_t*,jit_word_t,jit_int32_t,jit_int32_t);
663# define bxsubi(i0, r0, i1) _bxsubi(_jit, i0, r0, i1)
664static jit_word_t _bxsubi(jit_state_t*,jit_word_t,jit_int32_t,jit_word_t);
665# define bxsubr_u(i0, r0, r1) _bxsubr_u(_jit, i0, r0, r1)
666static jit_word_t _bxsubr_u(jit_state_t*,jit_word_t,jit_int32_t,jit_int32_t);
667# define bxsubi_u(i0, r0, i1) _bxsubi_u(_jit, i0, r0, i1)
668static jit_word_t _bxsubi_u(jit_state_t*,jit_word_t,jit_int32_t,jit_word_t);
669# define callr(r0) _callr(_jit, r0)
670static void _callr(jit_state_t*, jit_int32_t);
671# define calli(i0) _calli(_jit, i0)
672static jit_word_t _calli(jit_state_t*, jit_word_t);
519a9ea1
PC		 673# if __X64
674# define calli_p(i0) _calli_p(_jit, i0)
675static jit_word_t _calli_p(jit_state_t*, jit_word_t);
676# else
677# define calli_p(i0) calli(i0)
678# endif
4a71579b
PC		 679# define jmpr(r0) _jmpr(_jit, r0)
680static void _jmpr(jit_state_t*, jit_int32_t);
681# define jmpi(i0) _jmpi(_jit, i0)
682static jit_word_t _jmpi(jit_state_t*, jit_word_t);
519a9ea1
PC		 683# if __X64
684# define jmpi_p(i0) _jmpi_p(_jit, i0)
685static jit_word_t _jmpi_p(jit_state_t*, jit_word_t);
686# else
687# define jmpi_p(i0) jmpi(i0)
688# endif
4a71579b
PC		 689# define jmpsi(i0) _jmpsi(_jit, i0)
690static void _jmpsi(jit_state_t*, jit_uint8_t);
691# define prolog(node) _prolog(_jit, node)
692static void _prolog(jit_state_t*, jit_node_t*);
693# define epilog(node) _epilog(_jit, node)
694static void _epilog(jit_state_t*, jit_node_t*);
695# define vastart(r0) _vastart(_jit, r0)
696static void _vastart(jit_state_t*, jit_int32_t);
697# define vaarg(r0, r1) _vaarg(_jit, r0, r1)
698static void _vaarg(jit_state_t*, jit_int32_t, jit_int32_t);
699# define vaarg_d(r0, r1, i0) _vaarg_d(_jit, r0, r1, i0)
700static void _vaarg_d(jit_state_t*, jit_int32_t, jit_int32_t, jit_bool_t);
701# define patch_at(node, instr, label) _patch_at(_jit, node, instr, label)
702static void _patch_at(jit_state_t*, jit_node_t*, jit_word_t, jit_word_t);
703# if !defined(HAVE_FFSL)
704# if __X32
705# define ffsl(i) __builtin_ffs(i)
706# else
707# define ffsl(l) __builtin_ffsl(l)
708# endif
709# endif
1f22b268 710# define jit_cmov_p() jit_cpu.cmov
4a71579b
PC		 711#endif
712
713#if CODE
714static void
715_rex(jit_state_t *_jit, jit_int32_t l, jit_int32_t w,
716 jit_int32_t r, jit_int32_t x, jit_int32_t b)
717{
718#if __X64
719 jit_int32_t v = 0x40 | (w << 3);
720
721 if (r != _NOREG)
722 v |= (r & 8) >> 1;
723 if (x != _NOREG)
724 v |= (x & 8) >> 2;
725 if (b != _NOREG)
726 v |= (b & 8) >> 3;
727 if (l || v != 0x40)
728 ic(v);
729#endif
730}
731
732static void
733_rx(jit_state_t *_jit, jit_int32_t rd, jit_int32_t md,
734 jit_int32_t rb, jit_int32_t ri, jit_int32_t ms)
735{
736 if (ri == _NOREG) {
737 if (rb == _NOREG) {
738#if __X32
739 mrm(0x00, r7(rd), 0x05);
740#else
741 mrm(0x00, r7(rd), 0x04);
742 sib(_SCL1, 0x04, 0x05);
743#endif
744 ii(md);
745 }
746 else if (r7(rb) == _RSP_REGNO) {
747 if (md == 0) {
748 mrm(0x00, r7(rd), 0x04);
749 sib(ms, 0x04, 0x04);
750 }
751 else if ((jit_int8_t)md == md) {
752 mrm(0x01, r7(rd), 0x04);
753 sib(ms, 0x04, 0x04);
754 ic(md);
755 }
756 else {
757 mrm(0x02, r7(rd), 0x04);
758 sib(ms, 0x04, 0x04);
759 ii(md);
760 }
761 }
762 else {
763 if (md == 0 && r7(rb) != _RBP_REGNO)
764 mrm(0x00, r7(rd), r7(rb));
765 else if ((jit_int8_t)md == md) {
766 mrm(0x01, r7(rd), r7(rb));
767 ic(md);
768 }
769 else {
770 mrm(0x02, r7(rd), r7(rb));
771 ii(md);
772 }
773 }
774 }
775 else if (rb == _NOREG) {
776 mrm(0x00, r7(rd), 0x04);
777 sib(ms, r7(ri), 0x05);
778 ii(md);
779 }
780 else if (r8(ri) != _RSP_REGNO) {
781 if (md == 0 && r7(rb) != _RBP_REGNO) {
782 mrm(0x00, r7(rd), 0x04);
783 sib(ms, r7(ri), r7(rb));
784 }
785 else if ((jit_int8_t)md == md) {
786 mrm(0x01, r7(rd), 0x04);
787 sib(ms, r7(ri), r7(rb));
788 ic(md);
789 }
790 else {
791 mrm(0x02, r7(rd), 0x04);
792 sib(ms, r7(ri), r7(rb));
793 ic(md);
794 }
795 }
796 else {
797 fprintf(stderr, "illegal index register");
798 abort();
799 }
800}
801
802static void
803_nop(jit_state_t *_jit, jit_int32_t count)
804{
805 switch (count) {
806 case 0:
807 break;
808 case 1: /* NOP */
809 ic(0x90); break;
810 case 2: /* 66 NOP */
811 ic(0x66); ic(0x90);
812 break;
813 case 3: /* NOP DWORD ptr [EAX] */
814 ic(0x0f); ic(0x1f); ic(0x00);
815 break;
816 case 4: /* NOP DWORD ptr [EAX + 00H] */
817 ic(0x0f); ic(0x1f); ic(0x40); ic(0x00);
818 break;
819 case 5: /* NOP DWORD ptr [EAX + EAX*1 + 00H] */
820 ic(0x0f); ic(0x1f); ic(0x44); ic(0x00);
821 ic(0x00);
822 break;
823 case 6: /* 66 NOP DWORD ptr [EAX + EAX*1 + 00H] */
824 ic(0x66); ic(0x0f); ic(0x1f); ic(0x44);
825 ic(0x00); ic(0x00);
826 break;
827 case 7: /* NOP DWORD ptr [EAX + 00000000H] */
828 ic(0x0f); ic(0x1f); ic(0x80); ii(0x0000);
829 break;
830 case 8: /* NOP DWORD ptr [EAX + EAX*1 + 00000000H] */
831 ic(0x0f); ic(0x1f); ic(0x84); ic(0x00);
832 ii(0x0000);
833 break;
834 case 9: /* 66 NOP DWORD ptr [EAX + EAX*1 + 00000000H] */
835 ic(0x66); ic(0x0f); ic(0x1f); ic(0x84);
836 ic(0x00); ii(0x0000);
837 break;
838 default:
839 abort();
840 }
841}
842
843static void
844_lea(jit_state_t *_jit, jit_int32_t md, jit_int32_t rb,
845 jit_int32_t ri, jit_int32_t ms, jit_int32_t rd)
846{
847 rex(0, WIDE, rd, ri, rb);
848 ic(0x8d);
849 rx(rd, md, rb, ri, ms);
850}
851
852static void
853_pushr(jit_state_t *_jit, jit_int32_t r0)
854{
855 rex(0, WIDE, 0, 0, r0);
856 ic(0x50 | r7(r0));
857}
858
859static void
860_popr(jit_state_t *_jit, jit_int32_t r0)
861{
862 rex(0, WIDE, 0, 0, r0);
863 ic(0x58 | r7(r0));
864}
865
866static void
867_xchgr(jit_state_t *_jit, jit_int32_t r0, jit_int32_t r1)
868{
869 rex(0, WIDE, r1, _NOREG, r0);
870 ic(0x87);
871 mrm(0x03, r7(r1), r7(r0));
872}
873
874static void
875_testr(jit_state_t *_jit, jit_int32_t r0, jit_int32_t r1)
876{
877 rex(0, WIDE, r1, _NOREG, r0);
878 ic(0x85);
879 mrm(0x03, r7(r1), r7(r0));
880}
881
882static void
883_testi(jit_state_t *_jit, jit_int32_t r0, jit_word_t i0)
884{
885 rex(0, WIDE, _NOREG, _NOREG, r0);
886 if (r0 == _RAX_REGNO)
887 ic(0xa9);
888 else {
889 ic(0xf7);
890 mrm(0x03, 0x00, r7(r0));
891 }
892 ii(i0);
893}
894
895static void
896_cc(jit_state_t *_jit, jit_int32_t code, jit_int32_t r0)
897{
898 rex(0, 0, _NOREG, _NOREG, r0);
899 ic(0x0f);
900 ic(0x90 | code);
901 mrm(0x03, 0x00, r7(r0));
902}
903
904static void
905_alur(jit_state_t *_jit, jit_int32_t code, jit_int32_t r0, jit_int32_t r1)
906{
907 rex(0, WIDE, r1, _NOREG, r0);
908 ic(code | 0x01);
909 mrm(0x03, r7(r1), r7(r0));
910}
911
912static void
913_alui(jit_state_t *_jit, jit_int32_t code, jit_int32_t r0, jit_word_t i0)
914{
915 jit_int32_t reg;
916 if (can_sign_extend_int_p(i0)) {
917 rex(0, WIDE, _NOREG, _NOREG, r0);
918 if ((jit_int8_t)i0 == i0) {
919 ic(0x83);
920 ic(0xc0 | code | r7(r0));
921 ic(i0);
922 }
923 else {
924 if (r0 == _RAX_REGNO)
925 ic(code | 0x05);
926 else {
927 ic(0x81);
928 ic(0xc0 | code | r7(r0));
929 }
930 ii(i0);
931 }
932 }
933 else {
934 reg = jit_get_reg(jit_class_gpr);
935 movi(rn(reg), i0);
936 alur(code, r0, rn(reg));
937 jit_unget_reg(reg);
938 }
939}
940
941static void
942_save(jit_state_t *_jit, jit_int32_t r0)
943{
944 if (!_jitc->function->regoff[r0]) {
945 _jitc->function->regoff[r0] = jit_allocai(sizeof(jit_word_t));
946 _jitc->again = 1;
947 }
948 assert(!jit_regset_tstbit(&_jitc->regsav, r0));
949 jit_regset_setbit(&_jitc->regsav, r0);
950 stxi(_jitc->function->regoff[r0], _RBP_REGNO, r0);
951}
952
953static void
954_load(jit_state_t *_jit, jit_int32_t r0)
955{
956 assert(_jitc->function->regoff[r0]);
957 assert(jit_regset_tstbit(&_jitc->regsav, r0));
958 jit_regset_clrbit(&_jitc->regsav, r0);
959 ldxi(r0, _RBP_REGNO, _jitc->function->regoff[r0]);
960}
961
962static void
963_addr(jit_state_t *_jit, jit_int32_t r0, jit_int32_t r1, jit_int32_t r2)
964{
965 if (r0 == r1)
966 iaddr(r0, r2);
967 else if (r0 == r2)
968 iaddr(r0, r1);
969 else
970 lea(0, r1, r2, _SCL1, r0);
971}
972
973static void
974_addi(jit_state_t *_jit, jit_int32_t r0, jit_int32_t r1, jit_word_t i0)
975{
976 jit_int32_t reg;
977 if (i0 == 0)
978 movr(r0, r1);
979#if USE_INC_DEC
980 else if (i0 == 1)
981 incr(r0, r1);
982 else if (i0 == -1)
983 decr(r0, r1);
984#endif
985 else if (can_sign_extend_int_p(i0)) {
986 if (r0 == r1)
987 iaddi(r0, i0);
988 else
989 lea(i0, r1, _NOREG, _SCL1, r0);
990 }
991 else if (r0 != r1) {
992 movi(r0, i0);
993 iaddr(r0, r1);
994 }
995 else {
996 reg = jit_get_reg(jit_class_gpr);
997 movi(rn(reg), i0);
998 iaddr(r0, rn(reg));
999 jit_unget_reg(reg);
1000 }
1001}
1002
1003static void
1004_addcr(jit_state_t *_jit, jit_int32_t r0, jit_int32_t r1, jit_int32_t r2)
1005{
1006 if (r0 == r2)
1007 iaddr(r0, r1);
1008 else {
1009 movr(r0, r1);
1010 iaddr(r0, r2);
1011 }
1012}
1013
1014static void
1015_addci(jit_state_t *_jit, jit_int32_t r0, jit_int32_t r1, jit_word_t i0)
1016{
1017 jit_int32_t reg;
1018 if (can_sign_extend_int_p(i0)) {
1019 movr(r0, r1);
1020 iaddi(r0, i0);
1021 }
1022 else if (r0 == r1) {
1023 reg = jit_get_reg(jit_class_gpr);
1024 movi(rn(reg), i0);
1025 iaddr(r0, rn(reg));
1026 jit_unget_reg(reg);
1027 }
1028 else {
1029 movi(r0, i0);
1030 iaddr(r0, r1);
1031 }
1032}
1033
1034static void
1035_addxr(jit_state_t *_jit, jit_int32_t r0, jit_int32_t r1, jit_int32_t r2)
1036{
1037 if (r0 == r2)
1038 iaddxr(r0, r1);
1039 else {
1040 movr(r0, r1);
1041 iaddxr(r0, r2);
1042 }
1043}
1044
1045static void
1046_addxi(jit_state_t *_jit, jit_int32_t r0, jit_int32_t r1, jit_word_t i0)
1047{
1048 jit_int32_t reg;
1049 if (can_sign_extend_int_p(i0)) {
1050 movr(r0, r1);
1051 iaddxi(r0, i0);
1052 }
1053 else if (r0 == r1) {
1054 reg = jit_get_reg(jit_class_gpr);
1055 movi(rn(reg), i0);
1056 iaddxr(r0, rn(reg));
1057 jit_unget_reg(reg);
1058 }
1059 else {
1060 movi(r0, i0);
1061 iaddxr(r0, r1);
1062 }
1063}
1064
1065static void
1066_subr(jit_state_t *_jit, jit_int32_t r0, jit_int32_t r1, jit_int32_t r2)
1067{
1068 if (r1 == r2)
1069 ixorr(r0, r0);
1070 else if (r0 == r2) {
1071 isubr(r0, r1);
1072 inegr(r0);
1073 }
1074 else {
1075 movr(r0, r1);
1076 isubr(r0, r2);
1077 }
1078}
1079
1080static void
1081_subi(jit_state_t *_jit, jit_int32_t r0, jit_int32_t r1, jit_word_t i0)
1082{
1083 jit_int32_t reg;
1084 if (i0 == 0)
1085 movr(r0, r1);
1086#if USE_INC_DEC
1087 else if (i0 == 1)
1088 decr(r0, r1);
1089 else if (i0 == -1)
1090 incr(r0, r1);
1091#endif
1092 else if (can_sign_extend_int_p(i0)) {
1093 if (r0 == r1)
1094 isubi(r0, i0);
1095 else
1096 lea(-i0, r1, _NOREG, _SCL1, r0);
1097 }
1098 else if (r0 != r1) {
1099 movi(r0, -i0);
1100 iaddr(r0, r1);
1101 }
1102 else {
1103 reg = jit_get_reg(jit_class_gpr);
1104 movi(rn(reg), i0);
1105 isubr(r0, rn(reg));
1106 jit_unget_reg(reg);
1107 }
1108}
1109
1110static void
1111_subcr(jit_state_t *_jit, jit_int32_t r0, jit_int32_t r1, jit_int32_t r2)
1112{
1113 jit_int32_t reg;
1114 if (r0 == r2 && r0 != r1) {
1115 reg = jit_get_reg(jit_class_gpr);
1116 movr(rn(reg), r0);
1117 movr(r0, r1);
1118 isubr(r0, rn(reg));
1119 jit_unget_reg(reg);
1120 }
1121 else {
1122 movr(r0, r1);
1123 isubr(r0, r2);
1124 }
1125}
1126
1127static void
1128_subci(jit_state_t *_jit, jit_int32_t r0, jit_int32_t r1, jit_word_t i0)
1129{
1130 jit_int32_t reg;
1131 movr(r0, r1);
1132 if (can_sign_extend_int_p(i0))
1133 isubi(r0, i0);
1134 else {
1135 reg = jit_get_reg(jit_class_gpr);
1136 movi(rn(reg), i0);
1137 isubr(r0, rn(reg));
1138 jit_unget_reg(reg);
1139 }
1140}
1141
1142static void
1143_subxr(jit_state_t *_jit, jit_int32_t r0, jit_int32_t r1, jit_int32_t r2)
1144{
1145 jit_int32_t reg;
1146 if (r0 == r2 && r0 != r1) {
1147 reg = jit_get_reg(jit_class_gpr);
1148 movr(rn(reg), r0);
1149 movr(r0, r1);
1150 isubxr(r0, rn(reg));
1151 jit_unget_reg(reg);
1152 }
1153 else {
1154 movr(r0, r1);
1155 isubxr(r0, r2);
1156 }
1157}
1158
1159static void
1160_subxi(jit_state_t *_jit, jit_int32_t r0, jit_int32_t r1, jit_word_t i0)
1161{
1162 jit_int32_t reg;
1163 movr(r0, r1);
1164 if (can_sign_extend_int_p(i0))
1165 isubxi(r0, i0);
1166 else {
1167 reg = jit_get_reg(jit_class_gpr);
1168 imovi(rn(reg), i0);
1169 isubxr(r0, rn(reg));
1170 jit_unget_reg(reg);
1171 }
1172}
1173
1174static void
1175_rsbi(jit_state_t *_jit, jit_int32_t r0, jit_int32_t r1, jit_word_t i0)
1176{
1177 subi(r0, r1, i0);
1178 negr(r0, r0);
1179}
1180
1181static void
1182_imulr(jit_state_t *_jit, jit_int32_t r0, jit_int32_t r1)
1183{
1184 rex(0, WIDE, r0, _NOREG, r1);
1185 ic(0x0f);
1186 ic(0xaf);
1187 mrm(0x03, r7(r0), r7(r1));
1188}
1189
1190static void
1191_imuli(jit_state_t *_jit, jit_int32_t r0, jit_int32_t r1, jit_word_t i0)
1192{
1193 jit_int32_t reg;
1194 if (can_sign_extend_int_p(i0)) {
1195 rex(0, WIDE, r0, _NOREG, r1);
1196 if ((jit_int8_t)i0 == i0) {
1197 ic(0x6b);
1198 mrm(0x03, r7(r0), r7(r1));
1199 ic(i0);
1200 }
1201 else {
1202 ic(0x69);
1203 mrm(0x03, r7(r0), r7(r1));
1204 ii(i0);
1205 }
1206 }
1207 else {
1208 reg = jit_get_reg(jit_class_gpr);
1209 movi(rn(reg), i0);
1210 imulr(r0, rn(reg));
1211 jit_unget_reg(reg);
1212 }
1213}
1214
1215static void
1216_mulr(jit_state_t *_jit, jit_int32_t r0, jit_int32_t r1, jit_int32_t r2)
1217{
1218 if (r0 == r1)
1219 imulr(r0, r2);
1220 else if (r0 == r2)
1221 imulr(r0, r1);
1222 else {
1223 movr(r0, r1);
1224 imulr(r0, r2);
1225 }
1226}
1227
1228static void
1229_muli(jit_state_t *_jit, jit_int32_t r0, jit_int32_t r1, jit_word_t i0)
1230{
1231 switch (i0) {
1232 case 0:
1233 ixorr(r0, r0);
1234 break;
1235 case 1:
1236 movr(r0, r1);
1237 break;
1238 case -1:
1239 negr(r0, r1);
1240 break;
1241 case 2:
1242 lea(0, _NOREG, r1, _SCL2, r0);
1243 break;
1244 case 4:
1245 lea(0, _NOREG, r1, _SCL4, r0);
1246 break;
1247 case 8:
1248 lea(0, _NOREG, r1, _SCL8, r0);
1249 break;
1250 default:
1251 if (i0 > 0 && !(i0 & (i0 - 1)))
1252 lshi(r0, r1, ffsl(i0) - 1);
1253 else if (can_sign_extend_int_p(i0))
1254 imuli(r0, r1, i0);
1255 else if (r0 != r1) {
1256 movi(r0, i0);
1257 imulr(r0, r1);
1258 }
1259 else
1260 imuli(r0, r0, i0);
1261 break;
1262 }
1263}
1264
1265#define savset(rn) \
1266 if (r0 != rn) { \
1267 sav |= 1 << rn; \
1268 if (r1 != rn && r2 != rn) \
1269 set |= 1 << rn; \
1270 }
1271#define isavset(rn) \
1272 if (r0 != rn) { \
1273 sav |= 1 << rn; \
1274 if (r1 != rn) \
1275 set |= 1 << rn; \
1276 }
1277#define qsavset(rn) \
1278 if (r0 != rn && r1 != rn) { \
1279 sav |= 1 << rn; \
1280 if (r2 != rn && r3 != rn) \
1281 set |= 1 << rn; \
1282 }
1283#define allocr(rn, rv) \
1284 if (set & (1 << rn)) \
1285 (void)jit_get_reg(rv|jit_class_gpr|jit_class_named); \
1286 if (sav & (1 << rn)) { \
1287 if ( jit_regset_tstbit(&_jitc->regsav, rv) || \
1288 !jit_regset_tstbit(&_jitc->reglive, rv)) \
1289 sav &= ~(1 << rn); \
1290 else \
1291 save(rv); \
1292 }
1293#define clear(rn, rv) \
1294 if (set & (1 << rn)) \
1295 jit_unget_reg(rv); \
1296 if (sav & (1 << rn)) \
1297 load(rv);
1298static void
1299_iqmulr(jit_state_t *_jit, jit_int32_t r0, jit_int32_t r1,
1300 jit_int32_t r2, jit_int32_t r3, jit_bool_t sign)
1301{
1302 jit_int32_t mul;
1303 jit_int32_t sav;
1304 jit_int32_t set;
1305
1306 sav = set = 0;
1307 qsavset(_RDX_REGNO);
1308 qsavset(_RAX_REGNO);
1309 allocr(_RDX_REGNO, _RDX);
1310 allocr(_RAX_REGNO, _RAX);
1311
1312 if (r3 == _RAX_REGNO)
1313 mul = r2;
1314 else {
1315 mul = r3;
1316 movr(_RAX_REGNO, r2);
1317 }
1318 if (sign)
1319 umulr(mul);
1320 else
1321 umulr_u(mul);
1322
1323 if (r0 == _RDX_REGNO && r1 == _RAX_REGNO)
1324 xchgr(_RAX_REGNO, _RDX_REGNO);
1325 else {
1326 if (r0 != _RDX_REGNO)
1327 movr(r0, _RAX_REGNO);
1328 movr(r1, _RDX_REGNO);
1329 if (r0 == _RDX_REGNO)
1330 movr(r0, _RAX_REGNO);
1331 }
1332
1333 clear(_RDX_REGNO, _RDX);
1334 clear(_RAX_REGNO, _RAX);
1335}
1336
1337static void
1338_iqmuli(jit_state_t *_jit, jit_int32_t r0, jit_int32_t r1,
1339 jit_int32_t r2, jit_word_t i0, jit_bool_t sign)
1340{
1341 jit_int32_t reg;
1342
1343 if (i0 == 0) {
1344 ixorr(r0, r0);
1345 ixorr(r1, r1);
1346 }
1347 else {
1348 reg = jit_get_reg(jit_class_gpr);
1349 movi(rn(reg), i0);
1350 if (sign)
1351 qmulr(r0, r1, r2, rn(reg));
1352 else
1353 qmulr_u(r0, r1, r2, rn(reg));
1354 jit_unget_reg(reg);
1355 }
1356}
1357
1358static void
1359_sign_extend_rdx_rax(jit_state_t *_jit)
1360{
1361 rex(0, WIDE, 0, 0, 0);
1362 ic(0x99);
1363}
1364
1365static void
1366_divremr(jit_state_t *_jit, jit_int32_t r0, jit_int32_t r1, jit_int32_t r2,
1367 jit_bool_t sign, jit_bool_t divide)
1368{
1369 jit_int32_t div;
1370 jit_int32_t reg;
1371 jit_int32_t set;
1372 jit_int32_t sav;
1373 jit_int32_t use;
1374
1375 sav = set = use = 0;
1376 savset(_RDX_REGNO);
1377 savset(_RAX_REGNO);
1378 allocr(_RDX_REGNO, _RDX);
1379 allocr(_RAX_REGNO, _RAX);
1380
1381 if (r2 == _RAX_REGNO) {
1382 if (r0 == _RAX_REGNO || r0 == _RDX_REGNO) {
1383 if ((reg = jit_get_reg(jit_class_gpr|jit_class_chk)) == JIT_NOREG)
1384 reg = jit_get_reg((r1 == _RCX_REGNO ? _RBX : _RCX) |
1385 jit_class_gpr|jit_class_named);
1386 use = 1;
1387 div = rn(reg);
1388 movr(div, _RAX_REGNO);
1389 if (r1 != _RAX_REGNO)
1390 movr(_RAX_REGNO, r1);
1391 }
1392 else {
1393 if (r0 == r1)
1394 xchgr(r0, _RAX_REGNO);
1395 else {
1396 if (r0 != _RAX_REGNO)
1397 movr(r0, _RAX_REGNO);
1398 if (r1 != _RAX_REGNO)
1399 movr(_RAX_REGNO, r1);
1400 }
1401 div = r0;
1402 }
1403 }
1404 else if (r2 == _RDX_REGNO) {
1405 if (r0 == _RAX_REGNO || r0 == _RDX_REGNO) {
1406 if ((reg = jit_get_reg(jit_class_gpr|jit_class_chk)) == JIT_NOREG)
1407 reg = jit_get_reg((r1 == _RCX_REGNO ? _RBX : _RCX) |
1408 jit_class_gpr|jit_class_named);
1409 use = 1;
1410 div = rn(reg);
1411 movr(div, _RDX_REGNO);
1412 if (r1 != _RAX_REGNO)
1413 movr(_RAX_REGNO, r1);
1414 }
1415 else {
1416 if (r1 != _RAX_REGNO)
1417 movr(_RAX_REGNO, r1);
1418 movr(r0, _RDX_REGNO);
1419 div = r0;
1420 }
1421 }
1422 else {
1423 if (r1 != _RAX_REGNO)
1424 movr(_RAX_REGNO, r1);
1425 div = r2;
1426 }
1427
1428 if (sign) {
1429 sign_extend_rdx_rax();
1430 idivr(div);
1431 }
1432 else {
1433 ixorr(_RDX_REGNO, _RDX_REGNO);
1434 idivr_u(div);
1435 }
1436
1437 if (use)
1438 jit_unget_reg(reg);
1439
1440 if (divide)
1441 movr(r0, _RAX_REGNO);
1442 else
1443 movr(r0, _RDX_REGNO);
1444
1445 clear(_RDX_REGNO, _RDX);
1446 clear(_RAX_REGNO, _RAX);
1447}
1448
1449static void
1450_divremi(jit_state_t *_jit, jit_int32_t r0, jit_int32_t r1, jit_word_t i0,
1451 jit_bool_t sign, jit_bool_t divide)
1452{
1453 jit_int32_t reg;
1454 jit_int32_t div;
1455 jit_int32_t sav;
1456 jit_int32_t set;
1457 jit_int32_t use;
1458
1459 if (divide) {
1460 switch (i0) {
1461 case 1:
1462 movr(r0, r1);
1463 return;
1464 case -1:
1465 if (sign) {
1466 negr(r0, r1);
1467 return;
1468 }
1469 break;
1470 default:
1471 if (i0 > 0 && !(i0 & (i0 - 1))) {
1472 movr(r0, r1);
1473 if (sign)
1474 rshi(r0, r0, ffsl(i0) - 1);
1475 else
1476 rshi_u(r0, r0, ffsl(i0) - 1);
1477 return;
1478 }
1479 break;
1480 }
1481 }
1482 else if (i0 == 1 || (sign && i0 == -1)) {
1483 ixorr(r0, r0);
1484 return;
1485 }
1486 else if (!sign && i0 > 0 && !(i0 & (i0 - 1))) {
1487 if (can_sign_extend_int_p(i0)) {
1488 movr(r0, r1);
1489 iandi(r0, i0 - 1);
1490 }
1491 else if (r0 != r1) {
1492 movi(r0, i0 - 1);
1493 iandr(r0, r1);
1494 }
1495 else {
1496 reg = jit_get_reg(jit_class_gpr);
1497 movi(rn(reg), i0 - 1);
1498 iandr(r0, rn(reg));
1499 jit_unget_reg(reg);
1500 }
1501 return;
1502 }
1503
1504 sav = set = use = 0;
1505 isavset(_RDX_REGNO);
1506 isavset(_RAX_REGNO);
1507 allocr(_RDX_REGNO, _RDX);
1508 allocr(_RAX_REGNO, _RAX);
1509
1510 if (r0 == _RAX_REGNO || r0 == _RDX_REGNO || r0 == r1) {
1511 if ((reg = jit_get_reg(jit_class_gpr|jit_class_chk)) == JIT_NOREG)
1512 reg = jit_get_reg((r1 == _RCX_REGNO ? _RBX : _RCX) |
1513 jit_class_gpr|jit_class_named);
1514 use = 1;
1515 div = rn(reg);
1516 }
1517 else
1518 div = r0;
1519
1520 movi(div, i0);
1521 movr(_RAX_REGNO, r1);
1522
1523 if (sign) {
1524 sign_extend_rdx_rax();
1525 idivr(div);
1526 }
1527 else {
1528 ixorr(_RDX_REGNO, _RDX_REGNO);
1529 idivr_u(div);
1530 }
1531
1532 if (use)
1533 jit_unget_reg(reg);
1534
1535 if (divide)
1536 movr(r0, _RAX_REGNO);
1537 else
1538 movr(r0, _RDX_REGNO);
1539
1540 clear(_RDX_REGNO, _RDX);
1541 clear(_RAX_REGNO, _RAX);
1542}
1543
1544static void
1545_iqdivr(jit_state_t *_jit, jit_int32_t r0, jit_int32_t r1,
1546 jit_int32_t r2, jit_int32_t r3, jit_bool_t sign)
1547{
1548 jit_int32_t div;
1549 jit_int32_t reg;
1550 jit_int32_t sav;
1551 jit_int32_t set;
1552 jit_int32_t use;
1553
1554 sav = set = use = 0;
1555 qsavset(_RDX_REGNO);
1556 qsavset(_RAX_REGNO);
1557 allocr(_RDX_REGNO, _RDX);
1558 allocr(_RAX_REGNO, _RAX);
1559 if (r3 == _RAX_REGNO) {
1560 if (r0 == _RAX_REGNO || r0 == _RDX_REGNO) {
1561 if ((reg = jit_get_reg(jit_class_gpr|jit_class_chk)) == JIT_NOREG)
1562 reg = jit_get_reg((r1 == _RCX_REGNO ? _RBX : _RCX) |
1563 jit_class_gpr|jit_class_named);
1564 use = 1;
1565 div = rn(reg);
1566 movr(div, _RAX_REGNO);
1567 if (r2 != _RAX_REGNO)
1568 movr(_RAX_REGNO, r2);
1569 }
1570 else {
1571 if (r0 == r2)
1572 xchgr(r0, _RAX_REGNO);
1573 else {
1574 if (r0 != _RAX_REGNO)
1575 movr(r0, _RAX_REGNO);
1576 if (r2 != _RAX_REGNO)
1577 movr(_RAX_REGNO, r2);
1578 }
1579 div = r0;
1580 }
1581 }
1582 else if (r3 == _RDX_REGNO) {
1583 if (r0 == _RAX_REGNO || r0 == _RDX_REGNO) {
1584 if ((reg = jit_get_reg(jit_class_gpr|jit_class_chk)) == JIT_NOREG)
1585 reg = jit_get_reg((r1 == _RCX_REGNO ? _RBX : _RCX) |
1586 jit_class_gpr|jit_class_named);
1587 use = 1;
1588 div = rn(reg);
1589 movr(div, _RDX_REGNO);
1590 if (r2 != _RAX_REGNO)
1591 movr(_RAX_REGNO, r2);
1592 }
1593 else {
1594 if (r2 != _RAX_REGNO)
1595 movr(_RAX_REGNO, r2);
1596 movr(r0, _RDX_REGNO);
1597 div = r0;
1598 }
1599 }
1600 else {
1601 if (r2 != _RAX_REGNO)
1602 movr(_RAX_REGNO, r2);
1603 div = r3;
1604 }
1605 if (sign) {
1606 sign_extend_rdx_rax();
1607 idivr(div);
1608 }
1609 else {
1610 ixorr(_RDX_REGNO, _RDX_REGNO);
1611 idivr_u(div);
1612 }
1613 if (use)
1614 jit_unget_reg(reg);
1615
1616 if (r0 == _RDX_REGNO && r1 == _RAX_REGNO)
1617 xchgr(_RAX_REGNO, _RDX_REGNO);
1618 else {
1619 if (r0 != _RDX_REGNO)
1620 movr(r0, _RAX_REGNO);
1621 movr(r1, _RDX_REGNO);
1622 if (r0 == _RDX_REGNO)
1623 movr(r0, _RAX_REGNO);
1624 }
1625
1626 clear(_RDX_REGNO, _RDX);
1627 clear(_RAX_REGNO, _RAX);
1628}
1629
1630static void
1631_iqdivi(jit_state_t *_jit, jit_int32_t r0, jit_int32_t r1,
1632 jit_int32_t r2, jit_word_t i0, jit_bool_t sign)
1633{
1634 jit_int32_t reg;
1635
1636 reg = jit_get_reg(jit_class_gpr);
1637 movi(rn(reg), i0);
1638 if (sign)
1639 qdivr(r0, r1, r2, rn(reg));
1640 else
1641 qdivr_u(r0, r1, r2, rn(reg));
1642 jit_unget_reg(reg);
1643}
1644#undef clear
1645#undef allocr
1646#undef savset
1647
1648static void
1649_andr(jit_state_t *_jit, jit_int32_t r0, jit_int32_t r1, jit_int32_t r2)
1650{
1651 if (r1 == r2)
1652 movr(r0, r1);
1653 else if (r0 == r1)
1654 iandr(r0, r2);
1655 else if (r0 == r2)
1656 iandr(r0, r1);
1657 else {
1658 movr(r0, r1);
1659 iandr(r0, r2);
1660 }
1661}
1662
1663static void
1664_andi(jit_state_t *_jit, jit_int32_t r0, jit_int32_t r1, jit_word_t i0)
1665{
1666 jit_int32_t reg;
1667
1668 if (i0 == 0)
1669 ixorr(r0, r0);
1670 else if (i0 == -1)
1671 movr(r0, r1);
1672 else if (r0 == r1) {
1673 if (can_sign_extend_int_p(i0))
1674 iandi(r0, i0);
1675 else {
1676 reg = jit_get_reg(jit_class_gpr);
1677 movi(rn(reg), i0);
1678 iandr(r0, rn(reg));
1679 jit_unget_reg(reg);
1680 }
1681 }
1682 else {
1683 movi(r0, i0);
1684 iandr(r0, r1);
1685 }
1686}
1687
1688static void
1689_orr(jit_state_t *_jit, jit_int32_t r0, jit_int32_t r1, jit_int32_t r2)
1690{
1691 if (r1 == r2)
1692 movr(r0, r1);
1693 else if (r0 == r1)
1694 iorr(r0, r2);
1695 else if (r0 == r2)
1696 iorr(r0, r1);
1697 else {
1698 movr(r0, r1);
1699 iorr(r0, r2);
1700 }
1701}
1702
1703static void
1704_ori(jit_state_t *_jit, jit_int32_t r0, jit_int32_t r1, jit_word_t i0)
1705{
1706 jit_int32_t reg;
1707 if (i0 == 0)
1708 movr(r0, r1);
1709 else if (i0 == -1)
1710 movi(r0, -1);
1711 else if (can_sign_extend_int_p(i0)) {
1712 movr(r0, r1);
1713 iori(r0, i0);
1714 }
1715 else if (r0 != r1) {
1716 movi(r0, i0);
1717 iorr(r0, r1);
1718 }
1719 else {
1720 reg = jit_get_reg(jit_class_gpr);
1721 movi(rn(reg), i0);
1722 iorr(r0, rn(reg));
1723 jit_unget_reg(reg);
1724 }
1725}
1726
1727static void
1728_xorr(jit_state_t *_jit, jit_int32_t r0, jit_int32_t r1, jit_int32_t r2)
1729{
1730 if (r1 == r2)
1731 ixorr(r0, r0);
1732 else if (r0 == r1)
1733 ixorr(r0, r2);
1734 else if (r0 == r2)
1735 ixorr(r0, r1);
1736 else {
1737 movr(r0, r1);
1738 ixorr(r0, r2);
1739 }
1740}
1741
1742static void
1743_xori(jit_state_t *_jit, jit_int32_t r0, jit_int32_t r1, jit_word_t i0)
1744{
1745 jit_int32_t reg;
1746 if (i0 == 0)
1747 movr(r0, r1);
1748 else if (i0 == -1)
1749 comr(r0, r1);
1750 else if (can_sign_extend_int_p(i0)) {
1751 movr(r0, r1);
1752 ixori(r0, i0);
1753 }
1754 else if (r0 != r1) {
1755 movi(r0, i0);
1756 ixorr(r0, r1);
1757 }
1758 else {
1759 reg = jit_get_reg(jit_class_gpr);
1760 movi(rn(reg), i0);
1761 ixorr(r0, rn(reg));
1762 jit_unget_reg(reg);
1763 }
1764}
1765
1766static void
1767_irotshr(jit_state_t *_jit, jit_int32_t code, jit_int32_t r0)
1768{
1769 rex(0, WIDE, _RCX_REGNO, _NOREG, r0);
1770 ic(0xd3);
1771 mrm(0x03, code, r7(r0));
1772}
1773
1774static void
1775_rotshr(jit_state_t *_jit, jit_int32_t code,
1776 jit_int32_t r0, jit_int32_t r1, jit_int32_t r2)
1777{
1778 jit_int32_t reg;
1779 jit_int32_t use;
1780
1781 if (r0 == _RCX_REGNO) {
1782 reg = jit_get_reg(jit_class_gpr);
1783 movr(rn(reg), r1);
1784 if (r2 != _RCX_REGNO)
1785 movr(_RCX_REGNO, r2);
1786 irotshr(code, rn(reg));
1787 movr(_RCX_REGNO, rn(reg));
1788 jit_unget_reg(reg);
1789 }
1790 else if (r2 != _RCX_REGNO) {
1791 use = !jit_reg_free_p(_RCX);
1792 if (use) {
1793 reg = jit_get_reg(jit_class_gpr);
1794 movr(rn(reg), _RCX_REGNO);
1795 }
1796 else
1797 reg = 0;
1798 if (r1 == _RCX_REGNO) {
1799 if (r0 == r2)
1800 xchgr(r0, _RCX_REGNO);
1801 else {
1802 movr(r0, r1);
1803 movr(_RCX_REGNO, r2);
1804 }
1805 }
1806 else {
1807 movr(_RCX_REGNO, r2);
1808 movr(r0, r1);
1809 }
1810 irotshr(code, r0);
1811 if (use) {
1812 movr(_RCX_REGNO, rn(reg));
1813 jit_unget_reg(reg);
1814 }
1815 }
1816 else {
1817 movr(r0, r1);
1818 irotshr(code, r0);
1819 }
1820}
1821
1822static void
1823_irotshi(jit_state_t *_jit, jit_int32_t code, jit_int32_t r0, jit_word_t i0)
1824{
1825 rex(0, WIDE, _NOREG, _NOREG, r0);
1826 if (i0 == 1) {
1827 ic(0xd1);
1828 mrm(0x03, code, r7(r0));
1829 }
1830 else {
1831 ic(0xc1);
1832 mrm(0x03, code, r7(r0));
1833 ic(i0);
1834 }
1835}
1836
1837static void
1838_rotshi(jit_state_t *_jit, jit_int32_t code,
1839 jit_int32_t r0, jit_int32_t r1, jit_word_t i0)
1840{
1841 movr(r0, r1);
1842 if (i0)
1843 irotshi(code, r0, i0);
1844}
1845
1846static void
1847_lshi(jit_state_t *_jit, jit_int32_t r0, jit_int32_t r1, jit_word_t i0)
1848{
1849 if (i0 == 0)
1850 movr(r0, r1);
1851 else if (i0 <= 3)
1852 lea(0, _NOREG, r1, i0 == 1 ? _SCL2 : i0 == 2 ? _SCL4 : _SCL8, r0);
1853 else
1854 rotshi(X86_SHL, r0, r1, i0);
1855}
1856
1857static void
1858_unr(jit_state_t *_jit, jit_int32_t code, jit_int32_t r0)
1859{
1860 rex(0, WIDE, _NOREG, _NOREG, r0);
1861 ic(0xf7);
1862 mrm(0x03, code, r7(r0));
1863}
1864
1865static void
1866_negr(jit_state_t *_jit, jit_int32_t r0, jit_int32_t r1)
1867{
1868 if (r0 == r1)
1869 inegr(r0);
1870 else {
1871 ixorr(r0, r0);
1872 isubr(r0, r1);
1873 }
1874}
1875
1876static void
1877_comr(jit_state_t *_jit, jit_int32_t r0, jit_int32_t r1)
1878{
1879 movr(r0, r1);
1880 icomr(r0);
1881}
1882
1883#if USE_INC_DEC
1884static void
1885_incr(jit_state_t *_jit, jit_int32_t r0, jit_int32_t r1)
1886{
1887 movr(r0, r1);
1888# if __X64
1889 rex(0, WIDE, _NOREG, _NOREG, r0);
1890 ic(0xff);
1891 ic(0xc0 | r7(r0));
1892# else
1893 ic(0x40 | r7(r0));
1894# endif
1895}
1896
1897static void
1898_decr(jit_state_t *_jit, jit_int32_t r0, jit_int32_t r1)
1899{
1900 movr(r0, r1);
1901# if __X64
1902 rex(0, WIDE, _NOREG, _NOREG, r0);
1903 ic(0xff);
1904 ic(0xc8 | r7(r0));
1905# else
1906 ic(0x48 | r7(r0));
1907# endif
1908}
1909#endif
1910
1911static void
1912_cr(jit_state_t *_jit,
1913 jit_int32_t code, jit_int32_t r0, jit_int32_t r1, jit_int32_t r2)
1914{
1915 jit_int32_t reg;
1916 jit_bool_t same;
1917 if (reg8_p(r0)) {
1918 same = r0 == r1 || r0 == r2;
1919 if (!same)
1920 ixorr(r0, r0);
1921 icmpr(r1, r2);
1922 if (same)
1923 imovi(r0, 0);
1924 cc(code, r0);
1925 }
1926 else {
1927 reg = jit_get_reg(jit_class_gpr|jit_class_rg8);
1928 ixorr(rn(reg), rn(reg));
1929 icmpr(r1, r2);
1930 cc(code, rn(reg));
1931 movr(r0, rn(reg));
1932 jit_unget_reg(reg);
1933 }
1934}
1935
1936static void
1937_ci(jit_state_t *_jit,
1938 jit_int32_t code, jit_int32_t r0, jit_int32_t r1, jit_word_t i0)
1939{
1940 jit_int32_t reg;
1941 jit_bool_t same;
1942 if (reg8_p(r0)) {
1943 same = r0 == r1;
1944 if (!same)
1945 ixorr(r0, r0);
1946 icmpi(r1, i0);
1947 if (same)
1948 imovi(r0, 0);
1949 cc(code, r0);
1950 }
1951 else {
1952 reg = jit_get_reg(jit_class_gpr|jit_class_rg8);
1953 ixorr(rn(reg), rn(reg));
1954 icmpi(r1, i0);
1955 cc(code, rn(reg));
1956 movr(r0, rn(reg));
1957 jit_unget_reg(reg);
1958 }
1959}
1960
1961static void
1962_ci0(jit_state_t *_jit, jit_int32_t code, jit_int32_t r0, jit_int32_t r1)
1963{
1964 jit_int32_t reg;
1965 jit_bool_t same;
1966 if (reg8_p(r0)) {
1967 same = r0 == r1;
1968 if (!same)
1969 ixorr(r0, r0);
1970 testr(r1, r1);
1971 if (same)
1972 imovi(r0, 0);
1973 cc(code, r0);
1974 }
1975 else {
1976 reg = jit_get_reg(jit_class_gpr|jit_class_rg8);
1977 ixorr(rn(reg), rn(reg));
1978 testr(r1, r1);
1979 cc(code, rn(reg));
1980 movr(r0, rn(reg));
1981 jit_unget_reg(reg);
1982 }
1983}
1984
1985static void
1986_ltr(jit_state_t *_jit, jit_int32_t r0, jit_int32_t r1, jit_int32_t r2)
1987{
1988 if (r1 == r2)
1989 movi(r0, 0);
1990 else
1991 cr(X86_CC_L, r0, r1, r2);
1992}
1993
1994static void
1995_lti(jit_state_t *_jit, jit_int32_t r0, jit_int32_t r1, jit_word_t i0)
1996{
1997 if (i0)
1998 ci(X86_CC_L, r0, r1, i0);
1999 else
2000 ci0(X86_CC_S, r0, r1);
2001}
2002
2003static void
2004_ltr_u(jit_state_t *_jit, jit_int32_t r0, jit_int32_t r1, jit_int32_t r2)
2005{
2006 if (r1 == r2)
2007 movi(r0, 0);
2008 else
2009 cr(X86_CC_B, r0, r1, r2);
2010}
2011
2012static void
2013_ler(jit_state_t *_jit, jit_int32_t r0, jit_int32_t r1, jit_int32_t r2)
2014{
2015 if (r1 == r2)
2016 movi(r0, 1);
2017 else
2018 cr(X86_CC_LE, r0, r1, r2);
2019}
2020
2021static void
2022_ler_u(jit_state_t *_jit, jit_int32_t r0, jit_int32_t r1, jit_int32_t r2)
2023{
2024 if (r1 == r2)
2025 movi(r0, 1);
2026 else
2027 cr(X86_CC_BE, r0, r1, r2);
2028}
2029
2030static void
2031_lei_u(jit_state_t *_jit, jit_int32_t r0, jit_int32_t r1, jit_word_t i0)
2032{
2033 if (i0)
2034 ci(X86_CC_BE, r0, r1, i0);
2035 else
2036 ci0(X86_CC_E, r0, r1);
2037}
2038
2039static void
2040_eqr(jit_state_t *_jit, jit_int32_t r0, jit_int32_t r1, jit_int32_t r2)
2041{
2042 if (r1 == r2)
2043 movi(r0, 1);
2044 else
2045 cr(X86_CC_E, r0, r1, r2);
2046}
2047
2048static void
2049_eqi(jit_state_t *_jit, jit_int32_t r0, jit_int32_t r1, jit_word_t i0)
2050{
2051 if (i0)
2052 ci(X86_CC_E, r0, r1, i0);
2053 else
2054 ci0(X86_CC_E, r0, r1);
2055}
2056
2057static void
2058_ger(jit_state_t *_jit, jit_int32_t r0, jit_int32_t r1, jit_int32_t r2)
2059{
2060 if (r1 == r2)
2061 movi(r0, 1);
2062 else
2063 cr(X86_CC_GE, r0, r1, r2);
2064}
2065
2066static void
2067_gei(jit_state_t *_jit, jit_int32_t r0, jit_int32_t r1, jit_word_t i0)
2068{
2069 if (i0)
2070 ci(X86_CC_GE, r0, r1, i0);
2071 else
2072 ci0(X86_CC_NS, r0, r1);
2073}
2074
2075static void
2076_ger_u(jit_state_t *_jit, jit_int32_t r0, jit_int32_t r1, jit_int32_t r2)
2077{
2078 if (r1 == r2)
2079 movi(r0, 1);
2080 else
2081 cr(X86_CC_AE, r0, r1, r2);
2082}
2083
2084static void
2085_gei_u(jit_state_t *_jit, jit_int32_t r0, jit_int32_t r1, jit_word_t i0)
2086{
2087 if (i0)
2088 ci(X86_CC_AE, r0, r1, i0);
2089 else
2090 ci0(X86_CC_NB, r0, r1);
2091}
2092
2093static void
2094_gtr(jit_state_t *_jit, jit_int32_t r0, jit_int32_t r1, jit_int32_t r2)
2095{
2096 if (r1 == r2)
2097 movi(r0, 0);
2098 else
2099 cr(X86_CC_G, r0, r1, r2);
2100}
2101
2102static void
2103_gtr_u(jit_state_t *_jit, jit_int32_t r0, jit_int32_t r1, jit_int32_t r2)
2104{
2105 if (r1 == r2)
2106 movi(r0, 0);
2107 else
2108 cr(X86_CC_A, r0, r1, r2);
2109}
2110
2111static void
2112_gti_u(jit_state_t *_jit, jit_int32_t r0, jit_int32_t r1, jit_word_t i0)
2113{
2114 if (i0)
2115 ci(X86_CC_A, r0, r1, i0);
2116 else
2117 ci0(X86_CC_NE, r0, r1);
2118}
2119
2120static void
2121_ner(jit_state_t *_jit, jit_int32_t r0, jit_int32_t r1, jit_int32_t r2)
2122{
2123 if (r1 == r2)
2124 movi(r0, 0);
2125 else
2126 cr(X86_CC_NE, r0, r1, r2);
2127}
2128
2129static void
2130_nei(jit_state_t *_jit, jit_int32_t r0, jit_int32_t r1, jit_word_t i0)
2131{
2132 if (i0)
2133 ci(X86_CC_NE, r0, r1, i0);
2134 else
2135 ci0(X86_CC_NE, r0, r1);
2136}
2137
2138static void
2139_movr(jit_state_t *_jit, jit_int32_t r0, jit_int32_t r1)
2140{
2141 if (r0 != r1) {
2142 rex(0, 1, r1, _NOREG, r0);
2143 ic(0x89);
2144 ic(0xc0 | (r1 << 3) | r7(r0));
2145 }
2146}
2147
2148static void
2149_imovi(jit_state_t *_jit, jit_int32_t r0, jit_word_t i0)
2150{
2151#if __X64
2152# if !__X64_32
2153 if (fits_uint32_p(i0)) {
2154# endif
2155 rex(0, 0, _NOREG, _NOREG, r0);
2156 ic(0xb8 | r7(r0));
2157 ii(i0);
2158# if !__X64_32
2159 }
2160 else {
2161 rex(0, 1, _NOREG, _NOREG, r0);
2162 ic(0xb8 | r7(r0));
2163 il(i0);
2164 }
2165# endif
2166#else
2167 ic(0xb8 | r7(r0));
2168 ii(i0);
2169#endif
2170}
2171
2172static void
2173_movi(jit_state_t *_jit, jit_int32_t r0, jit_word_t i0)
2174{
2175 if (i0)
2176 imovi(r0, i0);
2177 else
2178 ixorr(r0, r0);
2179}
2180
2181static jit_word_t
2182_movi_p(jit_state_t *_jit, jit_int32_t r0, jit_word_t i0)
2183{
2184 rex(0, WIDE, _NOREG, _NOREG, r0);
2185 ic(0xb8 | r7(r0));
2186 il(i0);
2187 return (_jit->pc.w);
2188}
2189
2190static void
2191_movcr(jit_state_t *_jit, jit_int32_t r0, jit_int32_t r1)
2192{
2193 rex(0, WIDE, r0, _NOREG, r1);
2194 ic(0x0f);
2195 ic(0xbe);
2196 mrm(0x03, r7(r0), r7(r1));
2197}
2198
2199static void
2200_movcr_u(jit_state_t *_jit, jit_int32_t r0, jit_int32_t r1)
2201{
2202 rex(0, WIDE, r0, _NOREG, r1);
2203 ic(0x0f);
2204 ic(0xb6);
2205 mrm(0x03, r7(r0), r7(r1));
2206}
2207
2208static void
2209_movsr(jit_state_t *_jit, jit_int32_t r0, jit_int32_t r1)
2210{
2211 rex(0, WIDE, r0, _NOREG, r1);
2212 ic(0x0f);
2213 ic(0xbf);
2214 mrm(0x03, r7(r0), r7(r1));
2215}
2216
2217static void
2218_movsr_u(jit_state_t *_jit, jit_int32_t r0, jit_int32_t r1)
2219{
2220 rex(0, WIDE, r0, _NOREG, r1);
2221 ic(0x0f);
2222 ic(0xb7);
2223 mrm(0x03, r7(r0), r7(r1));
2224}
2225
ba3814c1
PC		 2226static void
2227_casx(jit_state_t *_jit, jit_int32_t r0, jit_int32_t r1,
2228 jit_int32_t r2, jit_int32_t r3, jit_word_t i0)
2229{
2230 jit_int32_t save_rax, restore_rax;
2231 jit_int32_t ascasr_reg, ascasr_use;
2232 if (r0 != _RAX_REGNO) { /* result not in %rax */
2233 if (r2 != _RAX_REGNO) { /* old value not in %rax */
2234 save_rax = jit_get_reg(jit_class_gpr);
2235 movr(rn(save_rax), _RAX_REGNO);
2236 restore_rax = 1;
2237 }
2238 else
2239 restore_rax = 0;
2240 }
2241 else
2242 restore_rax = 0;
2243 if (r2 != _RAX_REGNO)
2244 movr(_RAX_REGNO, r2);
2245 if (r1 == _NOREG) { /* using immediate address */
2246 if (!can_sign_extend_int_p(i0)) {
2247 ascasr_reg = jit_get_reg(jit_class_gpr);
2248 if (ascasr_reg == _RAX) {
2249 ascasr_reg = jit_get_reg(jit_class_gpr);
2250 jit_unget_reg(_RAX);
2251 }
2252 ascasr_use = 1;
2253 movi(rn(ascasr_reg), i0);
2254 }
2255 else
2256 ascasr_use = 0;
2257 }
2258 else
2259 ascasr_use = 0;
2260 ic(0xf0); /* lock */
2261 if (ascasr_use)
2262 rex(0, WIDE, r3, _NOREG, rn(ascasr_reg));
2263 else
2264 rex(0, WIDE, r3, _NOREG, r1);
2265 ic(0x0f);
2266 ic(0xb1);
2267 if (r1 != _NOREG) /* casr */
2268 rx(r3, 0, r1, _NOREG, _SCL1);
2269 else { /* casi */
2270 if (ascasr_use)
2271 rx(r3, 0, rn(ascasr_reg), _NOREG, _SCL1); /* address in reg */
2272 else
2273 rx(r3, i0, _NOREG, _NOREG, _SCL1); /* address in offset */
2274 }
2275 cc(X86_CC_E, r0);
2276 if (r0 != _RAX_REGNO)
2277 movr(r0, _RAX_REGNO);
2278 if (restore_rax) {
2279 movr(_RAX_REGNO, rn(save_rax));
2280 jit_unget_reg(save_rax);
2281 }
2282 if (ascasr_use)
2283 jit_unget_reg(ascasr_reg);
2284}
2285
1f22b268
PC		 2286static void
2287_movnr(jit_state_t *_jit, jit_int32_t r0, jit_int32_t r1, jit_int32_t r2)
2288{
2289 assert(jit_cmov_p());
2290
2291 testr(r2, r2);
2292
2293 rex(0, WIDE, r0, _NOREG, r1);
2294 ic(0x0f);
2295 ic(0x45);
2296 mrm(0x03, r7(r0), r7(r1));
2297}
2298
2299static void
2300_movzr(jit_state_t *_jit, jit_int32_t r0, jit_int32_t r1, jit_int32_t r2)
2301{
2302 assert(jit_cmov_p());
2303
2304 testr(r2, r2);
2305
2306 rex(0, WIDE, r0, _NOREG, r1);
2307 ic(0x0f);
2308 ic(0x44);
2309 mrm(0x03, r7(r0), r7(r1));
2310}
2311
4a71579b
PC		 2312#if __X64
2313static void
2314_movir(jit_state_t *_jit, jit_int32_t r0, jit_int32_t r1)
2315{
2316 rex(0, 1, r0, _NOREG, r1);
2317 ic(0x63);
2318 mrm(0x03, r7(r0), r7(r1));
2319}
2320
2321static void
2322_movir_u(jit_state_t *_jit, jit_int32_t r0, jit_int32_t r1)
2323{
2324 rex(0, 0, r1, _NOREG, r0);
2325 ic(0x89);
2326 ic(0xc0 | (r1 << 3) | r7(r0));
2327}
2328#endif
2329
2330static void
40a44dcb 2331_bswapr_us(jit_state_t *_jit, jit_int32_t r0, jit_int32_t r1)
4a71579b
PC		 2332{
2333 extr_us(r0, r1);
2334 ic(0x66);
2335 rex(0, 0, _NOREG, _NOREG, r0);
2336 ic(0xc1);
2337 mrm(0x03, X86_ROR, r7(r0));
2338 ic(8);
2339}
2340
2341static void
40a44dcb 2342_bswapr_ui(jit_state_t *_jit, jit_int32_t r0, jit_int32_t r1)
4a71579b
PC		 2343{
2344 movr(r0, r1);
2345 rex(0, 0, _NOREG, _NOREG, r0);
2346 ic(0x0f);
2347 ic(0xc8 | r7(r0));
2348}
2349
2350#if __X64 && !__X64_32
2351static void
40a44dcb 2352_bswapr_ul(jit_state_t *_jit, jit_int32_t r0, jit_int32_t r1)
4a71579b
PC		 2353{
2354 movr(r0, r1);
2355 rex(0, 1, _NOREG, _NOREG, r0);
2356 ic(0x0f);
2357 ic(0xc8 | r7(r0));
2358}
2359#endif
2360
2361static void
2362_extr_c(jit_state_t *_jit, jit_int32_t r0, jit_int32_t r1)
2363{
2364 jit_int32_t reg;
2365 if (reg8_p(r1))
2366 movcr(r0, r1);
2367 else {
2368 reg = jit_get_reg(jit_class_gpr|jit_class_rg8);
2369 movr(rn(reg), r1);
2370 movcr(r0, rn(reg));
2371 jit_unget_reg(reg);
2372 }
2373}
2374
2375static void
2376_extr_uc(jit_state_t *_jit, jit_int32_t r0, jit_int32_t r1)
2377{
2378 jit_int32_t reg;
2379 if (reg8_p(r1))
2380 movcr_u(r0, r1);
2381 else {
2382 reg = jit_get_reg(jit_class_gpr|jit_class_rg8);
2383 movr(rn(reg), r1);
2384 movcr_u(r0, rn(reg));
2385 jit_unget_reg(reg);
2386 }
2387}
2388
2389static void
2390_ldr_c(jit_state_t *_jit, jit_int32_t r0, jit_int32_t r1)
2391{
2392 rex(0, WIDE, r0, _NOREG, r1);
2393 ic(0x0f);
2394 ic(0xbe);
2395 rx(r0, 0, r1, _NOREG, _SCL1);
2396}
2397
2398static void
2399_ldi_c(jit_state_t *_jit, jit_int32_t r0, jit_word_t i0)
2400{
2401 jit_int32_t reg;
2402 if (can_sign_extend_int_p(i0)) {
2403 rex(0, WIDE, r0, _NOREG, _NOREG);
2404 ic(0x0f);
2405 ic(0xbe);
2406 rx(r0, i0, _NOREG, _NOREG, _SCL1);
2407 }
2408 else {
2409 reg = jit_get_reg(jit_class_gpr);
2410 movi(rn(reg), i0);
2411 ldr_c(r0, rn(reg));
2412 jit_unget_reg(reg);
2413 }
2414}
2415
2416static void
2417_ldr_uc(jit_state_t *_jit, jit_int32_t r0, jit_int32_t r1)
2418{
2419 rex(0, WIDE, r0, _NOREG, r1);
2420 ic(0x0f);
2421 ic(0xb6);
2422 rx(r0, 0, r1, _NOREG, _SCL1);
2423}
2424
2425static void
2426_ldi_uc(jit_state_t *_jit, jit_int32_t r0, jit_word_t i0)
2427{
2428 jit_int32_t reg;
2429 if (can_sign_extend_int_p(i0)) {
2430 rex(0, WIDE, r0, _NOREG, _NOREG);
2431 ic(0x0f);
2432 ic(0xb6);
2433 rx(r0, i0, _NOREG, _NOREG, _SCL1);
2434 }
2435 else {
2436 reg = jit_get_reg(jit_class_gpr);
2437 movi(rn(reg), i0);
2438 ldr_uc(r0, rn(reg));
2439 jit_unget_reg(reg);
2440 }
2441}
2442
2443static void
2444_ldr_s(jit_state_t *_jit, jit_int32_t r0, jit_int32_t r1)
2445{
2446 rex(0, WIDE, r0, _NOREG, r1);
2447 ic(0x0f);
2448 ic(0xbf);
2449 rx(r0, 0, r1, _NOREG, _SCL1);
2450}
2451
2452static void
2453_ldi_s(jit_state_t *_jit, jit_int32_t r0, jit_word_t i0)
2454{
2455 jit_int32_t reg;
2456 if (can_sign_extend_int_p(i0)) {
2457 rex(0, WIDE, r0, _NOREG, _NOREG);
2458 ic(0x0f);
2459 ic(0xbf);
2460 rx(r0, i0, _NOREG, _NOREG, _SCL1);
2461 }
2462 else {
2463 reg = jit_get_reg(jit_class_gpr);
2464 movi(rn(reg), i0);
2465 ldr_s(r0, rn(reg));
2466 jit_unget_reg(reg);
2467 }
2468}
2469
2470static void
2471_ldr_us(jit_state_t *_jit, jit_int32_t r0, jit_int32_t r1)
2472{
2473 rex(0, WIDE, r0, _NOREG, r1);
2474 ic(0x0f);
2475 ic(0xb7);
2476 rx(r0, 0, r1, _NOREG, _SCL1);
2477}
2478
2479static void
2480_ldi_us(jit_state_t *_jit, jit_int32_t r0, jit_word_t i0)
2481{
2482 jit_int32_t reg;
2483 if (can_sign_extend_int_p(i0)) {
2484 rex(0, WIDE, r0, _NOREG, _NOREG);
2485 ic(0x0f);
2486 ic(0xb7);
2487 rx(r0, i0, _NOREG, _NOREG, _SCL1);
2488 }
2489 else {
2490 reg = jit_get_reg(jit_class_gpr);
2491 movi(rn(reg), i0);
2492 ldr_us(r0, rn(reg));
2493 jit_unget_reg(reg);
2494 }
2495}
2496
2497#if __X32 || !__X64_32
2498static void
2499_ldr_i(jit_state_t *_jit, jit_int32_t r0, jit_int32_t r1)
2500{
2501#if __X64
2502 rex(0, WIDE, r0, _NOREG, r1);
2503 ic(0x63);
2504#else
2505 ic(0x8b);
2506#endif
2507 rx(r0, 0, r1, _NOREG, _SCL1);
2508}
2509
2510static void
2511_ldi_i(jit_state_t *_jit, jit_int32_t r0, jit_word_t i0)
2512{
2513 jit_int32_t reg;
2514 if (can_sign_extend_int_p(i0)) {
2515#if __X64
2516 rex(0, WIDE, r0, _NOREG, _NOREG);
2517 ic(0x63);
2518#else
2519 ic(0x8b);
2520#endif
2521 rx(r0, i0, _NOREG, _NOREG, _SCL1);
2522 }
2523 else {
2524 reg = jit_get_reg(jit_class_gpr);
2525 movi(rn(reg), i0);
2526 ldr_i(r0, rn(reg));
2527 jit_unget_reg(reg);
2528 }
2529}
2530#endif
2531
2532#if __X64
2533static void
2534_ldr_ui(jit_state_t *_jit, jit_int32_t r0, jit_int32_t r1)
2535{
2536 rex(0, 0, r0, _NOREG, r1);
2537 ic(0x63);
2538 rx(r0, 0, r1, _NOREG, _SCL1);
2539}
2540
2541static void
2542_ldi_ui(jit_state_t *_jit, jit_int32_t r0, jit_word_t i0)
2543{
2544 jit_int32_t reg;
2545 if (can_sign_extend_int_p(i0)) {
2546 rex(0, 0, r0, _NOREG, _NOREG);
2547 ic(0x63);
2548 rx(r0, i0, _NOREG, _NOREG, _SCL1);
2549 }
2550 else {
2551 reg = jit_get_reg(jit_class_gpr);
2552 movi(rn(reg), i0);
2553 ldr_ui(r0, rn(reg));
2554 jit_unget_reg(reg);
2555 }
2556}
2557
2558# if !__X64_32
2559static void
2560_ldr_l(jit_state_t *_jit, jit_int32_t r0, jit_int32_t r1)
2561{
2562 rex(0, 1, r0, _NOREG, r1);
2563 ic(0x8b);
2564 rx(r0, 0, r1, _NOREG, _SCL1);
2565}
2566
2567static void
2568_ldi_l(jit_state_t *_jit, jit_int32_t r0, jit_word_t i0)
2569{
2570 jit_int32_t reg;
2571 if (can_sign_extend_int_p(i0)) {
2572 rex(0, 1, r0, _NOREG, _NOREG);
2573 ic(0x8b);
2574 rx(r0, i0, _NOREG, _NOREG, _SCL1);
2575 }
2576 else {
2577 reg = jit_get_reg(jit_class_gpr);
2578 movi(rn(reg), i0);
2579 ldr_l(r0, rn(reg));
2580 jit_unget_reg(reg);
2581 }
2582}
2583# endif
2584#endif
2585
2586static void
2587_ldxr_c(jit_state_t *_jit, jit_int32_t r0, jit_int32_t r1, jit_int32_t r2)
2588{
2589#if __X64_32
2590 addr(r0, r1, r2);
2591 ldr_c(r0, r0);
2592#else
2593 rex(0, WIDE, r0, r1, r2);
2594 ic(0x0f);
2595 ic(0xbe);
2596 rx(r0, 0, r2, r1, _SCL1);
2597#endif
2598}
2599
2600static void
2601_ldxi_c(jit_state_t *_jit, jit_int32_t r0, jit_int32_t r1, jit_word_t i0)
2602{
2603 jit_int32_t reg;
2604 if (can_sign_extend_int_p(i0)) {
2605 rex(0, WIDE, r0, _NOREG, r1);
2606 ic(0x0f);
2607 ic(0xbe);
2608 rx(r0, i0, r1, _NOREG, _SCL1);
2609 }
2610 else {
2611 reg = jit_get_reg(jit_class_gpr);
2612 movi(rn(reg), i0);
2613 ldxr_c(r0, r1, rn(reg));
2614 jit_unget_reg(reg);
2615 }
2616}
2617
2618static void
2619_ldxr_uc(jit_state_t *_jit, jit_int32_t r0, jit_int32_t r1, jit_int32_t r2)
2620{
2621#if __X64_32
2622 addr(r0, r1, r2);
2623 ldr_uc(r0, r0);
2624#else
2625 rex(0, WIDE, r0, r1, r2);
2626 ic(0x0f);
2627 ic(0xb6);
2628 rx(r0, 0, r2, r1, _SCL1);
2629#endif
2630}
2631
2632static void
2633_ldxi_uc(jit_state_t *_jit, jit_int32_t r0, jit_int32_t r1, jit_word_t i0)
2634{
2635 jit_int32_t reg;
2636 if (can_sign_extend_int_p(i0)) {
2637 rex(0, WIDE, r0, _NOREG, r1);
2638 ic(0x0f);
2639 ic(0xb6);
2640 rx(r0, i0, r1, _NOREG, _SCL1);
2641 }
2642 else {
2643 reg = jit_get_reg(jit_class_gpr);
2644 movi(rn(reg), i0);
2645 ldxr_uc(r0, r1, rn(reg));
2646 jit_unget_reg(reg);
2647 }
2648}
2649
2650static void
2651_ldxr_s(jit_state_t *_jit, jit_int32_t r0, jit_int32_t r1, jit_int32_t r2)
2652{
2653#if __X64_32
2654 addr(r0, r1, r2);
2655 ldr_s(r0, r0);
2656#else
2657 rex(0, WIDE, r0, r1, r2);
2658 ic(0x0f);
2659 ic(0xbf);
2660 rx(r0, 0, r2, r1, _SCL1);
2661#endif
2662}
2663
2664static void
2665_ldxi_s(jit_state_t *_jit, jit_int32_t r0, jit_int32_t r1, jit_word_t i0)
2666{
2667 jit_int32_t reg;
2668 if (can_sign_extend_int_p(i0)) {
2669 rex(0, WIDE, r0, _NOREG, r1);
2670 ic(0x0f);
2671 ic(0xbf);
2672 rx(r0, i0, r1, _NOREG, _SCL1);
2673 }
2674 else {
2675 reg = jit_get_reg(jit_class_gpr);
2676 movi(rn(reg), i0);
2677 ldxr_s(r0, r1, rn(reg));
2678 jit_unget_reg(reg);
2679 }
2680}
2681
2682static void
2683_ldxr_us(jit_state_t *_jit, jit_int32_t r0, jit_int32_t r1, jit_int32_t r2)
2684{
2685#if __X64_32
2686 addr(r0, r1, r2);
2687 ldr_us(r0, r0);
2688#else
2689 rex(0, WIDE, r0, r1, r2);
2690 ic(0x0f);
2691 ic(0xb7);
2692 rx(r0, 0, r2, r1, _SCL1);
2693#endif
2694}
2695
2696static void
2697_ldxi_us(jit_state_t *_jit, jit_int32_t r0, jit_int32_t r1, jit_word_t i0)
2698{
2699 jit_int32_t reg;
2700 if (can_sign_extend_int_p(i0)) {
2701 rex(0, WIDE, r0, _NOREG, r1);
2702 ic(0x0f);
2703 ic(0xb7);
2704 rx(r0, i0, r1, _NOREG, _SCL1);
2705 }
2706 else {
2707 reg = jit_get_reg(jit_class_gpr);
2708 movi(rn(reg), i0);
2709 ldxr_us(r0, r1, rn(reg));
2710 jit_unget_reg(reg);
2711 }
2712}
2713
2714#if __X64 || !__X64_32
2715static void
2716_ldxr_i(jit_state_t *_jit, jit_int32_t r0, jit_int32_t r1, jit_int32_t r2)
2717{
2718#if __X64
2719 rex(0, WIDE, r0, r1, r2);
2720 ic(0x63);
2721#else
2722 ic(0x8b);
2723#endif
2724 rx(r0, 0, r2, r1, _SCL1);
2725}
2726
2727static void
2728_ldxi_i(jit_state_t *_jit, jit_int32_t r0, jit_int32_t r1, jit_word_t i0)
2729{
2730 jit_int32_t reg;
2731 if (can_sign_extend_int_p(i0)) {
2732#if __X64
2733 rex(0, WIDE, r0, _NOREG, r1);
2734 ic(0x63);
2735#else
2736 ic(0x8b);
2737#endif
2738 rx(r0, i0, r1, _NOREG, _SCL1);
2739 }
2740 else {
2741 reg = jit_get_reg(jit_class_gpr);
2742 movi(rn(reg), i0);
2743 ldxr_i(r0, r1, rn(reg));
2744 jit_unget_reg(reg);
2745 }
2746}
2747#endif
2748
2749#if __X64
2750static void
2751_ldxr_ui(jit_state_t *_jit, jit_int32_t r0, jit_int32_t r1, jit_int32_t r2)
2752{
2753#if __X64_32
2754 addr(r0, r1, r2);
2755 /* to avoid confusion with macro renames */
2756 _ldr_ui(_jit, r0, r0);
2757#else
2758 rex(0, 0, r0, r1, r2);
2759 ic(0x8b);
2760 rx(r0, 0, r2, r1, _SCL1);
2761#endif
2762}
2763
2764static void
2765_ldxi_ui(jit_state_t *_jit, jit_int32_t r0, jit_int32_t r1, jit_word_t i0)
2766{
2767 jit_int32_t reg;
2768 if (can_sign_extend_int_p(i0)) {
2769 rex(0, 0, r0, _NOREG, r1);
2770 ic(0x8b);
2771 rx(r0, i0, r1, _NOREG, _SCL1);
2772 }
2773 else {
2774 reg = jit_get_reg(jit_class_gpr);
2775 movi(rn(reg), i0);
2776 ldxr_ui(r0, r1, rn(reg));
2777 jit_unget_reg(reg);
2778 }
2779}
2780
2781# if !__X64_32
2782static void
2783_ldxr_l(jit_state_t *_jit, jit_int32_t r0, jit_int32_t r1, jit_int32_t r2)
2784{
2785 rex(0, 1, r0, r1, r2);
2786 ic(0x8b);
2787 rx(r0, 0, r2, r1, _SCL1);
2788}
2789
2790static void
2791_ldxi_l(jit_state_t *_jit, jit_int32_t r0, jit_int32_t r1, jit_word_t i0)
2792{
2793 jit_int32_t reg;
2794 if (can_sign_extend_int_p(i0)) {
2795 rex(0, 1, r0, _NOREG, r1);
2796 ic(0x8b);
2797 rx(r0, i0, r1, _NOREG, _SCL1);
2798 }
2799 else {
2800 reg = jit_get_reg(jit_class_gpr);
2801 movi(rn(reg), i0);
2802 ldxr_l(r0, r1, rn(reg));
2803 jit_unget_reg(reg);
2804 }
2805}
2806# endif
2807#endif
2808
2809static void
2810_str_c(jit_state_t *_jit, jit_int32_t r0, jit_int32_t r1)
2811{
2812 jit_int32_t reg;
2813 if (reg8_p(r1)) {
2814 rex(0, 0, r1, _NOREG, r0);
2815 ic(0x88);
2816 rx(r1, 0, r0, _NOREG, _SCL1);
2817 }
2818 else {
2819 reg = jit_get_reg(jit_class_gpr|jit_class_rg8);
2820 movr(rn(reg), r1);
2821 rex(0, 0, rn(reg), _NOREG, r0);
2822 ic(0x88);
2823 rx(rn(reg), 0, r0, _NOREG, _SCL1);
2824 jit_unget_reg(reg);
2825 }
2826}
2827
2828static void
2829_sti_c(jit_state_t *_jit, jit_word_t i0, jit_int32_t r0)
2830{
2831 jit_int32_t reg;
2832 if (can_sign_extend_int_p(i0)) {
2833 if (reg8_p(r0)) {
2834 rex(0, 0, r0, _NOREG, _NOREG);
2835 ic(0x88);
2836 rx(r0, i0, _NOREG, _NOREG, _SCL1);
2837 }
2838 else {
2839 reg = jit_get_reg(jit_class_gpr|jit_class_rg8);
2840 movr(rn(reg), r0);
2841 rex(0, 0, rn(reg), _NOREG, _NOREG);
2842 ic(0x88);
2843 rx(rn(reg), i0, _NOREG, _NOREG, _SCL1);
2844 jit_unget_reg(reg);
2845 }
2846 }
2847 else {
2848 reg = jit_get_reg(jit_class_gpr);
2849 movi(rn(reg), i0);
2850 str_c(rn(reg), r0);
2851 jit_unget_reg(reg);
2852 }
2853}
2854
2855static void
2856_str_s(jit_state_t *_jit, jit_int32_t r0, jit_int32_t r1)
2857{
2858 ic(0x66);
2859 rex(0, 0, r1, _NOREG, r0);
2860 ic(0x89);
2861 rx(r1, 0, r0, _NOREG, _SCL1);
2862}
2863
2864static void
2865_sti_s(jit_state_t *_jit, jit_word_t i0, jit_int32_t r0)
2866{
2867 jit_int32_t reg;
2868 if (can_sign_extend_int_p(i0)) {
2869 ic(0x66);
2870 rex(0, 0, r0, _NOREG, _NOREG);
2871 ic(0x89);
2872 rx(r0, i0, _NOREG, _NOREG, _SCL1);
2873 }
2874 else {
2875 reg = jit_get_reg(jit_class_gpr);
2876 movi(rn(reg), i0);
2877 str_s(rn(reg), r0);
2878 jit_unget_reg(reg);
2879 }
2880}
2881
2882static void
2883_str_i(jit_state_t *_jit, jit_int32_t r0, jit_int32_t r1)
2884{
2885 rex(0, 0, r1, _NOREG, r0);
2886 ic(0x89);
2887 rx(r1, 0, r0, _NOREG, _SCL1);
2888}
2889
2890static void
2891_sti_i(jit_state_t *_jit, jit_word_t i0, jit_int32_t r0)
2892{
2893 jit_int32_t reg;
2894 if (can_sign_extend_int_p(i0)) {
2895 rex(0, 0, r0, _NOREG, _NOREG);
2896 ic(0x89);
2897 rx(r0, i0, _NOREG, _NOREG, _SCL1);
2898 }
2899 else {
2900 reg = jit_get_reg(jit_class_gpr);
2901 movi(rn(reg), i0);
2902 str_i(rn(reg), r0);
2903 jit_unget_reg(reg);
2904 }
2905}
2906
2907#if __X64 && !__X64_32
2908static void
2909_str_l(jit_state_t *_jit, jit_int32_t r0, jit_int32_t r1)
2910{
2911 rex(0, 1, r1, _NOREG, r0);
2912 ic(0x89);
2913 rx(r1, 0, r0, _NOREG, _SCL1);
2914}
2915
2916static void
2917_sti_l(jit_state_t *_jit, jit_word_t i0, jit_int32_t r0)
2918{
2919 jit_int32_t reg;
2920 if (can_sign_extend_int_p(i0)) {
2921 rex(0, 1, r0, _NOREG, _NOREG);
2922 ic(0x89);
2923 rx(r0, i0, _NOREG, _NOREG, _SCL1);
2924 }
2925 else {
2926 reg = jit_get_reg(jit_class_gpr);
2927 movi(rn(reg), i0);
2928 str_l(rn(reg), r0);
2929 jit_unget_reg(reg);
2930 }
2931}
2932#endif
2933
2934static void
2935_stxr_c(jit_state_t *_jit, jit_int32_t r0, jit_int32_t r1, jit_int32_t r2)
2936{
2937 jit_int32_t reg;
2938#if __X64_32
2939 reg = jit_get_reg(jit_class_gpr);
2940 addr(rn(reg), r0, r1);
2941 str_c(rn(reg), r2);
2942 jit_unget_reg(reg);
2943#else
2944 if (reg8_p(r2)) {
2945 rex(0, 0, r2, r1, r0);
2946 ic(0x88);
2947 rx(r2, 0, r0, r1, _SCL1);
2948 }
2949 else {
2950 reg = jit_get_reg(jit_class_gpr|jit_class_rg8);
2951 movr(rn(reg), r2);
2952 rex(0, 0, rn(reg), r1, r0);
2953 ic(0x88);
2954 rx(rn(reg), 0, r0, r1, _SCL1);
2955 jit_unget_reg(reg);
2956 }
2957#endif
2958}
2959
2960static void
2961_stxi_c(jit_state_t *_jit, jit_word_t i0, jit_int32_t r0, jit_int32_t r1)
2962{
2963 jit_int32_t reg;
2964 if (can_sign_extend_int_p(i0)) {
2965 if (reg8_p(r1)) {
2966 rex(0, 0, r1, _NOREG, r0);
2967 ic(0x88);
2968 rx(r1, i0, r0, _NOREG, _SCL1);
2969 }
2970 else {
2971 reg = jit_get_reg(jit_class_gpr|jit_class_rg8);
2972 movr(rn(reg), r1);
2973 rex(0, 0, rn(reg), _NOREG, r0);
2974 ic(0x88);
2975 rx(rn(reg), i0, r0, _NOREG, _SCL1);
2976 jit_unget_reg(reg);
2977 }
2978 }
2979 else {
2980 reg = jit_get_reg(jit_class_gpr);
2981 movi(rn(reg), i0);
2982 stxr_c(rn(reg), r0, r1);
2983 jit_unget_reg(reg);
2984 }
2985}
2986
2987static void
2988_stxr_s(jit_state_t *_jit, jit_int32_t r0, jit_int32_t r1, jit_int32_t r2)
2989{
2990#if __X64_32
2991 jit_int32_t reg;
2992 reg = jit_get_reg(jit_class_gpr);
2993 addr(rn(reg), r0, r1);
2994 str_s(rn(reg), r2);
2995 jit_unget_reg(reg);
2996#else
2997 ic(0x66);
2998 rex(0, 0, r2, r1, r0);
2999 ic(0x89);
3000 rx(r2, 0, r0, r1, _SCL1);
3001#endif
3002}
3003
3004static void
3005_stxi_s(jit_state_t *_jit, jit_word_t i0, jit_int32_t r0, jit_int32_t r1)
3006{
3007 jit_int32_t reg;
3008 if (can_sign_extend_int_p(i0)) {
3009 ic(0x66);
3010 rex(0, 0, r1, _NOREG, r0);
3011 ic(0x89);
3012 rx(r1, i0, r0, _NOREG, _SCL1);
3013 }
3014 else {
3015 reg = jit_get_reg(jit_class_gpr);
3016 movi(rn(reg), i0);
3017 stxr_s(rn(reg), r0, r1);
3018 jit_unget_reg(reg);
3019 }
3020}
3021
3022static void
3023_stxr_i(jit_state_t *_jit, jit_int32_t r0, jit_int32_t r1, jit_int32_t r2)
3024{
3025#if __X64_32
3026 jit_int32_t reg;
3027 reg = jit_get_reg(jit_class_gpr);
3028 addr(rn(reg), r0, r1);
3029 str_i(rn(reg), r2);
3030 jit_unget_reg(reg);
3031#else
3032 rex(0, 0, r2, r1, r0);
3033 ic(0x89);
3034 rx(r2, 0, r0, r1, _SCL1);
3035#endif
3036}
3037
3038static void
3039_stxi_i(jit_state_t *_jit, jit_word_t i0, jit_int32_t r0, jit_int32_t r1)
3040{
3041 jit_int32_t reg;
3042 if (can_sign_extend_int_p(i0)) {
3043 rex(0, 0, r1, _NOREG, r0);
3044 ic(0x89);
3045 rx(r1, i0, r0, _NOREG, _SCL1);
3046 }
3047 else {
3048 reg = jit_get_reg(jit_class_gpr);
3049 movi(rn(reg), i0);
3050 stxr_i(rn(reg), r0, r1);
3051 jit_unget_reg(reg);
3052 }
3053}
3054
3055#if __X64 && !__X64_32
3056static void
3057_stxr_l(jit_state_t *_jit, jit_int32_t r0, jit_int32_t r1, jit_int32_t r2)
3058{
3059 rex(0, 1, r2, r1, r0);
3060 ic(0x89);
3061 rx(r2, 0, r0, r1, _SCL1);
3062}
3063
3064static void
3065_stxi_l(jit_state_t *_jit, jit_word_t i0, jit_int32_t r0, jit_int32_t r1)
3066{
3067 jit_int32_t reg;
3068 if (can_sign_extend_int_p(i0)) {
3069 rex(0, 1, r1, _NOREG, r0);
3070 ic(0x89);
3071 rx(r1, i0, r0, _NOREG, _SCL1);
3072 }
3073 else {
3074 reg = jit_get_reg(jit_class_gpr);
3075 movi(rn(reg), i0);
3076 stxr_l(rn(reg), r0, r1);
3077 jit_unget_reg(reg);
3078 }
3079}
3080#endif
3081
3082static void
3083_jccs(jit_state_t *_jit, jit_int32_t code, jit_word_t i0)
3084{
3085 jit_word_t w;
3086 ic(0x70 | code);
3087 w = i0 - (_jit->pc.w + 1);
3088 ic(w);
3089}
3090
3091static void
3092_jcc(jit_state_t *_jit, jit_int32_t code, jit_word_t i0)
3093{
3094 jit_word_t w;
3095 ic(0x0f);
3096 ic(0x80 | code);
3097 w = i0 - (_jit->pc.w + 4);
3098 ii(w);
3099}
3100
3101static void
3102_jcr(jit_state_t *_jit,
3103 jit_int32_t code, jit_word_t i0, jit_int32_t r0, jit_int32_t r1)
3104{
3105 alur(X86_CMP, r0, r1);
3106 jcc(code, i0);
3107}
3108
3109static void
3110_jci(jit_state_t *_jit,
3111 jit_int32_t code, jit_word_t i0, jit_int32_t r0, jit_word_t i1)
3112{
3113 alui(X86_CMP, r0, i1);
3114 jcc(code, i0);
3115}
3116
3117static void
3118_jci0(jit_state_t *_jit, jit_int32_t code, jit_word_t i0, jit_int32_t r0)
3119{
3120 testr(r0, r0);
3121 jcc(code, i0);
3122}
3123
3124static jit_word_t
3125_bltr(jit_state_t *_jit, jit_word_t i0, jit_int32_t r0, jit_int32_t r1)
3126{
3127 jcr(X86_CC_L, i0, r0, r1);
3128 return (_jit->pc.w);
3129}
3130
3131static jit_word_t
3132_blti(jit_state_t *_jit, jit_word_t i0, jit_int32_t r0, jit_word_t i1)
3133{
3134 if (i1) jci (X86_CC_L, i0, r0, i1);
3135 else jci0(X86_CC_S, i0, r0);
3136 return (_jit->pc.w);
3137}
3138
3139static jit_word_t
3140_bltr_u(jit_state_t *_jit, jit_word_t i0, jit_int32_t r0, jit_int32_t r1)
3141{
3142 jcr(X86_CC_B, i0, r0, r1);
3143 return (_jit->pc.w);
3144}
3145
3146static jit_word_t
3147_blti_u(jit_state_t *_jit, jit_word_t i0, jit_int32_t r0, jit_word_t i1)
3148{
3149 if (i1) jci (X86_CC_B, i0, r0, i1);
3150 else jci0(X86_CC_B, i0, r0);
3151 return (_jit->pc.w);
3152}
3153
3154static jit_word_t
3155_bler(jit_state_t *_jit, jit_word_t i0, jit_int32_t r0, jit_int32_t r1)
3156{
3157 if (r0 == r1) jmpi(i0);
3158 else jcr (X86_CC_LE, i0, r0, r1);
3159 return (_jit->pc.w);
3160}
3161
3162static jit_word_t
3163_blei(jit_state_t *_jit, jit_word_t i0, jit_int32_t r0, jit_word_t i1)
3164{
3165 if (i1) jci (X86_CC_LE, i0, r0, i1);
3166 else jci0(X86_CC_LE, i0, r0);
3167 return (_jit->pc.w);
3168}
3169
3170static jit_word_t
3171_bler_u(jit_state_t *_jit, jit_word_t i0, jit_int32_t r0, jit_int32_t r1)
3172{
3173 if (r0 == r1) jmpi(i0);
3174 else jcr (X86_CC_BE, i0, r0, r1);
3175 return (_jit->pc.w);
3176}
3177
3178static jit_word_t
3179_blei_u(jit_state_t *_jit, jit_word_t i0, jit_int32_t r0, jit_word_t i1)
3180{
3181 if (i1) jci (X86_CC_BE, i0, r0, i1);
3182 else jci0(X86_CC_BE, i0, r0);
3183 return (_jit->pc.w);
3184}
3185
3186static jit_word_t
3187_beqr(jit_state_t *_jit, jit_word_t i0, jit_int32_t r0, jit_int32_t r1)
3188{
3189 if (r0 == r1) jmpi(i0);
3190 else jcr (X86_CC_E, i0, r0, r1);
3191 return (_jit->pc.w);
3192}
3193
3194static jit_word_t
3195_beqi(jit_state_t *_jit, jit_word_t i0, jit_int32_t r0, jit_word_t i1)
3196{
3197 if (i1) jci (X86_CC_E, i0, r0, i1);
3198 else jci0(X86_CC_E, i0, r0);
3199 return (_jit->pc.w);
3200}
3201
3202static jit_word_t
3203_bger(jit_state_t *_jit, jit_word_t i0, jit_int32_t r0, jit_int32_t r1)
3204{
3205 if (r0 == r1) jmpi(i0);
3206 else jcr (X86_CC_GE, i0, r0, r1);
3207 return (_jit->pc.w);
3208}
3209
3210static jit_word_t
3211_bgei(jit_state_t *_jit, jit_word_t i0, jit_int32_t r0, jit_word_t i1)
3212{
3213 if (i1) jci (X86_CC_GE, i0, r0, i1);
3214 else jci0(X86_CC_NS, i0, r0);
3215 return (_jit->pc.w);
3216}
3217
3218static jit_word_t
3219_bger_u(jit_state_t *_jit, jit_word_t i0, jit_int32_t r0, jit_int32_t r1)
3220{
3221 if (r0 == r1) jmpi(i0);
3222 else jcr (X86_CC_AE, i0, r0, r1);
3223 return (_jit->pc.w);
3224}
3225
3226static jit_word_t
3227_bgei_u(jit_state_t *_jit, jit_word_t i0, jit_int32_t r0, jit_word_t i1)
3228{
3229 if (i1) jci (X86_CC_AE, i0, r0, i1);
3230 else jmpi(i0);
3231 return (_jit->pc.w);
3232}
3233
3234static jit_word_t
3235_bgtr(jit_state_t *_jit, jit_word_t i0, jit_int32_t r0, jit_int32_t r1)
3236{
3237 jcr(X86_CC_G, i0, r0, r1);
3238 return (_jit->pc.w);
3239}
3240
3241static jit_word_t
3242_bgti(jit_state_t *_jit, jit_word_t i0, jit_int32_t r0, jit_word_t i1)
3243{
3244 jci(X86_CC_G, i0, r0, i1);
3245 return (_jit->pc.w);
3246}
3247
3248static jit_word_t
3249_bgtr_u(jit_state_t *_jit, jit_word_t i0, jit_int32_t r0, jit_int32_t r1)
3250{
3251 jcr(X86_CC_A, i0, r0, r1);
3252 return (_jit->pc.w);
3253}
3254
3255static jit_word_t
3256_bgti_u(jit_state_t *_jit, jit_word_t i0, jit_int32_t r0, jit_word_t i1)
3257{
3258 if (i1) jci (X86_CC_A, i0, r0, i1);
3259 else jci0(X86_CC_NE, i0, r0);
3260 return (_jit->pc.w);
3261}
3262
3263static jit_word_t
3264_bner(jit_state_t *_jit, jit_word_t i0, jit_int32_t r0, jit_int32_t r1)
3265{
3266 jcr(X86_CC_NE, i0, r0, r1);
3267 return (_jit->pc.w);
3268}
3269
3270static jit_word_t
3271_bnei(jit_state_t *_jit, jit_word_t i0, jit_int32_t r0, jit_word_t i1)
3272{
3273 if (i1) jci (X86_CC_NE, i0, r0, i1);
3274 else jci0(X86_CC_NE, i0, r0);
3275 return (_jit->pc.w);
3276}
3277
3278static jit_word_t
3279_bmsr(jit_state_t *_jit, jit_word_t i0, jit_int32_t r0, jit_int32_t r1)
3280{
3281 testr(r0, r1);
3282 jnz(i0);
3283 return (_jit->pc.w);
3284}
3285
3286static jit_word_t
3287_bmsi(jit_state_t *_jit, jit_word_t i0, jit_int32_t r0, jit_word_t i1)
3288{
3289 jit_int32_t reg;
3290 if (can_zero_extend_int_p(i1))
3291 testi(r0, i1);
3292 else {
3293 reg = jit_get_reg(jit_class_gpr);
3294 movi(rn(reg), i1);
3295 testr(r0, rn(reg));
3296 jit_unget_reg(reg);
3297 }
3298 jnz(i0);
3299 return (_jit->pc.w);
3300}
3301
3302static jit_word_t
3303_bmcr(jit_state_t *_jit, jit_word_t i0, jit_int32_t r0, jit_int32_t r1)
3304{
3305 testr(r0, r1);
3306 jz(i0);
3307 return (_jit->pc.w);
3308}
3309
3310static jit_word_t
3311_bmci(jit_state_t *_jit, jit_word_t i0, jit_int32_t r0, jit_word_t i1)
3312{
3313 jit_int32_t reg;
3314 if (can_zero_extend_int_p(i1))
3315 testi(r0, i1);
3316 else {
3317 reg = jit_get_reg(jit_class_gpr);
3318 movi(rn(reg), i1);
3319 testr(r0, rn(reg));
3320 jit_unget_reg(reg);
3321 }
3322 jz(i0);
3323 return (_jit->pc.w);
3324}
3325
3326static jit_word_t
3327_boaddr(jit_state_t *_jit, jit_word_t i0, jit_int32_t r0, jit_int32_t r1)
3328{
3329 iaddr(r0, r1);
3330 jo(i0);
3331 return (_jit->pc.w);
3332}
3333
3334static jit_word_t
3335_boaddi(jit_state_t *_jit, jit_word_t i0, jit_int32_t r0, jit_word_t i1)
3336{
3337 jit_int32_t reg;
3338 if (can_sign_extend_int_p(i1)) {
3339 iaddi(r0, i1);
3340 jo(i0);
3341 return (_jit->pc.w);
3342 }
3343 reg = jit_get_reg(jit_class_gpr|jit_class_nospill);
3344 movi(rn(reg), i1);
3345 jit_unget_reg(reg);
3346 return (boaddr(i0, r0, rn(reg)));
3347}
3348
3349static jit_word_t
3350_boaddr_u(jit_state_t *_jit, jit_word_t i0, jit_int32_t r0, jit_int32_t r1)
3351{
3352 iaddr(r0, r1);
3353 jc(i0);
3354 return (_jit->pc.w);
3355}
3356
3357static jit_word_t
3358_boaddi_u(jit_state_t *_jit, jit_word_t i0, jit_int32_t r0, jit_word_t i1)
3359{
3360 jit_int32_t reg;
3361 if (can_sign_extend_int_p(i1)) {
3362 iaddi(r0, i1);
3363 jc(i0);
3364 return (_jit->pc.w);
3365 }
3366 reg = jit_get_reg(jit_class_gpr|jit_class_nospill);
3367 movi(rn(reg), i1);
3368 jit_unget_reg(reg);
3369 return (boaddr_u(i0, r0, rn(reg)));
3370}
3371
3372static jit_word_t
3373_bxaddr(jit_state_t *_jit, jit_word_t i0, jit_int32_t r0, jit_int32_t r1)
3374{
3375 iaddr(r0, r1);
3376 jno(i0);
3377 return (_jit->pc.w);
3378}
3379
3380static jit_word_t
3381_bxaddi(jit_state_t *_jit, jit_word_t i0, jit_int32_t r0, jit_word_t i1)
3382{
3383 jit_int32_t reg;
3384 if (can_sign_extend_int_p(i1)) {
3385 iaddi(r0, i1);
3386 jno(i0);
3387 return (_jit->pc.w);
3388 }
3389 reg = jit_get_reg(jit_class_gpr|jit_class_nospill);
3390 movi(rn(reg), i1);
3391 jit_unget_reg(reg);
3392 return (bxaddr(i0, r0, rn(reg)));
3393}
3394
3395static jit_word_t
3396_bxaddr_u(jit_state_t *_jit, jit_word_t i0, jit_int32_t r0, jit_int32_t r1)
3397{
3398 iaddr(r0, r1);
3399 jnc(i0);
3400 return (_jit->pc.w);
3401}
3402
3403static jit_word_t
3404_bxaddi_u(jit_state_t *_jit, jit_word_t i0, jit_int32_t r0, jit_word_t i1)
3405{
3406 jit_int32_t reg;
3407 if (can_sign_extend_int_p(i1)) {
3408 iaddi(r0, i1);
3409 jnc(i0);
3410 return (_jit->pc.w);
3411 }
3412 reg = jit_get_reg(jit_class_gpr|jit_class_nospill);
3413 movi(rn(reg), i1);
3414 jit_unget_reg(reg);
3415 return (bxaddr_u(i0, r0, rn(reg)));
3416}
3417
3418static jit_word_t
3419_bosubr(jit_state_t *_jit, jit_word_t i0, jit_int32_t r0, jit_int32_t r1)
3420{
3421 isubr(r0, r1);
3422 jo(i0);
3423 return (_jit->pc.w);
3424}
3425
3426static jit_word_t
3427_bosubi(jit_state_t *_jit, jit_word_t i0, jit_int32_t r0, jit_word_t i1)
3428{
3429 jit_int32_t reg;
3430 if (can_sign_extend_int_p(i1)) {
3431 isubi(r0, i1);
3432 jo(i0);
3433 return (_jit->pc.w);
3434 }
3435 reg = jit_get_reg(jit_class_gpr|jit_class_nospill);
3436 movi(rn(reg), i1);
3437 jit_unget_reg(reg);
3438 return (bosubr(i0, r0, rn(reg)));
3439}
3440
3441static jit_word_t
3442_bosubr_u(jit_state_t *_jit, jit_word_t i0, jit_int32_t r0, jit_int32_t r1)
3443{
3444 isubr(r0, r1);
3445 jc(i0);
3446 return (_jit->pc.w);
3447}
3448
3449static jit_word_t
3450_bosubi_u(jit_state_t *_jit, jit_word_t i0, jit_int32_t r0, jit_word_t i1)
3451{
3452 jit_int32_t reg;
3453 if (can_sign_extend_int_p(i1)) {
3454 isubi(r0, i1);
3455 jc(i0);
3456 return (_jit->pc.w);
3457 }
3458 reg = jit_get_reg(jit_class_gpr|jit_class_nospill);
3459 movi(rn(reg), i1);
3460 jit_unget_reg(reg);
3461 return (bosubr_u(i0, r0, rn(reg)));
3462}
3463
3464static jit_word_t
3465_bxsubr(jit_state_t *_jit, jit_word_t i0, jit_int32_t r0, jit_int32_t r1)
3466{
3467 isubr(r0, r1);
3468 jno(i0);
3469 return (_jit->pc.w);
3470}
3471
3472static jit_word_t
3473_bxsubi(jit_state_t *_jit, jit_word_t i0, jit_int32_t r0, jit_word_t i1)
3474{
3475 jit_int32_t reg;
3476 if (can_sign_extend_int_p(i1)) {
3477 isubi(r0, i1);
3478 jno(i0);
3479 return (_jit->pc.w);
3480 }
3481 reg = jit_get_reg(jit_class_gpr|jit_class_nospill);
3482 movi(rn(reg), i1);
3483 jit_unget_reg(reg);
3484 return (bxsubr(i0, r0, rn(reg)));
3485}
3486
3487static jit_word_t
3488_bxsubr_u(jit_state_t *_jit, jit_word_t i0, jit_int32_t r0, jit_int32_t r1)
3489{
3490 isubr(r0, r1);
3491 jnc(i0);
3492 return (_jit->pc.w);
3493}
3494
3495static jit_word_t
3496_bxsubi_u(jit_state_t *_jit, jit_word_t i0, jit_int32_t r0, jit_word_t i1)
3497{
3498 jit_int32_t reg;
3499 if (can_sign_extend_int_p(i1)) {
3500 isubi(r0, i1);
3501 jnc(i0);
3502 return (_jit->pc.w);
3503 }
3504 reg = jit_get_reg(jit_class_gpr|jit_class_nospill);
3505 movi(rn(reg), i1);
3506 jit_unget_reg(reg);
3507 return (bxsubr_u(i0, r0, rn(reg)));
3508}
3509
3510static void
3511_callr(jit_state_t *_jit, jit_int32_t r0)
3512{
3513 rex(0, 0, _NOREG, _NOREG, r0);
3514 ic(0xff);
3515 mrm(0x03, 0x02, r7(r0));
3516}
3517
3518static jit_word_t
3519_calli(jit_state_t *_jit, jit_word_t i0)
3520{
3521 jit_word_t word;
519a9ea1 3522 jit_word_t w;
4a71579b 3523#if __X64
519a9ea1
PC		 3524 w = i0 - (_jit->pc.w + 5);
3525 if ((jit_int32_t)w == w) {
3526#endif
3527 ic(0xe8);
3528 w = i0 - (_jit->pc.w + 4);
3529 ii(w);
3530 word = _jit->pc.w;
3531#if __X64
3532 }
3533 else
3534 word = calli_p(i0);
3535#endif
3536 return (word);
3537}
4a71579b 3538
519a9ea1
PC		 3539#if __X64
3540static jit_word_t
3541_calli_p(jit_state_t *_jit, jit_word_t i0)
3542{
3543 jit_word_t word;
3544 jit_int32_t reg;
4a71579b
PC		 3545 reg = jit_get_reg(jit_class_gpr);
3546 word = movi_p(rn(reg), i0);
3547 callr(rn(reg));
3548 jit_unget_reg(reg);
4a71579b
PC		 3549 return (word);
3550}
519a9ea1 3551#endif
4a71579b
PC		 3552
3553static void
3554_jmpr(jit_state_t *_jit, jit_int32_t r0)
3555{
519a9ea1 3556 rex(0, 0, _NOREG, _NOREG, r0);
4a71579b
PC		 3557 ic(0xff);
3558 mrm(0x03, 0x04, r7(r0));
3559}
3560
3561static jit_word_t
3562_jmpi(jit_state_t *_jit, jit_word_t i0)
3563{
519a9ea1 3564 jit_word_t word;
4a71579b 3565 jit_word_t w;
519a9ea1
PC		 3566#if __X64
3567 w = i0 - (_jit->pc.w + 5);
3568 if ((jit_int32_t)w == w) {
3569#endif
3570 ic(0xe9);
3571 w = i0 - (_jit->pc.w + 4);
3572 ii(w);
3573 word = _jit->pc.w;
3574#if __X64
3575 }
3576 else
3577 word = jmpi_p(i0);
3578#endif
3579 return (word);
4a71579b
PC		 3580}
3581
519a9ea1
PC		 3582#if __X64
3583static jit_word_t
3584_jmpi_p(jit_state_t *_jit, jit_word_t i0)
3585{
3586 jit_word_t word;
3587 jit_int32_t reg;
3588 reg = jit_get_reg(jit_class_gpr|jit_class_nospill);
3589 word = movi_p(rn(reg), i0);
3590 jmpr(rn(reg));
3591 jit_unget_reg(reg);
3592 return (word);
3593}
3594#endif
3595
4a71579b
PC		 3596static void
3597_jmpsi(jit_state_t *_jit, jit_uint8_t i0)
3598{
3599 ic(0xeb);
3600 ic(i0);
3601}
3602
3603static void
3604_prolog(jit_state_t *_jit, jit_node_t *node)
3605{
3606 jit_int32_t reg;
3607 if (_jitc->function->define_frame || _jitc->function->assume_frame) {
3608 jit_int32_t frame = -_jitc->function->frame;
3609 assert(_jitc->function->self.aoff >= frame);
3610 if (_jitc->function->assume_frame)
3611 return;
3612 _jitc->function->self.aoff = frame;
3613 }
3614 if (_jitc->function->allocar)
3615 _jitc->function->self.aoff &= -16;
3616#if __X64 && (__CYGWIN__ || _WIN32)
3617 _jitc->function->stack = (((/* first 32 bytes must be allocated */
3618 (_jitc->function->self.alen > 32 ?
3619 _jitc->function->self.alen : 32) -
3620 /* align stack at 16 bytes */
3621 _jitc->function->self.aoff) + 15) & -16) +
3622 stack_adjust;
3623#else
3624 _jitc->function->stack = (((_jitc->function->self.alen -
3625 _jitc->function->self.aoff) + 15) & -16) +
3626 stack_adjust;
3627#endif
3628 subi(_RSP_REGNO, _RSP_REGNO, stack_framesize - REAL_WORDSIZE);
3629 /* callee save registers */
3630#if __X32
3631 if (jit_regset_tstbit(&_jitc->function->regset, _RDI))
3632 stxi(12, _RSP_REGNO, _RDI_REGNO);
3633 if (jit_regset_tstbit(&_jitc->function->regset, _RSI))
3634 stxi( 8, _RSP_REGNO, _RSI_REGNO);
3635 if (jit_regset_tstbit(&_jitc->function->regset, _RBX))
3636 stxi( 4, _RSP_REGNO, _RBX_REGNO);
3637#else
3638# if __CYGWIN__ || _WIN32
3639 if (jit_regset_tstbit(&_jitc->function->regset, _XMM15))
3640 sse_stxi_d(136, _RSP_REGNO, _XMM15_REGNO);
3641 if (jit_regset_tstbit(&_jitc->function->regset, _XMM14))
3642 sse_stxi_d(128, _RSP_REGNO, _XMM14_REGNO);
3643 if (jit_regset_tstbit(&_jitc->function->regset, _XMM13))
3644 sse_stxi_d(120, _RSP_REGNO, _XMM13_REGNO);
3645 if (jit_regset_tstbit(&_jitc->function->regset, _XMM12))
3646 sse_stxi_d(112, _RSP_REGNO, _XMM12_REGNO);
3647 if (jit_regset_tstbit(&_jitc->function->regset, _XMM11))
3648 sse_stxi_d(104, _RSP_REGNO, _XMM11_REGNO);
3649 if (jit_regset_tstbit(&_jitc->function->regset, _XMM10))
3650 sse_stxi_d(96, _RSP_REGNO, _XMM10_REGNO);
3651 if (jit_regset_tstbit(&_jitc->function->regset, _XMM9))
3652 sse_stxi_d(88, _RSP_REGNO, _XMM9_REGNO);
3653 if (jit_regset_tstbit(&_jitc->function->regset, _XMM8))
3654 sse_stxi_d(80, _RSP_REGNO, _XMM8_REGNO);
3655 if (jit_regset_tstbit(&_jitc->function->regset, _XMM7))
3656 sse_stxi_d(72, _RSP_REGNO, _XMM7_REGNO);
3657 if (jit_regset_tstbit(&_jitc->function->regset, _XMM6))
3658 sse_stxi_d(64, _RSP_REGNO, _XMM6_REGNO);
3659 if (jit_regset_tstbit(&_jitc->function->regset, _R15))
3660 stxi(56, _RSP_REGNO, _R15_REGNO);
3661 if (jit_regset_tstbit(&_jitc->function->regset, _R14))
3662 stxi(48, _RSP_REGNO, _R14_REGNO);
3663 if (jit_regset_tstbit(&_jitc->function->regset, _R13))
3664 stxi(40, _RSP_REGNO, _R13_REGNO);
3665 if (jit_regset_tstbit(&_jitc->function->regset, _R12))
3666 stxi(32, _RSP_REGNO, _R12_REGNO);
3667 if (jit_regset_tstbit(&_jitc->function->regset, _RSI))
3668 stxi(24, _RSP_REGNO, _RSI_REGNO);
3669 if (jit_regset_tstbit(&_jitc->function->regset, _RDI))
3670 stxi(16, _RSP_REGNO, _RDI_REGNO);
3671 if (jit_regset_tstbit(&_jitc->function->regset, _RBX))
3672 stxi( 8, _RSP_REGNO, _RBX_REGNO);
3673# else
3674 if (jit_regset_tstbit(&_jitc->function->regset, _RBX))
3675 stxi(40, _RSP_REGNO, _RBX_REGNO);
3676 if (jit_regset_tstbit(&_jitc->function->regset, _R12))
3677 stxi(32, _RSP_REGNO, _R12_REGNO);
3678 if (jit_regset_tstbit(&_jitc->function->regset, _R13))
3679 stxi(24, _RSP_REGNO, _R13_REGNO);
3680 if (jit_regset_tstbit(&_jitc->function->regset, _R14))
3681 stxi(16, _RSP_REGNO, _R14_REGNO);
3682 if (jit_regset_tstbit(&_jitc->function->regset, _R15))
3683 stxi( 8, _RSP_REGNO, _R15_REGNO);
3684# endif
3685#endif
3686 stxi(0, _RSP_REGNO, _RBP_REGNO);
3687 movr(_RBP_REGNO, _RSP_REGNO);
3688
3689 /* alloca */
3690 subi(_RSP_REGNO, _RSP_REGNO, _jitc->function->stack);
3691 if (_jitc->function->allocar) {
3692 reg = jit_get_reg(jit_class_gpr);
3693 movi(rn(reg), _jitc->function->self.aoff);
3694 stxi_i(_jitc->function->aoffoff, _RBP_REGNO, rn(reg));
3695 jit_unget_reg(reg);
3696 }
3697
3698#if __X64 && !(__CYGWIN__ || _WIN32)
3699 if (_jitc->function->self.call & jit_call_varargs) {
3700 jit_word_t nofp_code;
3701
3702 /* Save gp registers in the save area, if any is a vararg */
3703 for (reg = first_gp_from_offset(_jitc->function->vagp);
3704 jit_arg_reg_p(reg); ++reg)
3705 stxi(_jitc->function->vaoff + first_gp_offset +
3706 reg * 8, _RBP_REGNO, rn(JIT_RA0 - reg));
3707
3708 reg = first_fp_from_offset(_jitc->function->vafp);
3709 if (jit_arg_f_reg_p(reg)) {
3710 /* Skip over if no float registers were passed as argument */
3711 /* test %al, %al */
3712 ic(0x84);
3713 ic(0xc0);
3714 jes(0);
3715 nofp_code = _jit->pc.w;
3716
3717 /* Save fp registers in the save area, if any is a vararg */
3718 /* Note that the full 16 byte xmm is not saved, because
3719 * lightning only handles float and double, and, while
3720 * attempting to provide a va_list compatible pointer as
3721 * jit_va_start return, does not guarantee it (on all ports). */
3722 for (; jit_arg_f_reg_p(reg); ++reg)
3723 sse_stxi_d(_jitc->function->vaoff + first_fp_offset +
3724 reg * va_fp_increment, _RBP_REGNO, rn(_XMM0 - reg));
3725
3726 patch_rel_char(nofp_code, _jit->pc.w);
3727 }
3728 }
3729#endif
3730}
3731
3732static void
3733_epilog(jit_state_t *_jit, jit_node_t *node)
3734{
3735 if (_jitc->function->assume_frame)
3736 return;
3737 /* callee save registers */
3738 movr(_RSP_REGNO, _RBP_REGNO);
3739#if __X32
3740 if (jit_regset_tstbit(&_jitc->function->regset, _RDI))
3741 ldxi(_RDI_REGNO, _RSP_REGNO, 12);
3742 if (jit_regset_tstbit(&_jitc->function->regset, _RSI))
3743 ldxi(_RSI_REGNO, _RSP_REGNO, 8);
3744 if (jit_regset_tstbit(&_jitc->function->regset, _RBX))
3745 ldxi(_RBX_REGNO, _RSP_REGNO, 4);
3746#else
3747# if __CYGWIN__ || _WIN32
3748 if (jit_regset_tstbit(&_jitc->function->regset, _XMM15))
3749 sse_ldxi_d(_XMM15_REGNO, _RSP_REGNO, 136);
3750 if (jit_regset_tstbit(&_jitc->function->regset, _XMM14))
3751 sse_ldxi_d(_XMM14_REGNO, _RSP_REGNO, 128);
3752 if (jit_regset_tstbit(&_jitc->function->regset, _XMM13))
3753 sse_ldxi_d(_XMM13_REGNO, _RSP_REGNO, 120);
3754 if (jit_regset_tstbit(&_jitc->function->regset, _XMM12))
3755 sse_ldxi_d(_XMM12_REGNO, _RSP_REGNO, 112);
3756 if (jit_regset_tstbit(&_jitc->function->regset, _XMM11))
3757 sse_ldxi_d(_XMM11_REGNO, _RSP_REGNO, 104);
3758 if (jit_regset_tstbit(&_jitc->function->regset, _XMM10))
3759 sse_ldxi_d(_XMM10_REGNO, _RSP_REGNO, 96);
3760 if (jit_regset_tstbit(&_jitc->function->regset, _XMM9))
3761 sse_ldxi_d(_XMM9_REGNO, _RSP_REGNO, 88);
3762 if (jit_regset_tstbit(&_jitc->function->regset, _XMM8))
3763 sse_ldxi_d(_XMM8_REGNO, _RSP_REGNO, 80);
3764 if (jit_regset_tstbit(&_jitc->function->regset, _XMM7))
3765 sse_ldxi_d(_XMM7_REGNO, _RSP_REGNO, 72);
3766 if (jit_regset_tstbit(&_jitc->function->regset, _XMM6))
3767 sse_ldxi_d(_XMM6_REGNO, _RSP_REGNO, 64);
3768 if (jit_regset_tstbit(&_jitc->function->regset, _R15))
3769 ldxi(_R15_REGNO, _RSP_REGNO, 56);
3770 if (jit_regset_tstbit(&_jitc->function->regset, _R14))
3771 ldxi(_R14_REGNO, _RSP_REGNO, 48);
3772 if (jit_regset_tstbit(&_jitc->function->regset, _R13))
3773 ldxi(_R13_REGNO, _RSP_REGNO, 40);
3774 if (jit_regset_tstbit(&_jitc->function->regset, _R12))
3775 ldxi(_R12_REGNO, _RSP_REGNO, 32);
3776 if (jit_regset_tstbit(&_jitc->function->regset, _RSI))
3777 ldxi(_RSI_REGNO, _RSP_REGNO, 24);
3778 if (jit_regset_tstbit(&_jitc->function->regset, _RDI))
3779 ldxi(_RDI_REGNO, _RSP_REGNO, 16);
3780 if (jit_regset_tstbit(&_jitc->function->regset, _RBX))
3781 ldxi(_RBX_REGNO, _RSP_REGNO, 8);
3782# else
3783 if (jit_regset_tstbit(&_jitc->function->regset, _RBX))
3784 ldxi(_RBX_REGNO, _RSP_REGNO, 40);
3785 if (jit_regset_tstbit(&_jitc->function->regset, _R12))
3786 ldxi(_R12_REGNO, _RSP_REGNO, 32);
3787 if (jit_regset_tstbit(&_jitc->function->regset, _R13))
3788 ldxi(_R13_REGNO, _RSP_REGNO, 24);
3789 if (jit_regset_tstbit(&_jitc->function->regset, _R14))
3790 ldxi(_R14_REGNO, _RSP_REGNO, 16);
3791 if (jit_regset_tstbit(&_jitc->function->regset, _R15))
3792 ldxi(_R15_REGNO, _RSP_REGNO, 8);
3793# endif
3794#endif
3795 ldxi(_RBP_REGNO, _RSP_REGNO, 0);
3796 addi(_RSP_REGNO, _RSP_REGNO, stack_framesize - REAL_WORDSIZE);
3797
3798 ic(0xc3);
3799}
3800
3801static void
3802_vastart(jit_state_t *_jit, jit_int32_t r0)
3803{
3804#if __X32 || __CYGWIN__ || _WIN32
3805 assert(_jitc->function->self.call & jit_call_varargs);
3806 addi(r0, _RBP_REGNO, _jitc->function->self.size);
3807#else
3808 jit_int32_t reg;
3809
3810 assert(_jitc->function->self.call & jit_call_varargs);
3811
3812 /* Return jit_va_list_t in the register argument */
3813 addi(r0, _RBP_REGNO, _jitc->function->vaoff);
3814 reg = jit_get_reg(jit_class_gpr);
3815
3816 /* Initialize gp offset in the save area. */
3817 movi(rn(reg), _jitc->function->vagp);
3818 stxi_i(offsetof(jit_va_list_t, gpoff), r0, rn(reg));
3819
3820 /* Initialize fp offset in the save area. */
3821 movi(rn(reg), _jitc->function->vafp);
3822 stxi_i(offsetof(jit_va_list_t, fpoff), r0, rn(reg));
3823
3824 /* Initialize overflow pointer to the first stack argument. */
3825 addi(rn(reg), _RBP_REGNO, _jitc->function->self.size);
3826 stxi(offsetof(jit_va_list_t, over), r0, rn(reg));
3827
3828 /* Initialize register save area pointer. */
3829 addi(rn(reg), r0, first_gp_offset);
3830 stxi(offsetof(jit_va_list_t, save), r0, rn(reg));
3831
3832 jit_unget_reg(reg);
3833#endif
3834}
3835
3836static void
3837_vaarg(jit_state_t *_jit, jit_int32_t r0, jit_int32_t r1)
3838{
3839#if __X32 || __CYGWIN__ || _WIN32
3840 assert(_jitc->function->self.call & jit_call_varargs);
3841 ldr(r0, r1);
3842 addi(r1, r1, va_gp_increment);
3843#else
3844 jit_int32_t rg0;
3845 jit_int32_t rg1;
3846 jit_word_t ge_code;
3847 jit_word_t lt_code;
3848
3849 assert(_jitc->function->self.call & jit_call_varargs);
3850
3851 rg0 = jit_get_reg(jit_class_gpr);
3852 rg1 = jit_get_reg(jit_class_gpr);
3853
3854 /* Load the gp offset in save area in the first temporary. */
3855 ldxi_i(rn(rg0), r1, offsetof(jit_va_list_t, gpoff));
3856
3857 /* Jump over if there are no remaining arguments in the save area. */
3858 icmpi(rn(rg0), va_gp_max_offset);
3859 jaes(0);
3860 ge_code = _jit->pc.w;
3861
3862 /* Load the save area pointer in the second temporary. */
3863 ldxi(rn(rg1), r1, offsetof(jit_va_list_t, save));
3864
3865 /* Load the vararg argument in the first argument. */
3866 ldxr(r0, rn(rg1), rn(rg0));
3867
3868 /* Update the gp offset. */
3869 addi(rn(rg0), rn(rg0), 8);
3870 stxi_i(offsetof(jit_va_list_t, gpoff), r1, rn(rg0));
3871
3872 /* Will only need one temporary register below. */
3873 jit_unget_reg(rg1);
3874
3875 /* Jump over overflow code. */
3876 jmpsi(0);
3877 lt_code = _jit->pc.w;
3878
3879 /* Where to land if argument is in overflow area. */
3880 patch_rel_char(ge_code, _jit->pc.w);
3881
3882 /* Load overflow pointer. */
3883 ldxi(rn(rg0), r1, offsetof(jit_va_list_t, over));
3884
3885 /* Load argument. */
3886 ldr(r0, rn(rg0));
3887
3888 /* Update overflow pointer. */
3889 addi(rn(rg0), rn(rg0), va_gp_increment);
3890 stxi(offsetof(jit_va_list_t, over), r1, rn(rg0));
3891
3892 /* Where to land if argument is in save area. */
3893 patch_rel_char(lt_code, _jit->pc.w);
3894
3895 jit_unget_reg(rg0);
3896#endif
3897}
3898
3899/* The x87 boolean argument tells if will put the result in a x87
3900 * register if non false, in a sse register otherwise. */
3901static void
3902_vaarg_d(jit_state_t *_jit, jit_int32_t r0, jit_int32_t r1, jit_bool_t x87)
3903{
3904#if __X32 || __CYGWIN__ || _WIN32
3905 assert(_jitc->function->self.call & jit_call_varargs);
3906 if (x87)
3907 x87_ldr_d(r0, r1);
3908 else
3909 sse_ldr_d(r0, r1);
3910 addi(r1, r1, 8);
3911#else
3912 jit_int32_t rg0;
3913 jit_int32_t rg1;
3914 jit_word_t ge_code;
3915 jit_word_t lt_code;
3916
3917 assert(_jitc->function->self.call & jit_call_varargs);
3918
3919 rg0 = jit_get_reg(jit_class_gpr);
3920 rg1 = jit_get_reg(jit_class_gpr);
3921
3922 /* Load the fp offset in save area in the first temporary. */
3923 ldxi_i(rn(rg0), r1, offsetof(jit_va_list_t, fpoff));
3924
3925 /* Jump over if there are no remaining arguments in the save area. */
3926 icmpi(rn(rg0), va_fp_max_offset);
3927 jaes(0);
3928 ge_code = _jit->pc.w;
3929
3930 /* Load the save area pointer in the second temporary. */
3931 ldxi(rn(rg1), r1, offsetof(jit_va_list_t, save));
3932
3933 /* Load the vararg argument in the first argument. */
3934 if (x87)
3935 x87_ldxr_d(r0, rn(rg1), rn(rg0));
3936 else
3937 sse_ldxr_d(r0, rn(rg1), rn(rg0));
3938
3939 /* Update the fp offset. */
3940 addi(rn(rg0), rn(rg0), va_fp_increment);
3941 stxi_i(offsetof(jit_va_list_t, fpoff), r1, rn(rg0));
3942
3943 /* Will only need one temporary register below. */
3944 jit_unget_reg(rg1);
3945
3946 /* Jump over overflow code. */
3947 jmpsi(0);
3948 lt_code = _jit->pc.w;
3949
3950 /* Where to land if argument is in overflow area. */
3951 patch_rel_char(ge_code, _jit->pc.w);
3952
3953 /* Load overflow pointer. */
3954 ldxi(rn(rg0), r1, offsetof(jit_va_list_t, over));
3955
3956 /* Load argument. */
3957 if (x87)
3958 x87_ldr_d(r0, rn(rg0));
3959 else
3960 sse_ldr_d(r0, rn(rg0));
3961
3962 /* Update overflow pointer. */
3963 addi(rn(rg0), rn(rg0), 8);
3964 stxi(offsetof(jit_va_list_t, over), r1, rn(rg0));
3965
3966 /* Where to land if argument is in save area. */
3967 patch_rel_char(lt_code, _jit->pc.w);
3968
3969 jit_unget_reg(rg0);
3970#endif
3971}
3972
3973static void
3974_patch_at(jit_state_t *_jit, jit_node_t *node,
3975 jit_word_t instr, jit_word_t label)
3976{
3977 switch (node->code) {
3978# if __X64
3979 case jit_code_calli:
519a9ea1 3980 case jit_code_jmpi:
4a71579b
PC		 3981# endif
3982 case jit_code_movi:
3983 patch_abs(instr, label);
3984 break;
3985 default:
3986 patch_rel(instr, label);
3987 break;
3988 }
3989}
3990#endif
ARM optimized PCSX fork