notaz.gp2x.de / pcsx_rearmed.git / blame
? search:
summary | shortlog | log | commit | commitdiff | tree
blob | blame (incremental) | history | HEAD
git subrepo pull (merge) --force deps/lightning
[pcsx_rearmed.git] / deps / lightning / lib / jit_x86-cpu.c
CommitLineData
4a71579b 1/*
c0c16242 2 * Copyright (C) 2012-2022 Free Software Foundation, Inc.
4a71579b
PC		 3 *
4 * This file is part of GNU lightning.
5 *
6 * GNU lightning is free software; you can redistribute it and/or modify it
7 * under the terms of the GNU Lesser General Public License as published
8 * by the Free Software Foundation; either version 3, or (at your option)
9 * any later version.
10 *
11 * GNU lightning is distributed in the hope that it will be useful, but
12 * WITHOUT ANY WARRANTY; without even the implied warranty of MERCHANTABILITY
13 * or FITNESS FOR A PARTICULAR PURPOSE. See the GNU Lesser General Public
14 * License for more details.
15 *
16 * Authors:
17 * Paulo Cesar Pereira de Andrade
18 */
19
20/* avoid using it due to partial stalls */
21#define USE_INC_DEC 0
22
23#if PROTO
24# if __X32 || __X64_32
25# define WIDE 0
26# define ldi(u, v) ldi_i(u, v)
27# define ldr(u, v) ldr_i(u, v)
28# define ldxr(u, v, w) ldxr_i(u, v, w)
29# define ldxi(u, v, w) ldxi_i(u, v, w)
30# define sti(u, v) sti_i(u, v)
31# define stxi(u, v, w) stxi_i(u, v, w)
32# define can_sign_extend_int_p(im) 1
33# define can_zero_extend_int_p(im) 1
34# define fits_uint32_p(im) 1
35# else
36# define WIDE 1
37# define ldi(u, v) ldi_l(u, v)
38# define ldr(u, v) ldr_l(u, v)
39# define ldxr(u, v, w) ldxr_l(u, v, w)
40# define ldxi(u, v, w) ldxi_l(u, v, w)
41# define sti(u, v) sti_l(u, v)
42# define stxi(u, v, w) stxi_l(u, v, w)
43# define can_sign_extend_int_p(im) \
44 (((im) >= 0 && (long long)(im) <= 0x7fffffffLL) || \
45 ((im) < 0 && (long long)(im) > -0x80000000LL))
46# define can_zero_extend_int_p(im) \
47 ((im) >= 0 && (im) < 0x80000000LL)
48# define fits_uint32_p(im) (((im) & 0xffffffff00000000LL) == 0)
49# endif
50# if __X32 || __CYGWIN__ || __X64_32 || _WIN32
51# define reg8_p(rn) \
52 ((rn) >= _RAX_REGNO && (rn) <= _RBX_REGNO)
53# else
54# define reg8_p(rn) 1
55# endif
56# define _RAX_REGNO 0
57# define _RCX_REGNO 1
58# define _RDX_REGNO 2
59# define _RBX_REGNO 3
60# define _RSP_REGNO 4
61# define _RBP_REGNO 5
62# define _RSI_REGNO 6
63# define _RDI_REGNO 7
64# define _R8_REGNO 8
65# define _R9_REGNO 9
66# define _R10_REGNO 10
67# define _R11_REGNO 11
68# define _R12_REGNO 12
69# define _R13_REGNO 13
70# define _R14_REGNO 14
71# define _R15_REGNO 15
72# define r7(reg) ((reg) & 7)
73# define r8(reg) ((reg) & 15)
74# define _SCL1 0x00
75# define _SCL2 0x01
76# define _SCL4 0x02
77# define _SCL8 0x03
78# define X86_ADD 0
79# define X86_OR 1 << 3
80# define X86_ADC 2 << 3
81# define X86_SBB 3 << 3
82# define X86_AND 4 << 3
83# define X86_SUB 5 << 3
84# define X86_XOR 6 << 3
85# define X86_CMP 7 << 3
86# define X86_ROL 0
87# define X86_ROR 1
88# define X86_RCL 2
89# define X86_RCR 3
90# define X86_SHL 4
91# define X86_SHR 5
92# define X86_SAR 7
93# define X86_NOT 2
94# define X86_NEG 3
95# define X86_MUL 4
96# define X86_IMUL 5
97# define X86_DIV 6
98# define X86_IDIV 7
99# define X86_CC_O 0x0
100# define X86_CC_NO 0x1
101# define X86_CC_NAE 0x2
102# define X86_CC_B 0x2
103# define X86_CC_C 0x2
104# define X86_CC_AE 0x3
105# define X86_CC_NB 0x3
106# define X86_CC_NC 0x3
107# define X86_CC_E 0x4
108# define X86_CC_Z 0x4
109# define X86_CC_NE 0x5
110# define X86_CC_NZ 0x5
111# define X86_CC_BE 0x6
112# define X86_CC_NA 0x6
113# define X86_CC_A 0x7
114# define X86_CC_NBE 0x7
115# define X86_CC_S 0x8
116# define X86_CC_NS 0x9
117# define X86_CC_P 0xa
118# define X86_CC_PE 0xa
119# define X86_CC_NP 0xb
120# define X86_CC_PO 0xb
121# define X86_CC_L 0xc
122# define X86_CC_NGE 0xc
123# define X86_CC_GE 0xd
124# define X86_CC_NL 0xd
125# define X86_CC_LE 0xe
126# define X86_CC_NG 0xe
127# define X86_CC_G 0xf
128# define X86_CC_NLE 0xf
129# define mrm(md, r, m) *_jit->pc.uc++ = (md<<6) | (r<<3) | m
130# define sib(sc, i, b) *_jit->pc.uc++ = (sc<<6) | (i<<3) | b
131# define ic(c) *_jit->pc.uc++ = c
132# define is(s) *_jit->pc.us++ = s
133# define ii(i) *_jit->pc.ui++ = i
134# if __X64 && !__X64_32
135# define il(l) *_jit->pc.ul++ = l
136# else
137# define il(l) ii(l)
138# endif
139# define patch_abs(instr, label) \
140 *(jit_word_t *)(instr - sizeof(jit_word_t)) = label
141# define patch_rel(instr, label) \
142 *(jit_int32_t *)(instr - 4) = label - instr
143# define patch_rel_char(instr, label) \
144 *(jit_int8_t *)(instr - 1) = label - instr
145# define rex(l, w, r, x, b) _rex(_jit, l, w, r, x, b)
146static void
147_rex(jit_state_t*,jit_int32_t,jit_int32_t,jit_int32_t,jit_int32_t,jit_int32_t);
148# define rx(rd, md, rb, ri, ms) _rx(_jit, rd, md, rb, ri, ms)
149static void
150_rx(jit_state_t*,jit_int32_t,jit_int32_t,jit_int32_t,jit_int32_t,jit_int32_t);
151# define nop(n) _nop(_jit, n)
152static void _nop(jit_state_t*, jit_int32_t);
153# define emms() is(0x770f)
154# define lea(md, rb, ri, ms, rd) _lea(_jit, md, rb, ri, ms, rd)
155static void
156_lea(jit_state_t*,jit_int32_t,jit_int32_t,jit_int32_t,jit_int32_t,jit_int32_t);
157# define pushr(r0) _pushr(_jit, r0)
158static void _pushr(jit_state_t*, jit_int32_t) maybe_unused;
159# define popr(r0) _popr(_jit, r0)
160static void _popr(jit_state_t*, jit_int32_t) maybe_unused;
161# define xchgr(r0, r1) _xchgr(_jit, r0, r1)
162static void _xchgr(jit_state_t*, jit_int32_t, jit_int32_t);
163# define testr(r0, r1) _testr(_jit, r0, r1)
164static void _testr(jit_state_t*, jit_int32_t, jit_int32_t);
165# define testi(r0, i0) _testi(_jit, r0, i0)
166static void _testi(jit_state_t*, jit_int32_t, jit_word_t);
167# define cc(code, r0) _cc(_jit, code, r0)
168static void _cc(jit_state_t*, jit_int32_t, jit_int32_t);
169# define icmpr(r0, r1) alur(X86_CMP, r0, r1)
170# define alur(code, r0, r1) _alur(_jit, code, r0, r1)
171static void _alur(jit_state_t*, jit_int32_t, jit_int32_t, jit_int32_t);
172# define icmpi(r0, i0) alui(X86_CMP, r0, i0)
173# define alui(code, r0, i0) _alui(_jit, code, r0, i0)
174static void _alui(jit_state_t*, jit_int32_t, jit_int32_t, jit_word_t);
175# define iaddr(r0, r1) alur(X86_ADD, r0, r1)
176# define save(r0) _save(_jit, r0)
177static void _save(jit_state_t*, jit_int32_t);
178# define load(r0) _load(_jit, r0)
179static void _load(jit_state_t*, jit_int32_t);
180# define addr(r0, r1, r2) _addr(_jit, r0, r1, r2)
181static void _addr(jit_state_t*, jit_int32_t, jit_int32_t, jit_int32_t);
182# define iaddi(r0, i0) alui(X86_ADD, r0, i0)
183# define addi(r0, r1, i0) _addi(_jit, r0, r1, i0)
184static void _addi(jit_state_t*, jit_int32_t, jit_int32_t, jit_word_t);
185#define addcr(r0, r1, r2) _addcr(_jit, r0, r1, r2)
186static void _addcr(jit_state_t*, jit_int32_t, jit_int32_t, jit_int32_t);
187#define addci(r0, r1, i0) _addci(_jit, r0, r1, i0)
188static void _addci(jit_state_t*, jit_int32_t, jit_int32_t, jit_word_t);
189# define iaddxr(r0, r1) alur(X86_ADC, r0, r1)
190# define addxr(r0, r1, r2) _addxr(_jit, r0, r1, r2)
191static void _addxr(jit_state_t*, jit_int32_t, jit_int32_t, jit_int32_t);
192# define iaddxi(r0, i0) alui(X86_ADC, r0, i0)
193# define addxi(r0, r1, i0) _addxi(_jit, r0, r1, i0)
194static void _addxi(jit_state_t*, jit_int32_t, jit_int32_t, jit_word_t);
195# define isubr(r0, r1) alur(X86_SUB, r0, r1)
196# define subr(r0, r1, r2) _subr(_jit, r0, r1, r2)
197static void _subr(jit_state_t*, jit_int32_t, jit_int32_t, jit_int32_t);
198# define isubi(r0, i0) alui(X86_SUB, r0, i0)
199# define subi(r0, r1, i0) _subi(_jit, r0, r1, i0)
200static void _subi(jit_state_t*, jit_int32_t, jit_int32_t, jit_word_t);
201# define subcr(r0, r1, r2) _subcr(_jit, r0, r1, r2)
202static void _subcr(jit_state_t*,jit_int32_t,jit_int32_t,jit_int32_t);
203# define subci(r0, r1, i0) _subci(_jit, r0, r1, i0)
204static void _subci(jit_state_t*,jit_int32_t,jit_int32_t,jit_word_t);
205# define isubxr(r0, r1) alur(X86_SBB, r0, r1)
206# define subxr(r0, r1, r2) _subxr(_jit, r0, r1, r2)
207static void _subxr(jit_state_t*,jit_int32_t,jit_int32_t,jit_int32_t);
208# define isubxi(r0, i0) alui(X86_SBB, r0, i0)
209# define subxi(r0, r1, i0) _subxi(_jit, r0, r1, i0)
210static void _subxi(jit_state_t*,jit_int32_t,jit_int32_t,jit_word_t);
211# define rsbi(r0, r1, i0) _rsbi(_jit, r0, r1, i0)
212static void _rsbi(jit_state_t*,jit_int32_t,jit_int32_t,jit_word_t);
213# define imulr(r0, r1) _imulr(_jit, r0, r1)
214static void _imulr(jit_state_t*, jit_int32_t, jit_int32_t);
215# define imuli(r0, r1, i0) _imuli(_jit, r0, r1, i0)
216static void _imuli(jit_state_t*, jit_int32_t, jit_int32_t, jit_word_t);
217# define mulr(r0, r1, r2) _mulr(_jit, r0, r1, r2)
218static void _mulr(jit_state_t*, jit_int32_t, jit_int32_t, jit_int32_t);
219# define muli(r0, r1, i0) _muli(_jit, r0, r1, i0)
220static void _muli(jit_state_t*, jit_int32_t, jit_int32_t, jit_word_t);
221# define umulr(r0) unr(X86_IMUL, r0)
222# define umulr_u(r0) unr(X86_MUL, r0)
223# define qmulr(r0, r1, r2, r3) _iqmulr(_jit, r0, r1, r2, r3, 1)
224# define qmulr_u(r0, r1, r2, r3) _iqmulr(_jit, r0, r1, r2, r3, 0)
225# define iqmulr(r0, r1, r2, r3, sign) _iqmulr(_jit, r0, r1, r2, r3, sign)
226static void _iqmulr(jit_state_t*, jit_int32_t, jit_int32_t,
227 jit_int32_t,jit_int32_t, jit_bool_t);
228# define qmuli(r0, r1, r2, i0) _iqmuli(_jit, r0, r1, r2, i0, 1)
229# define qmuli_u(r0, r1, r2, i0) _iqmuli(_jit, r0, r1, r2, i0, 0)
230# define iqmuli(r0, r1, r2, i0, sign) _iqmuli(_jit, r0, r1, r2, i0, sign)
231static void _iqmuli(jit_state_t*, jit_int32_t, jit_int32_t,
232 jit_int32_t,jit_word_t, jit_bool_t);
233# define sign_extend_rdx_rax() _sign_extend_rdx_rax(_jit)
234static void _sign_extend_rdx_rax(jit_state_t*);
235# define idivr(r0) unr(X86_IDIV, r0)
236# define idivr_u(r0) unr(X86_DIV, r0)
237# define divremr(r0, r1, r2, i0, i1) _divremr(_jit, r0, r1, r2, i0, i1)
238static void
239_divremr(jit_state_t*,jit_int32_t,jit_int32_t,jit_int32_t,
240 jit_bool_t,jit_bool_t);
241# define divremi(r0, r1, i0, i1, i2) _divremi(_jit, r0, r1, i0, i1, i2)
242static void
243_divremi(jit_state_t*,jit_int32_t,jit_int32_t,jit_word_t,jit_bool_t,jit_bool_t);
244# define divr(r0, r1, r2) divremr(r0, r1, r2, 1, 1)
245# define divi(r0, r1, i0) divremi(r0, r1, i0, 1, 1)
246# define divr_u(r0, r1, r2) divremr(r0, r1, r2, 0, 1)
247# define divi_u(r0, r1, i0) divremi(r0, r1, i0, 0, 1)
248# define qdivr(r0, r1, r2, r3) _iqdivr(_jit, r0, r1, r2, r3, 1)
249# define qdivr_u(r0, r1, r2, r3) _iqdivr(_jit, r0, r1, r2, r3, 0)
250# define iqdivr(r0, r1, r2, r3, sign) _iqdivr(_jit, r0, r1, r2, r3, sign)
251static void _iqdivr(jit_state_t*, jit_int32_t, jit_int32_t,
252 jit_int32_t,jit_int32_t, jit_bool_t);
253# define qdivi(r0, r1, r2, i0) _iqdivi(_jit, r0, r1, r2, i0, 1)
254# define qdivi_u(r0, r1, r2, i0) _iqdivi(_jit, r0, r1, r2, i0, 0)
255# define iqdivi(r0, r1, r2, i0, sign) _iqdivi(_jit, r0, r1, r2, i0, sign)
256static void _iqdivi(jit_state_t*, jit_int32_t, jit_int32_t,
257 jit_int32_t,jit_word_t, jit_bool_t);
258# define remr(r0, r1, r2) divremr(r0, r1, r2, 1, 0)
259# define remi(r0, r1, i0) divremi(r0, r1, i0, 1, 0)
260# define remr_u(r0, r1, r2) divremr(r0, r1, r2, 0, 0)
261# define remi_u(r0, r1, i0) divremi(r0, r1, i0, 0, 0)
262# define iandr(r0, r1) alur(X86_AND, r0, r1)
263# define andr(r0, r1, r2) _andr(_jit, r0, r1, r2)
264static void _andr(jit_state_t*,jit_int32_t,jit_int32_t,jit_int32_t);
265# define iandi(r0, i0) alui(X86_AND, r0, i0)
266# define andi(r0, r1, i0) _andi(_jit, r0, r1, i0)
267static void _andi(jit_state_t*, jit_int32_t,jit_int32_t,jit_word_t);
268# define iorr(r0, r1) alur(X86_OR, r0, r1)
269# define orr(r0, r1, r2) _orr(_jit, r0, r1, r2)
270static void _orr(jit_state_t*, jit_int32_t,jit_int32_t,jit_int32_t);
271# define iori(r0, i0) alui(X86_OR, r0, i0)
272# define ori(r0, r1, i0) _ori(_jit, r0, r1, i0)
273static void _ori(jit_state_t*, jit_int32_t,jit_int32_t,jit_word_t);
274# define ixorr(r0, r1) alur(X86_XOR, r0, r1)
275# define xorr(r0, r1, r2) _xorr(_jit, r0, r1, r2)
276static void _xorr(jit_state_t*, jit_int32_t,jit_int32_t,jit_int32_t);
277# define ixori(r0, i0) alui(X86_XOR, r0, i0)
278# define xori(r0, r1, i0) _xori(_jit, r0, r1, i0)
279static void _xori(jit_state_t*, jit_int32_t,jit_int32_t,jit_word_t);
280# define irotshr(code, r0) _irotshr(_jit, code, r0)
281static void _irotshr(jit_state_t*, jit_int32_t, jit_int32_t);
282# define rotshr(code, r0, r1, r2) _rotshr(_jit, code, r0, r1, r2)
283static void
284_rotshr(jit_state_t*,jit_int32_t,jit_int32_t,jit_int32_t,jit_int32_t);
285# define irotshi(code, r0, i0) _irotshi(_jit, code, r0, i0)
286static void _irotshi(jit_state_t*, jit_int32_t, jit_int32_t, jit_word_t);
287# define rotshi(code, r0, r1, i0) _rotshi(_jit, code, r0, r1, i0)
288static void
289_rotshi(jit_state_t*,jit_int32_t,jit_int32_t,jit_int32_t,jit_word_t);
290# define lshr(r0, r1, r2) rotshr(X86_SHL, r0, r1, r2)
291# define lshi(r0, r1, i0) _lshi(_jit, r0, r1, i0)
292static void _lshi(jit_state_t*, jit_int32_t, jit_int32_t, jit_word_t);
293# define rshr(r0, r1, r2) rotshr(X86_SAR, r0, r1, r2)
294# define rshi(r0, r1, i0) rotshi(X86_SAR, r0, r1, i0)
295# define rshr_u(r0, r1, r2) rotshr(X86_SHR, r0, r1, r2)
296# define rshi_u(r0, r1, i0) rotshi(X86_SHR, r0, r1, i0)
297# define unr(code, r0) _unr(_jit, code, r0)
298static void _unr(jit_state_t*, jit_int32_t, jit_int32_t);
299# define inegr(r0) unr(X86_NEG, r0)
300# define negr(r0, r1) _negr(_jit, r0, r1)
301static void _negr(jit_state_t*, jit_int32_t, jit_int32_t);
302# define icomr(r0) unr(X86_NOT, r0)
303# define comr(r0, r1) _comr(_jit, r0, r1)
304static void _comr(jit_state_t*, jit_int32_t, jit_int32_t);
305# if USE_INC_DEC
306# define incr(r0, r1) _incr(_jit, r0, r1)
307static void _incr(jit_state_t*, jit_int32_t, jit_int32_t);
308# define decr(r0, r1) _decr(_jit, r0, r1)
309static void _decr(jit_state_t*, jit_int32_t, jit_int32_t);
310# endif
311# define cr(code, r0, r1, r2) _cr(_jit, code, r0, r1, r2)
312static void
313_cr(jit_state_t*, jit_int32_t, jit_int32_t, jit_int32_t, jit_int32_t);
314# define ci(code, r0, r1, i0) _ci(_jit, code, r0, r1, i0)
315static void
316_ci(jit_state_t *_jit, jit_int32_t, jit_int32_t, jit_int32_t, jit_word_t);
317# define ci0(code, r0, r1) _ci0(_jit, code, r0, r1)
318static void _ci0(jit_state_t*, jit_int32_t, jit_int32_t, jit_int32_t);
319# define ltr(r0, r1, r2) _ltr(_jit, r0, r1, r2)
320static void _ltr(jit_state_t*, jit_int32_t, jit_int32_t, jit_int32_t);
321# define lti(r0, r1, i0) _lti(_jit, r0, r1, i0)
322static void _lti(jit_state_t*, jit_int32_t, jit_int32_t, jit_word_t);
323# define ltr_u(r0, r1, r2) _ltr_u(_jit, r0, r1, r2)
324static void _ltr_u(jit_state_t*, jit_int32_t, jit_int32_t, jit_int32_t);
325# define lti_u(r0, r1, i0) ci(X86_CC_B, r0, r1, i0)
326# define ler(r0, r1, r2) _ler(_jit, r0, r1, r2)
327static void _ler(jit_state_t*, jit_int32_t, jit_int32_t, jit_int32_t);
328# define lei(r0, r1, i0) ci(X86_CC_LE, r0, r1, i0)
329# define ler_u(r0, r1, r2) _ler_u(_jit, r0, r1, r2)
330static void _ler_u(jit_state_t*, jit_int32_t, jit_int32_t, jit_int32_t);
331# define lei_u(r0, r1, i0) _lei_u(_jit, r0, r1, i0)
332static void _lei_u(jit_state_t*, jit_int32_t, jit_int32_t, jit_word_t);
333# define eqr(r0, r1, r2) _eqr(_jit, r0, r1, r2)
334static void _eqr(jit_state_t*, jit_int32_t, jit_int32_t, jit_int32_t);
335# define eqi(r0, r1, i0) _eqi(_jit, r0, r1, i0)
336static void _eqi(jit_state_t*, jit_int32_t, jit_int32_t, jit_word_t);
337# define ger(r0, r1, r2) _ger(_jit, r0, r1, r2)
338static void _ger(jit_state_t*, jit_int32_t, jit_int32_t, jit_int32_t);
339# define gei(r0, r1, i0) _gei(_jit, r0, r1, i0)
340static void _gei(jit_state_t*, jit_int32_t, jit_int32_t, jit_word_t);
341# define ger_u(r0, r1, r2) _ger_u(_jit, r0, r1, r2)
342static void _ger_u(jit_state_t*, jit_int32_t, jit_int32_t, jit_int32_t);
343# define gei_u(r0, r1, i0) _gei_u(_jit, r0, r1, i0)
344static void _gei_u(jit_state_t*, jit_int32_t, jit_int32_t, jit_word_t);
345# define gtr(r0, r1, r2) _gtr(_jit, r0, r1, r2)
346static void _gtr(jit_state_t*, jit_int32_t, jit_int32_t, jit_int32_t);
347# define gti(r0, r1, i0) _ci(_jit, X86_CC_G, r0, r1, i0)
348# define gtr_u(r0, r1, r2) _gtr_u(_jit, r0, r1, r2)
349static void _gtr_u(jit_state_t*, jit_int32_t, jit_int32_t, jit_int32_t);
350# define gti_u(r0, r1, i0) _gti_u(_jit, r0, r1, i0)
351static void _gti_u(jit_state_t*, jit_int32_t, jit_int32_t, jit_word_t);
352# define ner(r0, r1, r2) _ner(_jit, r0, r1, r2)
353static void _ner(jit_state_t*, jit_int32_t, jit_int32_t, jit_int32_t);
354# define nei(r0, r1, i0) _nei(_jit, r0, r1, i0)
355static void _nei(jit_state_t*, jit_int32_t, jit_int32_t, jit_word_t);
356# define movr(r0, r1) _movr(_jit, r0, r1)
357static void _movr(jit_state_t*, jit_int32_t, jit_int32_t);
358# define imovi(r0, i0) _imovi(_jit, r0, i0)
359static void _imovi(jit_state_t*, jit_int32_t, jit_word_t);
360# define movi(r0, i0) _movi(_jit, r0, i0)
361static void _movi(jit_state_t*, jit_int32_t, jit_word_t);
362# define movi_p(r0, i0) _movi_p(_jit, r0, i0)
363static jit_word_t _movi_p(jit_state_t*, jit_int32_t, jit_word_t);
364# define movcr(r0, r1) _movcr(_jit, r0, r1)
365static void _movcr(jit_state_t*,jit_int32_t,jit_int32_t);
366# define movcr_u(r0, r1) _movcr_u(_jit, r0, r1)
367static void _movcr_u(jit_state_t*,jit_int32_t,jit_int32_t);
368# define movsr(r0, r1) _movsr(_jit, r0, r1)
369static void _movsr(jit_state_t*,jit_int32_t,jit_int32_t);
370# define movsr_u(r0, r1) _movsr_u(_jit, r0, r1)
371static void _movsr_u(jit_state_t*,jit_int32_t,jit_int32_t);
ba3814c1
PC		 372# define casx(r0, r1, r2, r3, i0) _casx(_jit, r0, r1, r2, r3, i0)
373static void _casx(jit_state_t *_jit,jit_int32_t,jit_int32_t,
374 jit_int32_t,jit_int32_t,jit_word_t);
375#define casr(r0, r1, r2, r3) casx(r0, r1, r2, r3, 0)
376#define casi(r0, i0, r1, r2) casx(r0, _NOREG, r1, r2, i0)
1f22b268
PC		 377#define movnr(r0, r1, r2) _movnr(_jit, r0, r1, r2)
378static void _movnr(jit_state_t*, jit_int32_t, jit_int32_t, jit_int32_t);
379#define movzr(r0, r1, r2) _movzr(_jit, r0, r1, r2)
380static void _movzr(jit_state_t*, jit_int32_t, jit_int32_t, jit_int32_t);
4a71579b
PC		 381# if __X64 && !__X64_32
382# define movir(r0, r1) _movir(_jit, r0, r1)
383static void _movir(jit_state_t*,jit_int32_t,jit_int32_t);
384# define movir_u(r0, r1) _movir_u(_jit, r0, r1)
385static void _movir_u(jit_state_t*,jit_int32_t,jit_int32_t);
386# endif
40a44dcb
PC		 387# define bswapr_us(r0, r1) _bswapr_us(_jit, r0, r1)
388static void _bswapr_us(jit_state_t*,jit_int32_t,jit_int32_t);
389# define bswapr_ui(r0, r1) _bswapr_ui(_jit, r0, r1)
390static void _bswapr_ui(jit_state_t*,jit_int32_t,jit_int32_t);
4a71579b 391# if __X64 && !__X64_32
40a44dcb
PC		 392#define bswapr_ul(r0, r1) _bswapr_ul(_jit, r0, r1)
393static void _bswapr_ul(jit_state_t*,jit_int32_t,jit_int32_t);
4a71579b
PC		 394#endif
395# define extr_c(r0, r1) _extr_c(_jit, r0, r1)
396static void _extr_c(jit_state_t*,jit_int32_t,jit_int32_t);
397# define extr_uc(r0, r1) _extr_uc(_jit, r0, r1)
398static void _extr_uc(jit_state_t*,jit_int32_t,jit_int32_t);
399# define extr_s(r0, r1) movsr(r0, r1)
400# define extr_us(r0, r1) movsr_u(r0, r1)
401# if __X64 && !__X64_32
402# define extr_i(r0, r1) movir(r0, r1)
403# define extr_ui(r0, r1) movir_u(r0, r1)
404# endif
405# define ldr_c(r0, r1) _ldr_c(_jit, r0, r1)
406static void _ldr_c(jit_state_t*, jit_int32_t, jit_int32_t);
407# define ldi_c(r0, i0) _ldi_c(_jit, r0, i0)
408static void _ldi_c(jit_state_t*, jit_int32_t, jit_word_t);
409# define ldr_uc(r0, r1) _ldr_uc(_jit, r0, r1)
410static void _ldr_uc(jit_state_t*, jit_int32_t, jit_int32_t);
411# define ldi_uc(r0, i0) _ldi_uc(_jit, r0, i0)
412static void _ldi_uc(jit_state_t*, jit_int32_t, jit_word_t);
413# define ldr_s(r0, r1) _ldr_s(_jit, r0, r1)
414static void _ldr_s(jit_state_t*, jit_int32_t, jit_int32_t);
415# define ldi_s(r0, i0) _ldi_s(_jit, r0, i0)
416static void _ldi_s(jit_state_t*, jit_int32_t, jit_word_t);
417# define ldr_us(r0, r1) _ldr_us(_jit, r0, r1)
418static void _ldr_us(jit_state_t*, jit_int32_t, jit_int32_t);
419# define ldi_us(r0, i0) _ldi_us(_jit, r0, i0)
420static void _ldi_us(jit_state_t*, jit_int32_t, jit_word_t);
421# if __X32 || !__X64_32
422# define ldr_i(r0, r1) _ldr_i(_jit, r0, r1)
423static void _ldr_i(jit_state_t*, jit_int32_t, jit_int32_t);
424# define ldi_i(r0, i0) _ldi_i(_jit, r0, i0)
425static void _ldi_i(jit_state_t*, jit_int32_t, jit_word_t);
426# endif
427# if __X64
428# if __X64_32
429# define ldr_i(r0, r1) _ldr_ui(_jit, r0, r1)
430# define ldi_i(r0, i0) _ldi_ui(_jit, r0, i0)
431# else
432# define ldr_ui(r0, r1) _ldr_ui(_jit, r0, r1)
433# define ldi_ui(r0, i0) _ldi_ui(_jit, r0, i0)
434# endif
435static void _ldr_ui(jit_state_t*, jit_int32_t, jit_int32_t);
436static void _ldi_ui(jit_state_t*, jit_int32_t, jit_word_t);
437# if !__X64_32
438# define ldr_l(r0, r1) _ldr_l(_jit, r0, r1)
439static void _ldr_l(jit_state_t*, jit_int32_t, jit_int32_t);
440# define ldi_l(r0, i0) _ldi_l(_jit, r0, i0)
441static void _ldi_l(jit_state_t*, jit_int32_t, jit_word_t);
442# endif
443# endif
444# define ldxr_c(r0, r1, r2) _ldxr_c(_jit, r0, r1, r2)
445static void _ldxr_c(jit_state_t*, jit_int32_t, jit_int32_t, jit_int32_t);
446# define ldxi_c(r0, r1, i0) _ldxi_c(_jit, r0, r1, i0)
447static void _ldxi_c(jit_state_t*, jit_int32_t, jit_int32_t, jit_word_t);
448# define ldxr_uc(r0, r1, r2) _ldxr_uc(_jit, r0, r1, r2)
449static void _ldxr_uc(jit_state_t*, jit_int32_t, jit_int32_t, jit_int32_t);
450# define ldxi_uc(r0, r1, i0) _ldxi_uc(_jit, r0, r1, i0)
451static void _ldxi_uc(jit_state_t*, jit_int32_t, jit_int32_t, jit_word_t);
452# define ldxr_s(r0, r1, r2) _ldxr_s(_jit, r0, r1, r2)
453static void _ldxr_s(jit_state_t*, jit_int32_t, jit_int32_t, jit_int32_t);
454# define ldxi_s(r0, r1, i0) _ldxi_s(_jit, r0, r1, i0)
455static void _ldxi_s(jit_state_t*, jit_int32_t, jit_int32_t, jit_word_t);
456# define ldxr_us(r0, r1, r2) _ldxr_us(_jit, r0, r1, r2)
457static void _ldxr_us(jit_state_t*, jit_int32_t, jit_int32_t, jit_int32_t);
458# define ldxi_us(r0, r1, i0) _ldxi_us(_jit, r0, r1, i0)
459static void _ldxi_us(jit_state_t*, jit_int32_t, jit_int32_t, jit_word_t);
460# if __X32 || !__X64_32
461# define ldxr_i(r0, r1, r2) _ldxr_i(_jit, r0, r1, r2)
462static void _ldxr_i(jit_state_t*, jit_int32_t, jit_int32_t, jit_int32_t);
463# define ldxi_i(r0, r1, i0) _ldxi_i(_jit, r0, r1, i0)
464static void _ldxi_i(jit_state_t*, jit_int32_t, jit_int32_t, jit_word_t);
465# endif
466# if __X64
467# if __X64_32
468# define ldxr_i(r0, r1, r2) _ldxr_ui(_jit, r0, r1, r2)
469# define ldxi_i(r0, r1, i0) _ldxi_ui(_jit, r0, r1, i0)
470# else
471# define ldxr_ui(r0, r1, r2) _ldxr_ui(_jit, r0, r1, r2)
472# define ldxi_ui(r0, r1, i0) _ldxi_ui(_jit, r0, r1, i0)
473# endif
474static void _ldxr_ui(jit_state_t*, jit_int32_t, jit_int32_t, jit_int32_t);
475static void _ldxi_ui(jit_state_t*, jit_int32_t, jit_int32_t, jit_word_t);
476# if !__X64_32
477# define ldxr_l(r0, r1, r2) _ldxr_l(_jit, r0, r1, r2)
478static void _ldxr_l(jit_state_t*, jit_int32_t, jit_int32_t, jit_int32_t);
479# define ldxi_l(r0, r1, i0) _ldxi_l(_jit, r0, r1, i0)
480static void _ldxi_l(jit_state_t*, jit_int32_t, jit_int32_t, jit_word_t);
481# endif
482# endif
483# define str_c(r0, r1) _str_c(_jit, r0, r1)
484static void _str_c(jit_state_t*, jit_int32_t, jit_int32_t);
485# define sti_c(i0, r0) _sti_c(_jit, i0, r0)
486static void _sti_c(jit_state_t*, jit_word_t, jit_int32_t);
487# define str_s(r0, r1) _str_s(_jit, r0, r1)
488static void _str_s(jit_state_t*, jit_int32_t, jit_int32_t);
489# define sti_s(i0, r0) _sti_s(_jit, i0, r0)
490static void _sti_s(jit_state_t*, jit_word_t, jit_int32_t);
491# define str_i(r0, r1) _str_i(_jit, r0, r1)
492static void _str_i(jit_state_t*, jit_int32_t, jit_int32_t);
493# define sti_i(i0, r0) _sti_i(_jit, i0, r0)
494static void _sti_i(jit_state_t*, jit_word_t, jit_int32_t);
495# if __X64 && !__X64_32
496# define str_l(r0, r1) _str_l(_jit, r0, r1)
497static void _str_l(jit_state_t*, jit_int32_t, jit_int32_t);
498# define sti_l(i0, r0) _sti_l(_jit, i0, r0)
499static void _sti_l(jit_state_t*, jit_word_t, jit_int32_t);
500# endif
501# define stxr_c(r0, r1, r2) _stxr_c(_jit, r0, r1, r2)
502static void _stxr_c(jit_state_t*, jit_int32_t, jit_int32_t, jit_int32_t);
503# define stxi_c(i0, r0, r1) _stxi_c(_jit, i0, r0, r1)
504static void _stxi_c(jit_state_t*, jit_word_t, jit_int32_t, jit_int32_t);
505# define stxr_s(r0, r1, r2) _stxr_s(_jit, r0, r1, r2)
506static void _stxr_s(jit_state_t*, jit_int32_t, jit_int32_t, jit_int32_t);
507# define stxi_s(i0, r0, r1) _stxi_s(_jit, i0, r0, r1)
508static void _stxi_s(jit_state_t*, jit_word_t, jit_int32_t, jit_int32_t);
509# define stxr_i(r0, r1, r2) _stxr_i(_jit, r0, r1, r2)
510static void _stxr_i(jit_state_t*, jit_int32_t, jit_int32_t, jit_int32_t);
511# define stxi_i(i0, r0, r1) _stxi_i(_jit, i0, r0, r1)
512static void _stxi_i(jit_state_t*, jit_word_t, jit_int32_t, jit_int32_t);
513# if __X64 && !__X64_32
514# define stxr_l(r0, r1, r2) _stxr_l(_jit, r0, r1, r2)
515static void _stxr_l(jit_state_t*, jit_int32_t, jit_int32_t, jit_int32_t);
516# define stxi_l(i0, r0, r1) _stxi_l(_jit, i0, r0, r1)
517static void _stxi_l(jit_state_t*, jit_word_t, jit_int32_t, jit_int32_t);
518# endif
519# define jcc(code, i0) _jcc(_jit, code, i0)
520# define jo(i0) jcc(X86_CC_O, i0)
521# define jno(i0) jcc(X86_CC_NO, i0)
522# define jnae(i0) jcc(X86_CC_NAE, i0)
523# define jb(i0) jcc(X86_CC_B, i0)
524# define jc(i0) jcc(X86_CC_C, i0)
525# define jae(i0) jcc(X86_CC_AE, i0)
526# define jnb(i0) jcc(X86_CC_NB, i0)
527# define jnc(i0) jcc(X86_CC_NC, i0)
528# define je(i0) jcc(X86_CC_E, i0)
529# define jz(i0) jcc(X86_CC_Z, i0)
530# define jne(i0) jcc(X86_CC_NE, i0)
531# define jnz(i0) jcc(X86_CC_NZ, i0)
532# define jbe(i0) jcc(X86_CC_BE, i0)
533# define jna(i0) jcc(X86_CC_NA, i0)
534# define ja(i0) jcc(X86_CC_A, i0)
535# define jnbe(i0) jcc(X86_CC_NBE, i0)
536# define js(i0) jcc(X86_CC_S, i0)
537# define jns(i0) jcc(X86_CC_NS, i0)
538# define jp(i0) jcc(X86_CC_P, i0)
539# define jpe(i0) jcc(X86_CC_PE, i0)
540# define jnp(i0) jcc(X86_CC_NP, i0)
541# define jpo(i0) jcc(X86_CC_PO, i0)
542# define jl(i0) jcc(X86_CC_L, i0)
543# define jnge(i0) jcc(X86_CC_NGE, i0)
544# define jge(i0) jcc(X86_CC_GE, i0)
545# define jnl(i0) jcc(X86_CC_NL, i0)
546# define jle(i0) jcc(X86_CC_LE, i0)
547# define jng(i0) jcc(X86_CC_NG, i0)
548# define jg(i0) jcc(X86_CC_G, i0)
549# define jnle(i0) jcc(X86_CC_NLE, i0)
550static void _jcc(jit_state_t*, jit_int32_t, jit_word_t);
551# define jccs(code, i0) _jccs(_jit, code, i0)
552# define jos(i0) jccs(X86_CC_O, i0)
553# define jnos(i0) jccs(X86_CC_NO, i0)
554# define jnaes(i0) jccs(X86_CC_NAE, i0)
555# define jbs(i0) jccs(X86_CC_B, i0)
556# define jcs(i0) jccs(X86_CC_C, i0)
557# define jaes(i0) jccs(X86_CC_AE, i0)
558# define jnbs(i0) jccs(X86_CC_NB, i0)
559# define jncs(i0) jccs(X86_CC_NC, i0)
560# define jes(i0) jccs(X86_CC_E, i0)
561# define jzs(i0) jccs(X86_CC_Z, i0)
562# define jnes(i0) jccs(X86_CC_NE, i0)
563# define jnzs(i0) jccs(X86_CC_NZ, i0)
564# define jbes(i0) jccs(X86_CC_BE, i0)
565# define jnas(i0) jccs(X86_CC_NA, i0)
566# define jas(i0) jccs(X86_CC_A, i0)
567# define jnbes(i0) jccs(X86_CC_NBE, i0)
568# define jss(i0) jccs(X86_CC_S, i0)
569# define jnss(i0) jccs(X86_CC_NS, i0)
570# define jps(i0) jccs(X86_CC_P, i0)
571# define jpes(i0) jccs(X86_CC_PE, i0)
572# define jnps(i0) jccs(X86_CC_NP, i0)
573# define jpos(i0) jccs(X86_CC_PO, i0)
574# define jls(i0) jccs(X86_CC_L, i0)
575# define jnges(i0) jccs(X86_CC_NGE, i0)
576# define jges(i0) jccs(X86_CC_GE, i0)
577# define jnls(i0) jccs(X86_CC_NL, i0)
578# define jles(i0) jccs(X86_CC_LE, i0)
579# define jngs(i0) jccs(X86_CC_NG, i0)
580# define jgs(i0) jccs(X86_CC_G, i0)
581# define jnles(i0) jccs(X86_CC_NLE, i0)
582static void _jccs(jit_state_t*, jit_int32_t, jit_word_t);
583# define jcr(code, i0, r0, r1) _jcr(_jit, code, i0, r0, r1)
584static void _jcr(jit_state_t*,jit_int32_t,jit_word_t,jit_int32_t,jit_int32_t);
585# define jci(code, i0, r0, i1) _jci(_jit, code, i0, r0, i1)
586static void _jci(jit_state_t*,jit_int32_t,jit_word_t,jit_int32_t,jit_word_t);
587# define jci0(code, i0, r0) _jci0(_jit, code, i0, r0)
588static void _jci0(jit_state_t*, jit_int32_t, jit_word_t, jit_int32_t);
589# define bltr(i0, r0, r1) _bltr(_jit, i0, r0, r1)
590static jit_word_t _bltr(jit_state_t*, jit_word_t, jit_int32_t, jit_int32_t);
591# define blti(i0, r0, i1) _blti(_jit, i0, r0, i1)
592static jit_word_t _blti(jit_state_t*, jit_word_t, jit_int32_t, jit_word_t);
593# define bltr_u(i0, r0, r1) _bltr_u(_jit, i0, r0, r1)
594static jit_word_t _bltr_u(jit_state_t*, jit_word_t, jit_int32_t, jit_int32_t);
595# define blti_u(i0, r0, i1) _blti_u(_jit, i0, r0, i1)
596static jit_word_t _blti_u(jit_state_t*, jit_word_t, jit_int32_t, jit_word_t);
597# define bler(i0, r0, r1) _bler(_jit, i0, r0, r1)
598static jit_word_t _bler(jit_state_t*, jit_word_t, jit_int32_t, jit_int32_t);
599# define blei(i0, r0, i1) _blei(_jit, i0, r0, i1)
600static jit_word_t _blei(jit_state_t*, jit_word_t, jit_int32_t, jit_word_t);
601# define bler_u(i0, r0, r1) _bler_u(_jit, i0, r0, r1)
602static jit_word_t _bler_u(jit_state_t*, jit_word_t, jit_int32_t, jit_int32_t);
603# define blei_u(i0, r0, i1) _blei_u(_jit, i0, r0, i1)
604static jit_word_t _blei_u(jit_state_t*, jit_word_t, jit_int32_t, jit_word_t);
605# define beqr(i0, r0, r1) _beqr(_jit, i0, r0, r1)
606static jit_word_t _beqr(jit_state_t*, jit_word_t, jit_int32_t, jit_int32_t);
607# define beqi(i0, r0, i1) _beqi(_jit, i0, r0, i1)
608static jit_word_t _beqi(jit_state_t*, jit_word_t, jit_int32_t, jit_word_t);
609# define bger(i0, r0, r1) _bger(_jit, i0, r0, r1)
610static jit_word_t _bger(jit_state_t*, jit_word_t, jit_int32_t, jit_int32_t);
611# define bgei(i0, r0, i1) _bgei(_jit, i0, r0, i1)
612static jit_word_t _bgei(jit_state_t*, jit_word_t, jit_int32_t, jit_word_t);
613# define bger_u(i0, r0, r1) _bger_u(_jit, i0, r0, r1)
614static jit_word_t _bger_u(jit_state_t*, jit_word_t, jit_int32_t, jit_int32_t);
615# define bgei_u(i0, r0, i1) _bgei_u(_jit, i0, r0, i1)
616static jit_word_t _bgei_u(jit_state_t*, jit_word_t, jit_int32_t, jit_word_t);
617# define bgtr(i0, r0, r1) _bgtr(_jit, i0, r0, r1)
618static jit_word_t _bgtr(jit_state_t*, jit_word_t, jit_int32_t, jit_int32_t);
619# define bgti(i0, r0, i1) _bgti(_jit, i0, r0, i1)
620static jit_word_t _bgti(jit_state_t*, jit_word_t, jit_int32_t, jit_word_t);
621# define bgtr_u(i0, r0, r1) _bgtr_u(_jit, i0, r0, r1)
622static jit_word_t _bgtr_u(jit_state_t*, jit_word_t, jit_int32_t, jit_int32_t);
623# define bgti_u(i0, r0, i1) _bgti_u(_jit, i0, r0, i1)
624static jit_word_t _bgti_u(jit_state_t*, jit_word_t, jit_int32_t, jit_word_t);
625# define bner(i0, r0, r1) _bner(_jit, i0, r0, r1)
626static jit_word_t _bner(jit_state_t*, jit_word_t, jit_int32_t, jit_int32_t);
627# define bnei(i0, r0, i1) _bnei(_jit, i0, r0, i1)
628static jit_word_t _bnei(jit_state_t*, jit_word_t, jit_int32_t, jit_word_t);
629# define bmsr(i0, r0, r1) _bmsr(_jit, i0, r0, r1)
630static jit_word_t _bmsr(jit_state_t*,jit_word_t,jit_int32_t,jit_int32_t);
631# define bmsi(i0, r0, i1) _bmsi(_jit, i0, r0, i1)
632static jit_word_t _bmsi(jit_state_t*,jit_word_t,jit_int32_t,jit_word_t);
633# define bmcr(i0, r0, r1) _bmcr(_jit, i0, r0, r1)
634static jit_word_t _bmcr(jit_state_t*,jit_word_t,jit_int32_t,jit_int32_t);
635# define bmci(i0, r0, i1) _bmci(_jit, i0, r0, i1)
636static jit_word_t _bmci(jit_state_t*,jit_word_t,jit_int32_t,jit_word_t);
637# define boaddr(i0, r0, r1) _boaddr(_jit, i0, r0, r1)
638static jit_word_t _boaddr(jit_state_t*,jit_word_t,jit_int32_t,jit_int32_t);
639# define boaddi(i0, r0, i1) _boaddi(_jit, i0, r0, i1)
640static jit_word_t _boaddi(jit_state_t*,jit_word_t,jit_int32_t,jit_word_t);
641# define boaddr_u(i0, r0, r1) _boaddr_u(_jit, i0, r0, r1)
642static jit_word_t _boaddr_u(jit_state_t*,jit_word_t,jit_int32_t,jit_int32_t);
643# define boaddi_u(i0, r0, i1) _boaddi_u(_jit, i0, r0, i1)
644static jit_word_t _boaddi_u(jit_state_t*,jit_word_t,jit_int32_t,jit_word_t);
645# define bxaddr(i0, r0, r1) _bxaddr(_jit, i0, r0, r1)
646static jit_word_t _bxaddr(jit_state_t*,jit_word_t,jit_int32_t,jit_int32_t);
647# define bxaddi(i0, r0, i1) _bxaddi(_jit, i0, r0, i1)
648static jit_word_t _bxaddi(jit_state_t*,jit_word_t,jit_int32_t,jit_word_t);
649# define bxaddr_u(i0, r0, r1) _bxaddr_u(_jit, i0, r0, r1)
650static jit_word_t _bxaddr_u(jit_state_t*,jit_word_t,jit_int32_t,jit_int32_t);
651# define bxaddi_u(i0, r0, i1) _bxaddi_u(_jit, i0, r0, i1)
652static jit_word_t _bxaddi_u(jit_state_t*,jit_word_t,jit_int32_t,jit_word_t);
653# define bosubr(i0, r0, r1) _bosubr(_jit, i0, r0, r1)
654static jit_word_t _bosubr(jit_state_t*,jit_word_t,jit_int32_t,jit_int32_t);
655# define bosubi(i0, r0, i1) _bosubi(_jit, i0, r0, i1)
656static jit_word_t _bosubi(jit_state_t*,jit_word_t,jit_int32_t,jit_word_t);
657# define bosubr_u(i0, r0, r1) _bosubr_u(_jit, i0, r0, r1)
658static jit_word_t _bosubr_u(jit_state_t*,jit_word_t,jit_int32_t,jit_int32_t);
659# define bosubi_u(i0, r0, i1) _bosubi_u(_jit, i0, r0, i1)
660static jit_word_t _bosubi_u(jit_state_t*,jit_word_t,jit_int32_t,jit_word_t);
661# define bxsubr(i0, r0, r1) _bxsubr(_jit, i0, r0, r1)
662static jit_word_t _bxsubr(jit_state_t*,jit_word_t,jit_int32_t,jit_int32_t);
663# define bxsubi(i0, r0, i1) _bxsubi(_jit, i0, r0, i1)
664static jit_word_t _bxsubi(jit_state_t*,jit_word_t,jit_int32_t,jit_word_t);
665# define bxsubr_u(i0, r0, r1) _bxsubr_u(_jit, i0, r0, r1)
666static jit_word_t _bxsubr_u(jit_state_t*,jit_word_t,jit_int32_t,jit_int32_t);
667# define bxsubi_u(i0, r0, i1) _bxsubi_u(_jit, i0, r0, i1)
668static jit_word_t _bxsubi_u(jit_state_t*,jit_word_t,jit_int32_t,jit_word_t);
669# define callr(r0) _callr(_jit, r0)
670static void _callr(jit_state_t*, jit_int32_t);
671# define calli(i0) _calli(_jit, i0)
672static jit_word_t _calli(jit_state_t*, jit_word_t);
519a9ea1
PC		 673# if __X64
674# define calli_p(i0) _calli_p(_jit, i0)
675static jit_word_t _calli_p(jit_state_t*, jit_word_t);
676# else
677# define calli_p(i0) calli(i0)
678# endif
4a71579b
PC		 679# define jmpr(r0) _jmpr(_jit, r0)
680static void _jmpr(jit_state_t*, jit_int32_t);
681# define jmpi(i0) _jmpi(_jit, i0)
682static jit_word_t _jmpi(jit_state_t*, jit_word_t);
519a9ea1
PC		 683# if __X64
684# define jmpi_p(i0) _jmpi_p(_jit, i0)
685static jit_word_t _jmpi_p(jit_state_t*, jit_word_t);
686# else
687# define jmpi_p(i0) jmpi(i0)
688# endif
4a71579b
PC		 689# define jmpsi(i0) _jmpsi(_jit, i0)
690static void _jmpsi(jit_state_t*, jit_uint8_t);
691# define prolog(node) _prolog(_jit, node)
692static void _prolog(jit_state_t*, jit_node_t*);
693# define epilog(node) _epilog(_jit, node)
694static void _epilog(jit_state_t*, jit_node_t*);
695# define vastart(r0) _vastart(_jit, r0)
696static void _vastart(jit_state_t*, jit_int32_t);
697# define vaarg(r0, r1) _vaarg(_jit, r0, r1)
698static void _vaarg(jit_state_t*, jit_int32_t, jit_int32_t);
699# define vaarg_d(r0, r1, i0) _vaarg_d(_jit, r0, r1, i0)
700static void _vaarg_d(jit_state_t*, jit_int32_t, jit_int32_t, jit_bool_t);
701# define patch_at(node, instr, label) _patch_at(_jit, node, instr, label)
702static void _patch_at(jit_state_t*, jit_node_t*, jit_word_t, jit_word_t);
703# if !defined(HAVE_FFSL)
704# if __X32
705# define ffsl(i) __builtin_ffs(i)
706# else
707# define ffsl(l) __builtin_ffsl(l)
708# endif
709# endif
1f22b268 710# define jit_cmov_p() jit_cpu.cmov
4a71579b
PC		 711#endif
712
713#if CODE
714static void
715_rex(jit_state_t *_jit, jit_int32_t l, jit_int32_t w,
716 jit_int32_t r, jit_int32_t x, jit_int32_t b)
717{
718#if __X64
719 jit_int32_t v = 0x40 | (w << 3);
720
721 if (r != _NOREG)
722 v |= (r & 8) >> 1;
723 if (x != _NOREG)
724 v |= (x & 8) >> 2;
725 if (b != _NOREG)
726 v |= (b & 8) >> 3;
727 if (l || v != 0x40)
728 ic(v);
729#endif
730}
731
732static void
733_rx(jit_state_t *_jit, jit_int32_t rd, jit_int32_t md,
734 jit_int32_t rb, jit_int32_t ri, jit_int32_t ms)
735{
736 if (ri == _NOREG) {
737 if (rb == _NOREG) {
738#if __X32
739 mrm(0x00, r7(rd), 0x05);
740#else
741 mrm(0x00, r7(rd), 0x04);
742 sib(_SCL1, 0x04, 0x05);
743#endif
744 ii(md);
745 }
746 else if (r7(rb) == _RSP_REGNO) {
747 if (md == 0) {
748 mrm(0x00, r7(rd), 0x04);
749 sib(ms, 0x04, 0x04);
750 }
751 else if ((jit_int8_t)md == md) {
752 mrm(0x01, r7(rd), 0x04);
753 sib(ms, 0x04, 0x04);
754 ic(md);
755 }
756 else {
757 mrm(0x02, r7(rd), 0x04);
758 sib(ms, 0x04, 0x04);
759 ii(md);
760 }
761 }
762 else {
763 if (md == 0 && r7(rb) != _RBP_REGNO)
764 mrm(0x00, r7(rd), r7(rb));
765 else if ((jit_int8_t)md == md) {
766 mrm(0x01, r7(rd), r7(rb));
767 ic(md);
768 }
769 else {
770 mrm(0x02, r7(rd), r7(rb));
771 ii(md);
772 }
773 }
774 }
775 else if (rb == _NOREG) {
776 mrm(0x00, r7(rd), 0x04);
777 sib(ms, r7(ri), 0x05);
778 ii(md);
779 }
780 else if (r8(ri) != _RSP_REGNO) {
781 if (md == 0 && r7(rb) != _RBP_REGNO) {
782 mrm(0x00, r7(rd), 0x04);
783 sib(ms, r7(ri), r7(rb));
784 }
785 else if ((jit_int8_t)md == md) {
786 mrm(0x01, r7(rd), 0x04);
787 sib(ms, r7(ri), r7(rb));
788 ic(md);
789 }
790 else {
791 mrm(0x02, r7(rd), 0x04);
792 sib(ms, r7(ri), r7(rb));
793 ic(md);
794 }
795 }
796 else {
797 fprintf(stderr, "illegal index register");
798 abort();
799 }
800}
801
802static void
803_nop(jit_state_t *_jit, jit_int32_t count)
804{
c0c16242
PC		 805 jit_int32_t i;
806 while (count) {
807 if (count > 9)
808 i = 9;
809 else
810 i = count;
811 switch (i) {
812 case 0:
813 break;
814 case 1: /* NOP */
815 ic(0x90); break;
816 case 2: /* 66 NOP */
817 ic(0x66); ic(0x90);
818 break;
819 case 3: /* NOP DWORD ptr [EAX] */
820 ic(0x0f); ic(0x1f); ic(0x00);
821 break;
822 case 4: /* NOP DWORD ptr [EAX + 00H] */
823 ic(0x0f); ic(0x1f); ic(0x40); ic(0x00);
824 break;
825 case 5: /* NOP DWORD ptr [EAX + EAX*1 + 00H] */
826 ic(0x0f); ic(0x1f); ic(0x44); ic(0x00);
827 ic(0x00);
828 break;
829 case 6: /* 66 NOP DWORD ptr [EAX + EAX*1 + 00H] */
830 ic(0x66); ic(0x0f); ic(0x1f); ic(0x44);
831 ic(0x00); ic(0x00);
832 break;
833 case 7: /* NOP DWORD ptr [EAX + 00000000H] */
834 ic(0x0f); ic(0x1f); ic(0x80); ii(0x0000);
835 break;
836 case 8: /* NOP DWORD ptr [EAX + EAX*1 + 00000000H] */
837 ic(0x0f); ic(0x1f); ic(0x84); ic(0x00);
838 ii(0x0000);
839 break;
840 case 9: /* 66 NOP DWORD ptr [EAX + EAX*1 + 00000000H] */
841 ic(0x66); ic(0x0f); ic(0x1f); ic(0x84);
842 ic(0x00); ii(0x0000);
843 break;
844 }
845 count -= i;
4a71579b
PC		 846 }
847}
4a71579b
PC		 848static void
849_lea(jit_state_t *_jit, jit_int32_t md, jit_int32_t rb,
850 jit_int32_t ri, jit_int32_t ms, jit_int32_t rd)
851{
852 rex(0, WIDE, rd, ri, rb);
853 ic(0x8d);
854 rx(rd, md, rb, ri, ms);
855}
856
857static void
858_pushr(jit_state_t *_jit, jit_int32_t r0)
859{
860 rex(0, WIDE, 0, 0, r0);
861 ic(0x50 | r7(r0));
862}
863
864static void
865_popr(jit_state_t *_jit, jit_int32_t r0)
866{
867 rex(0, WIDE, 0, 0, r0);
868 ic(0x58 | r7(r0));
869}
870
871static void
872_xchgr(jit_state_t *_jit, jit_int32_t r0, jit_int32_t r1)
873{
874 rex(0, WIDE, r1, _NOREG, r0);
875 ic(0x87);
876 mrm(0x03, r7(r1), r7(r0));
877}
878
879static void
880_testr(jit_state_t *_jit, jit_int32_t r0, jit_int32_t r1)
881{
882 rex(0, WIDE, r1, _NOREG, r0);
883 ic(0x85);
884 mrm(0x03, r7(r1), r7(r0));
885}
886
887static void
888_testi(jit_state_t *_jit, jit_int32_t r0, jit_word_t i0)
889{
890 rex(0, WIDE, _NOREG, _NOREG, r0);
891 if (r0 == _RAX_REGNO)
892 ic(0xa9);
893 else {
894 ic(0xf7);
895 mrm(0x03, 0x00, r7(r0));
896 }
897 ii(i0);
898}
899
900static void
901_cc(jit_state_t *_jit, jit_int32_t code, jit_int32_t r0)
902{
903 rex(0, 0, _NOREG, _NOREG, r0);
904 ic(0x0f);
905 ic(0x90 | code);
906 mrm(0x03, 0x00, r7(r0));
907}
908
909static void
910_alur(jit_state_t *_jit, jit_int32_t code, jit_int32_t r0, jit_int32_t r1)
911{
912 rex(0, WIDE, r1, _NOREG, r0);
913 ic(code | 0x01);
914 mrm(0x03, r7(r1), r7(r0));
915}
916
917static void
918_alui(jit_state_t *_jit, jit_int32_t code, jit_int32_t r0, jit_word_t i0)
919{
920 jit_int32_t reg;
921 if (can_sign_extend_int_p(i0)) {
922 rex(0, WIDE, _NOREG, _NOREG, r0);
923 if ((jit_int8_t)i0 == i0) {
924 ic(0x83);
925 ic(0xc0 | code | r7(r0));
926 ic(i0);
927 }
928 else {
929 if (r0 == _RAX_REGNO)
930 ic(code | 0x05);
931 else {
932 ic(0x81);
933 ic(0xc0 | code | r7(r0));
934 }
935 ii(i0);
936 }
937 }
938 else {
939 reg = jit_get_reg(jit_class_gpr);
940 movi(rn(reg), i0);
941 alur(code, r0, rn(reg));
942 jit_unget_reg(reg);
943 }
944}
945
946static void
947_save(jit_state_t *_jit, jit_int32_t r0)
948{
949 if (!_jitc->function->regoff[r0]) {
950 _jitc->function->regoff[r0] = jit_allocai(sizeof(jit_word_t));
951 _jitc->again = 1;
952 }
953 assert(!jit_regset_tstbit(&_jitc->regsav, r0));
954 jit_regset_setbit(&_jitc->regsav, r0);
955 stxi(_jitc->function->regoff[r0], _RBP_REGNO, r0);
956}
957
958static void
959_load(jit_state_t *_jit, jit_int32_t r0)
960{
961 assert(_jitc->function->regoff[r0]);
962 assert(jit_regset_tstbit(&_jitc->regsav, r0));
963 jit_regset_clrbit(&_jitc->regsav, r0);
964 ldxi(r0, _RBP_REGNO, _jitc->function->regoff[r0]);
965}
966
967static void
968_addr(jit_state_t *_jit, jit_int32_t r0, jit_int32_t r1, jit_int32_t r2)
969{
970 if (r0 == r1)
971 iaddr(r0, r2);
972 else if (r0 == r2)
973 iaddr(r0, r1);
974 else
975 lea(0, r1, r2, _SCL1, r0);
976}
977
978static void
979_addi(jit_state_t *_jit, jit_int32_t r0, jit_int32_t r1, jit_word_t i0)
980{
981 jit_int32_t reg;
982 if (i0 == 0)
983 movr(r0, r1);
984#if USE_INC_DEC
985 else if (i0 == 1)
986 incr(r0, r1);
987 else if (i0 == -1)
988 decr(r0, r1);
989#endif
990 else if (can_sign_extend_int_p(i0)) {
991 if (r0 == r1)
992 iaddi(r0, i0);
993 else
994 lea(i0, r1, _NOREG, _SCL1, r0);
995 }
996 else if (r0 != r1) {
997 movi(r0, i0);
998 iaddr(r0, r1);
999 }
1000 else {
1001 reg = jit_get_reg(jit_class_gpr);
1002 movi(rn(reg), i0);
1003 iaddr(r0, rn(reg));
1004 jit_unget_reg(reg);
1005 }
1006}
1007
1008static void
1009_addcr(jit_state_t *_jit, jit_int32_t r0, jit_int32_t r1, jit_int32_t r2)
1010{
1011 if (r0 == r2)
1012 iaddr(r0, r1);
1013 else {
1014 movr(r0, r1);
1015 iaddr(r0, r2);
1016 }
1017}
1018
1019static void
1020_addci(jit_state_t *_jit, jit_int32_t r0, jit_int32_t r1, jit_word_t i0)
1021{
1022 jit_int32_t reg;
1023 if (can_sign_extend_int_p(i0)) {
1024 movr(r0, r1);
1025 iaddi(r0, i0);
1026 }
1027 else if (r0 == r1) {
1028 reg = jit_get_reg(jit_class_gpr);
1029 movi(rn(reg), i0);
1030 iaddr(r0, rn(reg));
1031 jit_unget_reg(reg);
1032 }
1033 else {
1034 movi(r0, i0);
1035 iaddr(r0, r1);
1036 }
1037}
1038
1039static void
1040_addxr(jit_state_t *_jit, jit_int32_t r0, jit_int32_t r1, jit_int32_t r2)
1041{
1042 if (r0 == r2)
1043 iaddxr(r0, r1);
1044 else {
1045 movr(r0, r1);
1046 iaddxr(r0, r2);
1047 }
1048}
1049
1050static void
1051_addxi(jit_state_t *_jit, jit_int32_t r0, jit_int32_t r1, jit_word_t i0)
1052{
1053 jit_int32_t reg;
1054 if (can_sign_extend_int_p(i0)) {
1055 movr(r0, r1);
1056 iaddxi(r0, i0);
1057 }
1058 else if (r0 == r1) {
1059 reg = jit_get_reg(jit_class_gpr);
1060 movi(rn(reg), i0);
1061 iaddxr(r0, rn(reg));
1062 jit_unget_reg(reg);
1063 }
1064 else {
1065 movi(r0, i0);
1066 iaddxr(r0, r1);
1067 }
1068}
1069
1070static void
1071_subr(jit_state_t *_jit, jit_int32_t r0, jit_int32_t r1, jit_int32_t r2)
1072{
1073 if (r1 == r2)
1074 ixorr(r0, r0);
1075 else if (r0 == r2) {
1076 isubr(r0, r1);
1077 inegr(r0);
1078 }
1079 else {
1080 movr(r0, r1);
1081 isubr(r0, r2);
1082 }
1083}
1084
1085static void
1086_subi(jit_state_t *_jit, jit_int32_t r0, jit_int32_t r1, jit_word_t i0)
1087{
1088 jit_int32_t reg;
1089 if (i0 == 0)
1090 movr(r0, r1);
1091#if USE_INC_DEC
1092 else if (i0 == 1)
1093 decr(r0, r1);
1094 else if (i0 == -1)
1095 incr(r0, r1);
1096#endif
1097 else if (can_sign_extend_int_p(i0)) {
1098 if (r0 == r1)
1099 isubi(r0, i0);
1100 else
1101 lea(-i0, r1, _NOREG, _SCL1, r0);
1102 }
1103 else if (r0 != r1) {
1104 movi(r0, -i0);
1105 iaddr(r0, r1);
1106 }
1107 else {
1108 reg = jit_get_reg(jit_class_gpr);
1109 movi(rn(reg), i0);
1110 isubr(r0, rn(reg));
1111 jit_unget_reg(reg);
1112 }
1113}
1114
1115static void
1116_subcr(jit_state_t *_jit, jit_int32_t r0, jit_int32_t r1, jit_int32_t r2)
1117{
1118 jit_int32_t reg;
1119 if (r0 == r2 && r0 != r1) {
1120 reg = jit_get_reg(jit_class_gpr);
1121 movr(rn(reg), r0);
1122 movr(r0, r1);
1123 isubr(r0, rn(reg));
1124 jit_unget_reg(reg);
1125 }
1126 else {
1127 movr(r0, r1);
1128 isubr(r0, r2);
1129 }
1130}
1131
1132static void
1133_subci(jit_state_t *_jit, jit_int32_t r0, jit_int32_t r1, jit_word_t i0)
1134{
1135 jit_int32_t reg;
1136 movr(r0, r1);
1137 if (can_sign_extend_int_p(i0))
1138 isubi(r0, i0);
1139 else {
1140 reg = jit_get_reg(jit_class_gpr);
1141 movi(rn(reg), i0);
1142 isubr(r0, rn(reg));
1143 jit_unget_reg(reg);
1144 }
1145}
1146
1147static void
1148_subxr(jit_state_t *_jit, jit_int32_t r0, jit_int32_t r1, jit_int32_t r2)
1149{
1150 jit_int32_t reg;
1151 if (r0 == r2 && r0 != r1) {
1152 reg = jit_get_reg(jit_class_gpr);
1153 movr(rn(reg), r0);
1154 movr(r0, r1);
1155 isubxr(r0, rn(reg));
1156 jit_unget_reg(reg);
1157 }
1158 else {
1159 movr(r0, r1);
1160 isubxr(r0, r2);
1161 }
1162}
1163
1164static void
1165_subxi(jit_state_t *_jit, jit_int32_t r0, jit_int32_t r1, jit_word_t i0)
1166{
1167 jit_int32_t reg;
1168 movr(r0, r1);
1169 if (can_sign_extend_int_p(i0))
1170 isubxi(r0, i0);
1171 else {
1172 reg = jit_get_reg(jit_class_gpr);
1173 imovi(rn(reg), i0);
1174 isubxr(r0, rn(reg));
1175 jit_unget_reg(reg);
1176 }
1177}
1178
1179static void
1180_rsbi(jit_state_t *_jit, jit_int32_t r0, jit_int32_t r1, jit_word_t i0)
1181{
1182 subi(r0, r1, i0);
1183 negr(r0, r0);
1184}
1185
1186static void
1187_imulr(jit_state_t *_jit, jit_int32_t r0, jit_int32_t r1)
1188{
1189 rex(0, WIDE, r0, _NOREG, r1);
1190 ic(0x0f);
1191 ic(0xaf);
1192 mrm(0x03, r7(r0), r7(r1));
1193}
1194
1195static void
1196_imuli(jit_state_t *_jit, jit_int32_t r0, jit_int32_t r1, jit_word_t i0)
1197{
1198 jit_int32_t reg;
1199 if (can_sign_extend_int_p(i0)) {
1200 rex(0, WIDE, r0, _NOREG, r1);
1201 if ((jit_int8_t)i0 == i0) {
1202 ic(0x6b);
1203 mrm(0x03, r7(r0), r7(r1));
1204 ic(i0);
1205 }
1206 else {
1207 ic(0x69);
1208 mrm(0x03, r7(r0), r7(r1));
1209 ii(i0);
1210 }
1211 }
1212 else {
1213 reg = jit_get_reg(jit_class_gpr);
1214 movi(rn(reg), i0);
1215 imulr(r0, rn(reg));
1216 jit_unget_reg(reg);
1217 }
1218}
1219
1220static void
1221_mulr(jit_state_t *_jit, jit_int32_t r0, jit_int32_t r1, jit_int32_t r2)
1222{
1223 if (r0 == r1)
1224 imulr(r0, r2);
1225 else if (r0 == r2)
1226 imulr(r0, r1);
1227 else {
1228 movr(r0, r1);
1229 imulr(r0, r2);
1230 }
1231}
1232
1233static void
1234_muli(jit_state_t *_jit, jit_int32_t r0, jit_int32_t r1, jit_word_t i0)
1235{
1236 switch (i0) {
1237 case 0:
1238 ixorr(r0, r0);
1239 break;
1240 case 1:
1241 movr(r0, r1);
1242 break;
1243 case -1:
1244 negr(r0, r1);
1245 break;
1246 case 2:
1247 lea(0, _NOREG, r1, _SCL2, r0);
1248 break;
1249 case 4:
1250 lea(0, _NOREG, r1, _SCL4, r0);
1251 break;
1252 case 8:
1253 lea(0, _NOREG, r1, _SCL8, r0);
1254 break;
1255 default:
1256 if (i0 > 0 && !(i0 & (i0 - 1)))
1257 lshi(r0, r1, ffsl(i0) - 1);
1258 else if (can_sign_extend_int_p(i0))
1259 imuli(r0, r1, i0);
1260 else if (r0 != r1) {
1261 movi(r0, i0);
1262 imulr(r0, r1);
1263 }
1264 else
1265 imuli(r0, r0, i0);
1266 break;
1267 }
1268}
1269
1270#define savset(rn) \
1271 if (r0 != rn) { \
1272 sav |= 1 << rn; \
1273 if (r1 != rn && r2 != rn) \
1274 set |= 1 << rn; \
1275 }
1276#define isavset(rn) \
1277 if (r0 != rn) { \
1278 sav |= 1 << rn; \
1279 if (r1 != rn) \
1280 set |= 1 << rn; \
1281 }
1282#define qsavset(rn) \
1283 if (r0 != rn && r1 != rn) { \
1284 sav |= 1 << rn; \
1285 if (r2 != rn && r3 != rn) \
1286 set |= 1 << rn; \
1287 }
1288#define allocr(rn, rv) \
1289 if (set & (1 << rn)) \
1290 (void)jit_get_reg(rv|jit_class_gpr|jit_class_named); \
1291 if (sav & (1 << rn)) { \
1292 if ( jit_regset_tstbit(&_jitc->regsav, rv) || \
1293 !jit_regset_tstbit(&_jitc->reglive, rv)) \
1294 sav &= ~(1 << rn); \
1295 else \
1296 save(rv); \
1297 }
1298#define clear(rn, rv) \
1299 if (set & (1 << rn)) \
1300 jit_unget_reg(rv); \
1301 if (sav & (1 << rn)) \
1302 load(rv);
1303static void
1304_iqmulr(jit_state_t *_jit, jit_int32_t r0, jit_int32_t r1,
1305 jit_int32_t r2, jit_int32_t r3, jit_bool_t sign)
1306{
1307 jit_int32_t mul;
1308 jit_int32_t sav;
1309 jit_int32_t set;
1310
1311 sav = set = 0;
1312 qsavset(_RDX_REGNO);
1313 qsavset(_RAX_REGNO);
1314 allocr(_RDX_REGNO, _RDX);
1315 allocr(_RAX_REGNO, _RAX);
1316
1317 if (r3 == _RAX_REGNO)
1318 mul = r2;
1319 else {
1320 mul = r3;
1321 movr(_RAX_REGNO, r2);
1322 }
1323 if (sign)
1324 umulr(mul);
1325 else
1326 umulr_u(mul);
1327
1328 if (r0 == _RDX_REGNO && r1 == _RAX_REGNO)
1329 xchgr(_RAX_REGNO, _RDX_REGNO);
1330 else {
1331 if (r0 != _RDX_REGNO)
1332 movr(r0, _RAX_REGNO);
1333 movr(r1, _RDX_REGNO);
1334 if (r0 == _RDX_REGNO)
1335 movr(r0, _RAX_REGNO);
1336 }
1337
1338 clear(_RDX_REGNO, _RDX);
1339 clear(_RAX_REGNO, _RAX);
1340}
1341
1342static void
1343_iqmuli(jit_state_t *_jit, jit_int32_t r0, jit_int32_t r1,
1344 jit_int32_t r2, jit_word_t i0, jit_bool_t sign)
1345{
1346 jit_int32_t reg;
1347
1348 if (i0 == 0) {
1349 ixorr(r0, r0);
1350 ixorr(r1, r1);
1351 }
1352 else {
1353 reg = jit_get_reg(jit_class_gpr);
1354 movi(rn(reg), i0);
1355 if (sign)
1356 qmulr(r0, r1, r2, rn(reg));
1357 else
1358 qmulr_u(r0, r1, r2, rn(reg));
1359 jit_unget_reg(reg);
1360 }
1361}
1362
1363static void
1364_sign_extend_rdx_rax(jit_state_t *_jit)
1365{
1366 rex(0, WIDE, 0, 0, 0);
1367 ic(0x99);
1368}
1369
1370static void
1371_divremr(jit_state_t *_jit, jit_int32_t r0, jit_int32_t r1, jit_int32_t r2,
1372 jit_bool_t sign, jit_bool_t divide)
1373{
1374 jit_int32_t div;
1375 jit_int32_t reg;
1376 jit_int32_t set;
1377 jit_int32_t sav;
1378 jit_int32_t use;
1379
1380 sav = set = use = 0;
1381 savset(_RDX_REGNO);
1382 savset(_RAX_REGNO);
1383 allocr(_RDX_REGNO, _RDX);
1384 allocr(_RAX_REGNO, _RAX);
1385
1386 if (r2 == _RAX_REGNO) {
1387 if (r0 == _RAX_REGNO || r0 == _RDX_REGNO) {
1388 if ((reg = jit_get_reg(jit_class_gpr|jit_class_chk)) == JIT_NOREG)
1389 reg = jit_get_reg((r1 == _RCX_REGNO ? _RBX : _RCX) |
1390 jit_class_gpr|jit_class_named);
1391 use = 1;
1392 div = rn(reg);
1393 movr(div, _RAX_REGNO);
1394 if (r1 != _RAX_REGNO)
1395 movr(_RAX_REGNO, r1);
1396 }
1397 else {
1398 if (r0 == r1)
1399 xchgr(r0, _RAX_REGNO);
1400 else {
1401 if (r0 != _RAX_REGNO)
1402 movr(r0, _RAX_REGNO);
1403 if (r1 != _RAX_REGNO)
1404 movr(_RAX_REGNO, r1);
1405 }
1406 div = r0;
1407 }
1408 }
1409 else if (r2 == _RDX_REGNO) {
1410 if (r0 == _RAX_REGNO || r0 == _RDX_REGNO) {
1411 if ((reg = jit_get_reg(jit_class_gpr|jit_class_chk)) == JIT_NOREG)
1412 reg = jit_get_reg((r1 == _RCX_REGNO ? _RBX : _RCX) |
1413 jit_class_gpr|jit_class_named);
1414 use = 1;
1415 div = rn(reg);
1416 movr(div, _RDX_REGNO);
1417 if (r1 != _RAX_REGNO)
1418 movr(_RAX_REGNO, r1);
1419 }
1420 else {
1421 if (r1 != _RAX_REGNO)
1422 movr(_RAX_REGNO, r1);
1423 movr(r0, _RDX_REGNO);
1424 div = r0;
1425 }
1426 }
1427 else {
1428 if (r1 != _RAX_REGNO)
1429 movr(_RAX_REGNO, r1);
1430 div = r2;
1431 }
1432
1433 if (sign) {
1434 sign_extend_rdx_rax();
1435 idivr(div);
1436 }
1437 else {
1438 ixorr(_RDX_REGNO, _RDX_REGNO);
1439 idivr_u(div);
1440 }
1441
1442 if (use)
1443 jit_unget_reg(reg);
1444
1445 if (divide)
1446 movr(r0, _RAX_REGNO);
1447 else
1448 movr(r0, _RDX_REGNO);
1449
1450 clear(_RDX_REGNO, _RDX);
1451 clear(_RAX_REGNO, _RAX);
1452}
1453
1454static void
1455_divremi(jit_state_t *_jit, jit_int32_t r0, jit_int32_t r1, jit_word_t i0,
1456 jit_bool_t sign, jit_bool_t divide)
1457{
1458 jit_int32_t reg;
1459 jit_int32_t div;
1460 jit_int32_t sav;
1461 jit_int32_t set;
1462 jit_int32_t use;
1463
1464 if (divide) {
1465 switch (i0) {
1466 case 1:
1467 movr(r0, r1);
1468 return;
1469 case -1:
1470 if (sign) {
1471 negr(r0, r1);
1472 return;
1473 }
1474 break;
1475 default:
1476 if (i0 > 0 && !(i0 & (i0 - 1))) {
1477 movr(r0, r1);
1478 if (sign)
1479 rshi(r0, r0, ffsl(i0) - 1);
1480 else
1481 rshi_u(r0, r0, ffsl(i0) - 1);
1482 return;
1483 }
1484 break;
1485 }
1486 }
1487 else if (i0 == 1 || (sign && i0 == -1)) {
1488 ixorr(r0, r0);
1489 return;
1490 }
1491 else if (!sign && i0 > 0 && !(i0 & (i0 - 1))) {
1492 if (can_sign_extend_int_p(i0)) {
1493 movr(r0, r1);
1494 iandi(r0, i0 - 1);
1495 }
1496 else if (r0 != r1) {
1497 movi(r0, i0 - 1);
1498 iandr(r0, r1);
1499 }
1500 else {
1501 reg = jit_get_reg(jit_class_gpr);
1502 movi(rn(reg), i0 - 1);
1503 iandr(r0, rn(reg));
1504 jit_unget_reg(reg);
1505 }
1506 return;
1507 }
1508
1509 sav = set = use = 0;
1510 isavset(_RDX_REGNO);
1511 isavset(_RAX_REGNO);
1512 allocr(_RDX_REGNO, _RDX);
1513 allocr(_RAX_REGNO, _RAX);
1514
1515 if (r0 == _RAX_REGNO || r0 == _RDX_REGNO || r0 == r1) {
1516 if ((reg = jit_get_reg(jit_class_gpr|jit_class_chk)) == JIT_NOREG)
1517 reg = jit_get_reg((r1 == _RCX_REGNO ? _RBX : _RCX) |
1518 jit_class_gpr|jit_class_named);
1519 use = 1;
1520 div = rn(reg);
1521 }
1522 else
1523 div = r0;
1524
1525 movi(div, i0);
1526 movr(_RAX_REGNO, r1);
1527
1528 if (sign) {
1529 sign_extend_rdx_rax();
1530 idivr(div);
1531 }
1532 else {
1533 ixorr(_RDX_REGNO, _RDX_REGNO);
1534 idivr_u(div);
1535 }
1536
1537 if (use)
1538 jit_unget_reg(reg);
1539
1540 if (divide)
1541 movr(r0, _RAX_REGNO);
1542 else
1543 movr(r0, _RDX_REGNO);
1544
1545 clear(_RDX_REGNO, _RDX);
1546 clear(_RAX_REGNO, _RAX);
1547}
1548
1549static void
1550_iqdivr(jit_state_t *_jit, jit_int32_t r0, jit_int32_t r1,
1551 jit_int32_t r2, jit_int32_t r3, jit_bool_t sign)
1552{
1553 jit_int32_t div;
1554 jit_int32_t reg;
1555 jit_int32_t sav;
1556 jit_int32_t set;
1557 jit_int32_t use;
1558
1559 sav = set = use = 0;
1560 qsavset(_RDX_REGNO);
1561 qsavset(_RAX_REGNO);
1562 allocr(_RDX_REGNO, _RDX);
1563 allocr(_RAX_REGNO, _RAX);
1564 if (r3 == _RAX_REGNO) {
1565 if (r0 == _RAX_REGNO || r0 == _RDX_REGNO) {
1566 if ((reg = jit_get_reg(jit_class_gpr|jit_class_chk)) == JIT_NOREG)
1567 reg = jit_get_reg((r1 == _RCX_REGNO ? _RBX : _RCX) |
1568 jit_class_gpr|jit_class_named);
1569 use = 1;
1570 div = rn(reg);
1571 movr(div, _RAX_REGNO);
1572 if (r2 != _RAX_REGNO)
1573 movr(_RAX_REGNO, r2);
1574 }
1575 else {
1576 if (r0 == r2)
1577 xchgr(r0, _RAX_REGNO);
1578 else {
1579 if (r0 != _RAX_REGNO)
1580 movr(r0, _RAX_REGNO);
1581 if (r2 != _RAX_REGNO)
1582 movr(_RAX_REGNO, r2);
1583 }
1584 div = r0;
1585 }
1586 }
1587 else if (r3 == _RDX_REGNO) {
1588 if (r0 == _RAX_REGNO || r0 == _RDX_REGNO) {
1589 if ((reg = jit_get_reg(jit_class_gpr|jit_class_chk)) == JIT_NOREG)
1590 reg = jit_get_reg((r1 == _RCX_REGNO ? _RBX : _RCX) |
1591 jit_class_gpr|jit_class_named);
1592 use = 1;
1593 div = rn(reg);
1594 movr(div, _RDX_REGNO);
1595 if (r2 != _RAX_REGNO)
1596 movr(_RAX_REGNO, r2);
1597 }
1598 else {
1599 if (r2 != _RAX_REGNO)
1600 movr(_RAX_REGNO, r2);
1601 movr(r0, _RDX_REGNO);
1602 div = r0;
1603 }
1604 }
1605 else {
1606 if (r2 != _RAX_REGNO)
1607 movr(_RAX_REGNO, r2);
1608 div = r3;
1609 }
1610 if (sign) {
1611 sign_extend_rdx_rax();
1612 idivr(div);
1613 }
1614 else {
1615 ixorr(_RDX_REGNO, _RDX_REGNO);
1616 idivr_u(div);
1617 }
1618 if (use)
1619 jit_unget_reg(reg);
1620
1621 if (r0 == _RDX_REGNO && r1 == _RAX_REGNO)
1622 xchgr(_RAX_REGNO, _RDX_REGNO);
1623 else {
1624 if (r0 != _RDX_REGNO)
1625 movr(r0, _RAX_REGNO);
1626 movr(r1, _RDX_REGNO);
1627 if (r0 == _RDX_REGNO)
1628 movr(r0, _RAX_REGNO);
1629 }
1630
1631 clear(_RDX_REGNO, _RDX);
1632 clear(_RAX_REGNO, _RAX);
1633}
1634
1635static void
1636_iqdivi(jit_state_t *_jit, jit_int32_t r0, jit_int32_t r1,
1637 jit_int32_t r2, jit_word_t i0, jit_bool_t sign)
1638{
1639 jit_int32_t reg;
1640
1641 reg = jit_get_reg(jit_class_gpr);
1642 movi(rn(reg), i0);
1643 if (sign)
1644 qdivr(r0, r1, r2, rn(reg));
1645 else
1646 qdivr_u(r0, r1, r2, rn(reg));
1647 jit_unget_reg(reg);
1648}
1649#undef clear
1650#undef allocr
1651#undef savset
1652
1653static void
1654_andr(jit_state_t *_jit, jit_int32_t r0, jit_int32_t r1, jit_int32_t r2)
1655{
1656 if (r1 == r2)
1657 movr(r0, r1);
1658 else if (r0 == r1)
1659 iandr(r0, r2);
1660 else if (r0 == r2)
1661 iandr(r0, r1);
1662 else {
1663 movr(r0, r1);
1664 iandr(r0, r2);
1665 }
1666}
1667
1668static void
1669_andi(jit_state_t *_jit, jit_int32_t r0, jit_int32_t r1, jit_word_t i0)
1670{
1671 jit_int32_t reg;
1672
1673 if (i0 == 0)
1674 ixorr(r0, r0);
1675 else if (i0 == -1)
1676 movr(r0, r1);
1677 else if (r0 == r1) {
1678 if (can_sign_extend_int_p(i0))
1679 iandi(r0, i0);
1680 else {
1681 reg = jit_get_reg(jit_class_gpr);
1682 movi(rn(reg), i0);
1683 iandr(r0, rn(reg));
1684 jit_unget_reg(reg);
1685 }
1686 }
1687 else {
1688 movi(r0, i0);
1689 iandr(r0, r1);
1690 }
1691}
1692
1693static void
1694_orr(jit_state_t *_jit, jit_int32_t r0, jit_int32_t r1, jit_int32_t r2)
1695{
1696 if (r1 == r2)
1697 movr(r0, r1);
1698 else if (r0 == r1)
1699 iorr(r0, r2);
1700 else if (r0 == r2)
1701 iorr(r0, r1);
1702 else {
1703 movr(r0, r1);
1704 iorr(r0, r2);
1705 }
1706}
1707
1708static void
1709_ori(jit_state_t *_jit, jit_int32_t r0, jit_int32_t r1, jit_word_t i0)
1710{
1711 jit_int32_t reg;
1712 if (i0 == 0)
1713 movr(r0, r1);
1714 else if (i0 == -1)
1715 movi(r0, -1);
1716 else if (can_sign_extend_int_p(i0)) {
1717 movr(r0, r1);
1718 iori(r0, i0);
1719 }
1720 else if (r0 != r1) {
1721 movi(r0, i0);
1722 iorr(r0, r1);
1723 }
1724 else {
1725 reg = jit_get_reg(jit_class_gpr);
1726 movi(rn(reg), i0);
1727 iorr(r0, rn(reg));
1728 jit_unget_reg(reg);
1729 }
1730}
1731
1732static void
1733_xorr(jit_state_t *_jit, jit_int32_t r0, jit_int32_t r1, jit_int32_t r2)
1734{
1735 if (r1 == r2)
1736 ixorr(r0, r0);
1737 else if (r0 == r1)
1738 ixorr(r0, r2);
1739 else if (r0 == r2)
1740 ixorr(r0, r1);
1741 else {
1742 movr(r0, r1);
1743 ixorr(r0, r2);
1744 }
1745}
1746
1747static void
1748_xori(jit_state_t *_jit, jit_int32_t r0, jit_int32_t r1, jit_word_t i0)
1749{
1750 jit_int32_t reg;
1751 if (i0 == 0)
1752 movr(r0, r1);
1753 else if (i0 == -1)
1754 comr(r0, r1);
1755 else if (can_sign_extend_int_p(i0)) {
1756 movr(r0, r1);
1757 ixori(r0, i0);
1758 }
1759 else if (r0 != r1) {
1760 movi(r0, i0);
1761 ixorr(r0, r1);
1762 }
1763 else {
1764 reg = jit_get_reg(jit_class_gpr);
1765 movi(rn(reg), i0);
1766 ixorr(r0, rn(reg));
1767 jit_unget_reg(reg);
1768 }
1769}
1770
1771static void
1772_irotshr(jit_state_t *_jit, jit_int32_t code, jit_int32_t r0)
1773{
1774 rex(0, WIDE, _RCX_REGNO, _NOREG, r0);
1775 ic(0xd3);
1776 mrm(0x03, code, r7(r0));
1777}
1778
1779static void
1780_rotshr(jit_state_t *_jit, jit_int32_t code,
1781 jit_int32_t r0, jit_int32_t r1, jit_int32_t r2)
1782{
1783 jit_int32_t reg;
1784 jit_int32_t use;
1785
1786 if (r0 == _RCX_REGNO) {
1787 reg = jit_get_reg(jit_class_gpr);
1788 movr(rn(reg), r1);
1789 if (r2 != _RCX_REGNO)
1790 movr(_RCX_REGNO, r2);
1791 irotshr(code, rn(reg));
1792 movr(_RCX_REGNO, rn(reg));
1793 jit_unget_reg(reg);
1794 }
1795 else if (r2 != _RCX_REGNO) {
1796 use = !jit_reg_free_p(_RCX);
1797 if (use) {
1798 reg = jit_get_reg(jit_class_gpr);
1799 movr(rn(reg), _RCX_REGNO);
1800 }
1801 else
1802 reg = 0;
1803 if (r1 == _RCX_REGNO) {
1804 if (r0 == r2)
1805 xchgr(r0, _RCX_REGNO);
1806 else {
1807 movr(r0, r1);
1808 movr(_RCX_REGNO, r2);
1809 }
1810 }
1811 else {
1812 movr(_RCX_REGNO, r2);
1813 movr(r0, r1);
1814 }
1815 irotshr(code, r0);
1816 if (use) {
1817 movr(_RCX_REGNO, rn(reg));
1818 jit_unget_reg(reg);
1819 }
1820 }
1821 else {
1822 movr(r0, r1);
1823 irotshr(code, r0);
1824 }
1825}
1826
1827static void
1828_irotshi(jit_state_t *_jit, jit_int32_t code, jit_int32_t r0, jit_word_t i0)
1829{
1830 rex(0, WIDE, _NOREG, _NOREG, r0);
1831 if (i0 == 1) {
1832 ic(0xd1);
1833 mrm(0x03, code, r7(r0));
1834 }
1835 else {
1836 ic(0xc1);
1837 mrm(0x03, code, r7(r0));
1838 ic(i0);
1839 }
1840}
1841
1842static void
1843_rotshi(jit_state_t *_jit, jit_int32_t code,
1844 jit_int32_t r0, jit_int32_t r1, jit_word_t i0)
1845{
1846 movr(r0, r1);
1847 if (i0)
1848 irotshi(code, r0, i0);
1849}
1850
1851static void
1852_lshi(jit_state_t *_jit, jit_int32_t r0, jit_int32_t r1, jit_word_t i0)
1853{
1854 if (i0 == 0)
1855 movr(r0, r1);
1856 else if (i0 <= 3)
1857 lea(0, _NOREG, r1, i0 == 1 ? _SCL2 : i0 == 2 ? _SCL4 : _SCL8, r0);
1858 else
1859 rotshi(X86_SHL, r0, r1, i0);
1860}
1861
1862static void
1863_unr(jit_state_t *_jit, jit_int32_t code, jit_int32_t r0)
1864{
1865 rex(0, WIDE, _NOREG, _NOREG, r0);
1866 ic(0xf7);
1867 mrm(0x03, code, r7(r0));
1868}
1869
1870static void
1871_negr(jit_state_t *_jit, jit_int32_t r0, jit_int32_t r1)
1872{
1873 if (r0 == r1)
1874 inegr(r0);
1875 else {
1876 ixorr(r0, r0);
1877 isubr(r0, r1);
1878 }
1879}
1880
1881static void
1882_comr(jit_state_t *_jit, jit_int32_t r0, jit_int32_t r1)
1883{
1884 movr(r0, r1);
1885 icomr(r0);
1886}
1887
1888#if USE_INC_DEC
1889static void
1890_incr(jit_state_t *_jit, jit_int32_t r0, jit_int32_t r1)
1891{
1892 movr(r0, r1);
1893# if __X64
1894 rex(0, WIDE, _NOREG, _NOREG, r0);
1895 ic(0xff);
1896 ic(0xc0 | r7(r0));
1897# else
1898 ic(0x40 | r7(r0));
1899# endif
1900}
1901
1902static void
1903_decr(jit_state_t *_jit, jit_int32_t r0, jit_int32_t r1)
1904{
1905 movr(r0, r1);
1906# if __X64
1907 rex(0, WIDE, _NOREG, _NOREG, r0);
1908 ic(0xff);
1909 ic(0xc8 | r7(r0));
1910# else
1911 ic(0x48 | r7(r0));
1912# endif
1913}
1914#endif
1915
1916static void
1917_cr(jit_state_t *_jit,
1918 jit_int32_t code, jit_int32_t r0, jit_int32_t r1, jit_int32_t r2)
1919{
1920 jit_int32_t reg;
1921 jit_bool_t same;
1922 if (reg8_p(r0)) {
1923 same = r0 == r1 || r0 == r2;
1924 if (!same)
1925 ixorr(r0, r0);
1926 icmpr(r1, r2);
1927 if (same)
1928 imovi(r0, 0);
1929 cc(code, r0);
1930 }
1931 else {
1932 reg = jit_get_reg(jit_class_gpr|jit_class_rg8);
1933 ixorr(rn(reg), rn(reg));
1934 icmpr(r1, r2);
1935 cc(code, rn(reg));
1936 movr(r0, rn(reg));
1937 jit_unget_reg(reg);
1938 }
1939}
1940
1941static void
1942_ci(jit_state_t *_jit,
1943 jit_int32_t code, jit_int32_t r0, jit_int32_t r1, jit_word_t i0)
1944{
1945 jit_int32_t reg;
1946 jit_bool_t same;
1947 if (reg8_p(r0)) {
1948 same = r0 == r1;
1949 if (!same)
1950 ixorr(r0, r0);
1951 icmpi(r1, i0);
1952 if (same)
1953 imovi(r0, 0);
1954 cc(code, r0);
1955 }
1956 else {
1957 reg = jit_get_reg(jit_class_gpr|jit_class_rg8);
1958 ixorr(rn(reg), rn(reg));
1959 icmpi(r1, i0);
1960 cc(code, rn(reg));
1961 movr(r0, rn(reg));
1962 jit_unget_reg(reg);
1963 }
1964}
1965
1966static void
1967_ci0(jit_state_t *_jit, jit_int32_t code, jit_int32_t r0, jit_int32_t r1)
1968{
1969 jit_int32_t reg;
1970 jit_bool_t same;
1971 if (reg8_p(r0)) {
1972 same = r0 == r1;
1973 if (!same)
1974 ixorr(r0, r0);
1975 testr(r1, r1);
1976 if (same)
1977 imovi(r0, 0);
1978 cc(code, r0);
1979 }
1980 else {
1981 reg = jit_get_reg(jit_class_gpr|jit_class_rg8);
1982 ixorr(rn(reg), rn(reg));
1983 testr(r1, r1);
1984 cc(code, rn(reg));
1985 movr(r0, rn(reg));
1986 jit_unget_reg(reg);
1987 }
1988}
1989
1990static void
1991_ltr(jit_state_t *_jit, jit_int32_t r0, jit_int32_t r1, jit_int32_t r2)
1992{
1993 if (r1 == r2)
1994 movi(r0, 0);
1995 else
1996 cr(X86_CC_L, r0, r1, r2);
1997}
1998
1999static void
2000_lti(jit_state_t *_jit, jit_int32_t r0, jit_int32_t r1, jit_word_t i0)
2001{
2002 if (i0)
2003 ci(X86_CC_L, r0, r1, i0);
2004 else
2005 ci0(X86_CC_S, r0, r1);
2006}
2007
2008static void
2009_ltr_u(jit_state_t *_jit, jit_int32_t r0, jit_int32_t r1, jit_int32_t r2)
2010{
2011 if (r1 == r2)
2012 movi(r0, 0);
2013 else
2014 cr(X86_CC_B, r0, r1, r2);
2015}
2016
2017static void
2018_ler(jit_state_t *_jit, jit_int32_t r0, jit_int32_t r1, jit_int32_t r2)
2019{
2020 if (r1 == r2)
2021 movi(r0, 1);
2022 else
2023 cr(X86_CC_LE, r0, r1, r2);
2024}
2025
2026static void
2027_ler_u(jit_state_t *_jit, jit_int32_t r0, jit_int32_t r1, jit_int32_t r2)
2028{
2029 if (r1 == r2)
2030 movi(r0, 1);
2031 else
2032 cr(X86_CC_BE, r0, r1, r2);
2033}
2034
2035static void
2036_lei_u(jit_state_t *_jit, jit_int32_t r0, jit_int32_t r1, jit_word_t i0)
2037{
2038 if (i0)
2039 ci(X86_CC_BE, r0, r1, i0);
2040 else
2041 ci0(X86_CC_E, r0, r1);
2042}
2043
2044static void
2045_eqr(jit_state_t *_jit, jit_int32_t r0, jit_int32_t r1, jit_int32_t r2)
2046{
2047 if (r1 == r2)
2048 movi(r0, 1);
2049 else
2050 cr(X86_CC_E, r0, r1, r2);
2051}
2052
2053static void
2054_eqi(jit_state_t *_jit, jit_int32_t r0, jit_int32_t r1, jit_word_t i0)
2055{
2056 if (i0)
2057 ci(X86_CC_E, r0, r1, i0);
2058 else
2059 ci0(X86_CC_E, r0, r1);
2060}
2061
2062static void
2063_ger(jit_state_t *_jit, jit_int32_t r0, jit_int32_t r1, jit_int32_t r2)
2064{
2065 if (r1 == r2)
2066 movi(r0, 1);
2067 else
2068 cr(X86_CC_GE, r0, r1, r2);
2069}
2070
2071static void
2072_gei(jit_state_t *_jit, jit_int32_t r0, jit_int32_t r1, jit_word_t i0)
2073{
2074 if (i0)
2075 ci(X86_CC_GE, r0, r1, i0);
2076 else
2077 ci0(X86_CC_NS, r0, r1);
2078}
2079
2080static void
2081_ger_u(jit_state_t *_jit, jit_int32_t r0, jit_int32_t r1, jit_int32_t r2)
2082{
2083 if (r1 == r2)
2084 movi(r0, 1);
2085 else
2086 cr(X86_CC_AE, r0, r1, r2);
2087}
2088
2089static void
2090_gei_u(jit_state_t *_jit, jit_int32_t r0, jit_int32_t r1, jit_word_t i0)
2091{
2092 if (i0)
2093 ci(X86_CC_AE, r0, r1, i0);
2094 else
2095 ci0(X86_CC_NB, r0, r1);
2096}
2097
2098static void
2099_gtr(jit_state_t *_jit, jit_int32_t r0, jit_int32_t r1, jit_int32_t r2)
2100{
2101 if (r1 == r2)
2102 movi(r0, 0);
2103 else
2104 cr(X86_CC_G, r0, r1, r2);
2105}
2106
2107static void
2108_gtr_u(jit_state_t *_jit, jit_int32_t r0, jit_int32_t r1, jit_int32_t r2)
2109{
2110 if (r1 == r2)
2111 movi(r0, 0);
2112 else
2113 cr(X86_CC_A, r0, r1, r2);
2114}
2115
2116static void
2117_gti_u(jit_state_t *_jit, jit_int32_t r0, jit_int32_t r1, jit_word_t i0)
2118{
2119 if (i0)
2120 ci(X86_CC_A, r0, r1, i0);
2121 else
2122 ci0(X86_CC_NE, r0, r1);
2123}
2124
2125static void
2126_ner(jit_state_t *_jit, jit_int32_t r0, jit_int32_t r1, jit_int32_t r2)
2127{
2128 if (r1 == r2)
2129 movi(r0, 0);
2130 else
2131 cr(X86_CC_NE, r0, r1, r2);
2132}
2133
2134static void
2135_nei(jit_state_t *_jit, jit_int32_t r0, jit_int32_t r1, jit_word_t i0)
2136{
2137 if (i0)
2138 ci(X86_CC_NE, r0, r1, i0);
2139 else
2140 ci0(X86_CC_NE, r0, r1);
2141}
2142
2143static void
2144_movr(jit_state_t *_jit, jit_int32_t r0, jit_int32_t r1)
2145{
2146 if (r0 != r1) {
2147 rex(0, 1, r1, _NOREG, r0);
2148 ic(0x89);
2149 ic(0xc0 | (r1 << 3) | r7(r0));
2150 }
2151}
2152
2153static void
2154_imovi(jit_state_t *_jit, jit_int32_t r0, jit_word_t i0)
2155{
2156#if __X64
2157# if !__X64_32
2158 if (fits_uint32_p(i0)) {
2159# endif
2160 rex(0, 0, _NOREG, _NOREG, r0);
2161 ic(0xb8 | r7(r0));
2162 ii(i0);
2163# if !__X64_32
2164 }
2165 else {
2166 rex(0, 1, _NOREG, _NOREG, r0);
2167 ic(0xb8 | r7(r0));
2168 il(i0);
2169 }
2170# endif
2171#else
2172 ic(0xb8 | r7(r0));
2173 ii(i0);
2174#endif
2175}
2176
2177static void
2178_movi(jit_state_t *_jit, jit_int32_t r0, jit_word_t i0)
2179{
2180 if (i0)
2181 imovi(r0, i0);
2182 else
2183 ixorr(r0, r0);
2184}
2185
2186static jit_word_t
2187_movi_p(jit_state_t *_jit, jit_int32_t r0, jit_word_t i0)
2188{
2189 rex(0, WIDE, _NOREG, _NOREG, r0);
2190 ic(0xb8 | r7(r0));
2191 il(i0);
2192 return (_jit->pc.w);
2193}
2194
2195static void
2196_movcr(jit_state_t *_jit, jit_int32_t r0, jit_int32_t r1)
2197{
2198 rex(0, WIDE, r0, _NOREG, r1);
2199 ic(0x0f);
2200 ic(0xbe);
2201 mrm(0x03, r7(r0), r7(r1));
2202}
2203
2204static void
2205_movcr_u(jit_state_t *_jit, jit_int32_t r0, jit_int32_t r1)
2206{
2207 rex(0, WIDE, r0, _NOREG, r1);
2208 ic(0x0f);
2209 ic(0xb6);
2210 mrm(0x03, r7(r0), r7(r1));
2211}
2212
2213static void
2214_movsr(jit_state_t *_jit, jit_int32_t r0, jit_int32_t r1)
2215{
2216 rex(0, WIDE, r0, _NOREG, r1);
2217 ic(0x0f);
2218 ic(0xbf);
2219 mrm(0x03, r7(r0), r7(r1));
2220}
2221
2222static void
2223_movsr_u(jit_state_t *_jit, jit_int32_t r0, jit_int32_t r1)
2224{
2225 rex(0, WIDE, r0, _NOREG, r1);
2226 ic(0x0f);
2227 ic(0xb7);
2228 mrm(0x03, r7(r0), r7(r1));
2229}
2230
ba3814c1
PC		 2231static void
2232_casx(jit_state_t *_jit, jit_int32_t r0, jit_int32_t r1,
2233 jit_int32_t r2, jit_int32_t r3, jit_word_t i0)
2234{
2235 jit_int32_t save_rax, restore_rax;
2236 jit_int32_t ascasr_reg, ascasr_use;
2237 if (r0 != _RAX_REGNO) { /* result not in %rax */
2238 if (r2 != _RAX_REGNO) { /* old value not in %rax */
2239 save_rax = jit_get_reg(jit_class_gpr);
2240 movr(rn(save_rax), _RAX_REGNO);
2241 restore_rax = 1;
2242 }
2243 else
2244 restore_rax = 0;
2245 }
2246 else
2247 restore_rax = 0;
2248 if (r2 != _RAX_REGNO)
2249 movr(_RAX_REGNO, r2);
2250 if (r1 == _NOREG) { /* using immediate address */
2251 if (!can_sign_extend_int_p(i0)) {
2252 ascasr_reg = jit_get_reg(jit_class_gpr);
2253 if (ascasr_reg == _RAX) {
2254 ascasr_reg = jit_get_reg(jit_class_gpr);
2255 jit_unget_reg(_RAX);
2256 }
2257 ascasr_use = 1;
2258 movi(rn(ascasr_reg), i0);
2259 }
2260 else
2261 ascasr_use = 0;
2262 }
2263 else
2264 ascasr_use = 0;
2265 ic(0xf0); /* lock */
2266 if (ascasr_use)
2267 rex(0, WIDE, r3, _NOREG, rn(ascasr_reg));
2268 else
2269 rex(0, WIDE, r3, _NOREG, r1);
2270 ic(0x0f);
2271 ic(0xb1);
2272 if (r1 != _NOREG) /* casr */
2273 rx(r3, 0, r1, _NOREG, _SCL1);
2274 else { /* casi */
2275 if (ascasr_use)
2276 rx(r3, 0, rn(ascasr_reg), _NOREG, _SCL1); /* address in reg */
2277 else
2278 rx(r3, i0, _NOREG, _NOREG, _SCL1); /* address in offset */
2279 }
2280 cc(X86_CC_E, r0);
2281 if (r0 != _RAX_REGNO)
2282 movr(r0, _RAX_REGNO);
2283 if (restore_rax) {
2284 movr(_RAX_REGNO, rn(save_rax));
2285 jit_unget_reg(save_rax);
2286 }
2287 if (ascasr_use)
2288 jit_unget_reg(ascasr_reg);
2289}
2290
1f22b268
PC		 2291static void
2292_movnr(jit_state_t *_jit, jit_int32_t r0, jit_int32_t r1, jit_int32_t r2)
2293{
2294 assert(jit_cmov_p());
2295
2296 testr(r2, r2);
2297
2298 rex(0, WIDE, r0, _NOREG, r1);
2299 ic(0x0f);
2300 ic(0x45);
2301 mrm(0x03, r7(r0), r7(r1));
2302}
2303
2304static void
2305_movzr(jit_state_t *_jit, jit_int32_t r0, jit_int32_t r1, jit_int32_t r2)
2306{
2307 assert(jit_cmov_p());
2308
2309 testr(r2, r2);
2310
2311 rex(0, WIDE, r0, _NOREG, r1);
2312 ic(0x0f);
2313 ic(0x44);
2314 mrm(0x03, r7(r0), r7(r1));
2315}
2316
4a71579b
PC		 2317#if __X64
2318static void
2319_movir(jit_state_t *_jit, jit_int32_t r0, jit_int32_t r1)
2320{
2321 rex(0, 1, r0, _NOREG, r1);
2322 ic(0x63);
2323 mrm(0x03, r7(r0), r7(r1));
2324}
2325
2326static void
2327_movir_u(jit_state_t *_jit, jit_int32_t r0, jit_int32_t r1)
2328{
2329 rex(0, 0, r1, _NOREG, r0);
2330 ic(0x89);
2331 ic(0xc0 | (r1 << 3) | r7(r0));
2332}
2333#endif
2334
2335static void
40a44dcb 2336_bswapr_us(jit_state_t *_jit, jit_int32_t r0, jit_int32_t r1)
4a71579b
PC		 2337{
2338 extr_us(r0, r1);
2339 ic(0x66);
2340 rex(0, 0, _NOREG, _NOREG, r0);
2341 ic(0xc1);
2342 mrm(0x03, X86_ROR, r7(r0));
2343 ic(8);
2344}
2345
2346static void
40a44dcb 2347_bswapr_ui(jit_state_t *_jit, jit_int32_t r0, jit_int32_t r1)
4a71579b
PC		 2348{
2349 movr(r0, r1);
2350 rex(0, 0, _NOREG, _NOREG, r0);
2351 ic(0x0f);
2352 ic(0xc8 | r7(r0));
2353}
2354
2355#if __X64 && !__X64_32
2356static void
40a44dcb 2357_bswapr_ul(jit_state_t *_jit, jit_int32_t r0, jit_int32_t r1)
4a71579b
PC		 2358{
2359 movr(r0, r1);
2360 rex(0, 1, _NOREG, _NOREG, r0);
2361 ic(0x0f);
2362 ic(0xc8 | r7(r0));
2363}
2364#endif
2365
2366static void
2367_extr_c(jit_state_t *_jit, jit_int32_t r0, jit_int32_t r1)
2368{
2369 jit_int32_t reg;
2370 if (reg8_p(r1))
2371 movcr(r0, r1);
2372 else {
2373 reg = jit_get_reg(jit_class_gpr|jit_class_rg8);
2374 movr(rn(reg), r1);
2375 movcr(r0, rn(reg));
2376 jit_unget_reg(reg);
2377 }
2378}
2379
2380static void
2381_extr_uc(jit_state_t *_jit, jit_int32_t r0, jit_int32_t r1)
2382{
2383 jit_int32_t reg;
2384 if (reg8_p(r1))
2385 movcr_u(r0, r1);
2386 else {
2387 reg = jit_get_reg(jit_class_gpr|jit_class_rg8);
2388 movr(rn(reg), r1);
2389 movcr_u(r0, rn(reg));
2390 jit_unget_reg(reg);
2391 }
2392}
2393
2394static void
2395_ldr_c(jit_state_t *_jit, jit_int32_t r0, jit_int32_t r1)
2396{
2397 rex(0, WIDE, r0, _NOREG, r1);
2398 ic(0x0f);
2399 ic(0xbe);
2400 rx(r0, 0, r1, _NOREG, _SCL1);
2401}
2402
2403static void
2404_ldi_c(jit_state_t *_jit, jit_int32_t r0, jit_word_t i0)
2405{
2406 jit_int32_t reg;
2407 if (can_sign_extend_int_p(i0)) {
2408 rex(0, WIDE, r0, _NOREG, _NOREG);
2409 ic(0x0f);
2410 ic(0xbe);
2411 rx(r0, i0, _NOREG, _NOREG, _SCL1);
2412 }
2413 else {
2414 reg = jit_get_reg(jit_class_gpr);
2415 movi(rn(reg), i0);
2416 ldr_c(r0, rn(reg));
2417 jit_unget_reg(reg);
2418 }
2419}
2420
2421static void
2422_ldr_uc(jit_state_t *_jit, jit_int32_t r0, jit_int32_t r1)
2423{
2424 rex(0, WIDE, r0, _NOREG, r1);
2425 ic(0x0f);
2426 ic(0xb6);
2427 rx(r0, 0, r1, _NOREG, _SCL1);
2428}
2429
2430static void
2431_ldi_uc(jit_state_t *_jit, jit_int32_t r0, jit_word_t i0)
2432{
2433 jit_int32_t reg;
2434 if (can_sign_extend_int_p(i0)) {
2435 rex(0, WIDE, r0, _NOREG, _NOREG);
2436 ic(0x0f);
2437 ic(0xb6);
2438 rx(r0, i0, _NOREG, _NOREG, _SCL1);
2439 }
2440 else {
2441 reg = jit_get_reg(jit_class_gpr);
2442 movi(rn(reg), i0);
2443 ldr_uc(r0, rn(reg));
2444 jit_unget_reg(reg);
2445 }
2446}
2447
2448static void
2449_ldr_s(jit_state_t *_jit, jit_int32_t r0, jit_int32_t r1)
2450{
2451 rex(0, WIDE, r0, _NOREG, r1);
2452 ic(0x0f);
2453 ic(0xbf);
2454 rx(r0, 0, r1, _NOREG, _SCL1);
2455}
2456
2457static void
2458_ldi_s(jit_state_t *_jit, jit_int32_t r0, jit_word_t i0)
2459{
2460 jit_int32_t reg;
2461 if (can_sign_extend_int_p(i0)) {
2462 rex(0, WIDE, r0, _NOREG, _NOREG);
2463 ic(0x0f);
2464 ic(0xbf);
2465 rx(r0, i0, _NOREG, _NOREG, _SCL1);
2466 }
2467 else {
2468 reg = jit_get_reg(jit_class_gpr);
2469 movi(rn(reg), i0);
2470 ldr_s(r0, rn(reg));
2471 jit_unget_reg(reg);
2472 }
2473}
2474
2475static void
2476_ldr_us(jit_state_t *_jit, jit_int32_t r0, jit_int32_t r1)
2477{
2478 rex(0, WIDE, r0, _NOREG, r1);
2479 ic(0x0f);
2480 ic(0xb7);
2481 rx(r0, 0, r1, _NOREG, _SCL1);
2482}
2483
2484static void
2485_ldi_us(jit_state_t *_jit, jit_int32_t r0, jit_word_t i0)
2486{
2487 jit_int32_t reg;
2488 if (can_sign_extend_int_p(i0)) {
2489 rex(0, WIDE, r0, _NOREG, _NOREG);
2490 ic(0x0f);
2491 ic(0xb7);
2492 rx(r0, i0, _NOREG, _NOREG, _SCL1);
2493 }
2494 else {
2495 reg = jit_get_reg(jit_class_gpr);
2496 movi(rn(reg), i0);
2497 ldr_us(r0, rn(reg));
2498 jit_unget_reg(reg);
2499 }
2500}
2501
2502#if __X32 || !__X64_32
2503static void
2504_ldr_i(jit_state_t *_jit, jit_int32_t r0, jit_int32_t r1)
2505{
2506#if __X64
2507 rex(0, WIDE, r0, _NOREG, r1);
2508 ic(0x63);
2509#else
2510 ic(0x8b);
2511#endif
2512 rx(r0, 0, r1, _NOREG, _SCL1);
2513}
2514
2515static void
2516_ldi_i(jit_state_t *_jit, jit_int32_t r0, jit_word_t i0)
2517{
2518 jit_int32_t reg;
2519 if (can_sign_extend_int_p(i0)) {
2520#if __X64
2521 rex(0, WIDE, r0, _NOREG, _NOREG);
2522 ic(0x63);
2523#else
2524 ic(0x8b);
2525#endif
2526 rx(r0, i0, _NOREG, _NOREG, _SCL1);
2527 }
2528 else {
2529 reg = jit_get_reg(jit_class_gpr);
2530 movi(rn(reg), i0);
2531 ldr_i(r0, rn(reg));
2532 jit_unget_reg(reg);
2533 }
2534}
2535#endif
2536
2537#if __X64
2538static void
2539_ldr_ui(jit_state_t *_jit, jit_int32_t r0, jit_int32_t r1)
2540{
2541 rex(0, 0, r0, _NOREG, r1);
2542 ic(0x63);
2543 rx(r0, 0, r1, _NOREG, _SCL1);
2544}
2545
2546static void
2547_ldi_ui(jit_state_t *_jit, jit_int32_t r0, jit_word_t i0)
2548{
2549 jit_int32_t reg;
2550 if (can_sign_extend_int_p(i0)) {
2551 rex(0, 0, r0, _NOREG, _NOREG);
2552 ic(0x63);
2553 rx(r0, i0, _NOREG, _NOREG, _SCL1);
2554 }
2555 else {
2556 reg = jit_get_reg(jit_class_gpr);
2557 movi(rn(reg), i0);
2558 ldr_ui(r0, rn(reg));
2559 jit_unget_reg(reg);
2560 }
2561}
2562
2563# if !__X64_32
2564static void
2565_ldr_l(jit_state_t *_jit, jit_int32_t r0, jit_int32_t r1)
2566{
2567 rex(0, 1, r0, _NOREG, r1);
2568 ic(0x8b);
2569 rx(r0, 0, r1, _NOREG, _SCL1);
2570}
2571
2572static void
2573_ldi_l(jit_state_t *_jit, jit_int32_t r0, jit_word_t i0)
2574{
2575 jit_int32_t reg;
2576 if (can_sign_extend_int_p(i0)) {
2577 rex(0, 1, r0, _NOREG, _NOREG);
2578 ic(0x8b);
2579 rx(r0, i0, _NOREG, _NOREG, _SCL1);
2580 }
2581 else {
2582 reg = jit_get_reg(jit_class_gpr);
2583 movi(rn(reg), i0);
2584 ldr_l(r0, rn(reg));
2585 jit_unget_reg(reg);
2586 }
2587}
2588# endif
2589#endif
2590
2591static void
2592_ldxr_c(jit_state_t *_jit, jit_int32_t r0, jit_int32_t r1, jit_int32_t r2)
2593{
2594#if __X64_32
2595 addr(r0, r1, r2);
2596 ldr_c(r0, r0);
2597#else
2598 rex(0, WIDE, r0, r1, r2);
2599 ic(0x0f);
2600 ic(0xbe);
2601 rx(r0, 0, r2, r1, _SCL1);
2602#endif
2603}
2604
2605static void
2606_ldxi_c(jit_state_t *_jit, jit_int32_t r0, jit_int32_t r1, jit_word_t i0)
2607{
2608 jit_int32_t reg;
2609 if (can_sign_extend_int_p(i0)) {
2610 rex(0, WIDE, r0, _NOREG, r1);
2611 ic(0x0f);
2612 ic(0xbe);
2613 rx(r0, i0, r1, _NOREG, _SCL1);
2614 }
2615 else {
2616 reg = jit_get_reg(jit_class_gpr);
2617 movi(rn(reg), i0);
2618 ldxr_c(r0, r1, rn(reg));
2619 jit_unget_reg(reg);
2620 }
2621}
2622
2623static void
2624_ldxr_uc(jit_state_t *_jit, jit_int32_t r0, jit_int32_t r1, jit_int32_t r2)
2625{
2626#if __X64_32
2627 addr(r0, r1, r2);
2628 ldr_uc(r0, r0);
2629#else
2630 rex(0, WIDE, r0, r1, r2);
2631 ic(0x0f);
2632 ic(0xb6);
2633 rx(r0, 0, r2, r1, _SCL1);
2634#endif
2635}
2636
2637static void
2638_ldxi_uc(jit_state_t *_jit, jit_int32_t r0, jit_int32_t r1, jit_word_t i0)
2639{
2640 jit_int32_t reg;
2641 if (can_sign_extend_int_p(i0)) {
2642 rex(0, WIDE, r0, _NOREG, r1);
2643 ic(0x0f);
2644 ic(0xb6);
2645 rx(r0, i0, r1, _NOREG, _SCL1);
2646 }
2647 else {
2648 reg = jit_get_reg(jit_class_gpr);
2649 movi(rn(reg), i0);
2650 ldxr_uc(r0, r1, rn(reg));
2651 jit_unget_reg(reg);
2652 }
2653}
2654
2655static void
2656_ldxr_s(jit_state_t *_jit, jit_int32_t r0, jit_int32_t r1, jit_int32_t r2)
2657{
2658#if __X64_32
2659 addr(r0, r1, r2);
2660 ldr_s(r0, r0);
2661#else
2662 rex(0, WIDE, r0, r1, r2);
2663 ic(0x0f);
2664 ic(0xbf);
2665 rx(r0, 0, r2, r1, _SCL1);
2666#endif
2667}
2668
2669static void
2670_ldxi_s(jit_state_t *_jit, jit_int32_t r0, jit_int32_t r1, jit_word_t i0)
2671{
2672 jit_int32_t reg;
2673 if (can_sign_extend_int_p(i0)) {
2674 rex(0, WIDE, r0, _NOREG, r1);
2675 ic(0x0f);
2676 ic(0xbf);
2677 rx(r0, i0, r1, _NOREG, _SCL1);
2678 }
2679 else {
2680 reg = jit_get_reg(jit_class_gpr);
2681 movi(rn(reg), i0);
2682 ldxr_s(r0, r1, rn(reg));
2683 jit_unget_reg(reg);
2684 }
2685}
2686
2687static void
2688_ldxr_us(jit_state_t *_jit, jit_int32_t r0, jit_int32_t r1, jit_int32_t r2)
2689{
2690#if __X64_32
2691 addr(r0, r1, r2);
2692 ldr_us(r0, r0);
2693#else
2694 rex(0, WIDE, r0, r1, r2);
2695 ic(0x0f);
2696 ic(0xb7);
2697 rx(r0, 0, r2, r1, _SCL1);
2698#endif
2699}
2700
2701static void
2702_ldxi_us(jit_state_t *_jit, jit_int32_t r0, jit_int32_t r1, jit_word_t i0)
2703{
2704 jit_int32_t reg;
2705 if (can_sign_extend_int_p(i0)) {
2706 rex(0, WIDE, r0, _NOREG, r1);
2707 ic(0x0f);
2708 ic(0xb7);
2709 rx(r0, i0, r1, _NOREG, _SCL1);
2710 }
2711 else {
2712 reg = jit_get_reg(jit_class_gpr);
2713 movi(rn(reg), i0);
2714 ldxr_us(r0, r1, rn(reg));
2715 jit_unget_reg(reg);
2716 }
2717}
2718
2719#if __X64 || !__X64_32
2720static void
2721_ldxr_i(jit_state_t *_jit, jit_int32_t r0, jit_int32_t r1, jit_int32_t r2)
2722{
2723#if __X64
2724 rex(0, WIDE, r0, r1, r2);
2725 ic(0x63);
2726#else
2727 ic(0x8b);
2728#endif
2729 rx(r0, 0, r2, r1, _SCL1);
2730}
2731
2732static void
2733_ldxi_i(jit_state_t *_jit, jit_int32_t r0, jit_int32_t r1, jit_word_t i0)
2734{
2735 jit_int32_t reg;
2736 if (can_sign_extend_int_p(i0)) {
2737#if __X64
2738 rex(0, WIDE, r0, _NOREG, r1);
2739 ic(0x63);
2740#else
2741 ic(0x8b);
2742#endif
2743 rx(r0, i0, r1, _NOREG, _SCL1);
2744 }
2745 else {
2746 reg = jit_get_reg(jit_class_gpr);
2747 movi(rn(reg), i0);
2748 ldxr_i(r0, r1, rn(reg));
2749 jit_unget_reg(reg);
2750 }
2751}
2752#endif
2753
2754#if __X64
2755static void
2756_ldxr_ui(jit_state_t *_jit, jit_int32_t r0, jit_int32_t r1, jit_int32_t r2)
2757{
2758#if __X64_32
2759 addr(r0, r1, r2);
2760 /* to avoid confusion with macro renames */
2761 _ldr_ui(_jit, r0, r0);
2762#else
2763 rex(0, 0, r0, r1, r2);
2764 ic(0x8b);
2765 rx(r0, 0, r2, r1, _SCL1);
2766#endif
2767}
2768
2769static void
2770_ldxi_ui(jit_state_t *_jit, jit_int32_t r0, jit_int32_t r1, jit_word_t i0)
2771{
2772 jit_int32_t reg;
2773 if (can_sign_extend_int_p(i0)) {
2774 rex(0, 0, r0, _NOREG, r1);
2775 ic(0x8b);
2776 rx(r0, i0, r1, _NOREG, _SCL1);
2777 }
2778 else {
2779 reg = jit_get_reg(jit_class_gpr);
2780 movi(rn(reg), i0);
2781 ldxr_ui(r0, r1, rn(reg));
2782 jit_unget_reg(reg);
2783 }
2784}
2785
2786# if !__X64_32
2787static void
2788_ldxr_l(jit_state_t *_jit, jit_int32_t r0, jit_int32_t r1, jit_int32_t r2)
2789{
2790 rex(0, 1, r0, r1, r2);
2791 ic(0x8b);
2792 rx(r0, 0, r2, r1, _SCL1);
2793}
2794
2795static void
2796_ldxi_l(jit_state_t *_jit, jit_int32_t r0, jit_int32_t r1, jit_word_t i0)
2797{
2798 jit_int32_t reg;
2799 if (can_sign_extend_int_p(i0)) {
2800 rex(0, 1, r0, _NOREG, r1);
2801 ic(0x8b);
2802 rx(r0, i0, r1, _NOREG, _SCL1);
2803 }
2804 else {
2805 reg = jit_get_reg(jit_class_gpr);
2806 movi(rn(reg), i0);
2807 ldxr_l(r0, r1, rn(reg));
2808 jit_unget_reg(reg);
2809 }
2810}
2811# endif
2812#endif
2813
2814static void
2815_str_c(jit_state_t *_jit, jit_int32_t r0, jit_int32_t r1)
2816{
2817 jit_int32_t reg;
2818 if (reg8_p(r1)) {
2819 rex(0, 0, r1, _NOREG, r0);
2820 ic(0x88);
2821 rx(r1, 0, r0, _NOREG, _SCL1);
2822 }
2823 else {
2824 reg = jit_get_reg(jit_class_gpr|jit_class_rg8);
2825 movr(rn(reg), r1);
2826 rex(0, 0, rn(reg), _NOREG, r0);
2827 ic(0x88);
2828 rx(rn(reg), 0, r0, _NOREG, _SCL1);
2829 jit_unget_reg(reg);
2830 }
2831}
2832
2833static void
2834_sti_c(jit_state_t *_jit, jit_word_t i0, jit_int32_t r0)
2835{
2836 jit_int32_t reg;
2837 if (can_sign_extend_int_p(i0)) {
2838 if (reg8_p(r0)) {
2839 rex(0, 0, r0, _NOREG, _NOREG);
2840 ic(0x88);
2841 rx(r0, i0, _NOREG, _NOREG, _SCL1);
2842 }
2843 else {
2844 reg = jit_get_reg(jit_class_gpr|jit_class_rg8);
2845 movr(rn(reg), r0);
2846 rex(0, 0, rn(reg), _NOREG, _NOREG);
2847 ic(0x88);
2848 rx(rn(reg), i0, _NOREG, _NOREG, _SCL1);
2849 jit_unget_reg(reg);
2850 }
2851 }
2852 else {
2853 reg = jit_get_reg(jit_class_gpr);
2854 movi(rn(reg), i0);
2855 str_c(rn(reg), r0);
2856 jit_unget_reg(reg);
2857 }
2858}
2859
2860static void
2861_str_s(jit_state_t *_jit, jit_int32_t r0, jit_int32_t r1)
2862{
2863 ic(0x66);
2864 rex(0, 0, r1, _NOREG, r0);
2865 ic(0x89);
2866 rx(r1, 0, r0, _NOREG, _SCL1);
2867}
2868
2869static void
2870_sti_s(jit_state_t *_jit, jit_word_t i0, jit_int32_t r0)
2871{
2872 jit_int32_t reg;
2873 if (can_sign_extend_int_p(i0)) {
2874 ic(0x66);
2875 rex(0, 0, r0, _NOREG, _NOREG);
2876 ic(0x89);
2877 rx(r0, i0, _NOREG, _NOREG, _SCL1);
2878 }
2879 else {
2880 reg = jit_get_reg(jit_class_gpr);
2881 movi(rn(reg), i0);
2882 str_s(rn(reg), r0);
2883 jit_unget_reg(reg);
2884 }
2885}
2886
2887static void
2888_str_i(jit_state_t *_jit, jit_int32_t r0, jit_int32_t r1)
2889{
2890 rex(0, 0, r1, _NOREG, r0);
2891 ic(0x89);
2892 rx(r1, 0, r0, _NOREG, _SCL1);
2893}
2894
2895static void
2896_sti_i(jit_state_t *_jit, jit_word_t i0, jit_int32_t r0)
2897{
2898 jit_int32_t reg;
2899 if (can_sign_extend_int_p(i0)) {
2900 rex(0, 0, r0, _NOREG, _NOREG);
2901 ic(0x89);
2902 rx(r0, i0, _NOREG, _NOREG, _SCL1);
2903 }
2904 else {
2905 reg = jit_get_reg(jit_class_gpr);
2906 movi(rn(reg), i0);
2907 str_i(rn(reg), r0);
2908 jit_unget_reg(reg);
2909 }
2910}
2911
2912#if __X64 && !__X64_32
2913static void
2914_str_l(jit_state_t *_jit, jit_int32_t r0, jit_int32_t r1)
2915{
2916 rex(0, 1, r1, _NOREG, r0);
2917 ic(0x89);
2918 rx(r1, 0, r0, _NOREG, _SCL1);
2919}
2920
2921static void
2922_sti_l(jit_state_t *_jit, jit_word_t i0, jit_int32_t r0)
2923{
2924 jit_int32_t reg;
2925 if (can_sign_extend_int_p(i0)) {
2926 rex(0, 1, r0, _NOREG, _NOREG);
2927 ic(0x89);
2928 rx(r0, i0, _NOREG, _NOREG, _SCL1);
2929 }
2930 else {
2931 reg = jit_get_reg(jit_class_gpr);
2932 movi(rn(reg), i0);
2933 str_l(rn(reg), r0);
2934 jit_unget_reg(reg);
2935 }
2936}
2937#endif
2938
2939static void
2940_stxr_c(jit_state_t *_jit, jit_int32_t r0, jit_int32_t r1, jit_int32_t r2)
2941{
2942 jit_int32_t reg;
2943#if __X64_32
2944 reg = jit_get_reg(jit_class_gpr);
2945 addr(rn(reg), r0, r1);
2946 str_c(rn(reg), r2);
2947 jit_unget_reg(reg);
2948#else
2949 if (reg8_p(r2)) {
2950 rex(0, 0, r2, r1, r0);
2951 ic(0x88);
2952 rx(r2, 0, r0, r1, _SCL1);
2953 }
2954 else {
2955 reg = jit_get_reg(jit_class_gpr|jit_class_rg8);
2956 movr(rn(reg), r2);
2957 rex(0, 0, rn(reg), r1, r0);
2958 ic(0x88);
2959 rx(rn(reg), 0, r0, r1, _SCL1);
2960 jit_unget_reg(reg);
2961 }
2962#endif
2963}
2964
2965static void
2966_stxi_c(jit_state_t *_jit, jit_word_t i0, jit_int32_t r0, jit_int32_t r1)
2967{
2968 jit_int32_t reg;
2969 if (can_sign_extend_int_p(i0)) {
2970 if (reg8_p(r1)) {
2971 rex(0, 0, r1, _NOREG, r0);
2972 ic(0x88);
2973 rx(r1, i0, r0, _NOREG, _SCL1);
2974 }
2975 else {
2976 reg = jit_get_reg(jit_class_gpr|jit_class_rg8);
2977 movr(rn(reg), r1);
2978 rex(0, 0, rn(reg), _NOREG, r0);
2979 ic(0x88);
2980 rx(rn(reg), i0, r0, _NOREG, _SCL1);
2981 jit_unget_reg(reg);
2982 }
2983 }
2984 else {
2985 reg = jit_get_reg(jit_class_gpr);
2986 movi(rn(reg), i0);
2987 stxr_c(rn(reg), r0, r1);
2988 jit_unget_reg(reg);
2989 }
2990}
2991
2992static void
2993_stxr_s(jit_state_t *_jit, jit_int32_t r0, jit_int32_t r1, jit_int32_t r2)
2994{
2995#if __X64_32
2996 jit_int32_t reg;
2997 reg = jit_get_reg(jit_class_gpr);
2998 addr(rn(reg), r0, r1);
2999 str_s(rn(reg), r2);
3000 jit_unget_reg(reg);
3001#else
3002 ic(0x66);
3003 rex(0, 0, r2, r1, r0);
3004 ic(0x89);
3005 rx(r2, 0, r0, r1, _SCL1);
3006#endif
3007}
3008
3009static void
3010_stxi_s(jit_state_t *_jit, jit_word_t i0, jit_int32_t r0, jit_int32_t r1)
3011{
3012 jit_int32_t reg;
3013 if (can_sign_extend_int_p(i0)) {
3014 ic(0x66);
3015 rex(0, 0, r1, _NOREG, r0);
3016 ic(0x89);
3017 rx(r1, i0, r0, _NOREG, _SCL1);
3018 }
3019 else {
3020 reg = jit_get_reg(jit_class_gpr);
3021 movi(rn(reg), i0);
3022 stxr_s(rn(reg), r0, r1);
3023 jit_unget_reg(reg);
3024 }
3025}
3026
3027static void
3028_stxr_i(jit_state_t *_jit, jit_int32_t r0, jit_int32_t r1, jit_int32_t r2)
3029{
3030#if __X64_32
3031 jit_int32_t reg;
3032 reg = jit_get_reg(jit_class_gpr);
3033 addr(rn(reg), r0, r1);
3034 str_i(rn(reg), r2);
3035 jit_unget_reg(reg);
3036#else
3037 rex(0, 0, r2, r1, r0);
3038 ic(0x89);
3039 rx(r2, 0, r0, r1, _SCL1);
3040#endif
3041}
3042
3043static void
3044_stxi_i(jit_state_t *_jit, jit_word_t i0, jit_int32_t r0, jit_int32_t r1)
3045{
3046 jit_int32_t reg;
3047 if (can_sign_extend_int_p(i0)) {
3048 rex(0, 0, r1, _NOREG, r0);
3049 ic(0x89);
3050 rx(r1, i0, r0, _NOREG, _SCL1);
3051 }
3052 else {
3053 reg = jit_get_reg(jit_class_gpr);
3054 movi(rn(reg), i0);
3055 stxr_i(rn(reg), r0, r1);
3056 jit_unget_reg(reg);
3057 }
3058}
3059
3060#if __X64 && !__X64_32
3061static void
3062_stxr_l(jit_state_t *_jit, jit_int32_t r0, jit_int32_t r1, jit_int32_t r2)
3063{
3064 rex(0, 1, r2, r1, r0);
3065 ic(0x89);
3066 rx(r2, 0, r0, r1, _SCL1);
3067}
3068
3069static void
3070_stxi_l(jit_state_t *_jit, jit_word_t i0, jit_int32_t r0, jit_int32_t r1)
3071{
3072 jit_int32_t reg;
3073 if (can_sign_extend_int_p(i0)) {
3074 rex(0, 1, r1, _NOREG, r0);
3075 ic(0x89);
3076 rx(r1, i0, r0, _NOREG, _SCL1);
3077 }
3078 else {
3079 reg = jit_get_reg(jit_class_gpr);
3080 movi(rn(reg), i0);
3081 stxr_l(rn(reg), r0, r1);
3082 jit_unget_reg(reg);
3083 }
3084}
3085#endif
3086
3087static void
3088_jccs(jit_state_t *_jit, jit_int32_t code, jit_word_t i0)
3089{
3090 jit_word_t w;
3091 ic(0x70 | code);
3092 w = i0 - (_jit->pc.w + 1);
3093 ic(w);
3094}
3095
3096static void
3097_jcc(jit_state_t *_jit, jit_int32_t code, jit_word_t i0)
3098{
3099 jit_word_t w;
3100 ic(0x0f);
3101 ic(0x80 | code);
3102 w = i0 - (_jit->pc.w + 4);
3103 ii(w);
3104}
3105
3106static void
3107_jcr(jit_state_t *_jit,
3108 jit_int32_t code, jit_word_t i0, jit_int32_t r0, jit_int32_t r1)
3109{
3110 alur(X86_CMP, r0, r1);
3111 jcc(code, i0);
3112}
3113
3114static void
3115_jci(jit_state_t *_jit,
3116 jit_int32_t code, jit_word_t i0, jit_int32_t r0, jit_word_t i1)
3117{
3118 alui(X86_CMP, r0, i1);
3119 jcc(code, i0);
3120}
3121
3122static void
3123_jci0(jit_state_t *_jit, jit_int32_t code, jit_word_t i0, jit_int32_t r0)
3124{
3125 testr(r0, r0);
3126 jcc(code, i0);
3127}
3128
3129static jit_word_t
3130_bltr(jit_state_t *_jit, jit_word_t i0, jit_int32_t r0, jit_int32_t r1)
3131{
3132 jcr(X86_CC_L, i0, r0, r1);
3133 return (_jit->pc.w);
3134}
3135
3136static jit_word_t
3137_blti(jit_state_t *_jit, jit_word_t i0, jit_int32_t r0, jit_word_t i1)
3138{
3139 if (i1) jci (X86_CC_L, i0, r0, i1);
3140 else jci0(X86_CC_S, i0, r0);
3141 return (_jit->pc.w);
3142}
3143
3144static jit_word_t
3145_bltr_u(jit_state_t *_jit, jit_word_t i0, jit_int32_t r0, jit_int32_t r1)
3146{
3147 jcr(X86_CC_B, i0, r0, r1);
3148 return (_jit->pc.w);
3149}
3150
3151static jit_word_t
3152_blti_u(jit_state_t *_jit, jit_word_t i0, jit_int32_t r0, jit_word_t i1)
3153{
3154 if (i1) jci (X86_CC_B, i0, r0, i1);
3155 else jci0(X86_CC_B, i0, r0);
3156 return (_jit->pc.w);
3157}
3158
3159static jit_word_t
3160_bler(jit_state_t *_jit, jit_word_t i0, jit_int32_t r0, jit_int32_t r1)
3161{
3162 if (r0 == r1) jmpi(i0);
3163 else jcr (X86_CC_LE, i0, r0, r1);
3164 return (_jit->pc.w);
3165}
3166
3167static jit_word_t
3168_blei(jit_state_t *_jit, jit_word_t i0, jit_int32_t r0, jit_word_t i1)
3169{
3170 if (i1) jci (X86_CC_LE, i0, r0, i1);
3171 else jci0(X86_CC_LE, i0, r0);
3172 return (_jit->pc.w);
3173}
3174
3175static jit_word_t
3176_bler_u(jit_state_t *_jit, jit_word_t i0, jit_int32_t r0, jit_int32_t r1)
3177{
3178 if (r0 == r1) jmpi(i0);
3179 else jcr (X86_CC_BE, i0, r0, r1);
3180 return (_jit->pc.w);
3181}
3182
3183static jit_word_t
3184_blei_u(jit_state_t *_jit, jit_word_t i0, jit_int32_t r0, jit_word_t i1)
3185{
3186 if (i1) jci (X86_CC_BE, i0, r0, i1);
3187 else jci0(X86_CC_BE, i0, r0);
3188 return (_jit->pc.w);
3189}
3190
3191static jit_word_t
3192_beqr(jit_state_t *_jit, jit_word_t i0, jit_int32_t r0, jit_int32_t r1)
3193{
3194 if (r0 == r1) jmpi(i0);
3195 else jcr (X86_CC_E, i0, r0, r1);
3196 return (_jit->pc.w);
3197}
3198
3199static jit_word_t
3200_beqi(jit_state_t *_jit, jit_word_t i0, jit_int32_t r0, jit_word_t i1)
3201{
3202 if (i1) jci (X86_CC_E, i0, r0, i1);
3203 else jci0(X86_CC_E, i0, r0);
3204 return (_jit->pc.w);
3205}
3206
3207static jit_word_t
3208_bger(jit_state_t *_jit, jit_word_t i0, jit_int32_t r0, jit_int32_t r1)
3209{
3210 if (r0 == r1) jmpi(i0);
3211 else jcr (X86_CC_GE, i0, r0, r1);
3212 return (_jit->pc.w);
3213}
3214
3215static jit_word_t
3216_bgei(jit_state_t *_jit, jit_word_t i0, jit_int32_t r0, jit_word_t i1)
3217{
3218 if (i1) jci (X86_CC_GE, i0, r0, i1);
3219 else jci0(X86_CC_NS, i0, r0);
3220 return (_jit->pc.w);
3221}
3222
3223static jit_word_t
3224_bger_u(jit_state_t *_jit, jit_word_t i0, jit_int32_t r0, jit_int32_t r1)
3225{
3226 if (r0 == r1) jmpi(i0);
3227 else jcr (X86_CC_AE, i0, r0, r1);
3228 return (_jit->pc.w);
3229}
3230
3231static jit_word_t
3232_bgei_u(jit_state_t *_jit, jit_word_t i0, jit_int32_t r0, jit_word_t i1)
3233{
3234 if (i1) jci (X86_CC_AE, i0, r0, i1);
3235 else jmpi(i0);
3236 return (_jit->pc.w);
3237}
3238
3239static jit_word_t
3240_bgtr(jit_state_t *_jit, jit_word_t i0, jit_int32_t r0, jit_int32_t r1)
3241{
3242 jcr(X86_CC_G, i0, r0, r1);
3243 return (_jit->pc.w);
3244}
3245
3246static jit_word_t
3247_bgti(jit_state_t *_jit, jit_word_t i0, jit_int32_t r0, jit_word_t i1)
3248{
3249 jci(X86_CC_G, i0, r0, i1);
3250 return (_jit->pc.w);
3251}
3252
3253static jit_word_t
3254_bgtr_u(jit_state_t *_jit, jit_word_t i0, jit_int32_t r0, jit_int32_t r1)
3255{
3256 jcr(X86_CC_A, i0, r0, r1);
3257 return (_jit->pc.w);
3258}
3259
3260static jit_word_t
3261_bgti_u(jit_state_t *_jit, jit_word_t i0, jit_int32_t r0, jit_word_t i1)
3262{
3263 if (i1) jci (X86_CC_A, i0, r0, i1);
3264 else jci0(X86_CC_NE, i0, r0);
3265 return (_jit->pc.w);
3266}
3267
3268static jit_word_t
3269_bner(jit_state_t *_jit, jit_word_t i0, jit_int32_t r0, jit_int32_t r1)
3270{
3271 jcr(X86_CC_NE, i0, r0, r1);
3272 return (_jit->pc.w);
3273}
3274
3275static jit_word_t
3276_bnei(jit_state_t *_jit, jit_word_t i0, jit_int32_t r0, jit_word_t i1)
3277{
3278 if (i1) jci (X86_CC_NE, i0, r0, i1);
3279 else jci0(X86_CC_NE, i0, r0);
3280 return (_jit->pc.w);
3281}
3282
3283static jit_word_t
3284_bmsr(jit_state_t *_jit, jit_word_t i0, jit_int32_t r0, jit_int32_t r1)
3285{
3286 testr(r0, r1);
3287 jnz(i0);
3288 return (_jit->pc.w);
3289}
3290
3291static jit_word_t
3292_bmsi(jit_state_t *_jit, jit_word_t i0, jit_int32_t r0, jit_word_t i1)
3293{
3294 jit_int32_t reg;
3295 if (can_zero_extend_int_p(i1))
3296 testi(r0, i1);
3297 else {
3298 reg = jit_get_reg(jit_class_gpr);
3299 movi(rn(reg), i1);
3300 testr(r0, rn(reg));
3301 jit_unget_reg(reg);
3302 }
3303 jnz(i0);
3304 return (_jit->pc.w);
3305}
3306
3307static jit_word_t
3308_bmcr(jit_state_t *_jit, jit_word_t i0, jit_int32_t r0, jit_int32_t r1)
3309{
3310 testr(r0, r1);
3311 jz(i0);
3312 return (_jit->pc.w);
3313}
3314
3315static jit_word_t
3316_bmci(jit_state_t *_jit, jit_word_t i0, jit_int32_t r0, jit_word_t i1)
3317{
3318 jit_int32_t reg;
3319 if (can_zero_extend_int_p(i1))
3320 testi(r0, i1);
3321 else {
3322 reg = jit_get_reg(jit_class_gpr);
3323 movi(rn(reg), i1);
3324 testr(r0, rn(reg));
3325 jit_unget_reg(reg);
3326 }
3327 jz(i0);
3328 return (_jit->pc.w);
3329}
3330
3331static jit_word_t
3332_boaddr(jit_state_t *_jit, jit_word_t i0, jit_int32_t r0, jit_int32_t r1)
3333{
3334 iaddr(r0, r1);
3335 jo(i0);
3336 return (_jit->pc.w);
3337}
3338
3339static jit_word_t
3340_boaddi(jit_state_t *_jit, jit_word_t i0, jit_int32_t r0, jit_word_t i1)
3341{
3342 jit_int32_t reg;
3343 if (can_sign_extend_int_p(i1)) {
3344 iaddi(r0, i1);
3345 jo(i0);
3346 return (_jit->pc.w);
3347 }
3348 reg = jit_get_reg(jit_class_gpr|jit_class_nospill);
3349 movi(rn(reg), i1);
3350 jit_unget_reg(reg);
3351 return (boaddr(i0, r0, rn(reg)));
3352}
3353
3354static jit_word_t
3355_boaddr_u(jit_state_t *_jit, jit_word_t i0, jit_int32_t r0, jit_int32_t r1)
3356{
3357 iaddr(r0, r1);
3358 jc(i0);
3359 return (_jit->pc.w);
3360}
3361
3362static jit_word_t
3363_boaddi_u(jit_state_t *_jit, jit_word_t i0, jit_int32_t r0, jit_word_t i1)
3364{
3365 jit_int32_t reg;
3366 if (can_sign_extend_int_p(i1)) {
3367 iaddi(r0, i1);
3368 jc(i0);
3369 return (_jit->pc.w);
3370 }
3371 reg = jit_get_reg(jit_class_gpr|jit_class_nospill);
3372 movi(rn(reg), i1);
3373 jit_unget_reg(reg);
3374 return (boaddr_u(i0, r0, rn(reg)));
3375}
3376
3377static jit_word_t
3378_bxaddr(jit_state_t *_jit, jit_word_t i0, jit_int32_t r0, jit_int32_t r1)
3379{
3380 iaddr(r0, r1);
3381 jno(i0);
3382 return (_jit->pc.w);
3383}
3384
3385static jit_word_t
3386_bxaddi(jit_state_t *_jit, jit_word_t i0, jit_int32_t r0, jit_word_t i1)
3387{
3388 jit_int32_t reg;
3389 if (can_sign_extend_int_p(i1)) {
3390 iaddi(r0, i1);
3391 jno(i0);
3392 return (_jit->pc.w);
3393 }
3394 reg = jit_get_reg(jit_class_gpr|jit_class_nospill);
3395 movi(rn(reg), i1);
3396 jit_unget_reg(reg);
3397 return (bxaddr(i0, r0, rn(reg)));
3398}
3399
3400static jit_word_t
3401_bxaddr_u(jit_state_t *_jit, jit_word_t i0, jit_int32_t r0, jit_int32_t r1)
3402{
3403 iaddr(r0, r1);
3404 jnc(i0);
3405 return (_jit->pc.w);
3406}
3407
3408static jit_word_t
3409_bxaddi_u(jit_state_t *_jit, jit_word_t i0, jit_int32_t r0, jit_word_t i1)
3410{
3411 jit_int32_t reg;
3412 if (can_sign_extend_int_p(i1)) {
3413 iaddi(r0, i1);
3414 jnc(i0);
3415 return (_jit->pc.w);
3416 }
3417 reg = jit_get_reg(jit_class_gpr|jit_class_nospill);
3418 movi(rn(reg), i1);
3419 jit_unget_reg(reg);
3420 return (bxaddr_u(i0, r0, rn(reg)));
3421}
3422
3423static jit_word_t
3424_bosubr(jit_state_t *_jit, jit_word_t i0, jit_int32_t r0, jit_int32_t r1)
3425{
3426 isubr(r0, r1);
3427 jo(i0);
3428 return (_jit->pc.w);
3429}
3430
3431static jit_word_t
3432_bosubi(jit_state_t *_jit, jit_word_t i0, jit_int32_t r0, jit_word_t i1)
3433{
3434 jit_int32_t reg;
3435 if (can_sign_extend_int_p(i1)) {
3436 isubi(r0, i1);
3437 jo(i0);
3438 return (_jit->pc.w);
3439 }
3440 reg = jit_get_reg(jit_class_gpr|jit_class_nospill);
3441 movi(rn(reg), i1);
3442 jit_unget_reg(reg);
3443 return (bosubr(i0, r0, rn(reg)));
3444}
3445
3446static jit_word_t
3447_bosubr_u(jit_state_t *_jit, jit_word_t i0, jit_int32_t r0, jit_int32_t r1)
3448{
3449 isubr(r0, r1);
3450 jc(i0);
3451 return (_jit->pc.w);
3452}
3453
3454static jit_word_t
3455_bosubi_u(jit_state_t *_jit, jit_word_t i0, jit_int32_t r0, jit_word_t i1)
3456{
3457 jit_int32_t reg;
3458 if (can_sign_extend_int_p(i1)) {
3459 isubi(r0, i1);
3460 jc(i0);
3461 return (_jit->pc.w);
3462 }
3463 reg = jit_get_reg(jit_class_gpr|jit_class_nospill);
3464 movi(rn(reg), i1);
3465 jit_unget_reg(reg);
3466 return (bosubr_u(i0, r0, rn(reg)));
3467}
3468
3469static jit_word_t
3470_bxsubr(jit_state_t *_jit, jit_word_t i0, jit_int32_t r0, jit_int32_t r1)
3471{
3472 isubr(r0, r1);
3473 jno(i0);
3474 return (_jit->pc.w);
3475}
3476
3477static jit_word_t
3478_bxsubi(jit_state_t *_jit, jit_word_t i0, jit_int32_t r0, jit_word_t i1)
3479{
3480 jit_int32_t reg;
3481 if (can_sign_extend_int_p(i1)) {
3482 isubi(r0, i1);
3483 jno(i0);
3484 return (_jit->pc.w);
3485 }
3486 reg = jit_get_reg(jit_class_gpr|jit_class_nospill);
3487 movi(rn(reg), i1);
3488 jit_unget_reg(reg);
3489 return (bxsubr(i0, r0, rn(reg)));
3490}
3491
3492static jit_word_t
3493_bxsubr_u(jit_state_t *_jit, jit_word_t i0, jit_int32_t r0, jit_int32_t r1)
3494{
3495 isubr(r0, r1);
3496 jnc(i0);
3497 return (_jit->pc.w);
3498}
3499
3500static jit_word_t
3501_bxsubi_u(jit_state_t *_jit, jit_word_t i0, jit_int32_t r0, jit_word_t i1)
3502{
3503 jit_int32_t reg;
3504 if (can_sign_extend_int_p(i1)) {
3505 isubi(r0, i1);
3506 jnc(i0);
3507 return (_jit->pc.w);
3508 }
3509 reg = jit_get_reg(jit_class_gpr|jit_class_nospill);
3510 movi(rn(reg), i1);
3511 jit_unget_reg(reg);
3512 return (bxsubr_u(i0, r0, rn(reg)));
3513}
3514
3515static void
3516_callr(jit_state_t *_jit, jit_int32_t r0)
3517{
3518 rex(0, 0, _NOREG, _NOREG, r0);
3519 ic(0xff);
3520 mrm(0x03, 0x02, r7(r0));
3521}
3522
3523static jit_word_t
3524_calli(jit_state_t *_jit, jit_word_t i0)
3525{
3526 jit_word_t word;
519a9ea1 3527 jit_word_t w;
4a71579b 3528#if __X64
519a9ea1
PC		 3529 w = i0 - (_jit->pc.w + 5);
3530 if ((jit_int32_t)w == w) {
3531#endif
3532 ic(0xe8);
3533 w = i0 - (_jit->pc.w + 4);
3534 ii(w);
3535 word = _jit->pc.w;
3536#if __X64
3537 }
3538 else
3539 word = calli_p(i0);
3540#endif
3541 return (word);
3542}
4a71579b 3543
519a9ea1
PC		 3544#if __X64
3545static jit_word_t
3546_calli_p(jit_state_t *_jit, jit_word_t i0)
3547{
3548 jit_word_t word;
3549 jit_int32_t reg;
4a71579b
PC		 3550 reg = jit_get_reg(jit_class_gpr);
3551 word = movi_p(rn(reg), i0);
3552 callr(rn(reg));
3553 jit_unget_reg(reg);
4a71579b
PC		 3554 return (word);
3555}
519a9ea1 3556#endif
4a71579b
PC		 3557
3558static void
3559_jmpr(jit_state_t *_jit, jit_int32_t r0)
3560{
519a9ea1 3561 rex(0, 0, _NOREG, _NOREG, r0);
4a71579b
PC		 3562 ic(0xff);
3563 mrm(0x03, 0x04, r7(r0));
3564}
3565
3566static jit_word_t
3567_jmpi(jit_state_t *_jit, jit_word_t i0)
3568{
519a9ea1 3569 jit_word_t word;
4a71579b 3570 jit_word_t w;
519a9ea1
PC		 3571#if __X64
3572 w = i0 - (_jit->pc.w + 5);
3573 if ((jit_int32_t)w == w) {
3574#endif
3575 ic(0xe9);
3576 w = i0 - (_jit->pc.w + 4);
3577 ii(w);
3578 word = _jit->pc.w;
3579#if __X64
3580 }
3581 else
3582 word = jmpi_p(i0);
3583#endif
3584 return (word);
4a71579b
PC		 3585}
3586
519a9ea1
PC		 3587#if __X64
3588static jit_word_t
3589_jmpi_p(jit_state_t *_jit, jit_word_t i0)
3590{
3591 jit_word_t word;
3592 jit_int32_t reg;
3593 reg = jit_get_reg(jit_class_gpr|jit_class_nospill);
3594 word = movi_p(rn(reg), i0);
3595 jmpr(rn(reg));
3596 jit_unget_reg(reg);
3597 return (word);
3598}
3599#endif
3600
4a71579b
PC		 3601static void
3602_jmpsi(jit_state_t *_jit, jit_uint8_t i0)
3603{
3604 ic(0xeb);
3605 ic(i0);
3606}
3607
3608static void
3609_prolog(jit_state_t *_jit, jit_node_t *node)
3610{
3611 jit_int32_t reg;
3612 if (_jitc->function->define_frame || _jitc->function->assume_frame) {
3613 jit_int32_t frame = -_jitc->function->frame;
3614 assert(_jitc->function->self.aoff >= frame);
3615 if (_jitc->function->assume_frame)
3616 return;
3617 _jitc->function->self.aoff = frame;
3618 }
3619 if (_jitc->function->allocar)
3620 _jitc->function->self.aoff &= -16;
3621#if __X64 && (__CYGWIN__ || _WIN32)
3622 _jitc->function->stack = (((/* first 32 bytes must be allocated */
3623 (_jitc->function->self.alen > 32 ?
3624 _jitc->function->self.alen : 32) -
3625 /* align stack at 16 bytes */
3626 _jitc->function->self.aoff) + 15) & -16) +
3627 stack_adjust;
3628#else
3629 _jitc->function->stack = (((_jitc->function->self.alen -
3630 _jitc->function->self.aoff) + 15) & -16) +
3631 stack_adjust;
3632#endif
3633 subi(_RSP_REGNO, _RSP_REGNO, stack_framesize - REAL_WORDSIZE);
3634 /* callee save registers */
3635#if __X32
3636 if (jit_regset_tstbit(&_jitc->function->regset, _RDI))
3637 stxi(12, _RSP_REGNO, _RDI_REGNO);
3638 if (jit_regset_tstbit(&_jitc->function->regset, _RSI))
3639 stxi( 8, _RSP_REGNO, _RSI_REGNO);
3640 if (jit_regset_tstbit(&_jitc->function->regset, _RBX))
3641 stxi( 4, _RSP_REGNO, _RBX_REGNO);
3642#else
3643# if __CYGWIN__ || _WIN32
3644 if (jit_regset_tstbit(&_jitc->function->regset, _XMM15))
3645 sse_stxi_d(136, _RSP_REGNO, _XMM15_REGNO);
3646 if (jit_regset_tstbit(&_jitc->function->regset, _XMM14))
3647 sse_stxi_d(128, _RSP_REGNO, _XMM14_REGNO);
3648 if (jit_regset_tstbit(&_jitc->function->regset, _XMM13))
3649 sse_stxi_d(120, _RSP_REGNO, _XMM13_REGNO);
3650 if (jit_regset_tstbit(&_jitc->function->regset, _XMM12))
3651 sse_stxi_d(112, _RSP_REGNO, _XMM12_REGNO);
3652 if (jit_regset_tstbit(&_jitc->function->regset, _XMM11))
3653 sse_stxi_d(104, _RSP_REGNO, _XMM11_REGNO);
3654 if (jit_regset_tstbit(&_jitc->function->regset, _XMM10))
3655 sse_stxi_d(96, _RSP_REGNO, _XMM10_REGNO);
3656 if (jit_regset_tstbit(&_jitc->function->regset, _XMM9))
3657 sse_stxi_d(88, _RSP_REGNO, _XMM9_REGNO);
3658 if (jit_regset_tstbit(&_jitc->function->regset, _XMM8))
3659 sse_stxi_d(80, _RSP_REGNO, _XMM8_REGNO);
3660 if (jit_regset_tstbit(&_jitc->function->regset, _XMM7))
3661 sse_stxi_d(72, _RSP_REGNO, _XMM7_REGNO);
3662 if (jit_regset_tstbit(&_jitc->function->regset, _XMM6))
3663 sse_stxi_d(64, _RSP_REGNO, _XMM6_REGNO);
3664 if (jit_regset_tstbit(&_jitc->function->regset, _R15))
3665 stxi(56, _RSP_REGNO, _R15_REGNO);
3666 if (jit_regset_tstbit(&_jitc->function->regset, _R14))
3667 stxi(48, _RSP_REGNO, _R14_REGNO);
3668 if (jit_regset_tstbit(&_jitc->function->regset, _R13))
3669 stxi(40, _RSP_REGNO, _R13_REGNO);
3670 if (jit_regset_tstbit(&_jitc->function->regset, _R12))
3671 stxi(32, _RSP_REGNO, _R12_REGNO);
3672 if (jit_regset_tstbit(&_jitc->function->regset, _RSI))
3673 stxi(24, _RSP_REGNO, _RSI_REGNO);
3674 if (jit_regset_tstbit(&_jitc->function->regset, _RDI))
3675 stxi(16, _RSP_REGNO, _RDI_REGNO);
3676 if (jit_regset_tstbit(&_jitc->function->regset, _RBX))
3677 stxi( 8, _RSP_REGNO, _RBX_REGNO);
3678# else
3679 if (jit_regset_tstbit(&_jitc->function->regset, _RBX))
3680 stxi(40, _RSP_REGNO, _RBX_REGNO);
3681 if (jit_regset_tstbit(&_jitc->function->regset, _R12))
3682 stxi(32, _RSP_REGNO, _R12_REGNO);
3683 if (jit_regset_tstbit(&_jitc->function->regset, _R13))
3684 stxi(24, _RSP_REGNO, _R13_REGNO);
3685 if (jit_regset_tstbit(&_jitc->function->regset, _R14))
3686 stxi(16, _RSP_REGNO, _R14_REGNO);
3687 if (jit_regset_tstbit(&_jitc->function->regset, _R15))
3688 stxi( 8, _RSP_REGNO, _R15_REGNO);
3689# endif
3690#endif
3691 stxi(0, _RSP_REGNO, _RBP_REGNO);
3692 movr(_RBP_REGNO, _RSP_REGNO);
3693
3694 /* alloca */
3695 subi(_RSP_REGNO, _RSP_REGNO, _jitc->function->stack);
3696 if (_jitc->function->allocar) {
3697 reg = jit_get_reg(jit_class_gpr);
3698 movi(rn(reg), _jitc->function->self.aoff);
3699 stxi_i(_jitc->function->aoffoff, _RBP_REGNO, rn(reg));
3700 jit_unget_reg(reg);
3701 }
3702
3703#if __X64 && !(__CYGWIN__ || _WIN32)
3704 if (_jitc->function->self.call & jit_call_varargs) {
3705 jit_word_t nofp_code;
3706
3707 /* Save gp registers in the save area, if any is a vararg */
3708 for (reg = first_gp_from_offset(_jitc->function->vagp);
3709 jit_arg_reg_p(reg); ++reg)
3710 stxi(_jitc->function->vaoff + first_gp_offset +
3711 reg * 8, _RBP_REGNO, rn(JIT_RA0 - reg));
3712
3713 reg = first_fp_from_offset(_jitc->function->vafp);
3714 if (jit_arg_f_reg_p(reg)) {
3715 /* Skip over if no float registers were passed as argument */
3716 /* test %al, %al */
3717 ic(0x84);
3718 ic(0xc0);
3719 jes(0);
3720 nofp_code = _jit->pc.w;
3721
3722 /* Save fp registers in the save area, if any is a vararg */
3723 /* Note that the full 16 byte xmm is not saved, because
3724 * lightning only handles float and double, and, while
3725 * attempting to provide a va_list compatible pointer as
3726 * jit_va_start return, does not guarantee it (on all ports). */
3727 for (; jit_arg_f_reg_p(reg); ++reg)
3728 sse_stxi_d(_jitc->function->vaoff + first_fp_offset +
3729 reg * va_fp_increment, _RBP_REGNO, rn(_XMM0 - reg));
3730
3731 patch_rel_char(nofp_code, _jit->pc.w);
3732 }
3733 }
3734#endif
3735}
3736
3737static void
3738_epilog(jit_state_t *_jit, jit_node_t *node)
3739{
3740 if (_jitc->function->assume_frame)
3741 return;
3742 /* callee save registers */
3743 movr(_RSP_REGNO, _RBP_REGNO);
3744#if __X32
3745 if (jit_regset_tstbit(&_jitc->function->regset, _RDI))
3746 ldxi(_RDI_REGNO, _RSP_REGNO, 12);
3747 if (jit_regset_tstbit(&_jitc->function->regset, _RSI))
3748 ldxi(_RSI_REGNO, _RSP_REGNO, 8);
3749 if (jit_regset_tstbit(&_jitc->function->regset, _RBX))
3750 ldxi(_RBX_REGNO, _RSP_REGNO, 4);
3751#else
3752# if __CYGWIN__ || _WIN32
3753 if (jit_regset_tstbit(&_jitc->function->regset, _XMM15))
3754 sse_ldxi_d(_XMM15_REGNO, _RSP_REGNO, 136);
3755 if (jit_regset_tstbit(&_jitc->function->regset, _XMM14))
3756 sse_ldxi_d(_XMM14_REGNO, _RSP_REGNO, 128);
3757 if (jit_regset_tstbit(&_jitc->function->regset, _XMM13))
3758 sse_ldxi_d(_XMM13_REGNO, _RSP_REGNO, 120);
3759 if (jit_regset_tstbit(&_jitc->function->regset, _XMM12))
3760 sse_ldxi_d(_XMM12_REGNO, _RSP_REGNO, 112);
3761 if (jit_regset_tstbit(&_jitc->function->regset, _XMM11))
3762 sse_ldxi_d(_XMM11_REGNO, _RSP_REGNO, 104);
3763 if (jit_regset_tstbit(&_jitc->function->regset, _XMM10))
3764 sse_ldxi_d(_XMM10_REGNO, _RSP_REGNO, 96);
3765 if (jit_regset_tstbit(&_jitc->function->regset, _XMM9))
3766 sse_ldxi_d(_XMM9_REGNO, _RSP_REGNO, 88);
3767 if (jit_regset_tstbit(&_jitc->function->regset, _XMM8))
3768 sse_ldxi_d(_XMM8_REGNO, _RSP_REGNO, 80);
3769 if (jit_regset_tstbit(&_jitc->function->regset, _XMM7))
3770 sse_ldxi_d(_XMM7_REGNO, _RSP_REGNO, 72);
3771 if (jit_regset_tstbit(&_jitc->function->regset, _XMM6))
3772 sse_ldxi_d(_XMM6_REGNO, _RSP_REGNO, 64);
3773 if (jit_regset_tstbit(&_jitc->function->regset, _R15))
3774 ldxi(_R15_REGNO, _RSP_REGNO, 56);
3775 if (jit_regset_tstbit(&_jitc->function->regset, _R14))
3776 ldxi(_R14_REGNO, _RSP_REGNO, 48);
3777 if (jit_regset_tstbit(&_jitc->function->regset, _R13))
3778 ldxi(_R13_REGNO, _RSP_REGNO, 40);
3779 if (jit_regset_tstbit(&_jitc->function->regset, _R12))
3780 ldxi(_R12_REGNO, _RSP_REGNO, 32);
3781 if (jit_regset_tstbit(&_jitc->function->regset, _RSI))
3782 ldxi(_RSI_REGNO, _RSP_REGNO, 24);
3783 if (jit_regset_tstbit(&_jitc->function->regset, _RDI))
3784 ldxi(_RDI_REGNO, _RSP_REGNO, 16);
3785 if (jit_regset_tstbit(&_jitc->function->regset, _RBX))
3786 ldxi(_RBX_REGNO, _RSP_REGNO, 8);
3787# else
3788 if (jit_regset_tstbit(&_jitc->function->regset, _RBX))
3789 ldxi(_RBX_REGNO, _RSP_REGNO, 40);
3790 if (jit_regset_tstbit(&_jitc->function->regset, _R12))
3791 ldxi(_R12_REGNO, _RSP_REGNO, 32);
3792 if (jit_regset_tstbit(&_jitc->function->regset, _R13))
3793 ldxi(_R13_REGNO, _RSP_REGNO, 24);
3794 if (jit_regset_tstbit(&_jitc->function->regset, _R14))
3795 ldxi(_R14_REGNO, _RSP_REGNO, 16);
3796 if (jit_regset_tstbit(&_jitc->function->regset, _R15))
3797 ldxi(_R15_REGNO, _RSP_REGNO, 8);
3798# endif
3799#endif
3800 ldxi(_RBP_REGNO, _RSP_REGNO, 0);
3801 addi(_RSP_REGNO, _RSP_REGNO, stack_framesize - REAL_WORDSIZE);
3802
3803 ic(0xc3);
3804}
3805
3806static void
3807_vastart(jit_state_t *_jit, jit_int32_t r0)
3808{
3809#if __X32 || __CYGWIN__ || _WIN32
3810 assert(_jitc->function->self.call & jit_call_varargs);
3811 addi(r0, _RBP_REGNO, _jitc->function->self.size);
3812#else
3813 jit_int32_t reg;
3814
3815 assert(_jitc->function->self.call & jit_call_varargs);
3816
3817 /* Return jit_va_list_t in the register argument */
3818 addi(r0, _RBP_REGNO, _jitc->function->vaoff);
3819 reg = jit_get_reg(jit_class_gpr);
3820
3821 /* Initialize gp offset in the save area. */
3822 movi(rn(reg), _jitc->function->vagp);
3823 stxi_i(offsetof(jit_va_list_t, gpoff), r0, rn(reg));
3824
3825 /* Initialize fp offset in the save area. */
3826 movi(rn(reg), _jitc->function->vafp);
3827 stxi_i(offsetof(jit_va_list_t, fpoff), r0, rn(reg));
3828
3829 /* Initialize overflow pointer to the first stack argument. */
3830 addi(rn(reg), _RBP_REGNO, _jitc->function->self.size);
3831 stxi(offsetof(jit_va_list_t, over), r0, rn(reg));
3832
3833 /* Initialize register save area pointer. */
3834 addi(rn(reg), r0, first_gp_offset);
3835 stxi(offsetof(jit_va_list_t, save), r0, rn(reg));
3836
3837 jit_unget_reg(reg);
3838#endif
3839}
3840
3841static void
3842_vaarg(jit_state_t *_jit, jit_int32_t r0, jit_int32_t r1)
3843{
3844#if __X32 || __CYGWIN__ || _WIN32
3845 assert(_jitc->function->self.call & jit_call_varargs);
3846 ldr(r0, r1);
3847 addi(r1, r1, va_gp_increment);
3848#else
3849 jit_int32_t rg0;
3850 jit_int32_t rg1;
3851 jit_word_t ge_code;
3852 jit_word_t lt_code;
3853
3854 assert(_jitc->function->self.call & jit_call_varargs);
3855
3856 rg0 = jit_get_reg(jit_class_gpr);
3857 rg1 = jit_get_reg(jit_class_gpr);
3858
3859 /* Load the gp offset in save area in the first temporary. */
3860 ldxi_i(rn(rg0), r1, offsetof(jit_va_list_t, gpoff));
3861
3862 /* Jump over if there are no remaining arguments in the save area. */
3863 icmpi(rn(rg0), va_gp_max_offset);
3864 jaes(0);
3865 ge_code = _jit->pc.w;
3866
3867 /* Load the save area pointer in the second temporary. */
3868 ldxi(rn(rg1), r1, offsetof(jit_va_list_t, save));
3869
3870 /* Load the vararg argument in the first argument. */
3871 ldxr(r0, rn(rg1), rn(rg0));
3872
3873 /* Update the gp offset. */
3874 addi(rn(rg0), rn(rg0), 8);
3875 stxi_i(offsetof(jit_va_list_t, gpoff), r1, rn(rg0));
3876
3877 /* Will only need one temporary register below. */
3878 jit_unget_reg(rg1);
3879
3880 /* Jump over overflow code. */
3881 jmpsi(0);
3882 lt_code = _jit->pc.w;
3883
3884 /* Where to land if argument is in overflow area. */
3885 patch_rel_char(ge_code, _jit->pc.w);
3886
3887 /* Load overflow pointer. */
3888 ldxi(rn(rg0), r1, offsetof(jit_va_list_t, over));
3889
3890 /* Load argument. */
3891 ldr(r0, rn(rg0));
3892
3893 /* Update overflow pointer. */
3894 addi(rn(rg0), rn(rg0), va_gp_increment);
3895 stxi(offsetof(jit_va_list_t, over), r1, rn(rg0));
3896
3897 /* Where to land if argument is in save area. */
3898 patch_rel_char(lt_code, _jit->pc.w);
3899
3900 jit_unget_reg(rg0);
3901#endif
3902}
3903
3904/* The x87 boolean argument tells if will put the result in a x87
3905 * register if non false, in a sse register otherwise. */
3906static void
3907_vaarg_d(jit_state_t *_jit, jit_int32_t r0, jit_int32_t r1, jit_bool_t x87)
3908{
3909#if __X32 || __CYGWIN__ || _WIN32
3910 assert(_jitc->function->self.call & jit_call_varargs);
3911 if (x87)
3912 x87_ldr_d(r0, r1);
3913 else
3914 sse_ldr_d(r0, r1);
3915 addi(r1, r1, 8);
3916#else
3917 jit_int32_t rg0;
3918 jit_int32_t rg1;
3919 jit_word_t ge_code;
3920 jit_word_t lt_code;
3921
3922 assert(_jitc->function->self.call & jit_call_varargs);
3923
3924 rg0 = jit_get_reg(jit_class_gpr);
3925 rg1 = jit_get_reg(jit_class_gpr);
3926
3927 /* Load the fp offset in save area in the first temporary. */
3928 ldxi_i(rn(rg0), r1, offsetof(jit_va_list_t, fpoff));
3929
3930 /* Jump over if there are no remaining arguments in the save area. */
3931 icmpi(rn(rg0), va_fp_max_offset);
3932 jaes(0);
3933 ge_code = _jit->pc.w;
3934
3935 /* Load the save area pointer in the second temporary. */
3936 ldxi(rn(rg1), r1, offsetof(jit_va_list_t, save));
3937
3938 /* Load the vararg argument in the first argument. */
3939 if (x87)
3940 x87_ldxr_d(r0, rn(rg1), rn(rg0));
3941 else
3942 sse_ldxr_d(r0, rn(rg1), rn(rg0));
3943
3944 /* Update the fp offset. */
3945 addi(rn(rg0), rn(rg0), va_fp_increment);
3946 stxi_i(offsetof(jit_va_list_t, fpoff), r1, rn(rg0));
3947
3948 /* Will only need one temporary register below. */
3949 jit_unget_reg(rg1);
3950
3951 /* Jump over overflow code. */
3952 jmpsi(0);
3953 lt_code = _jit->pc.w;
3954
3955 /* Where to land if argument is in overflow area. */
3956 patch_rel_char(ge_code, _jit->pc.w);
3957
3958 /* Load overflow pointer. */
3959 ldxi(rn(rg0), r1, offsetof(jit_va_list_t, over));
3960
3961 /* Load argument. */
3962 if (x87)
3963 x87_ldr_d(r0, rn(rg0));
3964 else
3965 sse_ldr_d(r0, rn(rg0));
3966
3967 /* Update overflow pointer. */
3968 addi(rn(rg0), rn(rg0), 8);
3969 stxi(offsetof(jit_va_list_t, over), r1, rn(rg0));
3970
3971 /* Where to land if argument is in save area. */
3972 patch_rel_char(lt_code, _jit->pc.w);
3973
3974 jit_unget_reg(rg0);
3975#endif
3976}
3977
3978static void
3979_patch_at(jit_state_t *_jit, jit_node_t *node,
3980 jit_word_t instr, jit_word_t label)
3981{
3982 switch (node->code) {
3983# if __X64
3984 case jit_code_calli:
519a9ea1 3985 case jit_code_jmpi:
4a71579b
PC		 3986# endif
3987 case jit_code_movi:
3988 patch_abs(instr, label);
3989 break;
3990 default:
3991 patch_rel(instr, label);
3992 break;
3993 }
3994}
3995#endif
ARM optimized PCSX fork