notaz.gp2x.de / pcsx_rearmed.git / blame
? search:
summary | shortlog | log | commit | commitdiff | tree
blob | blame (incremental) | history | HEAD
lightning: Update to my own repository
[pcsx_rearmed.git] / deps / lightning / lib / jit_x86-cpu.c
CommitLineData
4a71579b
PC		 1/*
2 * Copyright (C) 2012-2019 Free Software Foundation, Inc.
3 *
4 * This file is part of GNU lightning.
5 *
6 * GNU lightning is free software; you can redistribute it and/or modify it
7 * under the terms of the GNU Lesser General Public License as published
8 * by the Free Software Foundation; either version 3, or (at your option)
9 * any later version.
10 *
11 * GNU lightning is distributed in the hope that it will be useful, but
12 * WITHOUT ANY WARRANTY; without even the implied warranty of MERCHANTABILITY
13 * or FITNESS FOR A PARTICULAR PURPOSE. See the GNU Lesser General Public
14 * License for more details.
15 *
16 * Authors:
17 * Paulo Cesar Pereira de Andrade
18 */
19
20/* avoid using it due to partial stalls */
21#define USE_INC_DEC 0
22
23#if PROTO
24# if __X32 || __X64_32
25# define WIDE 0
26# define ldi(u, v) ldi_i(u, v)
27# define ldr(u, v) ldr_i(u, v)
28# define ldxr(u, v, w) ldxr_i(u, v, w)
29# define ldxi(u, v, w) ldxi_i(u, v, w)
30# define sti(u, v) sti_i(u, v)
31# define stxi(u, v, w) stxi_i(u, v, w)
32# define can_sign_extend_int_p(im) 1
33# define can_zero_extend_int_p(im) 1
34# define fits_uint32_p(im) 1
35# else
36# define WIDE 1
37# define ldi(u, v) ldi_l(u, v)
38# define ldr(u, v) ldr_l(u, v)
39# define ldxr(u, v, w) ldxr_l(u, v, w)
40# define ldxi(u, v, w) ldxi_l(u, v, w)
41# define sti(u, v) sti_l(u, v)
42# define stxi(u, v, w) stxi_l(u, v, w)
43# define can_sign_extend_int_p(im) \
44 (((im) >= 0 && (long long)(im) <= 0x7fffffffLL) || \
45 ((im) < 0 && (long long)(im) > -0x80000000LL))
46# define can_zero_extend_int_p(im) \
47 ((im) >= 0 && (im) < 0x80000000LL)
48# define fits_uint32_p(im) (((im) & 0xffffffff00000000LL) == 0)
49# endif
50# if __X32 || __CYGWIN__ || __X64_32 || _WIN32
51# define reg8_p(rn) \
52 ((rn) >= _RAX_REGNO && (rn) <= _RBX_REGNO)
53# else
54# define reg8_p(rn) 1
55# endif
56# define _RAX_REGNO 0
57# define _RCX_REGNO 1
58# define _RDX_REGNO 2
59# define _RBX_REGNO 3
60# define _RSP_REGNO 4
61# define _RBP_REGNO 5
62# define _RSI_REGNO 6
63# define _RDI_REGNO 7
64# define _R8_REGNO 8
65# define _R9_REGNO 9
66# define _R10_REGNO 10
67# define _R11_REGNO 11
68# define _R12_REGNO 12
69# define _R13_REGNO 13
70# define _R14_REGNO 14
71# define _R15_REGNO 15
72# define r7(reg) ((reg) & 7)
73# define r8(reg) ((reg) & 15)
74# define _SCL1 0x00
75# define _SCL2 0x01
76# define _SCL4 0x02
77# define _SCL8 0x03
78# define X86_ADD 0
79# define X86_OR 1 << 3
80# define X86_ADC 2 << 3
81# define X86_SBB 3 << 3
82# define X86_AND 4 << 3
83# define X86_SUB 5 << 3
84# define X86_XOR 6 << 3
85# define X86_CMP 7 << 3
86# define X86_ROL 0
87# define X86_ROR 1
88# define X86_RCL 2
89# define X86_RCR 3
90# define X86_SHL 4
91# define X86_SHR 5
92# define X86_SAR 7
93# define X86_NOT 2
94# define X86_NEG 3
95# define X86_MUL 4
96# define X86_IMUL 5
97# define X86_DIV 6
98# define X86_IDIV 7
99# define X86_CC_O 0x0
100# define X86_CC_NO 0x1
101# define X86_CC_NAE 0x2
102# define X86_CC_B 0x2
103# define X86_CC_C 0x2
104# define X86_CC_AE 0x3
105# define X86_CC_NB 0x3
106# define X86_CC_NC 0x3
107# define X86_CC_E 0x4
108# define X86_CC_Z 0x4
109# define X86_CC_NE 0x5
110# define X86_CC_NZ 0x5
111# define X86_CC_BE 0x6
112# define X86_CC_NA 0x6
113# define X86_CC_A 0x7
114# define X86_CC_NBE 0x7
115# define X86_CC_S 0x8
116# define X86_CC_NS 0x9
117# define X86_CC_P 0xa
118# define X86_CC_PE 0xa
119# define X86_CC_NP 0xb
120# define X86_CC_PO 0xb
121# define X86_CC_L 0xc
122# define X86_CC_NGE 0xc
123# define X86_CC_GE 0xd
124# define X86_CC_NL 0xd
125# define X86_CC_LE 0xe
126# define X86_CC_NG 0xe
127# define X86_CC_G 0xf
128# define X86_CC_NLE 0xf
129# define mrm(md, r, m) *_jit->pc.uc++ = (md<<6) | (r<<3) | m
130# define sib(sc, i, b) *_jit->pc.uc++ = (sc<<6) | (i<<3) | b
131# define ic(c) *_jit->pc.uc++ = c
132# define is(s) *_jit->pc.us++ = s
133# define ii(i) *_jit->pc.ui++ = i
134# if __X64 && !__X64_32
135# define il(l) *_jit->pc.ul++ = l
136# else
137# define il(l) ii(l)
138# endif
139# define patch_abs(instr, label) \
140 *(jit_word_t *)(instr - sizeof(jit_word_t)) = label
141# define patch_rel(instr, label) \
142 *(jit_int32_t *)(instr - 4) = label - instr
143# define patch_rel_char(instr, label) \
144 *(jit_int8_t *)(instr - 1) = label - instr
145# define rex(l, w, r, x, b) _rex(_jit, l, w, r, x, b)
146static void
147_rex(jit_state_t*,jit_int32_t,jit_int32_t,jit_int32_t,jit_int32_t,jit_int32_t);
148# define rx(rd, md, rb, ri, ms) _rx(_jit, rd, md, rb, ri, ms)
149static void
150_rx(jit_state_t*,jit_int32_t,jit_int32_t,jit_int32_t,jit_int32_t,jit_int32_t);
151# define nop(n) _nop(_jit, n)
152static void _nop(jit_state_t*, jit_int32_t);
153# define emms() is(0x770f)
154# define lea(md, rb, ri, ms, rd) _lea(_jit, md, rb, ri, ms, rd)
155static void
156_lea(jit_state_t*,jit_int32_t,jit_int32_t,jit_int32_t,jit_int32_t,jit_int32_t);
157# define pushr(r0) _pushr(_jit, r0)
158static void _pushr(jit_state_t*, jit_int32_t) maybe_unused;
159# define popr(r0) _popr(_jit, r0)
160static void _popr(jit_state_t*, jit_int32_t) maybe_unused;
161# define xchgr(r0, r1) _xchgr(_jit, r0, r1)
162static void _xchgr(jit_state_t*, jit_int32_t, jit_int32_t);
163# define testr(r0, r1) _testr(_jit, r0, r1)
164static void _testr(jit_state_t*, jit_int32_t, jit_int32_t);
165# define testi(r0, i0) _testi(_jit, r0, i0)
166static void _testi(jit_state_t*, jit_int32_t, jit_word_t);
167# define cc(code, r0) _cc(_jit, code, r0)
168static void _cc(jit_state_t*, jit_int32_t, jit_int32_t);
169# define icmpr(r0, r1) alur(X86_CMP, r0, r1)
170# define alur(code, r0, r1) _alur(_jit, code, r0, r1)
171static void _alur(jit_state_t*, jit_int32_t, jit_int32_t, jit_int32_t);
172# define icmpi(r0, i0) alui(X86_CMP, r0, i0)
173# define alui(code, r0, i0) _alui(_jit, code, r0, i0)
174static void _alui(jit_state_t*, jit_int32_t, jit_int32_t, jit_word_t);
175# define iaddr(r0, r1) alur(X86_ADD, r0, r1)
176# define save(r0) _save(_jit, r0)
177static void _save(jit_state_t*, jit_int32_t);
178# define load(r0) _load(_jit, r0)
179static void _load(jit_state_t*, jit_int32_t);
180# define addr(r0, r1, r2) _addr(_jit, r0, r1, r2)
181static void _addr(jit_state_t*, jit_int32_t, jit_int32_t, jit_int32_t);
182# define iaddi(r0, i0) alui(X86_ADD, r0, i0)
183# define addi(r0, r1, i0) _addi(_jit, r0, r1, i0)
184static void _addi(jit_state_t*, jit_int32_t, jit_int32_t, jit_word_t);
185#define addcr(r0, r1, r2) _addcr(_jit, r0, r1, r2)
186static void _addcr(jit_state_t*, jit_int32_t, jit_int32_t, jit_int32_t);
187#define addci(r0, r1, i0) _addci(_jit, r0, r1, i0)
188static void _addci(jit_state_t*, jit_int32_t, jit_int32_t, jit_word_t);
189# define iaddxr(r0, r1) alur(X86_ADC, r0, r1)
190# define addxr(r0, r1, r2) _addxr(_jit, r0, r1, r2)
191static void _addxr(jit_state_t*, jit_int32_t, jit_int32_t, jit_int32_t);
192# define iaddxi(r0, i0) alui(X86_ADC, r0, i0)
193# define addxi(r0, r1, i0) _addxi(_jit, r0, r1, i0)
194static void _addxi(jit_state_t*, jit_int32_t, jit_int32_t, jit_word_t);
195# define isubr(r0, r1) alur(X86_SUB, r0, r1)
196# define subr(r0, r1, r2) _subr(_jit, r0, r1, r2)
197static void _subr(jit_state_t*, jit_int32_t, jit_int32_t, jit_int32_t);
198# define isubi(r0, i0) alui(X86_SUB, r0, i0)
199# define subi(r0, r1, i0) _subi(_jit, r0, r1, i0)
200static void _subi(jit_state_t*, jit_int32_t, jit_int32_t, jit_word_t);
201# define subcr(r0, r1, r2) _subcr(_jit, r0, r1, r2)
202static void _subcr(jit_state_t*,jit_int32_t,jit_int32_t,jit_int32_t);
203# define subci(r0, r1, i0) _subci(_jit, r0, r1, i0)
204static void _subci(jit_state_t*,jit_int32_t,jit_int32_t,jit_word_t);
205# define isubxr(r0, r1) alur(X86_SBB, r0, r1)
206# define subxr(r0, r1, r2) _subxr(_jit, r0, r1, r2)
207static void _subxr(jit_state_t*,jit_int32_t,jit_int32_t,jit_int32_t);
208# define isubxi(r0, i0) alui(X86_SBB, r0, i0)
209# define subxi(r0, r1, i0) _subxi(_jit, r0, r1, i0)
210static void _subxi(jit_state_t*,jit_int32_t,jit_int32_t,jit_word_t);
211# define rsbi(r0, r1, i0) _rsbi(_jit, r0, r1, i0)
212static void _rsbi(jit_state_t*,jit_int32_t,jit_int32_t,jit_word_t);
213# define imulr(r0, r1) _imulr(_jit, r0, r1)
214static void _imulr(jit_state_t*, jit_int32_t, jit_int32_t);
215# define imuli(r0, r1, i0) _imuli(_jit, r0, r1, i0)
216static void _imuli(jit_state_t*, jit_int32_t, jit_int32_t, jit_word_t);
217# define mulr(r0, r1, r2) _mulr(_jit, r0, r1, r2)
218static void _mulr(jit_state_t*, jit_int32_t, jit_int32_t, jit_int32_t);
219# define muli(r0, r1, i0) _muli(_jit, r0, r1, i0)
220static void _muli(jit_state_t*, jit_int32_t, jit_int32_t, jit_word_t);
221# define umulr(r0) unr(X86_IMUL, r0)
222# define umulr_u(r0) unr(X86_MUL, r0)
223# define qmulr(r0, r1, r2, r3) _iqmulr(_jit, r0, r1, r2, r3, 1)
224# define qmulr_u(r0, r1, r2, r3) _iqmulr(_jit, r0, r1, r2, r3, 0)
225# define iqmulr(r0, r1, r2, r3, sign) _iqmulr(_jit, r0, r1, r2, r3, sign)
226static void _iqmulr(jit_state_t*, jit_int32_t, jit_int32_t,
227 jit_int32_t,jit_int32_t, jit_bool_t);
228# define qmuli(r0, r1, r2, i0) _iqmuli(_jit, r0, r1, r2, i0, 1)
229# define qmuli_u(r0, r1, r2, i0) _iqmuli(_jit, r0, r1, r2, i0, 0)
230# define iqmuli(r0, r1, r2, i0, sign) _iqmuli(_jit, r0, r1, r2, i0, sign)
231static void _iqmuli(jit_state_t*, jit_int32_t, jit_int32_t,
232 jit_int32_t,jit_word_t, jit_bool_t);
233# define sign_extend_rdx_rax() _sign_extend_rdx_rax(_jit)
234static void _sign_extend_rdx_rax(jit_state_t*);
235# define idivr(r0) unr(X86_IDIV, r0)
236# define idivr_u(r0) unr(X86_DIV, r0)
237# define divremr(r0, r1, r2, i0, i1) _divremr(_jit, r0, r1, r2, i0, i1)
238static void
239_divremr(jit_state_t*,jit_int32_t,jit_int32_t,jit_int32_t,
240 jit_bool_t,jit_bool_t);
241# define divremi(r0, r1, i0, i1, i2) _divremi(_jit, r0, r1, i0, i1, i2)
242static void
243_divremi(jit_state_t*,jit_int32_t,jit_int32_t,jit_word_t,jit_bool_t,jit_bool_t);
244# define divr(r0, r1, r2) divremr(r0, r1, r2, 1, 1)
245# define divi(r0, r1, i0) divremi(r0, r1, i0, 1, 1)
246# define divr_u(r0, r1, r2) divremr(r0, r1, r2, 0, 1)
247# define divi_u(r0, r1, i0) divremi(r0, r1, i0, 0, 1)
248# define qdivr(r0, r1, r2, r3) _iqdivr(_jit, r0, r1, r2, r3, 1)
249# define qdivr_u(r0, r1, r2, r3) _iqdivr(_jit, r0, r1, r2, r3, 0)
250# define iqdivr(r0, r1, r2, r3, sign) _iqdivr(_jit, r0, r1, r2, r3, sign)
251static void _iqdivr(jit_state_t*, jit_int32_t, jit_int32_t,
252 jit_int32_t,jit_int32_t, jit_bool_t);
253# define qdivi(r0, r1, r2, i0) _iqdivi(_jit, r0, r1, r2, i0, 1)
254# define qdivi_u(r0, r1, r2, i0) _iqdivi(_jit, r0, r1, r2, i0, 0)
255# define iqdivi(r0, r1, r2, i0, sign) _iqdivi(_jit, r0, r1, r2, i0, sign)
256static void _iqdivi(jit_state_t*, jit_int32_t, jit_int32_t,
257 jit_int32_t,jit_word_t, jit_bool_t);
258# define remr(r0, r1, r2) divremr(r0, r1, r2, 1, 0)
259# define remi(r0, r1, i0) divremi(r0, r1, i0, 1, 0)
260# define remr_u(r0, r1, r2) divremr(r0, r1, r2, 0, 0)
261# define remi_u(r0, r1, i0) divremi(r0, r1, i0, 0, 0)
262# define iandr(r0, r1) alur(X86_AND, r0, r1)
263# define andr(r0, r1, r2) _andr(_jit, r0, r1, r2)
264static void _andr(jit_state_t*,jit_int32_t,jit_int32_t,jit_int32_t);
265# define iandi(r0, i0) alui(X86_AND, r0, i0)
266# define andi(r0, r1, i0) _andi(_jit, r0, r1, i0)
267static void _andi(jit_state_t*, jit_int32_t,jit_int32_t,jit_word_t);
268# define iorr(r0, r1) alur(X86_OR, r0, r1)
269# define orr(r0, r1, r2) _orr(_jit, r0, r1, r2)
270static void _orr(jit_state_t*, jit_int32_t,jit_int32_t,jit_int32_t);
271# define iori(r0, i0) alui(X86_OR, r0, i0)
272# define ori(r0, r1, i0) _ori(_jit, r0, r1, i0)
273static void _ori(jit_state_t*, jit_int32_t,jit_int32_t,jit_word_t);
274# define ixorr(r0, r1) alur(X86_XOR, r0, r1)
275# define xorr(r0, r1, r2) _xorr(_jit, r0, r1, r2)
276static void _xorr(jit_state_t*, jit_int32_t,jit_int32_t,jit_int32_t);
277# define ixori(r0, i0) alui(X86_XOR, r0, i0)
278# define xori(r0, r1, i0) _xori(_jit, r0, r1, i0)
279static void _xori(jit_state_t*, jit_int32_t,jit_int32_t,jit_word_t);
280# define irotshr(code, r0) _irotshr(_jit, code, r0)
281static void _irotshr(jit_state_t*, jit_int32_t, jit_int32_t);
282# define rotshr(code, r0, r1, r2) _rotshr(_jit, code, r0, r1, r2)
283static void
284_rotshr(jit_state_t*,jit_int32_t,jit_int32_t,jit_int32_t,jit_int32_t);
285# define irotshi(code, r0, i0) _irotshi(_jit, code, r0, i0)
286static void _irotshi(jit_state_t*, jit_int32_t, jit_int32_t, jit_word_t);
287# define rotshi(code, r0, r1, i0) _rotshi(_jit, code, r0, r1, i0)
288static void
289_rotshi(jit_state_t*,jit_int32_t,jit_int32_t,jit_int32_t,jit_word_t);
290# define lshr(r0, r1, r2) rotshr(X86_SHL, r0, r1, r2)
291# define lshi(r0, r1, i0) _lshi(_jit, r0, r1, i0)
292static void _lshi(jit_state_t*, jit_int32_t, jit_int32_t, jit_word_t);
293# define rshr(r0, r1, r2) rotshr(X86_SAR, r0, r1, r2)
294# define rshi(r0, r1, i0) rotshi(X86_SAR, r0, r1, i0)
295# define rshr_u(r0, r1, r2) rotshr(X86_SHR, r0, r1, r2)
296# define rshi_u(r0, r1, i0) rotshi(X86_SHR, r0, r1, i0)
297# define unr(code, r0) _unr(_jit, code, r0)
298static void _unr(jit_state_t*, jit_int32_t, jit_int32_t);
299# define inegr(r0) unr(X86_NEG, r0)
300# define negr(r0, r1) _negr(_jit, r0, r1)
301static void _negr(jit_state_t*, jit_int32_t, jit_int32_t);
302# define icomr(r0) unr(X86_NOT, r0)
303# define comr(r0, r1) _comr(_jit, r0, r1)
304static void _comr(jit_state_t*, jit_int32_t, jit_int32_t);
305# if USE_INC_DEC
306# define incr(r0, r1) _incr(_jit, r0, r1)
307static void _incr(jit_state_t*, jit_int32_t, jit_int32_t);
308# define decr(r0, r1) _decr(_jit, r0, r1)
309static void _decr(jit_state_t*, jit_int32_t, jit_int32_t);
310# endif
311# define cr(code, r0, r1, r2) _cr(_jit, code, r0, r1, r2)
312static void
313_cr(jit_state_t*, jit_int32_t, jit_int32_t, jit_int32_t, jit_int32_t);
314# define ci(code, r0, r1, i0) _ci(_jit, code, r0, r1, i0)
315static void
316_ci(jit_state_t *_jit, jit_int32_t, jit_int32_t, jit_int32_t, jit_word_t);
317# define ci0(code, r0, r1) _ci0(_jit, code, r0, r1)
318static void _ci0(jit_state_t*, jit_int32_t, jit_int32_t, jit_int32_t);
319# define ltr(r0, r1, r2) _ltr(_jit, r0, r1, r2)
320static void _ltr(jit_state_t*, jit_int32_t, jit_int32_t, jit_int32_t);
321# define lti(r0, r1, i0) _lti(_jit, r0, r1, i0)
322static void _lti(jit_state_t*, jit_int32_t, jit_int32_t, jit_word_t);
323# define ltr_u(r0, r1, r2) _ltr_u(_jit, r0, r1, r2)
324static void _ltr_u(jit_state_t*, jit_int32_t, jit_int32_t, jit_int32_t);
325# define lti_u(r0, r1, i0) ci(X86_CC_B, r0, r1, i0)
326# define ler(r0, r1, r2) _ler(_jit, r0, r1, r2)
327static void _ler(jit_state_t*, jit_int32_t, jit_int32_t, jit_int32_t);
328# define lei(r0, r1, i0) ci(X86_CC_LE, r0, r1, i0)
329# define ler_u(r0, r1, r2) _ler_u(_jit, r0, r1, r2)
330static void _ler_u(jit_state_t*, jit_int32_t, jit_int32_t, jit_int32_t);
331# define lei_u(r0, r1, i0) _lei_u(_jit, r0, r1, i0)
332static void _lei_u(jit_state_t*, jit_int32_t, jit_int32_t, jit_word_t);
333# define eqr(r0, r1, r2) _eqr(_jit, r0, r1, r2)
334static void _eqr(jit_state_t*, jit_int32_t, jit_int32_t, jit_int32_t);
335# define eqi(r0, r1, i0) _eqi(_jit, r0, r1, i0)
336static void _eqi(jit_state_t*, jit_int32_t, jit_int32_t, jit_word_t);
337# define ger(r0, r1, r2) _ger(_jit, r0, r1, r2)
338static void _ger(jit_state_t*, jit_int32_t, jit_int32_t, jit_int32_t);
339# define gei(r0, r1, i0) _gei(_jit, r0, r1, i0)
340static void _gei(jit_state_t*, jit_int32_t, jit_int32_t, jit_word_t);
341# define ger_u(r0, r1, r2) _ger_u(_jit, r0, r1, r2)
342static void _ger_u(jit_state_t*, jit_int32_t, jit_int32_t, jit_int32_t);
343# define gei_u(r0, r1, i0) _gei_u(_jit, r0, r1, i0)
344static void _gei_u(jit_state_t*, jit_int32_t, jit_int32_t, jit_word_t);
345# define gtr(r0, r1, r2) _gtr(_jit, r0, r1, r2)
346static void _gtr(jit_state_t*, jit_int32_t, jit_int32_t, jit_int32_t);
347# define gti(r0, r1, i0) _ci(_jit, X86_CC_G, r0, r1, i0)
348# define gtr_u(r0, r1, r2) _gtr_u(_jit, r0, r1, r2)
349static void _gtr_u(jit_state_t*, jit_int32_t, jit_int32_t, jit_int32_t);
350# define gti_u(r0, r1, i0) _gti_u(_jit, r0, r1, i0)
351static void _gti_u(jit_state_t*, jit_int32_t, jit_int32_t, jit_word_t);
352# define ner(r0, r1, r2) _ner(_jit, r0, r1, r2)
353static void _ner(jit_state_t*, jit_int32_t, jit_int32_t, jit_int32_t);
354# define nei(r0, r1, i0) _nei(_jit, r0, r1, i0)
355static void _nei(jit_state_t*, jit_int32_t, jit_int32_t, jit_word_t);
356# define movr(r0, r1) _movr(_jit, r0, r1)
357static void _movr(jit_state_t*, jit_int32_t, jit_int32_t);
358# define imovi(r0, i0) _imovi(_jit, r0, i0)
359static void _imovi(jit_state_t*, jit_int32_t, jit_word_t);
360# define movi(r0, i0) _movi(_jit, r0, i0)
361static void _movi(jit_state_t*, jit_int32_t, jit_word_t);
362# define movi_p(r0, i0) _movi_p(_jit, r0, i0)
363static jit_word_t _movi_p(jit_state_t*, jit_int32_t, jit_word_t);
364# define movcr(r0, r1) _movcr(_jit, r0, r1)
365static void _movcr(jit_state_t*,jit_int32_t,jit_int32_t);
366# define movcr_u(r0, r1) _movcr_u(_jit, r0, r1)
367static void _movcr_u(jit_state_t*,jit_int32_t,jit_int32_t);
368# define movsr(r0, r1) _movsr(_jit, r0, r1)
369static void _movsr(jit_state_t*,jit_int32_t,jit_int32_t);
370# define movsr_u(r0, r1) _movsr_u(_jit, r0, r1)
371static void _movsr_u(jit_state_t*,jit_int32_t,jit_int32_t);
372# if __X64 && !__X64_32
373# define movir(r0, r1) _movir(_jit, r0, r1)
374static void _movir(jit_state_t*,jit_int32_t,jit_int32_t);
375# define movir_u(r0, r1) _movir_u(_jit, r0, r1)
376static void _movir_u(jit_state_t*,jit_int32_t,jit_int32_t);
377# endif
378# define htonr_us(r0, r1) _htonr_us(_jit, r0, r1)
379static void _htonr_us(jit_state_t*,jit_int32_t,jit_int32_t);
380# define htonr_ui(r0, r1) _htonr_ui(_jit, r0, r1)
381static void _htonr_ui(jit_state_t*,jit_int32_t,jit_int32_t);
382# if __X64 && !__X64_32
383#define htonr_ul(r0, r1) _htonr_ul(_jit, r0, r1)
384static void _htonr_ul(jit_state_t*,jit_int32_t,jit_int32_t);
385#endif
386# define extr_c(r0, r1) _extr_c(_jit, r0, r1)
387static void _extr_c(jit_state_t*,jit_int32_t,jit_int32_t);
388# define extr_uc(r0, r1) _extr_uc(_jit, r0, r1)
389static void _extr_uc(jit_state_t*,jit_int32_t,jit_int32_t);
390# define extr_s(r0, r1) movsr(r0, r1)
391# define extr_us(r0, r1) movsr_u(r0, r1)
392# if __X64 && !__X64_32
393# define extr_i(r0, r1) movir(r0, r1)
394# define extr_ui(r0, r1) movir_u(r0, r1)
395# endif
396# define ldr_c(r0, r1) _ldr_c(_jit, r0, r1)
397static void _ldr_c(jit_state_t*, jit_int32_t, jit_int32_t);
398# define ldi_c(r0, i0) _ldi_c(_jit, r0, i0)
399static void _ldi_c(jit_state_t*, jit_int32_t, jit_word_t);
400# define ldr_uc(r0, r1) _ldr_uc(_jit, r0, r1)
401static void _ldr_uc(jit_state_t*, jit_int32_t, jit_int32_t);
402# define ldi_uc(r0, i0) _ldi_uc(_jit, r0, i0)
403static void _ldi_uc(jit_state_t*, jit_int32_t, jit_word_t);
404# define ldr_s(r0, r1) _ldr_s(_jit, r0, r1)
405static void _ldr_s(jit_state_t*, jit_int32_t, jit_int32_t);
406# define ldi_s(r0, i0) _ldi_s(_jit, r0, i0)
407static void _ldi_s(jit_state_t*, jit_int32_t, jit_word_t);
408# define ldr_us(r0, r1) _ldr_us(_jit, r0, r1)
409static void _ldr_us(jit_state_t*, jit_int32_t, jit_int32_t);
410# define ldi_us(r0, i0) _ldi_us(_jit, r0, i0)
411static void _ldi_us(jit_state_t*, jit_int32_t, jit_word_t);
412# if __X32 || !__X64_32
413# define ldr_i(r0, r1) _ldr_i(_jit, r0, r1)
414static void _ldr_i(jit_state_t*, jit_int32_t, jit_int32_t);
415# define ldi_i(r0, i0) _ldi_i(_jit, r0, i0)
416static void _ldi_i(jit_state_t*, jit_int32_t, jit_word_t);
417# endif
418# if __X64
419# if __X64_32
420# define ldr_i(r0, r1) _ldr_ui(_jit, r0, r1)
421# define ldi_i(r0, i0) _ldi_ui(_jit, r0, i0)
422# else
423# define ldr_ui(r0, r1) _ldr_ui(_jit, r0, r1)
424# define ldi_ui(r0, i0) _ldi_ui(_jit, r0, i0)
425# endif
426static void _ldr_ui(jit_state_t*, jit_int32_t, jit_int32_t);
427static void _ldi_ui(jit_state_t*, jit_int32_t, jit_word_t);
428# if !__X64_32
429# define ldr_l(r0, r1) _ldr_l(_jit, r0, r1)
430static void _ldr_l(jit_state_t*, jit_int32_t, jit_int32_t);
431# define ldi_l(r0, i0) _ldi_l(_jit, r0, i0)
432static void _ldi_l(jit_state_t*, jit_int32_t, jit_word_t);
433# endif
434# endif
435# define ldxr_c(r0, r1, r2) _ldxr_c(_jit, r0, r1, r2)
436static void _ldxr_c(jit_state_t*, jit_int32_t, jit_int32_t, jit_int32_t);
437# define ldxi_c(r0, r1, i0) _ldxi_c(_jit, r0, r1, i0)
438static void _ldxi_c(jit_state_t*, jit_int32_t, jit_int32_t, jit_word_t);
439# define ldxr_uc(r0, r1, r2) _ldxr_uc(_jit, r0, r1, r2)
440static void _ldxr_uc(jit_state_t*, jit_int32_t, jit_int32_t, jit_int32_t);
441# define ldxi_uc(r0, r1, i0) _ldxi_uc(_jit, r0, r1, i0)
442static void _ldxi_uc(jit_state_t*, jit_int32_t, jit_int32_t, jit_word_t);
443# define ldxr_s(r0, r1, r2) _ldxr_s(_jit, r0, r1, r2)
444static void _ldxr_s(jit_state_t*, jit_int32_t, jit_int32_t, jit_int32_t);
445# define ldxi_s(r0, r1, i0) _ldxi_s(_jit, r0, r1, i0)
446static void _ldxi_s(jit_state_t*, jit_int32_t, jit_int32_t, jit_word_t);
447# define ldxr_us(r0, r1, r2) _ldxr_us(_jit, r0, r1, r2)
448static void _ldxr_us(jit_state_t*, jit_int32_t, jit_int32_t, jit_int32_t);
449# define ldxi_us(r0, r1, i0) _ldxi_us(_jit, r0, r1, i0)
450static void _ldxi_us(jit_state_t*, jit_int32_t, jit_int32_t, jit_word_t);
451# if __X32 || !__X64_32
452# define ldxr_i(r0, r1, r2) _ldxr_i(_jit, r0, r1, r2)
453static void _ldxr_i(jit_state_t*, jit_int32_t, jit_int32_t, jit_int32_t);
454# define ldxi_i(r0, r1, i0) _ldxi_i(_jit, r0, r1, i0)
455static void _ldxi_i(jit_state_t*, jit_int32_t, jit_int32_t, jit_word_t);
456# endif
457# if __X64
458# if __X64_32
459# define ldxr_i(r0, r1, r2) _ldxr_ui(_jit, r0, r1, r2)
460# define ldxi_i(r0, r1, i0) _ldxi_ui(_jit, r0, r1, i0)
461# else
462# define ldxr_ui(r0, r1, r2) _ldxr_ui(_jit, r0, r1, r2)
463# define ldxi_ui(r0, r1, i0) _ldxi_ui(_jit, r0, r1, i0)
464# endif
465static void _ldxr_ui(jit_state_t*, jit_int32_t, jit_int32_t, jit_int32_t);
466static void _ldxi_ui(jit_state_t*, jit_int32_t, jit_int32_t, jit_word_t);
467# if !__X64_32
468# define ldxr_l(r0, r1, r2) _ldxr_l(_jit, r0, r1, r2)
469static void _ldxr_l(jit_state_t*, jit_int32_t, jit_int32_t, jit_int32_t);
470# define ldxi_l(r0, r1, i0) _ldxi_l(_jit, r0, r1, i0)
471static void _ldxi_l(jit_state_t*, jit_int32_t, jit_int32_t, jit_word_t);
472# endif
473# endif
474# define str_c(r0, r1) _str_c(_jit, r0, r1)
475static void _str_c(jit_state_t*, jit_int32_t, jit_int32_t);
476# define sti_c(i0, r0) _sti_c(_jit, i0, r0)
477static void _sti_c(jit_state_t*, jit_word_t, jit_int32_t);
478# define str_s(r0, r1) _str_s(_jit, r0, r1)
479static void _str_s(jit_state_t*, jit_int32_t, jit_int32_t);
480# define sti_s(i0, r0) _sti_s(_jit, i0, r0)
481static void _sti_s(jit_state_t*, jit_word_t, jit_int32_t);
482# define str_i(r0, r1) _str_i(_jit, r0, r1)
483static void _str_i(jit_state_t*, jit_int32_t, jit_int32_t);
484# define sti_i(i0, r0) _sti_i(_jit, i0, r0)
485static void _sti_i(jit_state_t*, jit_word_t, jit_int32_t);
486# if __X64 && !__X64_32
487# define str_l(r0, r1) _str_l(_jit, r0, r1)
488static void _str_l(jit_state_t*, jit_int32_t, jit_int32_t);
489# define sti_l(i0, r0) _sti_l(_jit, i0, r0)
490static void _sti_l(jit_state_t*, jit_word_t, jit_int32_t);
491# endif
492# define stxr_c(r0, r1, r2) _stxr_c(_jit, r0, r1, r2)
493static void _stxr_c(jit_state_t*, jit_int32_t, jit_int32_t, jit_int32_t);
494# define stxi_c(i0, r0, r1) _stxi_c(_jit, i0, r0, r1)
495static void _stxi_c(jit_state_t*, jit_word_t, jit_int32_t, jit_int32_t);
496# define stxr_s(r0, r1, r2) _stxr_s(_jit, r0, r1, r2)
497static void _stxr_s(jit_state_t*, jit_int32_t, jit_int32_t, jit_int32_t);
498# define stxi_s(i0, r0, r1) _stxi_s(_jit, i0, r0, r1)
499static void _stxi_s(jit_state_t*, jit_word_t, jit_int32_t, jit_int32_t);
500# define stxr_i(r0, r1, r2) _stxr_i(_jit, r0, r1, r2)
501static void _stxr_i(jit_state_t*, jit_int32_t, jit_int32_t, jit_int32_t);
502# define stxi_i(i0, r0, r1) _stxi_i(_jit, i0, r0, r1)
503static void _stxi_i(jit_state_t*, jit_word_t, jit_int32_t, jit_int32_t);
504# if __X64 && !__X64_32
505# define stxr_l(r0, r1, r2) _stxr_l(_jit, r0, r1, r2)
506static void _stxr_l(jit_state_t*, jit_int32_t, jit_int32_t, jit_int32_t);
507# define stxi_l(i0, r0, r1) _stxi_l(_jit, i0, r0, r1)
508static void _stxi_l(jit_state_t*, jit_word_t, jit_int32_t, jit_int32_t);
509# endif
510# define jcc(code, i0) _jcc(_jit, code, i0)
511# define jo(i0) jcc(X86_CC_O, i0)
512# define jno(i0) jcc(X86_CC_NO, i0)
513# define jnae(i0) jcc(X86_CC_NAE, i0)
514# define jb(i0) jcc(X86_CC_B, i0)
515# define jc(i0) jcc(X86_CC_C, i0)
516# define jae(i0) jcc(X86_CC_AE, i0)
517# define jnb(i0) jcc(X86_CC_NB, i0)
518# define jnc(i0) jcc(X86_CC_NC, i0)
519# define je(i0) jcc(X86_CC_E, i0)
520# define jz(i0) jcc(X86_CC_Z, i0)
521# define jne(i0) jcc(X86_CC_NE, i0)
522# define jnz(i0) jcc(X86_CC_NZ, i0)
523# define jbe(i0) jcc(X86_CC_BE, i0)
524# define jna(i0) jcc(X86_CC_NA, i0)
525# define ja(i0) jcc(X86_CC_A, i0)
526# define jnbe(i0) jcc(X86_CC_NBE, i0)
527# define js(i0) jcc(X86_CC_S, i0)
528# define jns(i0) jcc(X86_CC_NS, i0)
529# define jp(i0) jcc(X86_CC_P, i0)
530# define jpe(i0) jcc(X86_CC_PE, i0)
531# define jnp(i0) jcc(X86_CC_NP, i0)
532# define jpo(i0) jcc(X86_CC_PO, i0)
533# define jl(i0) jcc(X86_CC_L, i0)
534# define jnge(i0) jcc(X86_CC_NGE, i0)
535# define jge(i0) jcc(X86_CC_GE, i0)
536# define jnl(i0) jcc(X86_CC_NL, i0)
537# define jle(i0) jcc(X86_CC_LE, i0)
538# define jng(i0) jcc(X86_CC_NG, i0)
539# define jg(i0) jcc(X86_CC_G, i0)
540# define jnle(i0) jcc(X86_CC_NLE, i0)
541static void _jcc(jit_state_t*, jit_int32_t, jit_word_t);
542# define jccs(code, i0) _jccs(_jit, code, i0)
543# define jos(i0) jccs(X86_CC_O, i0)
544# define jnos(i0) jccs(X86_CC_NO, i0)
545# define jnaes(i0) jccs(X86_CC_NAE, i0)
546# define jbs(i0) jccs(X86_CC_B, i0)
547# define jcs(i0) jccs(X86_CC_C, i0)
548# define jaes(i0) jccs(X86_CC_AE, i0)
549# define jnbs(i0) jccs(X86_CC_NB, i0)
550# define jncs(i0) jccs(X86_CC_NC, i0)
551# define jes(i0) jccs(X86_CC_E, i0)
552# define jzs(i0) jccs(X86_CC_Z, i0)
553# define jnes(i0) jccs(X86_CC_NE, i0)
554# define jnzs(i0) jccs(X86_CC_NZ, i0)
555# define jbes(i0) jccs(X86_CC_BE, i0)
556# define jnas(i0) jccs(X86_CC_NA, i0)
557# define jas(i0) jccs(X86_CC_A, i0)
558# define jnbes(i0) jccs(X86_CC_NBE, i0)
559# define jss(i0) jccs(X86_CC_S, i0)
560# define jnss(i0) jccs(X86_CC_NS, i0)
561# define jps(i0) jccs(X86_CC_P, i0)
562# define jpes(i0) jccs(X86_CC_PE, i0)
563# define jnps(i0) jccs(X86_CC_NP, i0)
564# define jpos(i0) jccs(X86_CC_PO, i0)
565# define jls(i0) jccs(X86_CC_L, i0)
566# define jnges(i0) jccs(X86_CC_NGE, i0)
567# define jges(i0) jccs(X86_CC_GE, i0)
568# define jnls(i0) jccs(X86_CC_NL, i0)
569# define jles(i0) jccs(X86_CC_LE, i0)
570# define jngs(i0) jccs(X86_CC_NG, i0)
571# define jgs(i0) jccs(X86_CC_G, i0)
572# define jnles(i0) jccs(X86_CC_NLE, i0)
573static void _jccs(jit_state_t*, jit_int32_t, jit_word_t);
574# define jcr(code, i0, r0, r1) _jcr(_jit, code, i0, r0, r1)
575static void _jcr(jit_state_t*,jit_int32_t,jit_word_t,jit_int32_t,jit_int32_t);
576# define jci(code, i0, r0, i1) _jci(_jit, code, i0, r0, i1)
577static void _jci(jit_state_t*,jit_int32_t,jit_word_t,jit_int32_t,jit_word_t);
578# define jci0(code, i0, r0) _jci0(_jit, code, i0, r0)
579static void _jci0(jit_state_t*, jit_int32_t, jit_word_t, jit_int32_t);
580# define bltr(i0, r0, r1) _bltr(_jit, i0, r0, r1)
581static jit_word_t _bltr(jit_state_t*, jit_word_t, jit_int32_t, jit_int32_t);
582# define blti(i0, r0, i1) _blti(_jit, i0, r0, i1)
583static jit_word_t _blti(jit_state_t*, jit_word_t, jit_int32_t, jit_word_t);
584# define bltr_u(i0, r0, r1) _bltr_u(_jit, i0, r0, r1)
585static jit_word_t _bltr_u(jit_state_t*, jit_word_t, jit_int32_t, jit_int32_t);
586# define blti_u(i0, r0, i1) _blti_u(_jit, i0, r0, i1)
587static jit_word_t _blti_u(jit_state_t*, jit_word_t, jit_int32_t, jit_word_t);
588# define bler(i0, r0, r1) _bler(_jit, i0, r0, r1)
589static jit_word_t _bler(jit_state_t*, jit_word_t, jit_int32_t, jit_int32_t);
590# define blei(i0, r0, i1) _blei(_jit, i0, r0, i1)
591static jit_word_t _blei(jit_state_t*, jit_word_t, jit_int32_t, jit_word_t);
592# define bler_u(i0, r0, r1) _bler_u(_jit, i0, r0, r1)
593static jit_word_t _bler_u(jit_state_t*, jit_word_t, jit_int32_t, jit_int32_t);
594# define blei_u(i0, r0, i1) _blei_u(_jit, i0, r0, i1)
595static jit_word_t _blei_u(jit_state_t*, jit_word_t, jit_int32_t, jit_word_t);
596# define beqr(i0, r0, r1) _beqr(_jit, i0, r0, r1)
597static jit_word_t _beqr(jit_state_t*, jit_word_t, jit_int32_t, jit_int32_t);
598# define beqi(i0, r0, i1) _beqi(_jit, i0, r0, i1)
599static jit_word_t _beqi(jit_state_t*, jit_word_t, jit_int32_t, jit_word_t);
600# define bger(i0, r0, r1) _bger(_jit, i0, r0, r1)
601static jit_word_t _bger(jit_state_t*, jit_word_t, jit_int32_t, jit_int32_t);
602# define bgei(i0, r0, i1) _bgei(_jit, i0, r0, i1)
603static jit_word_t _bgei(jit_state_t*, jit_word_t, jit_int32_t, jit_word_t);
604# define bger_u(i0, r0, r1) _bger_u(_jit, i0, r0, r1)
605static jit_word_t _bger_u(jit_state_t*, jit_word_t, jit_int32_t, jit_int32_t);
606# define bgei_u(i0, r0, i1) _bgei_u(_jit, i0, r0, i1)
607static jit_word_t _bgei_u(jit_state_t*, jit_word_t, jit_int32_t, jit_word_t);
608# define bgtr(i0, r0, r1) _bgtr(_jit, i0, r0, r1)
609static jit_word_t _bgtr(jit_state_t*, jit_word_t, jit_int32_t, jit_int32_t);
610# define bgti(i0, r0, i1) _bgti(_jit, i0, r0, i1)
611static jit_word_t _bgti(jit_state_t*, jit_word_t, jit_int32_t, jit_word_t);
612# define bgtr_u(i0, r0, r1) _bgtr_u(_jit, i0, r0, r1)
613static jit_word_t _bgtr_u(jit_state_t*, jit_word_t, jit_int32_t, jit_int32_t);
614# define bgti_u(i0, r0, i1) _bgti_u(_jit, i0, r0, i1)
615static jit_word_t _bgti_u(jit_state_t*, jit_word_t, jit_int32_t, jit_word_t);
616# define bner(i0, r0, r1) _bner(_jit, i0, r0, r1)
617static jit_word_t _bner(jit_state_t*, jit_word_t, jit_int32_t, jit_int32_t);
618# define bnei(i0, r0, i1) _bnei(_jit, i0, r0, i1)
619static jit_word_t _bnei(jit_state_t*, jit_word_t, jit_int32_t, jit_word_t);
620# define bmsr(i0, r0, r1) _bmsr(_jit, i0, r0, r1)
621static jit_word_t _bmsr(jit_state_t*,jit_word_t,jit_int32_t,jit_int32_t);
622# define bmsi(i0, r0, i1) _bmsi(_jit, i0, r0, i1)
623static jit_word_t _bmsi(jit_state_t*,jit_word_t,jit_int32_t,jit_word_t);
624# define bmcr(i0, r0, r1) _bmcr(_jit, i0, r0, r1)
625static jit_word_t _bmcr(jit_state_t*,jit_word_t,jit_int32_t,jit_int32_t);
626# define bmci(i0, r0, i1) _bmci(_jit, i0, r0, i1)
627static jit_word_t _bmci(jit_state_t*,jit_word_t,jit_int32_t,jit_word_t);
628# define boaddr(i0, r0, r1) _boaddr(_jit, i0, r0, r1)
629static jit_word_t _boaddr(jit_state_t*,jit_word_t,jit_int32_t,jit_int32_t);
630# define boaddi(i0, r0, i1) _boaddi(_jit, i0, r0, i1)
631static jit_word_t _boaddi(jit_state_t*,jit_word_t,jit_int32_t,jit_word_t);
632# define boaddr_u(i0, r0, r1) _boaddr_u(_jit, i0, r0, r1)
633static jit_word_t _boaddr_u(jit_state_t*,jit_word_t,jit_int32_t,jit_int32_t);
634# define boaddi_u(i0, r0, i1) _boaddi_u(_jit, i0, r0, i1)
635static jit_word_t _boaddi_u(jit_state_t*,jit_word_t,jit_int32_t,jit_word_t);
636# define bxaddr(i0, r0, r1) _bxaddr(_jit, i0, r0, r1)
637static jit_word_t _bxaddr(jit_state_t*,jit_word_t,jit_int32_t,jit_int32_t);
638# define bxaddi(i0, r0, i1) _bxaddi(_jit, i0, r0, i1)
639static jit_word_t _bxaddi(jit_state_t*,jit_word_t,jit_int32_t,jit_word_t);
640# define bxaddr_u(i0, r0, r1) _bxaddr_u(_jit, i0, r0, r1)
641static jit_word_t _bxaddr_u(jit_state_t*,jit_word_t,jit_int32_t,jit_int32_t);
642# define bxaddi_u(i0, r0, i1) _bxaddi_u(_jit, i0, r0, i1)
643static jit_word_t _bxaddi_u(jit_state_t*,jit_word_t,jit_int32_t,jit_word_t);
644# define bosubr(i0, r0, r1) _bosubr(_jit, i0, r0, r1)
645static jit_word_t _bosubr(jit_state_t*,jit_word_t,jit_int32_t,jit_int32_t);
646# define bosubi(i0, r0, i1) _bosubi(_jit, i0, r0, i1)
647static jit_word_t _bosubi(jit_state_t*,jit_word_t,jit_int32_t,jit_word_t);
648# define bosubr_u(i0, r0, r1) _bosubr_u(_jit, i0, r0, r1)
649static jit_word_t _bosubr_u(jit_state_t*,jit_word_t,jit_int32_t,jit_int32_t);
650# define bosubi_u(i0, r0, i1) _bosubi_u(_jit, i0, r0, i1)
651static jit_word_t _bosubi_u(jit_state_t*,jit_word_t,jit_int32_t,jit_word_t);
652# define bxsubr(i0, r0, r1) _bxsubr(_jit, i0, r0, r1)
653static jit_word_t _bxsubr(jit_state_t*,jit_word_t,jit_int32_t,jit_int32_t);
654# define bxsubi(i0, r0, i1) _bxsubi(_jit, i0, r0, i1)
655static jit_word_t _bxsubi(jit_state_t*,jit_word_t,jit_int32_t,jit_word_t);
656# define bxsubr_u(i0, r0, r1) _bxsubr_u(_jit, i0, r0, r1)
657static jit_word_t _bxsubr_u(jit_state_t*,jit_word_t,jit_int32_t,jit_int32_t);
658# define bxsubi_u(i0, r0, i1) _bxsubi_u(_jit, i0, r0, i1)
659static jit_word_t _bxsubi_u(jit_state_t*,jit_word_t,jit_int32_t,jit_word_t);
660# define callr(r0) _callr(_jit, r0)
661static void _callr(jit_state_t*, jit_int32_t);
662# define calli(i0) _calli(_jit, i0)
663static jit_word_t _calli(jit_state_t*, jit_word_t);
519a9ea1
PC		 664# if __X64
665# define calli_p(i0) _calli_p(_jit, i0)
666static jit_word_t _calli_p(jit_state_t*, jit_word_t);
667# else
668# define calli_p(i0) calli(i0)
669# endif
4a71579b
PC		 670# define jmpr(r0) _jmpr(_jit, r0)
671static void _jmpr(jit_state_t*, jit_int32_t);
672# define jmpi(i0) _jmpi(_jit, i0)
673static jit_word_t _jmpi(jit_state_t*, jit_word_t);
519a9ea1
PC		 674# if __X64
675# define jmpi_p(i0) _jmpi_p(_jit, i0)
676static jit_word_t _jmpi_p(jit_state_t*, jit_word_t);
677# else
678# define jmpi_p(i0) jmpi(i0)
679# endif
4a71579b
PC		 680# define jmpsi(i0) _jmpsi(_jit, i0)
681static void _jmpsi(jit_state_t*, jit_uint8_t);
682# define prolog(node) _prolog(_jit, node)
683static void _prolog(jit_state_t*, jit_node_t*);
684# define epilog(node) _epilog(_jit, node)
685static void _epilog(jit_state_t*, jit_node_t*);
686# define vastart(r0) _vastart(_jit, r0)
687static void _vastart(jit_state_t*, jit_int32_t);
688# define vaarg(r0, r1) _vaarg(_jit, r0, r1)
689static void _vaarg(jit_state_t*, jit_int32_t, jit_int32_t);
690# define vaarg_d(r0, r1, i0) _vaarg_d(_jit, r0, r1, i0)
691static void _vaarg_d(jit_state_t*, jit_int32_t, jit_int32_t, jit_bool_t);
692# define patch_at(node, instr, label) _patch_at(_jit, node, instr, label)
693static void _patch_at(jit_state_t*, jit_node_t*, jit_word_t, jit_word_t);
694# if !defined(HAVE_FFSL)
695# if __X32
696# define ffsl(i) __builtin_ffs(i)
697# else
698# define ffsl(l) __builtin_ffsl(l)
699# endif
700# endif
701#endif
702
703#if CODE
704static void
705_rex(jit_state_t *_jit, jit_int32_t l, jit_int32_t w,
706 jit_int32_t r, jit_int32_t x, jit_int32_t b)
707{
708#if __X64
709 jit_int32_t v = 0x40 | (w << 3);
710
711 if (r != _NOREG)
712 v |= (r & 8) >> 1;
713 if (x != _NOREG)
714 v |= (x & 8) >> 2;
715 if (b != _NOREG)
716 v |= (b & 8) >> 3;
717 if (l || v != 0x40)
718 ic(v);
719#endif
720}
721
722static void
723_rx(jit_state_t *_jit, jit_int32_t rd, jit_int32_t md,
724 jit_int32_t rb, jit_int32_t ri, jit_int32_t ms)
725{
726 if (ri == _NOREG) {
727 if (rb == _NOREG) {
728#if __X32
729 mrm(0x00, r7(rd), 0x05);
730#else
731 mrm(0x00, r7(rd), 0x04);
732 sib(_SCL1, 0x04, 0x05);
733#endif
734 ii(md);
735 }
736 else if (r7(rb) == _RSP_REGNO) {
737 if (md == 0) {
738 mrm(0x00, r7(rd), 0x04);
739 sib(ms, 0x04, 0x04);
740 }
741 else if ((jit_int8_t)md == md) {
742 mrm(0x01, r7(rd), 0x04);
743 sib(ms, 0x04, 0x04);
744 ic(md);
745 }
746 else {
747 mrm(0x02, r7(rd), 0x04);
748 sib(ms, 0x04, 0x04);
749 ii(md);
750 }
751 }
752 else {
753 if (md == 0 && r7(rb) != _RBP_REGNO)
754 mrm(0x00, r7(rd), r7(rb));
755 else if ((jit_int8_t)md == md) {
756 mrm(0x01, r7(rd), r7(rb));
757 ic(md);
758 }
759 else {
760 mrm(0x02, r7(rd), r7(rb));
761 ii(md);
762 }
763 }
764 }
765 else if (rb == _NOREG) {
766 mrm(0x00, r7(rd), 0x04);
767 sib(ms, r7(ri), 0x05);
768 ii(md);
769 }
770 else if (r8(ri) != _RSP_REGNO) {
771 if (md == 0 && r7(rb) != _RBP_REGNO) {
772 mrm(0x00, r7(rd), 0x04);
773 sib(ms, r7(ri), r7(rb));
774 }
775 else if ((jit_int8_t)md == md) {
776 mrm(0x01, r7(rd), 0x04);
777 sib(ms, r7(ri), r7(rb));
778 ic(md);
779 }
780 else {
781 mrm(0x02, r7(rd), 0x04);
782 sib(ms, r7(ri), r7(rb));
783 ic(md);
784 }
785 }
786 else {
787 fprintf(stderr, "illegal index register");
788 abort();
789 }
790}
791
792static void
793_nop(jit_state_t *_jit, jit_int32_t count)
794{
795 switch (count) {
796 case 0:
797 break;
798 case 1: /* NOP */
799 ic(0x90); break;
800 case 2: /* 66 NOP */
801 ic(0x66); ic(0x90);
802 break;
803 case 3: /* NOP DWORD ptr [EAX] */
804 ic(0x0f); ic(0x1f); ic(0x00);
805 break;
806 case 4: /* NOP DWORD ptr [EAX + 00H] */
807 ic(0x0f); ic(0x1f); ic(0x40); ic(0x00);
808 break;
809 case 5: /* NOP DWORD ptr [EAX + EAX*1 + 00H] */
810 ic(0x0f); ic(0x1f); ic(0x44); ic(0x00);
811 ic(0x00);
812 break;
813 case 6: /* 66 NOP DWORD ptr [EAX + EAX*1 + 00H] */
814 ic(0x66); ic(0x0f); ic(0x1f); ic(0x44);
815 ic(0x00); ic(0x00);
816 break;
817 case 7: /* NOP DWORD ptr [EAX + 00000000H] */
818 ic(0x0f); ic(0x1f); ic(0x80); ii(0x0000);
819 break;
820 case 8: /* NOP DWORD ptr [EAX + EAX*1 + 00000000H] */
821 ic(0x0f); ic(0x1f); ic(0x84); ic(0x00);
822 ii(0x0000);
823 break;
824 case 9: /* 66 NOP DWORD ptr [EAX + EAX*1 + 00000000H] */
825 ic(0x66); ic(0x0f); ic(0x1f); ic(0x84);
826 ic(0x00); ii(0x0000);
827 break;
828 default:
829 abort();
830 }
831}
832
833static void
834_lea(jit_state_t *_jit, jit_int32_t md, jit_int32_t rb,
835 jit_int32_t ri, jit_int32_t ms, jit_int32_t rd)
836{
837 rex(0, WIDE, rd, ri, rb);
838 ic(0x8d);
839 rx(rd, md, rb, ri, ms);
840}
841
842static void
843_pushr(jit_state_t *_jit, jit_int32_t r0)
844{
845 rex(0, WIDE, 0, 0, r0);
846 ic(0x50 | r7(r0));
847}
848
849static void
850_popr(jit_state_t *_jit, jit_int32_t r0)
851{
852 rex(0, WIDE, 0, 0, r0);
853 ic(0x58 | r7(r0));
854}
855
856static void
857_xchgr(jit_state_t *_jit, jit_int32_t r0, jit_int32_t r1)
858{
859 rex(0, WIDE, r1, _NOREG, r0);
860 ic(0x87);
861 mrm(0x03, r7(r1), r7(r0));
862}
863
864static void
865_testr(jit_state_t *_jit, jit_int32_t r0, jit_int32_t r1)
866{
867 rex(0, WIDE, r1, _NOREG, r0);
868 ic(0x85);
869 mrm(0x03, r7(r1), r7(r0));
870}
871
872static void
873_testi(jit_state_t *_jit, jit_int32_t r0, jit_word_t i0)
874{
875 rex(0, WIDE, _NOREG, _NOREG, r0);
876 if (r0 == _RAX_REGNO)
877 ic(0xa9);
878 else {
879 ic(0xf7);
880 mrm(0x03, 0x00, r7(r0));
881 }
882 ii(i0);
883}
884
885static void
886_cc(jit_state_t *_jit, jit_int32_t code, jit_int32_t r0)
887{
888 rex(0, 0, _NOREG, _NOREG, r0);
889 ic(0x0f);
890 ic(0x90 | code);
891 mrm(0x03, 0x00, r7(r0));
892}
893
894static void
895_alur(jit_state_t *_jit, jit_int32_t code, jit_int32_t r0, jit_int32_t r1)
896{
897 rex(0, WIDE, r1, _NOREG, r0);
898 ic(code | 0x01);
899 mrm(0x03, r7(r1), r7(r0));
900}
901
902static void
903_alui(jit_state_t *_jit, jit_int32_t code, jit_int32_t r0, jit_word_t i0)
904{
905 jit_int32_t reg;
906 if (can_sign_extend_int_p(i0)) {
907 rex(0, WIDE, _NOREG, _NOREG, r0);
908 if ((jit_int8_t)i0 == i0) {
909 ic(0x83);
910 ic(0xc0 | code | r7(r0));
911 ic(i0);
912 }
913 else {
914 if (r0 == _RAX_REGNO)
915 ic(code | 0x05);
916 else {
917 ic(0x81);
918 ic(0xc0 | code | r7(r0));
919 }
920 ii(i0);
921 }
922 }
923 else {
924 reg = jit_get_reg(jit_class_gpr);
925 movi(rn(reg), i0);
926 alur(code, r0, rn(reg));
927 jit_unget_reg(reg);
928 }
929}
930
931static void
932_save(jit_state_t *_jit, jit_int32_t r0)
933{
934 if (!_jitc->function->regoff[r0]) {
935 _jitc->function->regoff[r0] = jit_allocai(sizeof(jit_word_t));
936 _jitc->again = 1;
937 }
938 assert(!jit_regset_tstbit(&_jitc->regsav, r0));
939 jit_regset_setbit(&_jitc->regsav, r0);
940 stxi(_jitc->function->regoff[r0], _RBP_REGNO, r0);
941}
942
943static void
944_load(jit_state_t *_jit, jit_int32_t r0)
945{
946 assert(_jitc->function->regoff[r0]);
947 assert(jit_regset_tstbit(&_jitc->regsav, r0));
948 jit_regset_clrbit(&_jitc->regsav, r0);
949 ldxi(r0, _RBP_REGNO, _jitc->function->regoff[r0]);
950}
951
952static void
953_addr(jit_state_t *_jit, jit_int32_t r0, jit_int32_t r1, jit_int32_t r2)
954{
955 if (r0 == r1)
956 iaddr(r0, r2);
957 else if (r0 == r2)
958 iaddr(r0, r1);
959 else
960 lea(0, r1, r2, _SCL1, r0);
961}
962
963static void
964_addi(jit_state_t *_jit, jit_int32_t r0, jit_int32_t r1, jit_word_t i0)
965{
966 jit_int32_t reg;
967 if (i0 == 0)
968 movr(r0, r1);
969#if USE_INC_DEC
970 else if (i0 == 1)
971 incr(r0, r1);
972 else if (i0 == -1)
973 decr(r0, r1);
974#endif
975 else if (can_sign_extend_int_p(i0)) {
976 if (r0 == r1)
977 iaddi(r0, i0);
978 else
979 lea(i0, r1, _NOREG, _SCL1, r0);
980 }
981 else if (r0 != r1) {
982 movi(r0, i0);
983 iaddr(r0, r1);
984 }
985 else {
986 reg = jit_get_reg(jit_class_gpr);
987 movi(rn(reg), i0);
988 iaddr(r0, rn(reg));
989 jit_unget_reg(reg);
990 }
991}
992
993static void
994_addcr(jit_state_t *_jit, jit_int32_t r0, jit_int32_t r1, jit_int32_t r2)
995{
996 if (r0 == r2)
997 iaddr(r0, r1);
998 else {
999 movr(r0, r1);
1000 iaddr(r0, r2);
1001 }
1002}
1003
1004static void
1005_addci(jit_state_t *_jit, jit_int32_t r0, jit_int32_t r1, jit_word_t i0)
1006{
1007 jit_int32_t reg;
1008 if (can_sign_extend_int_p(i0)) {
1009 movr(r0, r1);
1010 iaddi(r0, i0);
1011 }
1012 else if (r0 == r1) {
1013 reg = jit_get_reg(jit_class_gpr);
1014 movi(rn(reg), i0);
1015 iaddr(r0, rn(reg));
1016 jit_unget_reg(reg);
1017 }
1018 else {
1019 movi(r0, i0);
1020 iaddr(r0, r1);
1021 }
1022}
1023
1024static void
1025_addxr(jit_state_t *_jit, jit_int32_t r0, jit_int32_t r1, jit_int32_t r2)
1026{
1027 if (r0 == r2)
1028 iaddxr(r0, r1);
1029 else {
1030 movr(r0, r1);
1031 iaddxr(r0, r2);
1032 }
1033}
1034
1035static void
1036_addxi(jit_state_t *_jit, jit_int32_t r0, jit_int32_t r1, jit_word_t i0)
1037{
1038 jit_int32_t reg;
1039 if (can_sign_extend_int_p(i0)) {
1040 movr(r0, r1);
1041 iaddxi(r0, i0);
1042 }
1043 else if (r0 == r1) {
1044 reg = jit_get_reg(jit_class_gpr);
1045 movi(rn(reg), i0);
1046 iaddxr(r0, rn(reg));
1047 jit_unget_reg(reg);
1048 }
1049 else {
1050 movi(r0, i0);
1051 iaddxr(r0, r1);
1052 }
1053}
1054
1055static void
1056_subr(jit_state_t *_jit, jit_int32_t r0, jit_int32_t r1, jit_int32_t r2)
1057{
1058 if (r1 == r2)
1059 ixorr(r0, r0);
1060 else if (r0 == r2) {
1061 isubr(r0, r1);
1062 inegr(r0);
1063 }
1064 else {
1065 movr(r0, r1);
1066 isubr(r0, r2);
1067 }
1068}
1069
1070static void
1071_subi(jit_state_t *_jit, jit_int32_t r0, jit_int32_t r1, jit_word_t i0)
1072{
1073 jit_int32_t reg;
1074 if (i0 == 0)
1075 movr(r0, r1);
1076#if USE_INC_DEC
1077 else if (i0 == 1)
1078 decr(r0, r1);
1079 else if (i0 == -1)
1080 incr(r0, r1);
1081#endif
1082 else if (can_sign_extend_int_p(i0)) {
1083 if (r0 == r1)
1084 isubi(r0, i0);
1085 else
1086 lea(-i0, r1, _NOREG, _SCL1, r0);
1087 }
1088 else if (r0 != r1) {
1089 movi(r0, -i0);
1090 iaddr(r0, r1);
1091 }
1092 else {
1093 reg = jit_get_reg(jit_class_gpr);
1094 movi(rn(reg), i0);
1095 isubr(r0, rn(reg));
1096 jit_unget_reg(reg);
1097 }
1098}
1099
1100static void
1101_subcr(jit_state_t *_jit, jit_int32_t r0, jit_int32_t r1, jit_int32_t r2)
1102{
1103 jit_int32_t reg;
1104 if (r0 == r2 && r0 != r1) {
1105 reg = jit_get_reg(jit_class_gpr);
1106 movr(rn(reg), r0);
1107 movr(r0, r1);
1108 isubr(r0, rn(reg));
1109 jit_unget_reg(reg);
1110 }
1111 else {
1112 movr(r0, r1);
1113 isubr(r0, r2);
1114 }
1115}
1116
1117static void
1118_subci(jit_state_t *_jit, jit_int32_t r0, jit_int32_t r1, jit_word_t i0)
1119{
1120 jit_int32_t reg;
1121 movr(r0, r1);
1122 if (can_sign_extend_int_p(i0))
1123 isubi(r0, i0);
1124 else {
1125 reg = jit_get_reg(jit_class_gpr);
1126 movi(rn(reg), i0);
1127 isubr(r0, rn(reg));
1128 jit_unget_reg(reg);
1129 }
1130}
1131
1132static void
1133_subxr(jit_state_t *_jit, jit_int32_t r0, jit_int32_t r1, jit_int32_t r2)
1134{
1135 jit_int32_t reg;
1136 if (r0 == r2 && r0 != r1) {
1137 reg = jit_get_reg(jit_class_gpr);
1138 movr(rn(reg), r0);
1139 movr(r0, r1);
1140 isubxr(r0, rn(reg));
1141 jit_unget_reg(reg);
1142 }
1143 else {
1144 movr(r0, r1);
1145 isubxr(r0, r2);
1146 }
1147}
1148
1149static void
1150_subxi(jit_state_t *_jit, jit_int32_t r0, jit_int32_t r1, jit_word_t i0)
1151{
1152 jit_int32_t reg;
1153 movr(r0, r1);
1154 if (can_sign_extend_int_p(i0))
1155 isubxi(r0, i0);
1156 else {
1157 reg = jit_get_reg(jit_class_gpr);
1158 imovi(rn(reg), i0);
1159 isubxr(r0, rn(reg));
1160 jit_unget_reg(reg);
1161 }
1162}
1163
1164static void
1165_rsbi(jit_state_t *_jit, jit_int32_t r0, jit_int32_t r1, jit_word_t i0)
1166{
1167 subi(r0, r1, i0);
1168 negr(r0, r0);
1169}
1170
1171static void
1172_imulr(jit_state_t *_jit, jit_int32_t r0, jit_int32_t r1)
1173{
1174 rex(0, WIDE, r0, _NOREG, r1);
1175 ic(0x0f);
1176 ic(0xaf);
1177 mrm(0x03, r7(r0), r7(r1));
1178}
1179
1180static void
1181_imuli(jit_state_t *_jit, jit_int32_t r0, jit_int32_t r1, jit_word_t i0)
1182{
1183 jit_int32_t reg;
1184 if (can_sign_extend_int_p(i0)) {
1185 rex(0, WIDE, r0, _NOREG, r1);
1186 if ((jit_int8_t)i0 == i0) {
1187 ic(0x6b);
1188 mrm(0x03, r7(r0), r7(r1));
1189 ic(i0);
1190 }
1191 else {
1192 ic(0x69);
1193 mrm(0x03, r7(r0), r7(r1));
1194 ii(i0);
1195 }
1196 }
1197 else {
1198 reg = jit_get_reg(jit_class_gpr);
1199 movi(rn(reg), i0);
1200 imulr(r0, rn(reg));
1201 jit_unget_reg(reg);
1202 }
1203}
1204
1205static void
1206_mulr(jit_state_t *_jit, jit_int32_t r0, jit_int32_t r1, jit_int32_t r2)
1207{
1208 if (r0 == r1)
1209 imulr(r0, r2);
1210 else if (r0 == r2)
1211 imulr(r0, r1);
1212 else {
1213 movr(r0, r1);
1214 imulr(r0, r2);
1215 }
1216}
1217
1218static void
1219_muli(jit_state_t *_jit, jit_int32_t r0, jit_int32_t r1, jit_word_t i0)
1220{
1221 switch (i0) {
1222 case 0:
1223 ixorr(r0, r0);
1224 break;
1225 case 1:
1226 movr(r0, r1);
1227 break;
1228 case -1:
1229 negr(r0, r1);
1230 break;
1231 case 2:
1232 lea(0, _NOREG, r1, _SCL2, r0);
1233 break;
1234 case 4:
1235 lea(0, _NOREG, r1, _SCL4, r0);
1236 break;
1237 case 8:
1238 lea(0, _NOREG, r1, _SCL8, r0);
1239 break;
1240 default:
1241 if (i0 > 0 && !(i0 & (i0 - 1)))
1242 lshi(r0, r1, ffsl(i0) - 1);
1243 else if (can_sign_extend_int_p(i0))
1244 imuli(r0, r1, i0);
1245 else if (r0 != r1) {
1246 movi(r0, i0);
1247 imulr(r0, r1);
1248 }
1249 else
1250 imuli(r0, r0, i0);
1251 break;
1252 }
1253}
1254
1255#define savset(rn) \
1256 if (r0 != rn) { \
1257 sav |= 1 << rn; \
1258 if (r1 != rn && r2 != rn) \
1259 set |= 1 << rn; \
1260 }
1261#define isavset(rn) \
1262 if (r0 != rn) { \
1263 sav |= 1 << rn; \
1264 if (r1 != rn) \
1265 set |= 1 << rn; \
1266 }
1267#define qsavset(rn) \
1268 if (r0 != rn && r1 != rn) { \
1269 sav |= 1 << rn; \
1270 if (r2 != rn && r3 != rn) \
1271 set |= 1 << rn; \
1272 }
1273#define allocr(rn, rv) \
1274 if (set & (1 << rn)) \
1275 (void)jit_get_reg(rv|jit_class_gpr|jit_class_named); \
1276 if (sav & (1 << rn)) { \
1277 if ( jit_regset_tstbit(&_jitc->regsav, rv) || \
1278 !jit_regset_tstbit(&_jitc->reglive, rv)) \
1279 sav &= ~(1 << rn); \
1280 else \
1281 save(rv); \
1282 }
1283#define clear(rn, rv) \
1284 if (set & (1 << rn)) \
1285 jit_unget_reg(rv); \
1286 if (sav & (1 << rn)) \
1287 load(rv);
1288static void
1289_iqmulr(jit_state_t *_jit, jit_int32_t r0, jit_int32_t r1,
1290 jit_int32_t r2, jit_int32_t r3, jit_bool_t sign)
1291{
1292 jit_int32_t mul;
1293 jit_int32_t sav;
1294 jit_int32_t set;
1295
1296 sav = set = 0;
1297 qsavset(_RDX_REGNO);
1298 qsavset(_RAX_REGNO);
1299 allocr(_RDX_REGNO, _RDX);
1300 allocr(_RAX_REGNO, _RAX);
1301
1302 if (r3 == _RAX_REGNO)
1303 mul = r2;
1304 else {
1305 mul = r3;
1306 movr(_RAX_REGNO, r2);
1307 }
1308 if (sign)
1309 umulr(mul);
1310 else
1311 umulr_u(mul);
1312
1313 if (r0 == _RDX_REGNO && r1 == _RAX_REGNO)
1314 xchgr(_RAX_REGNO, _RDX_REGNO);
1315 else {
1316 if (r0 != _RDX_REGNO)
1317 movr(r0, _RAX_REGNO);
1318 movr(r1, _RDX_REGNO);
1319 if (r0 == _RDX_REGNO)
1320 movr(r0, _RAX_REGNO);
1321 }
1322
1323 clear(_RDX_REGNO, _RDX);
1324 clear(_RAX_REGNO, _RAX);
1325}
1326
1327static void
1328_iqmuli(jit_state_t *_jit, jit_int32_t r0, jit_int32_t r1,
1329 jit_int32_t r2, jit_word_t i0, jit_bool_t sign)
1330{
1331 jit_int32_t reg;
1332
1333 if (i0 == 0) {
1334 ixorr(r0, r0);
1335 ixorr(r1, r1);
1336 }
1337 else {
1338 reg = jit_get_reg(jit_class_gpr);
1339 movi(rn(reg), i0);
1340 if (sign)
1341 qmulr(r0, r1, r2, rn(reg));
1342 else
1343 qmulr_u(r0, r1, r2, rn(reg));
1344 jit_unget_reg(reg);
1345 }
1346}
1347
1348static void
1349_sign_extend_rdx_rax(jit_state_t *_jit)
1350{
1351 rex(0, WIDE, 0, 0, 0);
1352 ic(0x99);
1353}
1354
1355static void
1356_divremr(jit_state_t *_jit, jit_int32_t r0, jit_int32_t r1, jit_int32_t r2,
1357 jit_bool_t sign, jit_bool_t divide)
1358{
1359 jit_int32_t div;
1360 jit_int32_t reg;
1361 jit_int32_t set;
1362 jit_int32_t sav;
1363 jit_int32_t use;
1364
1365 sav = set = use = 0;
1366 savset(_RDX_REGNO);
1367 savset(_RAX_REGNO);
1368 allocr(_RDX_REGNO, _RDX);
1369 allocr(_RAX_REGNO, _RAX);
1370
1371 if (r2 == _RAX_REGNO) {
1372 if (r0 == _RAX_REGNO || r0 == _RDX_REGNO) {
1373 if ((reg = jit_get_reg(jit_class_gpr|jit_class_chk)) == JIT_NOREG)
1374 reg = jit_get_reg((r1 == _RCX_REGNO ? _RBX : _RCX) |
1375 jit_class_gpr|jit_class_named);
1376 use = 1;
1377 div = rn(reg);
1378 movr(div, _RAX_REGNO);
1379 if (r1 != _RAX_REGNO)
1380 movr(_RAX_REGNO, r1);
1381 }
1382 else {
1383 if (r0 == r1)
1384 xchgr(r0, _RAX_REGNO);
1385 else {
1386 if (r0 != _RAX_REGNO)
1387 movr(r0, _RAX_REGNO);
1388 if (r1 != _RAX_REGNO)
1389 movr(_RAX_REGNO, r1);
1390 }
1391 div = r0;
1392 }
1393 }
1394 else if (r2 == _RDX_REGNO) {
1395 if (r0 == _RAX_REGNO || r0 == _RDX_REGNO) {
1396 if ((reg = jit_get_reg(jit_class_gpr|jit_class_chk)) == JIT_NOREG)
1397 reg = jit_get_reg((r1 == _RCX_REGNO ? _RBX : _RCX) |
1398 jit_class_gpr|jit_class_named);
1399 use = 1;
1400 div = rn(reg);
1401 movr(div, _RDX_REGNO);
1402 if (r1 != _RAX_REGNO)
1403 movr(_RAX_REGNO, r1);
1404 }
1405 else {
1406 if (r1 != _RAX_REGNO)
1407 movr(_RAX_REGNO, r1);
1408 movr(r0, _RDX_REGNO);
1409 div = r0;
1410 }
1411 }
1412 else {
1413 if (r1 != _RAX_REGNO)
1414 movr(_RAX_REGNO, r1);
1415 div = r2;
1416 }
1417
1418 if (sign) {
1419 sign_extend_rdx_rax();
1420 idivr(div);
1421 }
1422 else {
1423 ixorr(_RDX_REGNO, _RDX_REGNO);
1424 idivr_u(div);
1425 }
1426
1427 if (use)
1428 jit_unget_reg(reg);
1429
1430 if (divide)
1431 movr(r0, _RAX_REGNO);
1432 else
1433 movr(r0, _RDX_REGNO);
1434
1435 clear(_RDX_REGNO, _RDX);
1436 clear(_RAX_REGNO, _RAX);
1437}
1438
1439static void
1440_divremi(jit_state_t *_jit, jit_int32_t r0, jit_int32_t r1, jit_word_t i0,
1441 jit_bool_t sign, jit_bool_t divide)
1442{
1443 jit_int32_t reg;
1444 jit_int32_t div;
1445 jit_int32_t sav;
1446 jit_int32_t set;
1447 jit_int32_t use;
1448
1449 if (divide) {
1450 switch (i0) {
1451 case 1:
1452 movr(r0, r1);
1453 return;
1454 case -1:
1455 if (sign) {
1456 negr(r0, r1);
1457 return;
1458 }
1459 break;
1460 default:
1461 if (i0 > 0 && !(i0 & (i0 - 1))) {
1462 movr(r0, r1);
1463 if (sign)
1464 rshi(r0, r0, ffsl(i0) - 1);
1465 else
1466 rshi_u(r0, r0, ffsl(i0) - 1);
1467 return;
1468 }
1469 break;
1470 }
1471 }
1472 else if (i0 == 1 || (sign && i0 == -1)) {
1473 ixorr(r0, r0);
1474 return;
1475 }
1476 else if (!sign && i0 > 0 && !(i0 & (i0 - 1))) {
1477 if (can_sign_extend_int_p(i0)) {
1478 movr(r0, r1);
1479 iandi(r0, i0 - 1);
1480 }
1481 else if (r0 != r1) {
1482 movi(r0, i0 - 1);
1483 iandr(r0, r1);
1484 }
1485 else {
1486 reg = jit_get_reg(jit_class_gpr);
1487 movi(rn(reg), i0 - 1);
1488 iandr(r0, rn(reg));
1489 jit_unget_reg(reg);
1490 }
1491 return;
1492 }
1493
1494 sav = set = use = 0;
1495 isavset(_RDX_REGNO);
1496 isavset(_RAX_REGNO);
1497 allocr(_RDX_REGNO, _RDX);
1498 allocr(_RAX_REGNO, _RAX);
1499
1500 if (r0 == _RAX_REGNO || r0 == _RDX_REGNO || r0 == r1) {
1501 if ((reg = jit_get_reg(jit_class_gpr|jit_class_chk)) == JIT_NOREG)
1502 reg = jit_get_reg((r1 == _RCX_REGNO ? _RBX : _RCX) |
1503 jit_class_gpr|jit_class_named);
1504 use = 1;
1505 div = rn(reg);
1506 }
1507 else
1508 div = r0;
1509
1510 movi(div, i0);
1511 movr(_RAX_REGNO, r1);
1512
1513 if (sign) {
1514 sign_extend_rdx_rax();
1515 idivr(div);
1516 }
1517 else {
1518 ixorr(_RDX_REGNO, _RDX_REGNO);
1519 idivr_u(div);
1520 }
1521
1522 if (use)
1523 jit_unget_reg(reg);
1524
1525 if (divide)
1526 movr(r0, _RAX_REGNO);
1527 else
1528 movr(r0, _RDX_REGNO);
1529
1530 clear(_RDX_REGNO, _RDX);
1531 clear(_RAX_REGNO, _RAX);
1532}
1533
1534static void
1535_iqdivr(jit_state_t *_jit, jit_int32_t r0, jit_int32_t r1,
1536 jit_int32_t r2, jit_int32_t r3, jit_bool_t sign)
1537{
1538 jit_int32_t div;
1539 jit_int32_t reg;
1540 jit_int32_t sav;
1541 jit_int32_t set;
1542 jit_int32_t use;
1543
1544 sav = set = use = 0;
1545 qsavset(_RDX_REGNO);
1546 qsavset(_RAX_REGNO);
1547 allocr(_RDX_REGNO, _RDX);
1548 allocr(_RAX_REGNO, _RAX);
1549 if (r3 == _RAX_REGNO) {
1550 if (r0 == _RAX_REGNO || r0 == _RDX_REGNO) {
1551 if ((reg = jit_get_reg(jit_class_gpr|jit_class_chk)) == JIT_NOREG)
1552 reg = jit_get_reg((r1 == _RCX_REGNO ? _RBX : _RCX) |
1553 jit_class_gpr|jit_class_named);
1554 use = 1;
1555 div = rn(reg);
1556 movr(div, _RAX_REGNO);
1557 if (r2 != _RAX_REGNO)
1558 movr(_RAX_REGNO, r2);
1559 }
1560 else {
1561 if (r0 == r2)
1562 xchgr(r0, _RAX_REGNO);
1563 else {
1564 if (r0 != _RAX_REGNO)
1565 movr(r0, _RAX_REGNO);
1566 if (r2 != _RAX_REGNO)
1567 movr(_RAX_REGNO, r2);
1568 }
1569 div = r0;
1570 }
1571 }
1572 else if (r3 == _RDX_REGNO) {
1573 if (r0 == _RAX_REGNO || r0 == _RDX_REGNO) {
1574 if ((reg = jit_get_reg(jit_class_gpr|jit_class_chk)) == JIT_NOREG)
1575 reg = jit_get_reg((r1 == _RCX_REGNO ? _RBX : _RCX) |
1576 jit_class_gpr|jit_class_named);
1577 use = 1;
1578 div = rn(reg);
1579 movr(div, _RDX_REGNO);
1580 if (r2 != _RAX_REGNO)
1581 movr(_RAX_REGNO, r2);
1582 }
1583 else {
1584 if (r2 != _RAX_REGNO)
1585 movr(_RAX_REGNO, r2);
1586 movr(r0, _RDX_REGNO);
1587 div = r0;
1588 }
1589 }
1590 else {
1591 if (r2 != _RAX_REGNO)
1592 movr(_RAX_REGNO, r2);
1593 div = r3;
1594 }
1595 if (sign) {
1596 sign_extend_rdx_rax();
1597 idivr(div);
1598 }
1599 else {
1600 ixorr(_RDX_REGNO, _RDX_REGNO);
1601 idivr_u(div);
1602 }
1603 if (use)
1604 jit_unget_reg(reg);
1605
1606 if (r0 == _RDX_REGNO && r1 == _RAX_REGNO)
1607 xchgr(_RAX_REGNO, _RDX_REGNO);
1608 else {
1609 if (r0 != _RDX_REGNO)
1610 movr(r0, _RAX_REGNO);
1611 movr(r1, _RDX_REGNO);
1612 if (r0 == _RDX_REGNO)
1613 movr(r0, _RAX_REGNO);
1614 }
1615
1616 clear(_RDX_REGNO, _RDX);
1617 clear(_RAX_REGNO, _RAX);
1618}
1619
1620static void
1621_iqdivi(jit_state_t *_jit, jit_int32_t r0, jit_int32_t r1,
1622 jit_int32_t r2, jit_word_t i0, jit_bool_t sign)
1623{
1624 jit_int32_t reg;
1625
1626 reg = jit_get_reg(jit_class_gpr);
1627 movi(rn(reg), i0);
1628 if (sign)
1629 qdivr(r0, r1, r2, rn(reg));
1630 else
1631 qdivr_u(r0, r1, r2, rn(reg));
1632 jit_unget_reg(reg);
1633}
1634#undef clear
1635#undef allocr
1636#undef savset
1637
1638static void
1639_andr(jit_state_t *_jit, jit_int32_t r0, jit_int32_t r1, jit_int32_t r2)
1640{
1641 if (r1 == r2)
1642 movr(r0, r1);
1643 else if (r0 == r1)
1644 iandr(r0, r2);
1645 else if (r0 == r2)
1646 iandr(r0, r1);
1647 else {
1648 movr(r0, r1);
1649 iandr(r0, r2);
1650 }
1651}
1652
1653static void
1654_andi(jit_state_t *_jit, jit_int32_t r0, jit_int32_t r1, jit_word_t i0)
1655{
1656 jit_int32_t reg;
1657
1658 if (i0 == 0)
1659 ixorr(r0, r0);
1660 else if (i0 == -1)
1661 movr(r0, r1);
1662 else if (r0 == r1) {
1663 if (can_sign_extend_int_p(i0))
1664 iandi(r0, i0);
1665 else {
1666 reg = jit_get_reg(jit_class_gpr);
1667 movi(rn(reg), i0);
1668 iandr(r0, rn(reg));
1669 jit_unget_reg(reg);
1670 }
1671 }
1672 else {
1673 movi(r0, i0);
1674 iandr(r0, r1);
1675 }
1676}
1677
1678static void
1679_orr(jit_state_t *_jit, jit_int32_t r0, jit_int32_t r1, jit_int32_t r2)
1680{
1681 if (r1 == r2)
1682 movr(r0, r1);
1683 else if (r0 == r1)
1684 iorr(r0, r2);
1685 else if (r0 == r2)
1686 iorr(r0, r1);
1687 else {
1688 movr(r0, r1);
1689 iorr(r0, r2);
1690 }
1691}
1692
1693static void
1694_ori(jit_state_t *_jit, jit_int32_t r0, jit_int32_t r1, jit_word_t i0)
1695{
1696 jit_int32_t reg;
1697 if (i0 == 0)
1698 movr(r0, r1);
1699 else if (i0 == -1)
1700 movi(r0, -1);
1701 else if (can_sign_extend_int_p(i0)) {
1702 movr(r0, r1);
1703 iori(r0, i0);
1704 }
1705 else if (r0 != r1) {
1706 movi(r0, i0);
1707 iorr(r0, r1);
1708 }
1709 else {
1710 reg = jit_get_reg(jit_class_gpr);
1711 movi(rn(reg), i0);
1712 iorr(r0, rn(reg));
1713 jit_unget_reg(reg);
1714 }
1715}
1716
1717static void
1718_xorr(jit_state_t *_jit, jit_int32_t r0, jit_int32_t r1, jit_int32_t r2)
1719{
1720 if (r1 == r2)
1721 ixorr(r0, r0);
1722 else if (r0 == r1)
1723 ixorr(r0, r2);
1724 else if (r0 == r2)
1725 ixorr(r0, r1);
1726 else {
1727 movr(r0, r1);
1728 ixorr(r0, r2);
1729 }
1730}
1731
1732static void
1733_xori(jit_state_t *_jit, jit_int32_t r0, jit_int32_t r1, jit_word_t i0)
1734{
1735 jit_int32_t reg;
1736 if (i0 == 0)
1737 movr(r0, r1);
1738 else if (i0 == -1)
1739 comr(r0, r1);
1740 else if (can_sign_extend_int_p(i0)) {
1741 movr(r0, r1);
1742 ixori(r0, i0);
1743 }
1744 else if (r0 != r1) {
1745 movi(r0, i0);
1746 ixorr(r0, r1);
1747 }
1748 else {
1749 reg = jit_get_reg(jit_class_gpr);
1750 movi(rn(reg), i0);
1751 ixorr(r0, rn(reg));
1752 jit_unget_reg(reg);
1753 }
1754}
1755
1756static void
1757_irotshr(jit_state_t *_jit, jit_int32_t code, jit_int32_t r0)
1758{
1759 rex(0, WIDE, _RCX_REGNO, _NOREG, r0);
1760 ic(0xd3);
1761 mrm(0x03, code, r7(r0));
1762}
1763
1764static void
1765_rotshr(jit_state_t *_jit, jit_int32_t code,
1766 jit_int32_t r0, jit_int32_t r1, jit_int32_t r2)
1767{
1768 jit_int32_t reg;
1769 jit_int32_t use;
1770
1771 if (r0 == _RCX_REGNO) {
1772 reg = jit_get_reg(jit_class_gpr);
1773 movr(rn(reg), r1);
1774 if (r2 != _RCX_REGNO)
1775 movr(_RCX_REGNO, r2);
1776 irotshr(code, rn(reg));
1777 movr(_RCX_REGNO, rn(reg));
1778 jit_unget_reg(reg);
1779 }
1780 else if (r2 != _RCX_REGNO) {
1781 use = !jit_reg_free_p(_RCX);
1782 if (use) {
1783 reg = jit_get_reg(jit_class_gpr);
1784 movr(rn(reg), _RCX_REGNO);
1785 }
1786 else
1787 reg = 0;
1788 if (r1 == _RCX_REGNO) {
1789 if (r0 == r2)
1790 xchgr(r0, _RCX_REGNO);
1791 else {
1792 movr(r0, r1);
1793 movr(_RCX_REGNO, r2);
1794 }
1795 }
1796 else {
1797 movr(_RCX_REGNO, r2);
1798 movr(r0, r1);
1799 }
1800 irotshr(code, r0);
1801 if (use) {
1802 movr(_RCX_REGNO, rn(reg));
1803 jit_unget_reg(reg);
1804 }
1805 }
1806 else {
1807 movr(r0, r1);
1808 irotshr(code, r0);
1809 }
1810}
1811
1812static void
1813_irotshi(jit_state_t *_jit, jit_int32_t code, jit_int32_t r0, jit_word_t i0)
1814{
1815 rex(0, WIDE, _NOREG, _NOREG, r0);
1816 if (i0 == 1) {
1817 ic(0xd1);
1818 mrm(0x03, code, r7(r0));
1819 }
1820 else {
1821 ic(0xc1);
1822 mrm(0x03, code, r7(r0));
1823 ic(i0);
1824 }
1825}
1826
1827static void
1828_rotshi(jit_state_t *_jit, jit_int32_t code,
1829 jit_int32_t r0, jit_int32_t r1, jit_word_t i0)
1830{
1831 movr(r0, r1);
1832 if (i0)
1833 irotshi(code, r0, i0);
1834}
1835
1836static void
1837_lshi(jit_state_t *_jit, jit_int32_t r0, jit_int32_t r1, jit_word_t i0)
1838{
1839 if (i0 == 0)
1840 movr(r0, r1);
1841 else if (i0 <= 3)
1842 lea(0, _NOREG, r1, i0 == 1 ? _SCL2 : i0 == 2 ? _SCL4 : _SCL8, r0);
1843 else
1844 rotshi(X86_SHL, r0, r1, i0);
1845}
1846
1847static void
1848_unr(jit_state_t *_jit, jit_int32_t code, jit_int32_t r0)
1849{
1850 rex(0, WIDE, _NOREG, _NOREG, r0);
1851 ic(0xf7);
1852 mrm(0x03, code, r7(r0));
1853}
1854
1855static void
1856_negr(jit_state_t *_jit, jit_int32_t r0, jit_int32_t r1)
1857{
1858 if (r0 == r1)
1859 inegr(r0);
1860 else {
1861 ixorr(r0, r0);
1862 isubr(r0, r1);
1863 }
1864}
1865
1866static void
1867_comr(jit_state_t *_jit, jit_int32_t r0, jit_int32_t r1)
1868{
1869 movr(r0, r1);
1870 icomr(r0);
1871}
1872
1873#if USE_INC_DEC
1874static void
1875_incr(jit_state_t *_jit, jit_int32_t r0, jit_int32_t r1)
1876{
1877 movr(r0, r1);
1878# if __X64
1879 rex(0, WIDE, _NOREG, _NOREG, r0);
1880 ic(0xff);
1881 ic(0xc0 | r7(r0));
1882# else
1883 ic(0x40 | r7(r0));
1884# endif
1885}
1886
1887static void
1888_decr(jit_state_t *_jit, jit_int32_t r0, jit_int32_t r1)
1889{
1890 movr(r0, r1);
1891# if __X64
1892 rex(0, WIDE, _NOREG, _NOREG, r0);
1893 ic(0xff);
1894 ic(0xc8 | r7(r0));
1895# else
1896 ic(0x48 | r7(r0));
1897# endif
1898}
1899#endif
1900
1901static void
1902_cr(jit_state_t *_jit,
1903 jit_int32_t code, jit_int32_t r0, jit_int32_t r1, jit_int32_t r2)
1904{
1905 jit_int32_t reg;
1906 jit_bool_t same;
1907 if (reg8_p(r0)) {
1908 same = r0 == r1 || r0 == r2;
1909 if (!same)
1910 ixorr(r0, r0);
1911 icmpr(r1, r2);
1912 if (same)
1913 imovi(r0, 0);
1914 cc(code, r0);
1915 }
1916 else {
1917 reg = jit_get_reg(jit_class_gpr|jit_class_rg8);
1918 ixorr(rn(reg), rn(reg));
1919 icmpr(r1, r2);
1920 cc(code, rn(reg));
1921 movr(r0, rn(reg));
1922 jit_unget_reg(reg);
1923 }
1924}
1925
1926static void
1927_ci(jit_state_t *_jit,
1928 jit_int32_t code, jit_int32_t r0, jit_int32_t r1, jit_word_t i0)
1929{
1930 jit_int32_t reg;
1931 jit_bool_t same;
1932 if (reg8_p(r0)) {
1933 same = r0 == r1;
1934 if (!same)
1935 ixorr(r0, r0);
1936 icmpi(r1, i0);
1937 if (same)
1938 imovi(r0, 0);
1939 cc(code, r0);
1940 }
1941 else {
1942 reg = jit_get_reg(jit_class_gpr|jit_class_rg8);
1943 ixorr(rn(reg), rn(reg));
1944 icmpi(r1, i0);
1945 cc(code, rn(reg));
1946 movr(r0, rn(reg));
1947 jit_unget_reg(reg);
1948 }
1949}
1950
1951static void
1952_ci0(jit_state_t *_jit, jit_int32_t code, jit_int32_t r0, jit_int32_t r1)
1953{
1954 jit_int32_t reg;
1955 jit_bool_t same;
1956 if (reg8_p(r0)) {
1957 same = r0 == r1;
1958 if (!same)
1959 ixorr(r0, r0);
1960 testr(r1, r1);
1961 if (same)
1962 imovi(r0, 0);
1963 cc(code, r0);
1964 }
1965 else {
1966 reg = jit_get_reg(jit_class_gpr|jit_class_rg8);
1967 ixorr(rn(reg), rn(reg));
1968 testr(r1, r1);
1969 cc(code, rn(reg));
1970 movr(r0, rn(reg));
1971 jit_unget_reg(reg);
1972 }
1973}
1974
1975static void
1976_ltr(jit_state_t *_jit, jit_int32_t r0, jit_int32_t r1, jit_int32_t r2)
1977{
1978 if (r1 == r2)
1979 movi(r0, 0);
1980 else
1981 cr(X86_CC_L, r0, r1, r2);
1982}
1983
1984static void
1985_lti(jit_state_t *_jit, jit_int32_t r0, jit_int32_t r1, jit_word_t i0)
1986{
1987 if (i0)
1988 ci(X86_CC_L, r0, r1, i0);
1989 else
1990 ci0(X86_CC_S, r0, r1);
1991}
1992
1993static void
1994_ltr_u(jit_state_t *_jit, jit_int32_t r0, jit_int32_t r1, jit_int32_t r2)
1995{
1996 if (r1 == r2)
1997 movi(r0, 0);
1998 else
1999 cr(X86_CC_B, r0, r1, r2);
2000}
2001
2002static void
2003_ler(jit_state_t *_jit, jit_int32_t r0, jit_int32_t r1, jit_int32_t r2)
2004{
2005 if (r1 == r2)
2006 movi(r0, 1);
2007 else
2008 cr(X86_CC_LE, r0, r1, r2);
2009}
2010
2011static void
2012_ler_u(jit_state_t *_jit, jit_int32_t r0, jit_int32_t r1, jit_int32_t r2)
2013{
2014 if (r1 == r2)
2015 movi(r0, 1);
2016 else
2017 cr(X86_CC_BE, r0, r1, r2);
2018}
2019
2020static void
2021_lei_u(jit_state_t *_jit, jit_int32_t r0, jit_int32_t r1, jit_word_t i0)
2022{
2023 if (i0)
2024 ci(X86_CC_BE, r0, r1, i0);
2025 else
2026 ci0(X86_CC_E, r0, r1);
2027}
2028
2029static void
2030_eqr(jit_state_t *_jit, jit_int32_t r0, jit_int32_t r1, jit_int32_t r2)
2031{
2032 if (r1 == r2)
2033 movi(r0, 1);
2034 else
2035 cr(X86_CC_E, r0, r1, r2);
2036}
2037
2038static void
2039_eqi(jit_state_t *_jit, jit_int32_t r0, jit_int32_t r1, jit_word_t i0)
2040{
2041 if (i0)
2042 ci(X86_CC_E, r0, r1, i0);
2043 else
2044 ci0(X86_CC_E, r0, r1);
2045}
2046
2047static void
2048_ger(jit_state_t *_jit, jit_int32_t r0, jit_int32_t r1, jit_int32_t r2)
2049{
2050 if (r1 == r2)
2051 movi(r0, 1);
2052 else
2053 cr(X86_CC_GE, r0, r1, r2);
2054}
2055
2056static void
2057_gei(jit_state_t *_jit, jit_int32_t r0, jit_int32_t r1, jit_word_t i0)
2058{
2059 if (i0)
2060 ci(X86_CC_GE, r0, r1, i0);
2061 else
2062 ci0(X86_CC_NS, r0, r1);
2063}
2064
2065static void
2066_ger_u(jit_state_t *_jit, jit_int32_t r0, jit_int32_t r1, jit_int32_t r2)
2067{
2068 if (r1 == r2)
2069 movi(r0, 1);
2070 else
2071 cr(X86_CC_AE, r0, r1, r2);
2072}
2073
2074static void
2075_gei_u(jit_state_t *_jit, jit_int32_t r0, jit_int32_t r1, jit_word_t i0)
2076{
2077 if (i0)
2078 ci(X86_CC_AE, r0, r1, i0);
2079 else
2080 ci0(X86_CC_NB, r0, r1);
2081}
2082
2083static void
2084_gtr(jit_state_t *_jit, jit_int32_t r0, jit_int32_t r1, jit_int32_t r2)
2085{
2086 if (r1 == r2)
2087 movi(r0, 0);
2088 else
2089 cr(X86_CC_G, r0, r1, r2);
2090}
2091
2092static void
2093_gtr_u(jit_state_t *_jit, jit_int32_t r0, jit_int32_t r1, jit_int32_t r2)
2094{
2095 if (r1 == r2)
2096 movi(r0, 0);
2097 else
2098 cr(X86_CC_A, r0, r1, r2);
2099}
2100
2101static void
2102_gti_u(jit_state_t *_jit, jit_int32_t r0, jit_int32_t r1, jit_word_t i0)
2103{
2104 if (i0)
2105 ci(X86_CC_A, r0, r1, i0);
2106 else
2107 ci0(X86_CC_NE, r0, r1);
2108}
2109
2110static void
2111_ner(jit_state_t *_jit, jit_int32_t r0, jit_int32_t r1, jit_int32_t r2)
2112{
2113 if (r1 == r2)
2114 movi(r0, 0);
2115 else
2116 cr(X86_CC_NE, r0, r1, r2);
2117}
2118
2119static void
2120_nei(jit_state_t *_jit, jit_int32_t r0, jit_int32_t r1, jit_word_t i0)
2121{
2122 if (i0)
2123 ci(X86_CC_NE, r0, r1, i0);
2124 else
2125 ci0(X86_CC_NE, r0, r1);
2126}
2127
2128static void
2129_movr(jit_state_t *_jit, jit_int32_t r0, jit_int32_t r1)
2130{
2131 if (r0 != r1) {
2132 rex(0, 1, r1, _NOREG, r0);
2133 ic(0x89);
2134 ic(0xc0 | (r1 << 3) | r7(r0));
2135 }
2136}
2137
2138static void
2139_imovi(jit_state_t *_jit, jit_int32_t r0, jit_word_t i0)
2140{
2141#if __X64
2142# if !__X64_32
2143 if (fits_uint32_p(i0)) {
2144# endif
2145 rex(0, 0, _NOREG, _NOREG, r0);
2146 ic(0xb8 | r7(r0));
2147 ii(i0);
2148# if !__X64_32
2149 }
2150 else {
2151 rex(0, 1, _NOREG, _NOREG, r0);
2152 ic(0xb8 | r7(r0));
2153 il(i0);
2154 }
2155# endif
2156#else
2157 ic(0xb8 | r7(r0));
2158 ii(i0);
2159#endif
2160}
2161
2162static void
2163_movi(jit_state_t *_jit, jit_int32_t r0, jit_word_t i0)
2164{
2165 if (i0)
2166 imovi(r0, i0);
2167 else
2168 ixorr(r0, r0);
2169}
2170
2171static jit_word_t
2172_movi_p(jit_state_t *_jit, jit_int32_t r0, jit_word_t i0)
2173{
2174 rex(0, WIDE, _NOREG, _NOREG, r0);
2175 ic(0xb8 | r7(r0));
2176 il(i0);
2177 return (_jit->pc.w);
2178}
2179
2180static void
2181_movcr(jit_state_t *_jit, jit_int32_t r0, jit_int32_t r1)
2182{
2183 rex(0, WIDE, r0, _NOREG, r1);
2184 ic(0x0f);
2185 ic(0xbe);
2186 mrm(0x03, r7(r0), r7(r1));
2187}
2188
2189static void
2190_movcr_u(jit_state_t *_jit, jit_int32_t r0, jit_int32_t r1)
2191{
2192 rex(0, WIDE, r0, _NOREG, r1);
2193 ic(0x0f);
2194 ic(0xb6);
2195 mrm(0x03, r7(r0), r7(r1));
2196}
2197
2198static void
2199_movsr(jit_state_t *_jit, jit_int32_t r0, jit_int32_t r1)
2200{
2201 rex(0, WIDE, r0, _NOREG, r1);
2202 ic(0x0f);
2203 ic(0xbf);
2204 mrm(0x03, r7(r0), r7(r1));
2205}
2206
2207static void
2208_movsr_u(jit_state_t *_jit, jit_int32_t r0, jit_int32_t r1)
2209{
2210 rex(0, WIDE, r0, _NOREG, r1);
2211 ic(0x0f);
2212 ic(0xb7);
2213 mrm(0x03, r7(r0), r7(r1));
2214}
2215
2216#if __X64
2217static void
2218_movir(jit_state_t *_jit, jit_int32_t r0, jit_int32_t r1)
2219{
2220 rex(0, 1, r0, _NOREG, r1);
2221 ic(0x63);
2222 mrm(0x03, r7(r0), r7(r1));
2223}
2224
2225static void
2226_movir_u(jit_state_t *_jit, jit_int32_t r0, jit_int32_t r1)
2227{
2228 rex(0, 0, r1, _NOREG, r0);
2229 ic(0x89);
2230 ic(0xc0 | (r1 << 3) | r7(r0));
2231}
2232#endif
2233
2234static void
2235_htonr_us(jit_state_t *_jit, jit_int32_t r0, jit_int32_t r1)
2236{
2237 extr_us(r0, r1);
2238 ic(0x66);
2239 rex(0, 0, _NOREG, _NOREG, r0);
2240 ic(0xc1);
2241 mrm(0x03, X86_ROR, r7(r0));
2242 ic(8);
2243}
2244
2245static void
2246_htonr_ui(jit_state_t *_jit, jit_int32_t r0, jit_int32_t r1)
2247{
2248 movr(r0, r1);
2249 rex(0, 0, _NOREG, _NOREG, r0);
2250 ic(0x0f);
2251 ic(0xc8 | r7(r0));
2252}
2253
2254#if __X64 && !__X64_32
2255static void
2256_htonr_ul(jit_state_t *_jit, jit_int32_t r0, jit_int32_t r1)
2257{
2258 movr(r0, r1);
2259 rex(0, 1, _NOREG, _NOREG, r0);
2260 ic(0x0f);
2261 ic(0xc8 | r7(r0));
2262}
2263#endif
2264
2265static void
2266_extr_c(jit_state_t *_jit, jit_int32_t r0, jit_int32_t r1)
2267{
2268 jit_int32_t reg;
2269 if (reg8_p(r1))
2270 movcr(r0, r1);
2271 else {
2272 reg = jit_get_reg(jit_class_gpr|jit_class_rg8);
2273 movr(rn(reg), r1);
2274 movcr(r0, rn(reg));
2275 jit_unget_reg(reg);
2276 }
2277}
2278
2279static void
2280_extr_uc(jit_state_t *_jit, jit_int32_t r0, jit_int32_t r1)
2281{
2282 jit_int32_t reg;
2283 if (reg8_p(r1))
2284 movcr_u(r0, r1);
2285 else {
2286 reg = jit_get_reg(jit_class_gpr|jit_class_rg8);
2287 movr(rn(reg), r1);
2288 movcr_u(r0, rn(reg));
2289 jit_unget_reg(reg);
2290 }
2291}
2292
2293static void
2294_ldr_c(jit_state_t *_jit, jit_int32_t r0, jit_int32_t r1)
2295{
2296 rex(0, WIDE, r0, _NOREG, r1);
2297 ic(0x0f);
2298 ic(0xbe);
2299 rx(r0, 0, r1, _NOREG, _SCL1);
2300}
2301
2302static void
2303_ldi_c(jit_state_t *_jit, jit_int32_t r0, jit_word_t i0)
2304{
2305 jit_int32_t reg;
2306 if (can_sign_extend_int_p(i0)) {
2307 rex(0, WIDE, r0, _NOREG, _NOREG);
2308 ic(0x0f);
2309 ic(0xbe);
2310 rx(r0, i0, _NOREG, _NOREG, _SCL1);
2311 }
2312 else {
2313 reg = jit_get_reg(jit_class_gpr);
2314 movi(rn(reg), i0);
2315 ldr_c(r0, rn(reg));
2316 jit_unget_reg(reg);
2317 }
2318}
2319
2320static void
2321_ldr_uc(jit_state_t *_jit, jit_int32_t r0, jit_int32_t r1)
2322{
2323 rex(0, WIDE, r0, _NOREG, r1);
2324 ic(0x0f);
2325 ic(0xb6);
2326 rx(r0, 0, r1, _NOREG, _SCL1);
2327}
2328
2329static void
2330_ldi_uc(jit_state_t *_jit, jit_int32_t r0, jit_word_t i0)
2331{
2332 jit_int32_t reg;
2333 if (can_sign_extend_int_p(i0)) {
2334 rex(0, WIDE, r0, _NOREG, _NOREG);
2335 ic(0x0f);
2336 ic(0xb6);
2337 rx(r0, i0, _NOREG, _NOREG, _SCL1);
2338 }
2339 else {
2340 reg = jit_get_reg(jit_class_gpr);
2341 movi(rn(reg), i0);
2342 ldr_uc(r0, rn(reg));
2343 jit_unget_reg(reg);
2344 }
2345}
2346
2347static void
2348_ldr_s(jit_state_t *_jit, jit_int32_t r0, jit_int32_t r1)
2349{
2350 rex(0, WIDE, r0, _NOREG, r1);
2351 ic(0x0f);
2352 ic(0xbf);
2353 rx(r0, 0, r1, _NOREG, _SCL1);
2354}
2355
2356static void
2357_ldi_s(jit_state_t *_jit, jit_int32_t r0, jit_word_t i0)
2358{
2359 jit_int32_t reg;
2360 if (can_sign_extend_int_p(i0)) {
2361 rex(0, WIDE, r0, _NOREG, _NOREG);
2362 ic(0x0f);
2363 ic(0xbf);
2364 rx(r0, i0, _NOREG, _NOREG, _SCL1);
2365 }
2366 else {
2367 reg = jit_get_reg(jit_class_gpr);
2368 movi(rn(reg), i0);
2369 ldr_s(r0, rn(reg));
2370 jit_unget_reg(reg);
2371 }
2372}
2373
2374static void
2375_ldr_us(jit_state_t *_jit, jit_int32_t r0, jit_int32_t r1)
2376{
2377 rex(0, WIDE, r0, _NOREG, r1);
2378 ic(0x0f);
2379 ic(0xb7);
2380 rx(r0, 0, r1, _NOREG, _SCL1);
2381}
2382
2383static void
2384_ldi_us(jit_state_t *_jit, jit_int32_t r0, jit_word_t i0)
2385{
2386 jit_int32_t reg;
2387 if (can_sign_extend_int_p(i0)) {
2388 rex(0, WIDE, r0, _NOREG, _NOREG);
2389 ic(0x0f);
2390 ic(0xb7);
2391 rx(r0, i0, _NOREG, _NOREG, _SCL1);
2392 }
2393 else {
2394 reg = jit_get_reg(jit_class_gpr);
2395 movi(rn(reg), i0);
2396 ldr_us(r0, rn(reg));
2397 jit_unget_reg(reg);
2398 }
2399}
2400
2401#if __X32 || !__X64_32
2402static void
2403_ldr_i(jit_state_t *_jit, jit_int32_t r0, jit_int32_t r1)
2404{
2405#if __X64
2406 rex(0, WIDE, r0, _NOREG, r1);
2407 ic(0x63);
2408#else
2409 ic(0x8b);
2410#endif
2411 rx(r0, 0, r1, _NOREG, _SCL1);
2412}
2413
2414static void
2415_ldi_i(jit_state_t *_jit, jit_int32_t r0, jit_word_t i0)
2416{
2417 jit_int32_t reg;
2418 if (can_sign_extend_int_p(i0)) {
2419#if __X64
2420 rex(0, WIDE, r0, _NOREG, _NOREG);
2421 ic(0x63);
2422#else
2423 ic(0x8b);
2424#endif
2425 rx(r0, i0, _NOREG, _NOREG, _SCL1);
2426 }
2427 else {
2428 reg = jit_get_reg(jit_class_gpr);
2429 movi(rn(reg), i0);
2430 ldr_i(r0, rn(reg));
2431 jit_unget_reg(reg);
2432 }
2433}
2434#endif
2435
2436#if __X64
2437static void
2438_ldr_ui(jit_state_t *_jit, jit_int32_t r0, jit_int32_t r1)
2439{
2440 rex(0, 0, r0, _NOREG, r1);
2441 ic(0x63);
2442 rx(r0, 0, r1, _NOREG, _SCL1);
2443}
2444
2445static void
2446_ldi_ui(jit_state_t *_jit, jit_int32_t r0, jit_word_t i0)
2447{
2448 jit_int32_t reg;
2449 if (can_sign_extend_int_p(i0)) {
2450 rex(0, 0, r0, _NOREG, _NOREG);
2451 ic(0x63);
2452 rx(r0, i0, _NOREG, _NOREG, _SCL1);
2453 }
2454 else {
2455 reg = jit_get_reg(jit_class_gpr);
2456 movi(rn(reg), i0);
2457 ldr_ui(r0, rn(reg));
2458 jit_unget_reg(reg);
2459 }
2460}
2461
2462# if !__X64_32
2463static void
2464_ldr_l(jit_state_t *_jit, jit_int32_t r0, jit_int32_t r1)
2465{
2466 rex(0, 1, r0, _NOREG, r1);
2467 ic(0x8b);
2468 rx(r0, 0, r1, _NOREG, _SCL1);
2469}
2470
2471static void
2472_ldi_l(jit_state_t *_jit, jit_int32_t r0, jit_word_t i0)
2473{
2474 jit_int32_t reg;
2475 if (can_sign_extend_int_p(i0)) {
2476 rex(0, 1, r0, _NOREG, _NOREG);
2477 ic(0x8b);
2478 rx(r0, i0, _NOREG, _NOREG, _SCL1);
2479 }
2480 else {
2481 reg = jit_get_reg(jit_class_gpr);
2482 movi(rn(reg), i0);
2483 ldr_l(r0, rn(reg));
2484 jit_unget_reg(reg);
2485 }
2486}
2487# endif
2488#endif
2489
2490static void
2491_ldxr_c(jit_state_t *_jit, jit_int32_t r0, jit_int32_t r1, jit_int32_t r2)
2492{
2493#if __X64_32
2494 addr(r0, r1, r2);
2495 ldr_c(r0, r0);
2496#else
2497 rex(0, WIDE, r0, r1, r2);
2498 ic(0x0f);
2499 ic(0xbe);
2500 rx(r0, 0, r2, r1, _SCL1);
2501#endif
2502}
2503
2504static void
2505_ldxi_c(jit_state_t *_jit, jit_int32_t r0, jit_int32_t r1, jit_word_t i0)
2506{
2507 jit_int32_t reg;
2508 if (can_sign_extend_int_p(i0)) {
2509 rex(0, WIDE, r0, _NOREG, r1);
2510 ic(0x0f);
2511 ic(0xbe);
2512 rx(r0, i0, r1, _NOREG, _SCL1);
2513 }
2514 else {
2515 reg = jit_get_reg(jit_class_gpr);
2516 movi(rn(reg), i0);
2517 ldxr_c(r0, r1, rn(reg));
2518 jit_unget_reg(reg);
2519 }
2520}
2521
2522static void
2523_ldxr_uc(jit_state_t *_jit, jit_int32_t r0, jit_int32_t r1, jit_int32_t r2)
2524{
2525#if __X64_32
2526 addr(r0, r1, r2);
2527 ldr_uc(r0, r0);
2528#else
2529 rex(0, WIDE, r0, r1, r2);
2530 ic(0x0f);
2531 ic(0xb6);
2532 rx(r0, 0, r2, r1, _SCL1);
2533#endif
2534}
2535
2536static void
2537_ldxi_uc(jit_state_t *_jit, jit_int32_t r0, jit_int32_t r1, jit_word_t i0)
2538{
2539 jit_int32_t reg;
2540 if (can_sign_extend_int_p(i0)) {
2541 rex(0, WIDE, r0, _NOREG, r1);
2542 ic(0x0f);
2543 ic(0xb6);
2544 rx(r0, i0, r1, _NOREG, _SCL1);
2545 }
2546 else {
2547 reg = jit_get_reg(jit_class_gpr);
2548 movi(rn(reg), i0);
2549 ldxr_uc(r0, r1, rn(reg));
2550 jit_unget_reg(reg);
2551 }
2552}
2553
2554static void
2555_ldxr_s(jit_state_t *_jit, jit_int32_t r0, jit_int32_t r1, jit_int32_t r2)
2556{
2557#if __X64_32
2558 addr(r0, r1, r2);
2559 ldr_s(r0, r0);
2560#else
2561 rex(0, WIDE, r0, r1, r2);
2562 ic(0x0f);
2563 ic(0xbf);
2564 rx(r0, 0, r2, r1, _SCL1);
2565#endif
2566}
2567
2568static void
2569_ldxi_s(jit_state_t *_jit, jit_int32_t r0, jit_int32_t r1, jit_word_t i0)
2570{
2571 jit_int32_t reg;
2572 if (can_sign_extend_int_p(i0)) {
2573 rex(0, WIDE, r0, _NOREG, r1);
2574 ic(0x0f);
2575 ic(0xbf);
2576 rx(r0, i0, r1, _NOREG, _SCL1);
2577 }
2578 else {
2579 reg = jit_get_reg(jit_class_gpr);
2580 movi(rn(reg), i0);
2581 ldxr_s(r0, r1, rn(reg));
2582 jit_unget_reg(reg);
2583 }
2584}
2585
2586static void
2587_ldxr_us(jit_state_t *_jit, jit_int32_t r0, jit_int32_t r1, jit_int32_t r2)
2588{
2589#if __X64_32
2590 addr(r0, r1, r2);
2591 ldr_us(r0, r0);
2592#else
2593 rex(0, WIDE, r0, r1, r2);
2594 ic(0x0f);
2595 ic(0xb7);
2596 rx(r0, 0, r2, r1, _SCL1);
2597#endif
2598}
2599
2600static void
2601_ldxi_us(jit_state_t *_jit, jit_int32_t r0, jit_int32_t r1, jit_word_t i0)
2602{
2603 jit_int32_t reg;
2604 if (can_sign_extend_int_p(i0)) {
2605 rex(0, WIDE, r0, _NOREG, r1);
2606 ic(0x0f);
2607 ic(0xb7);
2608 rx(r0, i0, r1, _NOREG, _SCL1);
2609 }
2610 else {
2611 reg = jit_get_reg(jit_class_gpr);
2612 movi(rn(reg), i0);
2613 ldxr_us(r0, r1, rn(reg));
2614 jit_unget_reg(reg);
2615 }
2616}
2617
2618#if __X64 || !__X64_32
2619static void
2620_ldxr_i(jit_state_t *_jit, jit_int32_t r0, jit_int32_t r1, jit_int32_t r2)
2621{
2622#if __X64
2623 rex(0, WIDE, r0, r1, r2);
2624 ic(0x63);
2625#else
2626 ic(0x8b);
2627#endif
2628 rx(r0, 0, r2, r1, _SCL1);
2629}
2630
2631static void
2632_ldxi_i(jit_state_t *_jit, jit_int32_t r0, jit_int32_t r1, jit_word_t i0)
2633{
2634 jit_int32_t reg;
2635 if (can_sign_extend_int_p(i0)) {
2636#if __X64
2637 rex(0, WIDE, r0, _NOREG, r1);
2638 ic(0x63);
2639#else
2640 ic(0x8b);
2641#endif
2642 rx(r0, i0, r1, _NOREG, _SCL1);
2643 }
2644 else {
2645 reg = jit_get_reg(jit_class_gpr);
2646 movi(rn(reg), i0);
2647 ldxr_i(r0, r1, rn(reg));
2648 jit_unget_reg(reg);
2649 }
2650}
2651#endif
2652
2653#if __X64
2654static void
2655_ldxr_ui(jit_state_t *_jit, jit_int32_t r0, jit_int32_t r1, jit_int32_t r2)
2656{
2657#if __X64_32
2658 addr(r0, r1, r2);
2659 /* to avoid confusion with macro renames */
2660 _ldr_ui(_jit, r0, r0);
2661#else
2662 rex(0, 0, r0, r1, r2);
2663 ic(0x8b);
2664 rx(r0, 0, r2, r1, _SCL1);
2665#endif
2666}
2667
2668static void
2669_ldxi_ui(jit_state_t *_jit, jit_int32_t r0, jit_int32_t r1, jit_word_t i0)
2670{
2671 jit_int32_t reg;
2672 if (can_sign_extend_int_p(i0)) {
2673 rex(0, 0, r0, _NOREG, r1);
2674 ic(0x8b);
2675 rx(r0, i0, r1, _NOREG, _SCL1);
2676 }
2677 else {
2678 reg = jit_get_reg(jit_class_gpr);
2679 movi(rn(reg), i0);
2680 ldxr_ui(r0, r1, rn(reg));
2681 jit_unget_reg(reg);
2682 }
2683}
2684
2685# if !__X64_32
2686static void
2687_ldxr_l(jit_state_t *_jit, jit_int32_t r0, jit_int32_t r1, jit_int32_t r2)
2688{
2689 rex(0, 1, r0, r1, r2);
2690 ic(0x8b);
2691 rx(r0, 0, r2, r1, _SCL1);
2692}
2693
2694static void
2695_ldxi_l(jit_state_t *_jit, jit_int32_t r0, jit_int32_t r1, jit_word_t i0)
2696{
2697 jit_int32_t reg;
2698 if (can_sign_extend_int_p(i0)) {
2699 rex(0, 1, r0, _NOREG, r1);
2700 ic(0x8b);
2701 rx(r0, i0, r1, _NOREG, _SCL1);
2702 }
2703 else {
2704 reg = jit_get_reg(jit_class_gpr);
2705 movi(rn(reg), i0);
2706 ldxr_l(r0, r1, rn(reg));
2707 jit_unget_reg(reg);
2708 }
2709}
2710# endif
2711#endif
2712
2713static void
2714_str_c(jit_state_t *_jit, jit_int32_t r0, jit_int32_t r1)
2715{
2716 jit_int32_t reg;
2717 if (reg8_p(r1)) {
2718 rex(0, 0, r1, _NOREG, r0);
2719 ic(0x88);
2720 rx(r1, 0, r0, _NOREG, _SCL1);
2721 }
2722 else {
2723 reg = jit_get_reg(jit_class_gpr|jit_class_rg8);
2724 movr(rn(reg), r1);
2725 rex(0, 0, rn(reg), _NOREG, r0);
2726 ic(0x88);
2727 rx(rn(reg), 0, r0, _NOREG, _SCL1);
2728 jit_unget_reg(reg);
2729 }
2730}
2731
2732static void
2733_sti_c(jit_state_t *_jit, jit_word_t i0, jit_int32_t r0)
2734{
2735 jit_int32_t reg;
2736 if (can_sign_extend_int_p(i0)) {
2737 if (reg8_p(r0)) {
2738 rex(0, 0, r0, _NOREG, _NOREG);
2739 ic(0x88);
2740 rx(r0, i0, _NOREG, _NOREG, _SCL1);
2741 }
2742 else {
2743 reg = jit_get_reg(jit_class_gpr|jit_class_rg8);
2744 movr(rn(reg), r0);
2745 rex(0, 0, rn(reg), _NOREG, _NOREG);
2746 ic(0x88);
2747 rx(rn(reg), i0, _NOREG, _NOREG, _SCL1);
2748 jit_unget_reg(reg);
2749 }
2750 }
2751 else {
2752 reg = jit_get_reg(jit_class_gpr);
2753 movi(rn(reg), i0);
2754 str_c(rn(reg), r0);
2755 jit_unget_reg(reg);
2756 }
2757}
2758
2759static void
2760_str_s(jit_state_t *_jit, jit_int32_t r0, jit_int32_t r1)
2761{
2762 ic(0x66);
2763 rex(0, 0, r1, _NOREG, r0);
2764 ic(0x89);
2765 rx(r1, 0, r0, _NOREG, _SCL1);
2766}
2767
2768static void
2769_sti_s(jit_state_t *_jit, jit_word_t i0, jit_int32_t r0)
2770{
2771 jit_int32_t reg;
2772 if (can_sign_extend_int_p(i0)) {
2773 ic(0x66);
2774 rex(0, 0, r0, _NOREG, _NOREG);
2775 ic(0x89);
2776 rx(r0, i0, _NOREG, _NOREG, _SCL1);
2777 }
2778 else {
2779 reg = jit_get_reg(jit_class_gpr);
2780 movi(rn(reg), i0);
2781 str_s(rn(reg), r0);
2782 jit_unget_reg(reg);
2783 }
2784}
2785
2786static void
2787_str_i(jit_state_t *_jit, jit_int32_t r0, jit_int32_t r1)
2788{
2789 rex(0, 0, r1, _NOREG, r0);
2790 ic(0x89);
2791 rx(r1, 0, r0, _NOREG, _SCL1);
2792}
2793
2794static void
2795_sti_i(jit_state_t *_jit, jit_word_t i0, jit_int32_t r0)
2796{
2797 jit_int32_t reg;
2798 if (can_sign_extend_int_p(i0)) {
2799 rex(0, 0, r0, _NOREG, _NOREG);
2800 ic(0x89);
2801 rx(r0, i0, _NOREG, _NOREG, _SCL1);
2802 }
2803 else {
2804 reg = jit_get_reg(jit_class_gpr);
2805 movi(rn(reg), i0);
2806 str_i(rn(reg), r0);
2807 jit_unget_reg(reg);
2808 }
2809}
2810
2811#if __X64 && !__X64_32
2812static void
2813_str_l(jit_state_t *_jit, jit_int32_t r0, jit_int32_t r1)
2814{
2815 rex(0, 1, r1, _NOREG, r0);
2816 ic(0x89);
2817 rx(r1, 0, r0, _NOREG, _SCL1);
2818}
2819
2820static void
2821_sti_l(jit_state_t *_jit, jit_word_t i0, jit_int32_t r0)
2822{
2823 jit_int32_t reg;
2824 if (can_sign_extend_int_p(i0)) {
2825 rex(0, 1, r0, _NOREG, _NOREG);
2826 ic(0x89);
2827 rx(r0, i0, _NOREG, _NOREG, _SCL1);
2828 }
2829 else {
2830 reg = jit_get_reg(jit_class_gpr);
2831 movi(rn(reg), i0);
2832 str_l(rn(reg), r0);
2833 jit_unget_reg(reg);
2834 }
2835}
2836#endif
2837
2838static void
2839_stxr_c(jit_state_t *_jit, jit_int32_t r0, jit_int32_t r1, jit_int32_t r2)
2840{
2841 jit_int32_t reg;
2842#if __X64_32
2843 reg = jit_get_reg(jit_class_gpr);
2844 addr(rn(reg), r0, r1);
2845 str_c(rn(reg), r2);
2846 jit_unget_reg(reg);
2847#else
2848 if (reg8_p(r2)) {
2849 rex(0, 0, r2, r1, r0);
2850 ic(0x88);
2851 rx(r2, 0, r0, r1, _SCL1);
2852 }
2853 else {
2854 reg = jit_get_reg(jit_class_gpr|jit_class_rg8);
2855 movr(rn(reg), r2);
2856 rex(0, 0, rn(reg), r1, r0);
2857 ic(0x88);
2858 rx(rn(reg), 0, r0, r1, _SCL1);
2859 jit_unget_reg(reg);
2860 }
2861#endif
2862}
2863
2864static void
2865_stxi_c(jit_state_t *_jit, jit_word_t i0, jit_int32_t r0, jit_int32_t r1)
2866{
2867 jit_int32_t reg;
2868 if (can_sign_extend_int_p(i0)) {
2869 if (reg8_p(r1)) {
2870 rex(0, 0, r1, _NOREG, r0);
2871 ic(0x88);
2872 rx(r1, i0, r0, _NOREG, _SCL1);
2873 }
2874 else {
2875 reg = jit_get_reg(jit_class_gpr|jit_class_rg8);
2876 movr(rn(reg), r1);
2877 rex(0, 0, rn(reg), _NOREG, r0);
2878 ic(0x88);
2879 rx(rn(reg), i0, r0, _NOREG, _SCL1);
2880 jit_unget_reg(reg);
2881 }
2882 }
2883 else {
2884 reg = jit_get_reg(jit_class_gpr);
2885 movi(rn(reg), i0);
2886 stxr_c(rn(reg), r0, r1);
2887 jit_unget_reg(reg);
2888 }
2889}
2890
2891static void
2892_stxr_s(jit_state_t *_jit, jit_int32_t r0, jit_int32_t r1, jit_int32_t r2)
2893{
2894#if __X64_32
2895 jit_int32_t reg;
2896 reg = jit_get_reg(jit_class_gpr);
2897 addr(rn(reg), r0, r1);
2898 str_s(rn(reg), r2);
2899 jit_unget_reg(reg);
2900#else
2901 ic(0x66);
2902 rex(0, 0, r2, r1, r0);
2903 ic(0x89);
2904 rx(r2, 0, r0, r1, _SCL1);
2905#endif
2906}
2907
2908static void
2909_stxi_s(jit_state_t *_jit, jit_word_t i0, jit_int32_t r0, jit_int32_t r1)
2910{
2911 jit_int32_t reg;
2912 if (can_sign_extend_int_p(i0)) {
2913 ic(0x66);
2914 rex(0, 0, r1, _NOREG, r0);
2915 ic(0x89);
2916 rx(r1, i0, r0, _NOREG, _SCL1);
2917 }
2918 else {
2919 reg = jit_get_reg(jit_class_gpr);
2920 movi(rn(reg), i0);
2921 stxr_s(rn(reg), r0, r1);
2922 jit_unget_reg(reg);
2923 }
2924}
2925
2926static void
2927_stxr_i(jit_state_t *_jit, jit_int32_t r0, jit_int32_t r1, jit_int32_t r2)
2928{
2929#if __X64_32
2930 jit_int32_t reg;
2931 reg = jit_get_reg(jit_class_gpr);
2932 addr(rn(reg), r0, r1);
2933 str_i(rn(reg), r2);
2934 jit_unget_reg(reg);
2935#else
2936 rex(0, 0, r2, r1, r0);
2937 ic(0x89);
2938 rx(r2, 0, r0, r1, _SCL1);
2939#endif
2940}
2941
2942static void
2943_stxi_i(jit_state_t *_jit, jit_word_t i0, jit_int32_t r0, jit_int32_t r1)
2944{
2945 jit_int32_t reg;
2946 if (can_sign_extend_int_p(i0)) {
2947 rex(0, 0, r1, _NOREG, r0);
2948 ic(0x89);
2949 rx(r1, i0, r0, _NOREG, _SCL1);
2950 }
2951 else {
2952 reg = jit_get_reg(jit_class_gpr);
2953 movi(rn(reg), i0);
2954 stxr_i(rn(reg), r0, r1);
2955 jit_unget_reg(reg);
2956 }
2957}
2958
2959#if __X64 && !__X64_32
2960static void
2961_stxr_l(jit_state_t *_jit, jit_int32_t r0, jit_int32_t r1, jit_int32_t r2)
2962{
2963 rex(0, 1, r2, r1, r0);
2964 ic(0x89);
2965 rx(r2, 0, r0, r1, _SCL1);
2966}
2967
2968static void
2969_stxi_l(jit_state_t *_jit, jit_word_t i0, jit_int32_t r0, jit_int32_t r1)
2970{
2971 jit_int32_t reg;
2972 if (can_sign_extend_int_p(i0)) {
2973 rex(0, 1, r1, _NOREG, r0);
2974 ic(0x89);
2975 rx(r1, i0, r0, _NOREG, _SCL1);
2976 }
2977 else {
2978 reg = jit_get_reg(jit_class_gpr);
2979 movi(rn(reg), i0);
2980 stxr_l(rn(reg), r0, r1);
2981 jit_unget_reg(reg);
2982 }
2983}
2984#endif
2985
2986static void
2987_jccs(jit_state_t *_jit, jit_int32_t code, jit_word_t i0)
2988{
2989 jit_word_t w;
2990 ic(0x70 | code);
2991 w = i0 - (_jit->pc.w + 1);
2992 ic(w);
2993}
2994
2995static void
2996_jcc(jit_state_t *_jit, jit_int32_t code, jit_word_t i0)
2997{
2998 jit_word_t w;
2999 ic(0x0f);
3000 ic(0x80 | code);
3001 w = i0 - (_jit->pc.w + 4);
3002 ii(w);
3003}
3004
3005static void
3006_jcr(jit_state_t *_jit,
3007 jit_int32_t code, jit_word_t i0, jit_int32_t r0, jit_int32_t r1)
3008{
3009 alur(X86_CMP, r0, r1);
3010 jcc(code, i0);
3011}
3012
3013static void
3014_jci(jit_state_t *_jit,
3015 jit_int32_t code, jit_word_t i0, jit_int32_t r0, jit_word_t i1)
3016{
3017 alui(X86_CMP, r0, i1);
3018 jcc(code, i0);
3019}
3020
3021static void
3022_jci0(jit_state_t *_jit, jit_int32_t code, jit_word_t i0, jit_int32_t r0)
3023{
3024 testr(r0, r0);
3025 jcc(code, i0);
3026}
3027
3028static jit_word_t
3029_bltr(jit_state_t *_jit, jit_word_t i0, jit_int32_t r0, jit_int32_t r1)
3030{
3031 jcr(X86_CC_L, i0, r0, r1);
3032 return (_jit->pc.w);
3033}
3034
3035static jit_word_t
3036_blti(jit_state_t *_jit, jit_word_t i0, jit_int32_t r0, jit_word_t i1)
3037{
3038 if (i1) jci (X86_CC_L, i0, r0, i1);
3039 else jci0(X86_CC_S, i0, r0);
3040 return (_jit->pc.w);
3041}
3042
3043static jit_word_t
3044_bltr_u(jit_state_t *_jit, jit_word_t i0, jit_int32_t r0, jit_int32_t r1)
3045{
3046 jcr(X86_CC_B, i0, r0, r1);
3047 return (_jit->pc.w);
3048}
3049
3050static jit_word_t
3051_blti_u(jit_state_t *_jit, jit_word_t i0, jit_int32_t r0, jit_word_t i1)
3052{
3053 if (i1) jci (X86_CC_B, i0, r0, i1);
3054 else jci0(X86_CC_B, i0, r0);
3055 return (_jit->pc.w);
3056}
3057
3058static jit_word_t
3059_bler(jit_state_t *_jit, jit_word_t i0, jit_int32_t r0, jit_int32_t r1)
3060{
3061 if (r0 == r1) jmpi(i0);
3062 else jcr (X86_CC_LE, i0, r0, r1);
3063 return (_jit->pc.w);
3064}
3065
3066static jit_word_t
3067_blei(jit_state_t *_jit, jit_word_t i0, jit_int32_t r0, jit_word_t i1)
3068{
3069 if (i1) jci (X86_CC_LE, i0, r0, i1);
3070 else jci0(X86_CC_LE, i0, r0);
3071 return (_jit->pc.w);
3072}
3073
3074static jit_word_t
3075_bler_u(jit_state_t *_jit, jit_word_t i0, jit_int32_t r0, jit_int32_t r1)
3076{
3077 if (r0 == r1) jmpi(i0);
3078 else jcr (X86_CC_BE, i0, r0, r1);
3079 return (_jit->pc.w);
3080}
3081
3082static jit_word_t
3083_blei_u(jit_state_t *_jit, jit_word_t i0, jit_int32_t r0, jit_word_t i1)
3084{
3085 if (i1) jci (X86_CC_BE, i0, r0, i1);
3086 else jci0(X86_CC_BE, i0, r0);
3087 return (_jit->pc.w);
3088}
3089
3090static jit_word_t
3091_beqr(jit_state_t *_jit, jit_word_t i0, jit_int32_t r0, jit_int32_t r1)
3092{
3093 if (r0 == r1) jmpi(i0);
3094 else jcr (X86_CC_E, i0, r0, r1);
3095 return (_jit->pc.w);
3096}
3097
3098static jit_word_t
3099_beqi(jit_state_t *_jit, jit_word_t i0, jit_int32_t r0, jit_word_t i1)
3100{
3101 if (i1) jci (X86_CC_E, i0, r0, i1);
3102 else jci0(X86_CC_E, i0, r0);
3103 return (_jit->pc.w);
3104}
3105
3106static jit_word_t
3107_bger(jit_state_t *_jit, jit_word_t i0, jit_int32_t r0, jit_int32_t r1)
3108{
3109 if (r0 == r1) jmpi(i0);
3110 else jcr (X86_CC_GE, i0, r0, r1);
3111 return (_jit->pc.w);
3112}
3113
3114static jit_word_t
3115_bgei(jit_state_t *_jit, jit_word_t i0, jit_int32_t r0, jit_word_t i1)
3116{
3117 if (i1) jci (X86_CC_GE, i0, r0, i1);
3118 else jci0(X86_CC_NS, i0, r0);
3119 return (_jit->pc.w);
3120}
3121
3122static jit_word_t
3123_bger_u(jit_state_t *_jit, jit_word_t i0, jit_int32_t r0, jit_int32_t r1)
3124{
3125 if (r0 == r1) jmpi(i0);
3126 else jcr (X86_CC_AE, i0, r0, r1);
3127 return (_jit->pc.w);
3128}
3129
3130static jit_word_t
3131_bgei_u(jit_state_t *_jit, jit_word_t i0, jit_int32_t r0, jit_word_t i1)
3132{
3133 if (i1) jci (X86_CC_AE, i0, r0, i1);
3134 else jmpi(i0);
3135 return (_jit->pc.w);
3136}
3137
3138static jit_word_t
3139_bgtr(jit_state_t *_jit, jit_word_t i0, jit_int32_t r0, jit_int32_t r1)
3140{
3141 jcr(X86_CC_G, i0, r0, r1);
3142 return (_jit->pc.w);
3143}
3144
3145static jit_word_t
3146_bgti(jit_state_t *_jit, jit_word_t i0, jit_int32_t r0, jit_word_t i1)
3147{
3148 jci(X86_CC_G, i0, r0, i1);
3149 return (_jit->pc.w);
3150}
3151
3152static jit_word_t
3153_bgtr_u(jit_state_t *_jit, jit_word_t i0, jit_int32_t r0, jit_int32_t r1)
3154{
3155 jcr(X86_CC_A, i0, r0, r1);
3156 return (_jit->pc.w);
3157}
3158
3159static jit_word_t
3160_bgti_u(jit_state_t *_jit, jit_word_t i0, jit_int32_t r0, jit_word_t i1)
3161{
3162 if (i1) jci (X86_CC_A, i0, r0, i1);
3163 else jci0(X86_CC_NE, i0, r0);
3164 return (_jit->pc.w);
3165}
3166
3167static jit_word_t
3168_bner(jit_state_t *_jit, jit_word_t i0, jit_int32_t r0, jit_int32_t r1)
3169{
3170 jcr(X86_CC_NE, i0, r0, r1);
3171 return (_jit->pc.w);
3172}
3173
3174static jit_word_t
3175_bnei(jit_state_t *_jit, jit_word_t i0, jit_int32_t r0, jit_word_t i1)
3176{
3177 if (i1) jci (X86_CC_NE, i0, r0, i1);
3178 else jci0(X86_CC_NE, i0, r0);
3179 return (_jit->pc.w);
3180}
3181
3182static jit_word_t
3183_bmsr(jit_state_t *_jit, jit_word_t i0, jit_int32_t r0, jit_int32_t r1)
3184{
3185 testr(r0, r1);
3186 jnz(i0);
3187 return (_jit->pc.w);
3188}
3189
3190static jit_word_t
3191_bmsi(jit_state_t *_jit, jit_word_t i0, jit_int32_t r0, jit_word_t i1)
3192{
3193 jit_int32_t reg;
3194 if (can_zero_extend_int_p(i1))
3195 testi(r0, i1);
3196 else {
3197 reg = jit_get_reg(jit_class_gpr);
3198 movi(rn(reg), i1);
3199 testr(r0, rn(reg));
3200 jit_unget_reg(reg);
3201 }
3202 jnz(i0);
3203 return (_jit->pc.w);
3204}
3205
3206static jit_word_t
3207_bmcr(jit_state_t *_jit, jit_word_t i0, jit_int32_t r0, jit_int32_t r1)
3208{
3209 testr(r0, r1);
3210 jz(i0);
3211 return (_jit->pc.w);
3212}
3213
3214static jit_word_t
3215_bmci(jit_state_t *_jit, jit_word_t i0, jit_int32_t r0, jit_word_t i1)
3216{
3217 jit_int32_t reg;
3218 if (can_zero_extend_int_p(i1))
3219 testi(r0, i1);
3220 else {
3221 reg = jit_get_reg(jit_class_gpr);
3222 movi(rn(reg), i1);
3223 testr(r0, rn(reg));
3224 jit_unget_reg(reg);
3225 }
3226 jz(i0);
3227 return (_jit->pc.w);
3228}
3229
3230static jit_word_t
3231_boaddr(jit_state_t *_jit, jit_word_t i0, jit_int32_t r0, jit_int32_t r1)
3232{
3233 iaddr(r0, r1);
3234 jo(i0);
3235 return (_jit->pc.w);
3236}
3237
3238static jit_word_t
3239_boaddi(jit_state_t *_jit, jit_word_t i0, jit_int32_t r0, jit_word_t i1)
3240{
3241 jit_int32_t reg;
3242 if (can_sign_extend_int_p(i1)) {
3243 iaddi(r0, i1);
3244 jo(i0);
3245 return (_jit->pc.w);
3246 }
3247 reg = jit_get_reg(jit_class_gpr|jit_class_nospill);
3248 movi(rn(reg), i1);
3249 jit_unget_reg(reg);
3250 return (boaddr(i0, r0, rn(reg)));
3251}
3252
3253static jit_word_t
3254_boaddr_u(jit_state_t *_jit, jit_word_t i0, jit_int32_t r0, jit_int32_t r1)
3255{
3256 iaddr(r0, r1);
3257 jc(i0);
3258 return (_jit->pc.w);
3259}
3260
3261static jit_word_t
3262_boaddi_u(jit_state_t *_jit, jit_word_t i0, jit_int32_t r0, jit_word_t i1)
3263{
3264 jit_int32_t reg;
3265 if (can_sign_extend_int_p(i1)) {
3266 iaddi(r0, i1);
3267 jc(i0);
3268 return (_jit->pc.w);
3269 }
3270 reg = jit_get_reg(jit_class_gpr|jit_class_nospill);
3271 movi(rn(reg), i1);
3272 jit_unget_reg(reg);
3273 return (boaddr_u(i0, r0, rn(reg)));
3274}
3275
3276static jit_word_t
3277_bxaddr(jit_state_t *_jit, jit_word_t i0, jit_int32_t r0, jit_int32_t r1)
3278{
3279 iaddr(r0, r1);
3280 jno(i0);
3281 return (_jit->pc.w);
3282}
3283
3284static jit_word_t
3285_bxaddi(jit_state_t *_jit, jit_word_t i0, jit_int32_t r0, jit_word_t i1)
3286{
3287 jit_int32_t reg;
3288 if (can_sign_extend_int_p(i1)) {
3289 iaddi(r0, i1);
3290 jno(i0);
3291 return (_jit->pc.w);
3292 }
3293 reg = jit_get_reg(jit_class_gpr|jit_class_nospill);
3294 movi(rn(reg), i1);
3295 jit_unget_reg(reg);
3296 return (bxaddr(i0, r0, rn(reg)));
3297}
3298
3299static jit_word_t
3300_bxaddr_u(jit_state_t *_jit, jit_word_t i0, jit_int32_t r0, jit_int32_t r1)
3301{
3302 iaddr(r0, r1);
3303 jnc(i0);
3304 return (_jit->pc.w);
3305}
3306
3307static jit_word_t
3308_bxaddi_u(jit_state_t *_jit, jit_word_t i0, jit_int32_t r0, jit_word_t i1)
3309{
3310 jit_int32_t reg;
3311 if (can_sign_extend_int_p(i1)) {
3312 iaddi(r0, i1);
3313 jnc(i0);
3314 return (_jit->pc.w);
3315 }
3316 reg = jit_get_reg(jit_class_gpr|jit_class_nospill);
3317 movi(rn(reg), i1);
3318 jit_unget_reg(reg);
3319 return (bxaddr_u(i0, r0, rn(reg)));
3320}
3321
3322static jit_word_t
3323_bosubr(jit_state_t *_jit, jit_word_t i0, jit_int32_t r0, jit_int32_t r1)
3324{
3325 isubr(r0, r1);
3326 jo(i0);
3327 return (_jit->pc.w);
3328}
3329
3330static jit_word_t
3331_bosubi(jit_state_t *_jit, jit_word_t i0, jit_int32_t r0, jit_word_t i1)
3332{
3333 jit_int32_t reg;
3334 if (can_sign_extend_int_p(i1)) {
3335 isubi(r0, i1);
3336 jo(i0);
3337 return (_jit->pc.w);
3338 }
3339 reg = jit_get_reg(jit_class_gpr|jit_class_nospill);
3340 movi(rn(reg), i1);
3341 jit_unget_reg(reg);
3342 return (bosubr(i0, r0, rn(reg)));
3343}
3344
3345static jit_word_t
3346_bosubr_u(jit_state_t *_jit, jit_word_t i0, jit_int32_t r0, jit_int32_t r1)
3347{
3348 isubr(r0, r1);
3349 jc(i0);
3350 return (_jit->pc.w);
3351}
3352
3353static jit_word_t
3354_bosubi_u(jit_state_t *_jit, jit_word_t i0, jit_int32_t r0, jit_word_t i1)
3355{
3356 jit_int32_t reg;
3357 if (can_sign_extend_int_p(i1)) {
3358 isubi(r0, i1);
3359 jc(i0);
3360 return (_jit->pc.w);
3361 }
3362 reg = jit_get_reg(jit_class_gpr|jit_class_nospill);
3363 movi(rn(reg), i1);
3364 jit_unget_reg(reg);
3365 return (bosubr_u(i0, r0, rn(reg)));
3366}
3367
3368static jit_word_t
3369_bxsubr(jit_state_t *_jit, jit_word_t i0, jit_int32_t r0, jit_int32_t r1)
3370{
3371 isubr(r0, r1);
3372 jno(i0);
3373 return (_jit->pc.w);
3374}
3375
3376static jit_word_t
3377_bxsubi(jit_state_t *_jit, jit_word_t i0, jit_int32_t r0, jit_word_t i1)
3378{
3379 jit_int32_t reg;
3380 if (can_sign_extend_int_p(i1)) {
3381 isubi(r0, i1);
3382 jno(i0);
3383 return (_jit->pc.w);
3384 }
3385 reg = jit_get_reg(jit_class_gpr|jit_class_nospill);
3386 movi(rn(reg), i1);
3387 jit_unget_reg(reg);
3388 return (bxsubr(i0, r0, rn(reg)));
3389}
3390
3391static jit_word_t
3392_bxsubr_u(jit_state_t *_jit, jit_word_t i0, jit_int32_t r0, jit_int32_t r1)
3393{
3394 isubr(r0, r1);
3395 jnc(i0);
3396 return (_jit->pc.w);
3397}
3398
3399static jit_word_t
3400_bxsubi_u(jit_state_t *_jit, jit_word_t i0, jit_int32_t r0, jit_word_t i1)
3401{
3402 jit_int32_t reg;
3403 if (can_sign_extend_int_p(i1)) {
3404 isubi(r0, i1);
3405 jnc(i0);
3406 return (_jit->pc.w);
3407 }
3408 reg = jit_get_reg(jit_class_gpr|jit_class_nospill);
3409 movi(rn(reg), i1);
3410 jit_unget_reg(reg);
3411 return (bxsubr_u(i0, r0, rn(reg)));
3412}
3413
3414static void
3415_callr(jit_state_t *_jit, jit_int32_t r0)
3416{
3417 rex(0, 0, _NOREG, _NOREG, r0);
3418 ic(0xff);
3419 mrm(0x03, 0x02, r7(r0));
3420}
3421
3422static jit_word_t
3423_calli(jit_state_t *_jit, jit_word_t i0)
3424{
3425 jit_word_t word;
519a9ea1 3426 jit_word_t w;
4a71579b 3427#if __X64
519a9ea1
PC		 3428 w = i0 - (_jit->pc.w + 5);
3429 if ((jit_int32_t)w == w) {
3430#endif
3431 ic(0xe8);
3432 w = i0 - (_jit->pc.w + 4);
3433 ii(w);
3434 word = _jit->pc.w;
3435#if __X64
3436 }
3437 else
3438 word = calli_p(i0);
3439#endif
3440 return (word);
3441}
4a71579b 3442
519a9ea1
PC		 3443#if __X64
3444static jit_word_t
3445_calli_p(jit_state_t *_jit, jit_word_t i0)
3446{
3447 jit_word_t word;
3448 jit_int32_t reg;
4a71579b
PC		 3449 reg = jit_get_reg(jit_class_gpr);
3450 word = movi_p(rn(reg), i0);
3451 callr(rn(reg));
3452 jit_unget_reg(reg);
4a71579b
PC		 3453 return (word);
3454}
519a9ea1 3455#endif
4a71579b
PC		 3456
3457static void
3458_jmpr(jit_state_t *_jit, jit_int32_t r0)
3459{
519a9ea1 3460 rex(0, 0, _NOREG, _NOREG, r0);
4a71579b
PC		 3461 ic(0xff);
3462 mrm(0x03, 0x04, r7(r0));
3463}
3464
3465static jit_word_t
3466_jmpi(jit_state_t *_jit, jit_word_t i0)
3467{
519a9ea1 3468 jit_word_t word;
4a71579b 3469 jit_word_t w;
519a9ea1
PC		 3470#if __X64
3471 w = i0 - (_jit->pc.w + 5);
3472 if ((jit_int32_t)w == w) {
3473#endif
3474 ic(0xe9);
3475 w = i0 - (_jit->pc.w + 4);
3476 ii(w);
3477 word = _jit->pc.w;
3478#if __X64
3479 }
3480 else
3481 word = jmpi_p(i0);
3482#endif
3483 return (word);
4a71579b
PC		 3484}
3485
519a9ea1
PC		 3486#if __X64
3487static jit_word_t
3488_jmpi_p(jit_state_t *_jit, jit_word_t i0)
3489{
3490 jit_word_t word;
3491 jit_int32_t reg;
3492 reg = jit_get_reg(jit_class_gpr|jit_class_nospill);
3493 word = movi_p(rn(reg), i0);
3494 jmpr(rn(reg));
3495 jit_unget_reg(reg);
3496 return (word);
3497}
3498#endif
3499
4a71579b
PC		 3500static void
3501_jmpsi(jit_state_t *_jit, jit_uint8_t i0)
3502{
3503 ic(0xeb);
3504 ic(i0);
3505}
3506
3507static void
3508_prolog(jit_state_t *_jit, jit_node_t *node)
3509{
3510 jit_int32_t reg;
3511 if (_jitc->function->define_frame || _jitc->function->assume_frame) {
3512 jit_int32_t frame = -_jitc->function->frame;
3513 assert(_jitc->function->self.aoff >= frame);
3514 if (_jitc->function->assume_frame)
3515 return;
3516 _jitc->function->self.aoff = frame;
3517 }
3518 if (_jitc->function->allocar)
3519 _jitc->function->self.aoff &= -16;
3520#if __X64 && (__CYGWIN__ || _WIN32)
3521 _jitc->function->stack = (((/* first 32 bytes must be allocated */
3522 (_jitc->function->self.alen > 32 ?
3523 _jitc->function->self.alen : 32) -
3524 /* align stack at 16 bytes */
3525 _jitc->function->self.aoff) + 15) & -16) +
3526 stack_adjust;
3527#else
3528 _jitc->function->stack = (((_jitc->function->self.alen -
3529 _jitc->function->self.aoff) + 15) & -16) +
3530 stack_adjust;
3531#endif
3532 subi(_RSP_REGNO, _RSP_REGNO, stack_framesize - REAL_WORDSIZE);
3533 /* callee save registers */
3534#if __X32
3535 if (jit_regset_tstbit(&_jitc->function->regset, _RDI))
3536 stxi(12, _RSP_REGNO, _RDI_REGNO);
3537 if (jit_regset_tstbit(&_jitc->function->regset, _RSI))
3538 stxi( 8, _RSP_REGNO, _RSI_REGNO);
3539 if (jit_regset_tstbit(&_jitc->function->regset, _RBX))
3540 stxi( 4, _RSP_REGNO, _RBX_REGNO);
3541#else
3542# if __CYGWIN__ || _WIN32
3543 if (jit_regset_tstbit(&_jitc->function->regset, _XMM15))
3544 sse_stxi_d(136, _RSP_REGNO, _XMM15_REGNO);
3545 if (jit_regset_tstbit(&_jitc->function->regset, _XMM14))
3546 sse_stxi_d(128, _RSP_REGNO, _XMM14_REGNO);
3547 if (jit_regset_tstbit(&_jitc->function->regset, _XMM13))
3548 sse_stxi_d(120, _RSP_REGNO, _XMM13_REGNO);
3549 if (jit_regset_tstbit(&_jitc->function->regset, _XMM12))
3550 sse_stxi_d(112, _RSP_REGNO, _XMM12_REGNO);
3551 if (jit_regset_tstbit(&_jitc->function->regset, _XMM11))
3552 sse_stxi_d(104, _RSP_REGNO, _XMM11_REGNO);
3553 if (jit_regset_tstbit(&_jitc->function->regset, _XMM10))
3554 sse_stxi_d(96, _RSP_REGNO, _XMM10_REGNO);
3555 if (jit_regset_tstbit(&_jitc->function->regset, _XMM9))
3556 sse_stxi_d(88, _RSP_REGNO, _XMM9_REGNO);
3557 if (jit_regset_tstbit(&_jitc->function->regset, _XMM8))
3558 sse_stxi_d(80, _RSP_REGNO, _XMM8_REGNO);
3559 if (jit_regset_tstbit(&_jitc->function->regset, _XMM7))
3560 sse_stxi_d(72, _RSP_REGNO, _XMM7_REGNO);
3561 if (jit_regset_tstbit(&_jitc->function->regset, _XMM6))
3562 sse_stxi_d(64, _RSP_REGNO, _XMM6_REGNO);
3563 if (jit_regset_tstbit(&_jitc->function->regset, _R15))
3564 stxi(56, _RSP_REGNO, _R15_REGNO);
3565 if (jit_regset_tstbit(&_jitc->function->regset, _R14))
3566 stxi(48, _RSP_REGNO, _R14_REGNO);
3567 if (jit_regset_tstbit(&_jitc->function->regset, _R13))
3568 stxi(40, _RSP_REGNO, _R13_REGNO);
3569 if (jit_regset_tstbit(&_jitc->function->regset, _R12))
3570 stxi(32, _RSP_REGNO, _R12_REGNO);
3571 if (jit_regset_tstbit(&_jitc->function->regset, _RSI))
3572 stxi(24, _RSP_REGNO, _RSI_REGNO);
3573 if (jit_regset_tstbit(&_jitc->function->regset, _RDI))
3574 stxi(16, _RSP_REGNO, _RDI_REGNO);
3575 if (jit_regset_tstbit(&_jitc->function->regset, _RBX))
3576 stxi( 8, _RSP_REGNO, _RBX_REGNO);
3577# else
3578 if (jit_regset_tstbit(&_jitc->function->regset, _RBX))
3579 stxi(40, _RSP_REGNO, _RBX_REGNO);
3580 if (jit_regset_tstbit(&_jitc->function->regset, _R12))
3581 stxi(32, _RSP_REGNO, _R12_REGNO);
3582 if (jit_regset_tstbit(&_jitc->function->regset, _R13))
3583 stxi(24, _RSP_REGNO, _R13_REGNO);
3584 if (jit_regset_tstbit(&_jitc->function->regset, _R14))
3585 stxi(16, _RSP_REGNO, _R14_REGNO);
3586 if (jit_regset_tstbit(&_jitc->function->regset, _R15))
3587 stxi( 8, _RSP_REGNO, _R15_REGNO);
3588# endif
3589#endif
3590 stxi(0, _RSP_REGNO, _RBP_REGNO);
3591 movr(_RBP_REGNO, _RSP_REGNO);
3592
3593 /* alloca */
3594 subi(_RSP_REGNO, _RSP_REGNO, _jitc->function->stack);
3595 if (_jitc->function->allocar) {
3596 reg = jit_get_reg(jit_class_gpr);
3597 movi(rn(reg), _jitc->function->self.aoff);
3598 stxi_i(_jitc->function->aoffoff, _RBP_REGNO, rn(reg));
3599 jit_unget_reg(reg);
3600 }
3601
3602#if __X64 && !(__CYGWIN__ || _WIN32)
3603 if (_jitc->function->self.call & jit_call_varargs) {
3604 jit_word_t nofp_code;
3605
3606 /* Save gp registers in the save area, if any is a vararg */
3607 for (reg = first_gp_from_offset(_jitc->function->vagp);
3608 jit_arg_reg_p(reg); ++reg)
3609 stxi(_jitc->function->vaoff + first_gp_offset +
3610 reg * 8, _RBP_REGNO, rn(JIT_RA0 - reg));
3611
3612 reg = first_fp_from_offset(_jitc->function->vafp);
3613 if (jit_arg_f_reg_p(reg)) {
3614 /* Skip over if no float registers were passed as argument */
3615 /* test %al, %al */
3616 ic(0x84);
3617 ic(0xc0);
3618 jes(0);
3619 nofp_code = _jit->pc.w;
3620
3621 /* Save fp registers in the save area, if any is a vararg */
3622 /* Note that the full 16 byte xmm is not saved, because
3623 * lightning only handles float and double, and, while
3624 * attempting to provide a va_list compatible pointer as
3625 * jit_va_start return, does not guarantee it (on all ports). */
3626 for (; jit_arg_f_reg_p(reg); ++reg)
3627 sse_stxi_d(_jitc->function->vaoff + first_fp_offset +
3628 reg * va_fp_increment, _RBP_REGNO, rn(_XMM0 - reg));
3629
3630 patch_rel_char(nofp_code, _jit->pc.w);
3631 }
3632 }
3633#endif
3634}
3635
3636static void
3637_epilog(jit_state_t *_jit, jit_node_t *node)
3638{
3639 if (_jitc->function->assume_frame)
3640 return;
3641 /* callee save registers */
3642 movr(_RSP_REGNO, _RBP_REGNO);
3643#if __X32
3644 if (jit_regset_tstbit(&_jitc->function->regset, _RDI))
3645 ldxi(_RDI_REGNO, _RSP_REGNO, 12);
3646 if (jit_regset_tstbit(&_jitc->function->regset, _RSI))
3647 ldxi(_RSI_REGNO, _RSP_REGNO, 8);
3648 if (jit_regset_tstbit(&_jitc->function->regset, _RBX))
3649 ldxi(_RBX_REGNO, _RSP_REGNO, 4);
3650#else
3651# if __CYGWIN__ || _WIN32
3652 if (jit_regset_tstbit(&_jitc->function->regset, _XMM15))
3653 sse_ldxi_d(_XMM15_REGNO, _RSP_REGNO, 136);
3654 if (jit_regset_tstbit(&_jitc->function->regset, _XMM14))
3655 sse_ldxi_d(_XMM14_REGNO, _RSP_REGNO, 128);
3656 if (jit_regset_tstbit(&_jitc->function->regset, _XMM13))
3657 sse_ldxi_d(_XMM13_REGNO, _RSP_REGNO, 120);
3658 if (jit_regset_tstbit(&_jitc->function->regset, _XMM12))
3659 sse_ldxi_d(_XMM12_REGNO, _RSP_REGNO, 112);
3660 if (jit_regset_tstbit(&_jitc->function->regset, _XMM11))
3661 sse_ldxi_d(_XMM11_REGNO, _RSP_REGNO, 104);
3662 if (jit_regset_tstbit(&_jitc->function->regset, _XMM10))
3663 sse_ldxi_d(_XMM10_REGNO, _RSP_REGNO, 96);
3664 if (jit_regset_tstbit(&_jitc->function->regset, _XMM9))
3665 sse_ldxi_d(_XMM9_REGNO, _RSP_REGNO, 88);
3666 if (jit_regset_tstbit(&_jitc->function->regset, _XMM8))
3667 sse_ldxi_d(_XMM8_REGNO, _RSP_REGNO, 80);
3668 if (jit_regset_tstbit(&_jitc->function->regset, _XMM7))
3669 sse_ldxi_d(_XMM7_REGNO, _RSP_REGNO, 72);
3670 if (jit_regset_tstbit(&_jitc->function->regset, _XMM6))
3671 sse_ldxi_d(_XMM6_REGNO, _RSP_REGNO, 64);
3672 if (jit_regset_tstbit(&_jitc->function->regset, _R15))
3673 ldxi(_R15_REGNO, _RSP_REGNO, 56);
3674 if (jit_regset_tstbit(&_jitc->function->regset, _R14))
3675 ldxi(_R14_REGNO, _RSP_REGNO, 48);
3676 if (jit_regset_tstbit(&_jitc->function->regset, _R13))
3677 ldxi(_R13_REGNO, _RSP_REGNO, 40);
3678 if (jit_regset_tstbit(&_jitc->function->regset, _R12))
3679 ldxi(_R12_REGNO, _RSP_REGNO, 32);
3680 if (jit_regset_tstbit(&_jitc->function->regset, _RSI))
3681 ldxi(_RSI_REGNO, _RSP_REGNO, 24);
3682 if (jit_regset_tstbit(&_jitc->function->regset, _RDI))
3683 ldxi(_RDI_REGNO, _RSP_REGNO, 16);
3684 if (jit_regset_tstbit(&_jitc->function->regset, _RBX))
3685 ldxi(_RBX_REGNO, _RSP_REGNO, 8);
3686# else
3687 if (jit_regset_tstbit(&_jitc->function->regset, _RBX))
3688 ldxi(_RBX_REGNO, _RSP_REGNO, 40);
3689 if (jit_regset_tstbit(&_jitc->function->regset, _R12))
3690 ldxi(_R12_REGNO, _RSP_REGNO, 32);
3691 if (jit_regset_tstbit(&_jitc->function->regset, _R13))
3692 ldxi(_R13_REGNO, _RSP_REGNO, 24);
3693 if (jit_regset_tstbit(&_jitc->function->regset, _R14))
3694 ldxi(_R14_REGNO, _RSP_REGNO, 16);
3695 if (jit_regset_tstbit(&_jitc->function->regset, _R15))
3696 ldxi(_R15_REGNO, _RSP_REGNO, 8);
3697# endif
3698#endif
3699 ldxi(_RBP_REGNO, _RSP_REGNO, 0);
3700 addi(_RSP_REGNO, _RSP_REGNO, stack_framesize - REAL_WORDSIZE);
3701
3702 ic(0xc3);
3703}
3704
3705static void
3706_vastart(jit_state_t *_jit, jit_int32_t r0)
3707{
3708#if __X32 || __CYGWIN__ || _WIN32
3709 assert(_jitc->function->self.call & jit_call_varargs);
3710 addi(r0, _RBP_REGNO, _jitc->function->self.size);
3711#else
3712 jit_int32_t reg;
3713
3714 assert(_jitc->function->self.call & jit_call_varargs);
3715
3716 /* Return jit_va_list_t in the register argument */
3717 addi(r0, _RBP_REGNO, _jitc->function->vaoff);
3718 reg = jit_get_reg(jit_class_gpr);
3719
3720 /* Initialize gp offset in the save area. */
3721 movi(rn(reg), _jitc->function->vagp);
3722 stxi_i(offsetof(jit_va_list_t, gpoff), r0, rn(reg));
3723
3724 /* Initialize fp offset in the save area. */
3725 movi(rn(reg), _jitc->function->vafp);
3726 stxi_i(offsetof(jit_va_list_t, fpoff), r0, rn(reg));
3727
3728 /* Initialize overflow pointer to the first stack argument. */
3729 addi(rn(reg), _RBP_REGNO, _jitc->function->self.size);
3730 stxi(offsetof(jit_va_list_t, over), r0, rn(reg));
3731
3732 /* Initialize register save area pointer. */
3733 addi(rn(reg), r0, first_gp_offset);
3734 stxi(offsetof(jit_va_list_t, save), r0, rn(reg));
3735
3736 jit_unget_reg(reg);
3737#endif
3738}
3739
3740static void
3741_vaarg(jit_state_t *_jit, jit_int32_t r0, jit_int32_t r1)
3742{
3743#if __X32 || __CYGWIN__ || _WIN32
3744 assert(_jitc->function->self.call & jit_call_varargs);
3745 ldr(r0, r1);
3746 addi(r1, r1, va_gp_increment);
3747#else
3748 jit_int32_t rg0;
3749 jit_int32_t rg1;
3750 jit_word_t ge_code;
3751 jit_word_t lt_code;
3752
3753 assert(_jitc->function->self.call & jit_call_varargs);
3754
3755 rg0 = jit_get_reg(jit_class_gpr);
3756 rg1 = jit_get_reg(jit_class_gpr);
3757
3758 /* Load the gp offset in save area in the first temporary. */
3759 ldxi_i(rn(rg0), r1, offsetof(jit_va_list_t, gpoff));
3760
3761 /* Jump over if there are no remaining arguments in the save area. */
3762 icmpi(rn(rg0), va_gp_max_offset);
3763 jaes(0);
3764 ge_code = _jit->pc.w;
3765
3766 /* Load the save area pointer in the second temporary. */
3767 ldxi(rn(rg1), r1, offsetof(jit_va_list_t, save));
3768
3769 /* Load the vararg argument in the first argument. */
3770 ldxr(r0, rn(rg1), rn(rg0));
3771
3772 /* Update the gp offset. */
3773 addi(rn(rg0), rn(rg0), 8);
3774 stxi_i(offsetof(jit_va_list_t, gpoff), r1, rn(rg0));
3775
3776 /* Will only need one temporary register below. */
3777 jit_unget_reg(rg1);
3778
3779 /* Jump over overflow code. */
3780 jmpsi(0);
3781 lt_code = _jit->pc.w;
3782
3783 /* Where to land if argument is in overflow area. */
3784 patch_rel_char(ge_code, _jit->pc.w);
3785
3786 /* Load overflow pointer. */
3787 ldxi(rn(rg0), r1, offsetof(jit_va_list_t, over));
3788
3789 /* Load argument. */
3790 ldr(r0, rn(rg0));
3791
3792 /* Update overflow pointer. */
3793 addi(rn(rg0), rn(rg0), va_gp_increment);
3794 stxi(offsetof(jit_va_list_t, over), r1, rn(rg0));
3795
3796 /* Where to land if argument is in save area. */
3797 patch_rel_char(lt_code, _jit->pc.w);
3798
3799 jit_unget_reg(rg0);
3800#endif
3801}
3802
3803/* The x87 boolean argument tells if will put the result in a x87
3804 * register if non false, in a sse register otherwise. */
3805static void
3806_vaarg_d(jit_state_t *_jit, jit_int32_t r0, jit_int32_t r1, jit_bool_t x87)
3807{
3808#if __X32 || __CYGWIN__ || _WIN32
3809 assert(_jitc->function->self.call & jit_call_varargs);
3810 if (x87)
3811 x87_ldr_d(r0, r1);
3812 else
3813 sse_ldr_d(r0, r1);
3814 addi(r1, r1, 8);
3815#else
3816 jit_int32_t rg0;
3817 jit_int32_t rg1;
3818 jit_word_t ge_code;
3819 jit_word_t lt_code;
3820
3821 assert(_jitc->function->self.call & jit_call_varargs);
3822
3823 rg0 = jit_get_reg(jit_class_gpr);
3824 rg1 = jit_get_reg(jit_class_gpr);
3825
3826 /* Load the fp offset in save area in the first temporary. */
3827 ldxi_i(rn(rg0), r1, offsetof(jit_va_list_t, fpoff));
3828
3829 /* Jump over if there are no remaining arguments in the save area. */
3830 icmpi(rn(rg0), va_fp_max_offset);
3831 jaes(0);
3832 ge_code = _jit->pc.w;
3833
3834 /* Load the save area pointer in the second temporary. */
3835 ldxi(rn(rg1), r1, offsetof(jit_va_list_t, save));
3836
3837 /* Load the vararg argument in the first argument. */
3838 if (x87)
3839 x87_ldxr_d(r0, rn(rg1), rn(rg0));
3840 else
3841 sse_ldxr_d(r0, rn(rg1), rn(rg0));
3842
3843 /* Update the fp offset. */
3844 addi(rn(rg0), rn(rg0), va_fp_increment);
3845 stxi_i(offsetof(jit_va_list_t, fpoff), r1, rn(rg0));
3846
3847 /* Will only need one temporary register below. */
3848 jit_unget_reg(rg1);
3849
3850 /* Jump over overflow code. */
3851 jmpsi(0);
3852 lt_code = _jit->pc.w;
3853
3854 /* Where to land if argument is in overflow area. */
3855 patch_rel_char(ge_code, _jit->pc.w);
3856
3857 /* Load overflow pointer. */
3858 ldxi(rn(rg0), r1, offsetof(jit_va_list_t, over));
3859
3860 /* Load argument. */
3861 if (x87)
3862 x87_ldr_d(r0, rn(rg0));
3863 else
3864 sse_ldr_d(r0, rn(rg0));
3865
3866 /* Update overflow pointer. */
3867 addi(rn(rg0), rn(rg0), 8);
3868 stxi(offsetof(jit_va_list_t, over), r1, rn(rg0));
3869
3870 /* Where to land if argument is in save area. */
3871 patch_rel_char(lt_code, _jit->pc.w);
3872
3873 jit_unget_reg(rg0);
3874#endif
3875}
3876
3877static void
3878_patch_at(jit_state_t *_jit, jit_node_t *node,
3879 jit_word_t instr, jit_word_t label)
3880{
3881 switch (node->code) {
3882# if __X64
3883 case jit_code_calli:
519a9ea1 3884 case jit_code_jmpi:
4a71579b
PC		 3885# endif
3886 case jit_code_movi:
3887 patch_abs(instr, label);
3888 break;
3889 default:
3890 patch_rel(instr, label);
3891 break;
3892 }
3893}
3894#endif
ARM optimized PCSX fork