notaz.gp2x.de / pcsx_rearmed.git / blame
? search:
summary | shortlog | log | commit | commitdiff | tree
blob | blame (incremental) | history | HEAD
Merge pull request #542 from gameblabla/mdec_fix
[pcsx_rearmed.git] / deps / lightning / lib / jit_x86-cpu.c
CommitLineData
4a71579b
PC		 1/*
2 * Copyright (C) 2012-2019 Free Software Foundation, Inc.
3 *
4 * This file is part of GNU lightning.
5 *
6 * GNU lightning is free software; you can redistribute it and/or modify it
7 * under the terms of the GNU Lesser General Public License as published
8 * by the Free Software Foundation; either version 3, or (at your option)
9 * any later version.
10 *
11 * GNU lightning is distributed in the hope that it will be useful, but
12 * WITHOUT ANY WARRANTY; without even the implied warranty of MERCHANTABILITY
13 * or FITNESS FOR A PARTICULAR PURPOSE. See the GNU Lesser General Public
14 * License for more details.
15 *
16 * Authors:
17 * Paulo Cesar Pereira de Andrade
18 */
19
20/* avoid using it due to partial stalls */
21#define USE_INC_DEC 0
22
23#if PROTO
24# if __X32 || __X64_32
25# define WIDE 0
26# define ldi(u, v) ldi_i(u, v)
27# define ldr(u, v) ldr_i(u, v)
28# define ldxr(u, v, w) ldxr_i(u, v, w)
29# define ldxi(u, v, w) ldxi_i(u, v, w)
30# define sti(u, v) sti_i(u, v)
31# define stxi(u, v, w) stxi_i(u, v, w)
32# define can_sign_extend_int_p(im) 1
33# define can_zero_extend_int_p(im) 1
34# define fits_uint32_p(im) 1
35# else
36# define WIDE 1
37# define ldi(u, v) ldi_l(u, v)
38# define ldr(u, v) ldr_l(u, v)
39# define ldxr(u, v, w) ldxr_l(u, v, w)
40# define ldxi(u, v, w) ldxi_l(u, v, w)
41# define sti(u, v) sti_l(u, v)
42# define stxi(u, v, w) stxi_l(u, v, w)
43# define can_sign_extend_int_p(im) \
44 (((im) >= 0 && (long long)(im) <= 0x7fffffffLL) || \
45 ((im) < 0 && (long long)(im) > -0x80000000LL))
46# define can_zero_extend_int_p(im) \
47 ((im) >= 0 && (im) < 0x80000000LL)
48# define fits_uint32_p(im) (((im) & 0xffffffff00000000LL) == 0)
49# endif
50# if __X32 || __CYGWIN__ || __X64_32 || _WIN32
51# define reg8_p(rn) \
52 ((rn) >= _RAX_REGNO && (rn) <= _RBX_REGNO)
53# else
54# define reg8_p(rn) 1
55# endif
56# define _RAX_REGNO 0
57# define _RCX_REGNO 1
58# define _RDX_REGNO 2
59# define _RBX_REGNO 3
60# define _RSP_REGNO 4
61# define _RBP_REGNO 5
62# define _RSI_REGNO 6
63# define _RDI_REGNO 7
64# define _R8_REGNO 8
65# define _R9_REGNO 9
66# define _R10_REGNO 10
67# define _R11_REGNO 11
68# define _R12_REGNO 12
69# define _R13_REGNO 13
70# define _R14_REGNO 14
71# define _R15_REGNO 15
72# define r7(reg) ((reg) & 7)
73# define r8(reg) ((reg) & 15)
74# define _SCL1 0x00
75# define _SCL2 0x01
76# define _SCL4 0x02
77# define _SCL8 0x03
78# define X86_ADD 0
79# define X86_OR 1 << 3
80# define X86_ADC 2 << 3
81# define X86_SBB 3 << 3
82# define X86_AND 4 << 3
83# define X86_SUB 5 << 3
84# define X86_XOR 6 << 3
85# define X86_CMP 7 << 3
86# define X86_ROL 0
87# define X86_ROR 1
88# define X86_RCL 2
89# define X86_RCR 3
90# define X86_SHL 4
91# define X86_SHR 5
92# define X86_SAR 7
93# define X86_NOT 2
94# define X86_NEG 3
95# define X86_MUL 4
96# define X86_IMUL 5
97# define X86_DIV 6
98# define X86_IDIV 7
99# define X86_CC_O 0x0
100# define X86_CC_NO 0x1
101# define X86_CC_NAE 0x2
102# define X86_CC_B 0x2
103# define X86_CC_C 0x2
104# define X86_CC_AE 0x3
105# define X86_CC_NB 0x3
106# define X86_CC_NC 0x3
107# define X86_CC_E 0x4
108# define X86_CC_Z 0x4
109# define X86_CC_NE 0x5
110# define X86_CC_NZ 0x5
111# define X86_CC_BE 0x6
112# define X86_CC_NA 0x6
113# define X86_CC_A 0x7
114# define X86_CC_NBE 0x7
115# define X86_CC_S 0x8
116# define X86_CC_NS 0x9
117# define X86_CC_P 0xa
118# define X86_CC_PE 0xa
119# define X86_CC_NP 0xb
120# define X86_CC_PO 0xb
121# define X86_CC_L 0xc
122# define X86_CC_NGE 0xc
123# define X86_CC_GE 0xd
124# define X86_CC_NL 0xd
125# define X86_CC_LE 0xe
126# define X86_CC_NG 0xe
127# define X86_CC_G 0xf
128# define X86_CC_NLE 0xf
129# define mrm(md, r, m) *_jit->pc.uc++ = (md<<6) | (r<<3) | m
130# define sib(sc, i, b) *_jit->pc.uc++ = (sc<<6) | (i<<3) | b
131# define ic(c) *_jit->pc.uc++ = c
132# define is(s) *_jit->pc.us++ = s
133# define ii(i) *_jit->pc.ui++ = i
134# if __X64 && !__X64_32
135# define il(l) *_jit->pc.ul++ = l
136# else
137# define il(l) ii(l)
138# endif
139# define patch_abs(instr, label) \
140 *(jit_word_t *)(instr - sizeof(jit_word_t)) = label
141# define patch_rel(instr, label) \
142 *(jit_int32_t *)(instr - 4) = label - instr
143# define patch_rel_char(instr, label) \
144 *(jit_int8_t *)(instr - 1) = label - instr
145# define rex(l, w, r, x, b) _rex(_jit, l, w, r, x, b)
146static void
147_rex(jit_state_t*,jit_int32_t,jit_int32_t,jit_int32_t,jit_int32_t,jit_int32_t);
148# define rx(rd, md, rb, ri, ms) _rx(_jit, rd, md, rb, ri, ms)
149static void
150_rx(jit_state_t*,jit_int32_t,jit_int32_t,jit_int32_t,jit_int32_t,jit_int32_t);
151# define nop(n) _nop(_jit, n)
152static void _nop(jit_state_t*, jit_int32_t);
153# define emms() is(0x770f)
154# define lea(md, rb, ri, ms, rd) _lea(_jit, md, rb, ri, ms, rd)
155static void
156_lea(jit_state_t*,jit_int32_t,jit_int32_t,jit_int32_t,jit_int32_t,jit_int32_t);
157# define pushr(r0) _pushr(_jit, r0)
158static void _pushr(jit_state_t*, jit_int32_t) maybe_unused;
159# define popr(r0) _popr(_jit, r0)
160static void _popr(jit_state_t*, jit_int32_t) maybe_unused;
161# define xchgr(r0, r1) _xchgr(_jit, r0, r1)
162static void _xchgr(jit_state_t*, jit_int32_t, jit_int32_t);
163# define testr(r0, r1) _testr(_jit, r0, r1)
164static void _testr(jit_state_t*, jit_int32_t, jit_int32_t);
165# define testi(r0, i0) _testi(_jit, r0, i0)
166static void _testi(jit_state_t*, jit_int32_t, jit_word_t);
167# define cc(code, r0) _cc(_jit, code, r0)
168static void _cc(jit_state_t*, jit_int32_t, jit_int32_t);
169# define icmpr(r0, r1) alur(X86_CMP, r0, r1)
170# define alur(code, r0, r1) _alur(_jit, code, r0, r1)
171static void _alur(jit_state_t*, jit_int32_t, jit_int32_t, jit_int32_t);
172# define icmpi(r0, i0) alui(X86_CMP, r0, i0)
173# define alui(code, r0, i0) _alui(_jit, code, r0, i0)
174static void _alui(jit_state_t*, jit_int32_t, jit_int32_t, jit_word_t);
175# define iaddr(r0, r1) alur(X86_ADD, r0, r1)
176# define save(r0) _save(_jit, r0)
177static void _save(jit_state_t*, jit_int32_t);
178# define load(r0) _load(_jit, r0)
179static void _load(jit_state_t*, jit_int32_t);
180# define addr(r0, r1, r2) _addr(_jit, r0, r1, r2)
181static void _addr(jit_state_t*, jit_int32_t, jit_int32_t, jit_int32_t);
182# define iaddi(r0, i0) alui(X86_ADD, r0, i0)
183# define addi(r0, r1, i0) _addi(_jit, r0, r1, i0)
184static void _addi(jit_state_t*, jit_int32_t, jit_int32_t, jit_word_t);
185#define addcr(r0, r1, r2) _addcr(_jit, r0, r1, r2)
186static void _addcr(jit_state_t*, jit_int32_t, jit_int32_t, jit_int32_t);
187#define addci(r0, r1, i0) _addci(_jit, r0, r1, i0)
188static void _addci(jit_state_t*, jit_int32_t, jit_int32_t, jit_word_t);
189# define iaddxr(r0, r1) alur(X86_ADC, r0, r1)
190# define addxr(r0, r1, r2) _addxr(_jit, r0, r1, r2)
191static void _addxr(jit_state_t*, jit_int32_t, jit_int32_t, jit_int32_t);
192# define iaddxi(r0, i0) alui(X86_ADC, r0, i0)
193# define addxi(r0, r1, i0) _addxi(_jit, r0, r1, i0)
194static void _addxi(jit_state_t*, jit_int32_t, jit_int32_t, jit_word_t);
195# define isubr(r0, r1) alur(X86_SUB, r0, r1)
196# define subr(r0, r1, r2) _subr(_jit, r0, r1, r2)
197static void _subr(jit_state_t*, jit_int32_t, jit_int32_t, jit_int32_t);
198# define isubi(r0, i0) alui(X86_SUB, r0, i0)
199# define subi(r0, r1, i0) _subi(_jit, r0, r1, i0)
200static void _subi(jit_state_t*, jit_int32_t, jit_int32_t, jit_word_t);
201# define subcr(r0, r1, r2) _subcr(_jit, r0, r1, r2)
202static void _subcr(jit_state_t*,jit_int32_t,jit_int32_t,jit_int32_t);
203# define subci(r0, r1, i0) _subci(_jit, r0, r1, i0)
204static void _subci(jit_state_t*,jit_int32_t,jit_int32_t,jit_word_t);
205# define isubxr(r0, r1) alur(X86_SBB, r0, r1)
206# define subxr(r0, r1, r2) _subxr(_jit, r0, r1, r2)
207static void _subxr(jit_state_t*,jit_int32_t,jit_int32_t,jit_int32_t);
208# define isubxi(r0, i0) alui(X86_SBB, r0, i0)
209# define subxi(r0, r1, i0) _subxi(_jit, r0, r1, i0)
210static void _subxi(jit_state_t*,jit_int32_t,jit_int32_t,jit_word_t);
211# define rsbi(r0, r1, i0) _rsbi(_jit, r0, r1, i0)
212static void _rsbi(jit_state_t*,jit_int32_t,jit_int32_t,jit_word_t);
213# define imulr(r0, r1) _imulr(_jit, r0, r1)
214static void _imulr(jit_state_t*, jit_int32_t, jit_int32_t);
215# define imuli(r0, r1, i0) _imuli(_jit, r0, r1, i0)
216static void _imuli(jit_state_t*, jit_int32_t, jit_int32_t, jit_word_t);
217# define mulr(r0, r1, r2) _mulr(_jit, r0, r1, r2)
218static void _mulr(jit_state_t*, jit_int32_t, jit_int32_t, jit_int32_t);
219# define muli(r0, r1, i0) _muli(_jit, r0, r1, i0)
220static void _muli(jit_state_t*, jit_int32_t, jit_int32_t, jit_word_t);
221# define umulr(r0) unr(X86_IMUL, r0)
222# define umulr_u(r0) unr(X86_MUL, r0)
223# define qmulr(r0, r1, r2, r3) _iqmulr(_jit, r0, r1, r2, r3, 1)
224# define qmulr_u(r0, r1, r2, r3) _iqmulr(_jit, r0, r1, r2, r3, 0)
225# define iqmulr(r0, r1, r2, r3, sign) _iqmulr(_jit, r0, r1, r2, r3, sign)
226static void _iqmulr(jit_state_t*, jit_int32_t, jit_int32_t,
227 jit_int32_t,jit_int32_t, jit_bool_t);
228# define qmuli(r0, r1, r2, i0) _iqmuli(_jit, r0, r1, r2, i0, 1)
229# define qmuli_u(r0, r1, r2, i0) _iqmuli(_jit, r0, r1, r2, i0, 0)
230# define iqmuli(r0, r1, r2, i0, sign) _iqmuli(_jit, r0, r1, r2, i0, sign)
231static void _iqmuli(jit_state_t*, jit_int32_t, jit_int32_t,
232 jit_int32_t,jit_word_t, jit_bool_t);
233# define sign_extend_rdx_rax() _sign_extend_rdx_rax(_jit)
234static void _sign_extend_rdx_rax(jit_state_t*);
235# define idivr(r0) unr(X86_IDIV, r0)
236# define idivr_u(r0) unr(X86_DIV, r0)
237# define divremr(r0, r1, r2, i0, i1) _divremr(_jit, r0, r1, r2, i0, i1)
238static void
239_divremr(jit_state_t*,jit_int32_t,jit_int32_t,jit_int32_t,
240 jit_bool_t,jit_bool_t);
241# define divremi(r0, r1, i0, i1, i2) _divremi(_jit, r0, r1, i0, i1, i2)
242static void
243_divremi(jit_state_t*,jit_int32_t,jit_int32_t,jit_word_t,jit_bool_t,jit_bool_t);
244# define divr(r0, r1, r2) divremr(r0, r1, r2, 1, 1)
245# define divi(r0, r1, i0) divremi(r0, r1, i0, 1, 1)
246# define divr_u(r0, r1, r2) divremr(r0, r1, r2, 0, 1)
247# define divi_u(r0, r1, i0) divremi(r0, r1, i0, 0, 1)
248# define qdivr(r0, r1, r2, r3) _iqdivr(_jit, r0, r1, r2, r3, 1)
249# define qdivr_u(r0, r1, r2, r3) _iqdivr(_jit, r0, r1, r2, r3, 0)
250# define iqdivr(r0, r1, r2, r3, sign) _iqdivr(_jit, r0, r1, r2, r3, sign)
251static void _iqdivr(jit_state_t*, jit_int32_t, jit_int32_t,
252 jit_int32_t,jit_int32_t, jit_bool_t);
253# define qdivi(r0, r1, r2, i0) _iqdivi(_jit, r0, r1, r2, i0, 1)
254# define qdivi_u(r0, r1, r2, i0) _iqdivi(_jit, r0, r1, r2, i0, 0)
255# define iqdivi(r0, r1, r2, i0, sign) _iqdivi(_jit, r0, r1, r2, i0, sign)
256static void _iqdivi(jit_state_t*, jit_int32_t, jit_int32_t,
257 jit_int32_t,jit_word_t, jit_bool_t);
258# define remr(r0, r1, r2) divremr(r0, r1, r2, 1, 0)
259# define remi(r0, r1, i0) divremi(r0, r1, i0, 1, 0)
260# define remr_u(r0, r1, r2) divremr(r0, r1, r2, 0, 0)
261# define remi_u(r0, r1, i0) divremi(r0, r1, i0, 0, 0)
262# define iandr(r0, r1) alur(X86_AND, r0, r1)
263# define andr(r0, r1, r2) _andr(_jit, r0, r1, r2)
264static void _andr(jit_state_t*,jit_int32_t,jit_int32_t,jit_int32_t);
265# define iandi(r0, i0) alui(X86_AND, r0, i0)
266# define andi(r0, r1, i0) _andi(_jit, r0, r1, i0)
267static void _andi(jit_state_t*, jit_int32_t,jit_int32_t,jit_word_t);
268# define iorr(r0, r1) alur(X86_OR, r0, r1)
269# define orr(r0, r1, r2) _orr(_jit, r0, r1, r2)
270static void _orr(jit_state_t*, jit_int32_t,jit_int32_t,jit_int32_t);
271# define iori(r0, i0) alui(X86_OR, r0, i0)
272# define ori(r0, r1, i0) _ori(_jit, r0, r1, i0)
273static void _ori(jit_state_t*, jit_int32_t,jit_int32_t,jit_word_t);
274# define ixorr(r0, r1) alur(X86_XOR, r0, r1)
275# define xorr(r0, r1, r2) _xorr(_jit, r0, r1, r2)
276static void _xorr(jit_state_t*, jit_int32_t,jit_int32_t,jit_int32_t);
277# define ixori(r0, i0) alui(X86_XOR, r0, i0)
278# define xori(r0, r1, i0) _xori(_jit, r0, r1, i0)
279static void _xori(jit_state_t*, jit_int32_t,jit_int32_t,jit_word_t);
280# define irotshr(code, r0) _irotshr(_jit, code, r0)
281static void _irotshr(jit_state_t*, jit_int32_t, jit_int32_t);
282# define rotshr(code, r0, r1, r2) _rotshr(_jit, code, r0, r1, r2)
283static void
284_rotshr(jit_state_t*,jit_int32_t,jit_int32_t,jit_int32_t,jit_int32_t);
285# define irotshi(code, r0, i0) _irotshi(_jit, code, r0, i0)
286static void _irotshi(jit_state_t*, jit_int32_t, jit_int32_t, jit_word_t);
287# define rotshi(code, r0, r1, i0) _rotshi(_jit, code, r0, r1, i0)
288static void
289_rotshi(jit_state_t*,jit_int32_t,jit_int32_t,jit_int32_t,jit_word_t);
290# define lshr(r0, r1, r2) rotshr(X86_SHL, r0, r1, r2)
291# define lshi(r0, r1, i0) _lshi(_jit, r0, r1, i0)
292static void _lshi(jit_state_t*, jit_int32_t, jit_int32_t, jit_word_t);
293# define rshr(r0, r1, r2) rotshr(X86_SAR, r0, r1, r2)
294# define rshi(r0, r1, i0) rotshi(X86_SAR, r0, r1, i0)
295# define rshr_u(r0, r1, r2) rotshr(X86_SHR, r0, r1, r2)
296# define rshi_u(r0, r1, i0) rotshi(X86_SHR, r0, r1, i0)
297# define unr(code, r0) _unr(_jit, code, r0)
298static void _unr(jit_state_t*, jit_int32_t, jit_int32_t);
299# define inegr(r0) unr(X86_NEG, r0)
300# define negr(r0, r1) _negr(_jit, r0, r1)
301static void _negr(jit_state_t*, jit_int32_t, jit_int32_t);
302# define icomr(r0) unr(X86_NOT, r0)
303# define comr(r0, r1) _comr(_jit, r0, r1)
304static void _comr(jit_state_t*, jit_int32_t, jit_int32_t);
305# if USE_INC_DEC
306# define incr(r0, r1) _incr(_jit, r0, r1)
307static void _incr(jit_state_t*, jit_int32_t, jit_int32_t);
308# define decr(r0, r1) _decr(_jit, r0, r1)
309static void _decr(jit_state_t*, jit_int32_t, jit_int32_t);
310# endif
311# define cr(code, r0, r1, r2) _cr(_jit, code, r0, r1, r2)
312static void
313_cr(jit_state_t*, jit_int32_t, jit_int32_t, jit_int32_t, jit_int32_t);
314# define ci(code, r0, r1, i0) _ci(_jit, code, r0, r1, i0)
315static void
316_ci(jit_state_t *_jit, jit_int32_t, jit_int32_t, jit_int32_t, jit_word_t);
317# define ci0(code, r0, r1) _ci0(_jit, code, r0, r1)
318static void _ci0(jit_state_t*, jit_int32_t, jit_int32_t, jit_int32_t);
319# define ltr(r0, r1, r2) _ltr(_jit, r0, r1, r2)
320static void _ltr(jit_state_t*, jit_int32_t, jit_int32_t, jit_int32_t);
321# define lti(r0, r1, i0) _lti(_jit, r0, r1, i0)
322static void _lti(jit_state_t*, jit_int32_t, jit_int32_t, jit_word_t);
323# define ltr_u(r0, r1, r2) _ltr_u(_jit, r0, r1, r2)
324static void _ltr_u(jit_state_t*, jit_int32_t, jit_int32_t, jit_int32_t);
325# define lti_u(r0, r1, i0) ci(X86_CC_B, r0, r1, i0)
326# define ler(r0, r1, r2) _ler(_jit, r0, r1, r2)
327static void _ler(jit_state_t*, jit_int32_t, jit_int32_t, jit_int32_t);
328# define lei(r0, r1, i0) ci(X86_CC_LE, r0, r1, i0)
329# define ler_u(r0, r1, r2) _ler_u(_jit, r0, r1, r2)
330static void _ler_u(jit_state_t*, jit_int32_t, jit_int32_t, jit_int32_t);
331# define lei_u(r0, r1, i0) _lei_u(_jit, r0, r1, i0)
332static void _lei_u(jit_state_t*, jit_int32_t, jit_int32_t, jit_word_t);
333# define eqr(r0, r1, r2) _eqr(_jit, r0, r1, r2)
334static void _eqr(jit_state_t*, jit_int32_t, jit_int32_t, jit_int32_t);
335# define eqi(r0, r1, i0) _eqi(_jit, r0, r1, i0)
336static void _eqi(jit_state_t*, jit_int32_t, jit_int32_t, jit_word_t);
337# define ger(r0, r1, r2) _ger(_jit, r0, r1, r2)
338static void _ger(jit_state_t*, jit_int32_t, jit_int32_t, jit_int32_t);
339# define gei(r0, r1, i0) _gei(_jit, r0, r1, i0)
340static void _gei(jit_state_t*, jit_int32_t, jit_int32_t, jit_word_t);
341# define ger_u(r0, r1, r2) _ger_u(_jit, r0, r1, r2)
342static void _ger_u(jit_state_t*, jit_int32_t, jit_int32_t, jit_int32_t);
343# define gei_u(r0, r1, i0) _gei_u(_jit, r0, r1, i0)
344static void _gei_u(jit_state_t*, jit_int32_t, jit_int32_t, jit_word_t);
345# define gtr(r0, r1, r2) _gtr(_jit, r0, r1, r2)
346static void _gtr(jit_state_t*, jit_int32_t, jit_int32_t, jit_int32_t);
347# define gti(r0, r1, i0) _ci(_jit, X86_CC_G, r0, r1, i0)
348# define gtr_u(r0, r1, r2) _gtr_u(_jit, r0, r1, r2)
349static void _gtr_u(jit_state_t*, jit_int32_t, jit_int32_t, jit_int32_t);
350# define gti_u(r0, r1, i0) _gti_u(_jit, r0, r1, i0)
351static void _gti_u(jit_state_t*, jit_int32_t, jit_int32_t, jit_word_t);
352# define ner(r0, r1, r2) _ner(_jit, r0, r1, r2)
353static void _ner(jit_state_t*, jit_int32_t, jit_int32_t, jit_int32_t);
354# define nei(r0, r1, i0) _nei(_jit, r0, r1, i0)
355static void _nei(jit_state_t*, jit_int32_t, jit_int32_t, jit_word_t);
356# define movr(r0, r1) _movr(_jit, r0, r1)
357static void _movr(jit_state_t*, jit_int32_t, jit_int32_t);
358# define imovi(r0, i0) _imovi(_jit, r0, i0)
359static void _imovi(jit_state_t*, jit_int32_t, jit_word_t);
360# define movi(r0, i0) _movi(_jit, r0, i0)
361static void _movi(jit_state_t*, jit_int32_t, jit_word_t);
362# define movi_p(r0, i0) _movi_p(_jit, r0, i0)
363static jit_word_t _movi_p(jit_state_t*, jit_int32_t, jit_word_t);
364# define movcr(r0, r1) _movcr(_jit, r0, r1)
365static void _movcr(jit_state_t*,jit_int32_t,jit_int32_t);
366# define movcr_u(r0, r1) _movcr_u(_jit, r0, r1)
367static void _movcr_u(jit_state_t*,jit_int32_t,jit_int32_t);
368# define movsr(r0, r1) _movsr(_jit, r0, r1)
369static void _movsr(jit_state_t*,jit_int32_t,jit_int32_t);
370# define movsr_u(r0, r1) _movsr_u(_jit, r0, r1)
371static void _movsr_u(jit_state_t*,jit_int32_t,jit_int32_t);
372# if __X64 && !__X64_32
373# define movir(r0, r1) _movir(_jit, r0, r1)
374static void _movir(jit_state_t*,jit_int32_t,jit_int32_t);
375# define movir_u(r0, r1) _movir_u(_jit, r0, r1)
376static void _movir_u(jit_state_t*,jit_int32_t,jit_int32_t);
377# endif
378# define htonr_us(r0, r1) _htonr_us(_jit, r0, r1)
379static void _htonr_us(jit_state_t*,jit_int32_t,jit_int32_t);
380# define htonr_ui(r0, r1) _htonr_ui(_jit, r0, r1)
381static void _htonr_ui(jit_state_t*,jit_int32_t,jit_int32_t);
382# if __X64 && !__X64_32
383#define htonr_ul(r0, r1) _htonr_ul(_jit, r0, r1)
384static void _htonr_ul(jit_state_t*,jit_int32_t,jit_int32_t);
385#endif
386# define extr_c(r0, r1) _extr_c(_jit, r0, r1)
387static void _extr_c(jit_state_t*,jit_int32_t,jit_int32_t);
388# define extr_uc(r0, r1) _extr_uc(_jit, r0, r1)
389static void _extr_uc(jit_state_t*,jit_int32_t,jit_int32_t);
390# define extr_s(r0, r1) movsr(r0, r1)
391# define extr_us(r0, r1) movsr_u(r0, r1)
392# if __X64 && !__X64_32
393# define extr_i(r0, r1) movir(r0, r1)
394# define extr_ui(r0, r1) movir_u(r0, r1)
395# endif
396# define ldr_c(r0, r1) _ldr_c(_jit, r0, r1)
397static void _ldr_c(jit_state_t*, jit_int32_t, jit_int32_t);
398# define ldi_c(r0, i0) _ldi_c(_jit, r0, i0)
399static void _ldi_c(jit_state_t*, jit_int32_t, jit_word_t);
400# define ldr_uc(r0, r1) _ldr_uc(_jit, r0, r1)
401static void _ldr_uc(jit_state_t*, jit_int32_t, jit_int32_t);
402# define ldi_uc(r0, i0) _ldi_uc(_jit, r0, i0)
403static void _ldi_uc(jit_state_t*, jit_int32_t, jit_word_t);
404# define ldr_s(r0, r1) _ldr_s(_jit, r0, r1)
405static void _ldr_s(jit_state_t*, jit_int32_t, jit_int32_t);
406# define ldi_s(r0, i0) _ldi_s(_jit, r0, i0)
407static void _ldi_s(jit_state_t*, jit_int32_t, jit_word_t);
408# define ldr_us(r0, r1) _ldr_us(_jit, r0, r1)
409static void _ldr_us(jit_state_t*, jit_int32_t, jit_int32_t);
410# define ldi_us(r0, i0) _ldi_us(_jit, r0, i0)
411static void _ldi_us(jit_state_t*, jit_int32_t, jit_word_t);
412# if __X32 || !__X64_32
413# define ldr_i(r0, r1) _ldr_i(_jit, r0, r1)
414static void _ldr_i(jit_state_t*, jit_int32_t, jit_int32_t);
415# define ldi_i(r0, i0) _ldi_i(_jit, r0, i0)
416static void _ldi_i(jit_state_t*, jit_int32_t, jit_word_t);
417# endif
418# if __X64
419# if __X64_32
420# define ldr_i(r0, r1) _ldr_ui(_jit, r0, r1)
421# define ldi_i(r0, i0) _ldi_ui(_jit, r0, i0)
422# else
423# define ldr_ui(r0, r1) _ldr_ui(_jit, r0, r1)
424# define ldi_ui(r0, i0) _ldi_ui(_jit, r0, i0)
425# endif
426static void _ldr_ui(jit_state_t*, jit_int32_t, jit_int32_t);
427static void _ldi_ui(jit_state_t*, jit_int32_t, jit_word_t);
428# if !__X64_32
429# define ldr_l(r0, r1) _ldr_l(_jit, r0, r1)
430static void _ldr_l(jit_state_t*, jit_int32_t, jit_int32_t);
431# define ldi_l(r0, i0) _ldi_l(_jit, r0, i0)
432static void _ldi_l(jit_state_t*, jit_int32_t, jit_word_t);
433# endif
434# endif
435# define ldxr_c(r0, r1, r2) _ldxr_c(_jit, r0, r1, r2)
436static void _ldxr_c(jit_state_t*, jit_int32_t, jit_int32_t, jit_int32_t);
437# define ldxi_c(r0, r1, i0) _ldxi_c(_jit, r0, r1, i0)
438static void _ldxi_c(jit_state_t*, jit_int32_t, jit_int32_t, jit_word_t);
439# define ldxr_uc(r0, r1, r2) _ldxr_uc(_jit, r0, r1, r2)
440static void _ldxr_uc(jit_state_t*, jit_int32_t, jit_int32_t, jit_int32_t);
441# define ldxi_uc(r0, r1, i0) _ldxi_uc(_jit, r0, r1, i0)
442static void _ldxi_uc(jit_state_t*, jit_int32_t, jit_int32_t, jit_word_t);
443# define ldxr_s(r0, r1, r2) _ldxr_s(_jit, r0, r1, r2)
444static void _ldxr_s(jit_state_t*, jit_int32_t, jit_int32_t, jit_int32_t);
445# define ldxi_s(r0, r1, i0) _ldxi_s(_jit, r0, r1, i0)
446static void _ldxi_s(jit_state_t*, jit_int32_t, jit_int32_t, jit_word_t);
447# define ldxr_us(r0, r1, r2) _ldxr_us(_jit, r0, r1, r2)
448static void _ldxr_us(jit_state_t*, jit_int32_t, jit_int32_t, jit_int32_t);
449# define ldxi_us(r0, r1, i0) _ldxi_us(_jit, r0, r1, i0)
450static void _ldxi_us(jit_state_t*, jit_int32_t, jit_int32_t, jit_word_t);
451# if __X32 || !__X64_32
452# define ldxr_i(r0, r1, r2) _ldxr_i(_jit, r0, r1, r2)
453static void _ldxr_i(jit_state_t*, jit_int32_t, jit_int32_t, jit_int32_t);
454# define ldxi_i(r0, r1, i0) _ldxi_i(_jit, r0, r1, i0)
455static void _ldxi_i(jit_state_t*, jit_int32_t, jit_int32_t, jit_word_t);
456# endif
457# if __X64
458# if __X64_32
459# define ldxr_i(r0, r1, r2) _ldxr_ui(_jit, r0, r1, r2)
460# define ldxi_i(r0, r1, i0) _ldxi_ui(_jit, r0, r1, i0)
461# else
462# define ldxr_ui(r0, r1, r2) _ldxr_ui(_jit, r0, r1, r2)
463# define ldxi_ui(r0, r1, i0) _ldxi_ui(_jit, r0, r1, i0)
464# endif
465static void _ldxr_ui(jit_state_t*, jit_int32_t, jit_int32_t, jit_int32_t);
466static void _ldxi_ui(jit_state_t*, jit_int32_t, jit_int32_t, jit_word_t);
467# if !__X64_32
468# define ldxr_l(r0, r1, r2) _ldxr_l(_jit, r0, r1, r2)
469static void _ldxr_l(jit_state_t*, jit_int32_t, jit_int32_t, jit_int32_t);
470# define ldxi_l(r0, r1, i0) _ldxi_l(_jit, r0, r1, i0)
471static void _ldxi_l(jit_state_t*, jit_int32_t, jit_int32_t, jit_word_t);
472# endif
473# endif
474# define str_c(r0, r1) _str_c(_jit, r0, r1)
475static void _str_c(jit_state_t*, jit_int32_t, jit_int32_t);
476# define sti_c(i0, r0) _sti_c(_jit, i0, r0)
477static void _sti_c(jit_state_t*, jit_word_t, jit_int32_t);
478# define str_s(r0, r1) _str_s(_jit, r0, r1)
479static void _str_s(jit_state_t*, jit_int32_t, jit_int32_t);
480# define sti_s(i0, r0) _sti_s(_jit, i0, r0)
481static void _sti_s(jit_state_t*, jit_word_t, jit_int32_t);
482# define str_i(r0, r1) _str_i(_jit, r0, r1)
483static void _str_i(jit_state_t*, jit_int32_t, jit_int32_t);
484# define sti_i(i0, r0) _sti_i(_jit, i0, r0)
485static void _sti_i(jit_state_t*, jit_word_t, jit_int32_t);
486# if __X64 && !__X64_32
487# define str_l(r0, r1) _str_l(_jit, r0, r1)
488static void _str_l(jit_state_t*, jit_int32_t, jit_int32_t);
489# define sti_l(i0, r0) _sti_l(_jit, i0, r0)
490static void _sti_l(jit_state_t*, jit_word_t, jit_int32_t);
491# endif
492# define stxr_c(r0, r1, r2) _stxr_c(_jit, r0, r1, r2)
493static void _stxr_c(jit_state_t*, jit_int32_t, jit_int32_t, jit_int32_t);
494# define stxi_c(i0, r0, r1) _stxi_c(_jit, i0, r0, r1)
495static void _stxi_c(jit_state_t*, jit_word_t, jit_int32_t, jit_int32_t);
496# define stxr_s(r0, r1, r2) _stxr_s(_jit, r0, r1, r2)
497static void _stxr_s(jit_state_t*, jit_int32_t, jit_int32_t, jit_int32_t);
498# define stxi_s(i0, r0, r1) _stxi_s(_jit, i0, r0, r1)
499static void _stxi_s(jit_state_t*, jit_word_t, jit_int32_t, jit_int32_t);
500# define stxr_i(r0, r1, r2) _stxr_i(_jit, r0, r1, r2)
501static void _stxr_i(jit_state_t*, jit_int32_t, jit_int32_t, jit_int32_t);
502# define stxi_i(i0, r0, r1) _stxi_i(_jit, i0, r0, r1)
503static void _stxi_i(jit_state_t*, jit_word_t, jit_int32_t, jit_int32_t);
504# if __X64 && !__X64_32
505# define stxr_l(r0, r1, r2) _stxr_l(_jit, r0, r1, r2)
506static void _stxr_l(jit_state_t*, jit_int32_t, jit_int32_t, jit_int32_t);
507# define stxi_l(i0, r0, r1) _stxi_l(_jit, i0, r0, r1)
508static void _stxi_l(jit_state_t*, jit_word_t, jit_int32_t, jit_int32_t);
509# endif
510# define jcc(code, i0) _jcc(_jit, code, i0)
511# define jo(i0) jcc(X86_CC_O, i0)
512# define jno(i0) jcc(X86_CC_NO, i0)
513# define jnae(i0) jcc(X86_CC_NAE, i0)
514# define jb(i0) jcc(X86_CC_B, i0)
515# define jc(i0) jcc(X86_CC_C, i0)
516# define jae(i0) jcc(X86_CC_AE, i0)
517# define jnb(i0) jcc(X86_CC_NB, i0)
518# define jnc(i0) jcc(X86_CC_NC, i0)
519# define je(i0) jcc(X86_CC_E, i0)
520# define jz(i0) jcc(X86_CC_Z, i0)
521# define jne(i0) jcc(X86_CC_NE, i0)
522# define jnz(i0) jcc(X86_CC_NZ, i0)
523# define jbe(i0) jcc(X86_CC_BE, i0)
524# define jna(i0) jcc(X86_CC_NA, i0)
525# define ja(i0) jcc(X86_CC_A, i0)
526# define jnbe(i0) jcc(X86_CC_NBE, i0)
527# define js(i0) jcc(X86_CC_S, i0)
528# define jns(i0) jcc(X86_CC_NS, i0)
529# define jp(i0) jcc(X86_CC_P, i0)
530# define jpe(i0) jcc(X86_CC_PE, i0)
531# define jnp(i0) jcc(X86_CC_NP, i0)
532# define jpo(i0) jcc(X86_CC_PO, i0)
533# define jl(i0) jcc(X86_CC_L, i0)
534# define jnge(i0) jcc(X86_CC_NGE, i0)
535# define jge(i0) jcc(X86_CC_GE, i0)
536# define jnl(i0) jcc(X86_CC_NL, i0)
537# define jle(i0) jcc(X86_CC_LE, i0)
538# define jng(i0) jcc(X86_CC_NG, i0)
539# define jg(i0) jcc(X86_CC_G, i0)
540# define jnle(i0) jcc(X86_CC_NLE, i0)
541static void _jcc(jit_state_t*, jit_int32_t, jit_word_t);
542# define jccs(code, i0) _jccs(_jit, code, i0)
543# define jos(i0) jccs(X86_CC_O, i0)
544# define jnos(i0) jccs(X86_CC_NO, i0)
545# define jnaes(i0) jccs(X86_CC_NAE, i0)
546# define jbs(i0) jccs(X86_CC_B, i0)
547# define jcs(i0) jccs(X86_CC_C, i0)
548# define jaes(i0) jccs(X86_CC_AE, i0)
549# define jnbs(i0) jccs(X86_CC_NB, i0)
550# define jncs(i0) jccs(X86_CC_NC, i0)
551# define jes(i0) jccs(X86_CC_E, i0)
552# define jzs(i0) jccs(X86_CC_Z, i0)
553# define jnes(i0) jccs(X86_CC_NE, i0)
554# define jnzs(i0) jccs(X86_CC_NZ, i0)
555# define jbes(i0) jccs(X86_CC_BE, i0)
556# define jnas(i0) jccs(X86_CC_NA, i0)
557# define jas(i0) jccs(X86_CC_A, i0)
558# define jnbes(i0) jccs(X86_CC_NBE, i0)
559# define jss(i0) jccs(X86_CC_S, i0)
560# define jnss(i0) jccs(X86_CC_NS, i0)
561# define jps(i0) jccs(X86_CC_P, i0)
562# define jpes(i0) jccs(X86_CC_PE, i0)
563# define jnps(i0) jccs(X86_CC_NP, i0)
564# define jpos(i0) jccs(X86_CC_PO, i0)
565# define jls(i0) jccs(X86_CC_L, i0)
566# define jnges(i0) jccs(X86_CC_NGE, i0)
567# define jges(i0) jccs(X86_CC_GE, i0)
568# define jnls(i0) jccs(X86_CC_NL, i0)
569# define jles(i0) jccs(X86_CC_LE, i0)
570# define jngs(i0) jccs(X86_CC_NG, i0)
571# define jgs(i0) jccs(X86_CC_G, i0)
572# define jnles(i0) jccs(X86_CC_NLE, i0)
573static void _jccs(jit_state_t*, jit_int32_t, jit_word_t);
574# define jcr(code, i0, r0, r1) _jcr(_jit, code, i0, r0, r1)
575static void _jcr(jit_state_t*,jit_int32_t,jit_word_t,jit_int32_t,jit_int32_t);
576# define jci(code, i0, r0, i1) _jci(_jit, code, i0, r0, i1)
577static void _jci(jit_state_t*,jit_int32_t,jit_word_t,jit_int32_t,jit_word_t);
578# define jci0(code, i0, r0) _jci0(_jit, code, i0, r0)
579static void _jci0(jit_state_t*, jit_int32_t, jit_word_t, jit_int32_t);
580# define bltr(i0, r0, r1) _bltr(_jit, i0, r0, r1)
581static jit_word_t _bltr(jit_state_t*, jit_word_t, jit_int32_t, jit_int32_t);
582# define blti(i0, r0, i1) _blti(_jit, i0, r0, i1)
583static jit_word_t _blti(jit_state_t*, jit_word_t, jit_int32_t, jit_word_t);
584# define bltr_u(i0, r0, r1) _bltr_u(_jit, i0, r0, r1)
585static jit_word_t _bltr_u(jit_state_t*, jit_word_t, jit_int32_t, jit_int32_t);
586# define blti_u(i0, r0, i1) _blti_u(_jit, i0, r0, i1)
587static jit_word_t _blti_u(jit_state_t*, jit_word_t, jit_int32_t, jit_word_t);
588# define bler(i0, r0, r1) _bler(_jit, i0, r0, r1)
589static jit_word_t _bler(jit_state_t*, jit_word_t, jit_int32_t, jit_int32_t);
590# define blei(i0, r0, i1) _blei(_jit, i0, r0, i1)
591static jit_word_t _blei(jit_state_t*, jit_word_t, jit_int32_t, jit_word_t);
592# define bler_u(i0, r0, r1) _bler_u(_jit, i0, r0, r1)
593static jit_word_t _bler_u(jit_state_t*, jit_word_t, jit_int32_t, jit_int32_t);
594# define blei_u(i0, r0, i1) _blei_u(_jit, i0, r0, i1)
595static jit_word_t _blei_u(jit_state_t*, jit_word_t, jit_int32_t, jit_word_t);
596# define beqr(i0, r0, r1) _beqr(_jit, i0, r0, r1)
597static jit_word_t _beqr(jit_state_t*, jit_word_t, jit_int32_t, jit_int32_t);
598# define beqi(i0, r0, i1) _beqi(_jit, i0, r0, i1)
599static jit_word_t _beqi(jit_state_t*, jit_word_t, jit_int32_t, jit_word_t);
600# define bger(i0, r0, r1) _bger(_jit, i0, r0, r1)
601static jit_word_t _bger(jit_state_t*, jit_word_t, jit_int32_t, jit_int32_t);
602# define bgei(i0, r0, i1) _bgei(_jit, i0, r0, i1)
603static jit_word_t _bgei(jit_state_t*, jit_word_t, jit_int32_t, jit_word_t);
604# define bger_u(i0, r0, r1) _bger_u(_jit, i0, r0, r1)
605static jit_word_t _bger_u(jit_state_t*, jit_word_t, jit_int32_t, jit_int32_t);
606# define bgei_u(i0, r0, i1) _bgei_u(_jit, i0, r0, i1)
607static jit_word_t _bgei_u(jit_state_t*, jit_word_t, jit_int32_t, jit_word_t);
608# define bgtr(i0, r0, r1) _bgtr(_jit, i0, r0, r1)
609static jit_word_t _bgtr(jit_state_t*, jit_word_t, jit_int32_t, jit_int32_t);
610# define bgti(i0, r0, i1) _bgti(_jit, i0, r0, i1)
611static jit_word_t _bgti(jit_state_t*, jit_word_t, jit_int32_t, jit_word_t);
612# define bgtr_u(i0, r0, r1) _bgtr_u(_jit, i0, r0, r1)
613static jit_word_t _bgtr_u(jit_state_t*, jit_word_t, jit_int32_t, jit_int32_t);
614# define bgti_u(i0, r0, i1) _bgti_u(_jit, i0, r0, i1)
615static jit_word_t _bgti_u(jit_state_t*, jit_word_t, jit_int32_t, jit_word_t);
616# define bner(i0, r0, r1) _bner(_jit, i0, r0, r1)
617static jit_word_t _bner(jit_state_t*, jit_word_t, jit_int32_t, jit_int32_t);
618# define bnei(i0, r0, i1) _bnei(_jit, i0, r0, i1)
619static jit_word_t _bnei(jit_state_t*, jit_word_t, jit_int32_t, jit_word_t);
620# define bmsr(i0, r0, r1) _bmsr(_jit, i0, r0, r1)
621static jit_word_t _bmsr(jit_state_t*,jit_word_t,jit_int32_t,jit_int32_t);
622# define bmsi(i0, r0, i1) _bmsi(_jit, i0, r0, i1)
623static jit_word_t _bmsi(jit_state_t*,jit_word_t,jit_int32_t,jit_word_t);
624# define bmcr(i0, r0, r1) _bmcr(_jit, i0, r0, r1)
625static jit_word_t _bmcr(jit_state_t*,jit_word_t,jit_int32_t,jit_int32_t);
626# define bmci(i0, r0, i1) _bmci(_jit, i0, r0, i1)
627static jit_word_t _bmci(jit_state_t*,jit_word_t,jit_int32_t,jit_word_t);
628# define boaddr(i0, r0, r1) _boaddr(_jit, i0, r0, r1)
629static jit_word_t _boaddr(jit_state_t*,jit_word_t,jit_int32_t,jit_int32_t);
630# define boaddi(i0, r0, i1) _boaddi(_jit, i0, r0, i1)
631static jit_word_t _boaddi(jit_state_t*,jit_word_t,jit_int32_t,jit_word_t);
632# define boaddr_u(i0, r0, r1) _boaddr_u(_jit, i0, r0, r1)
633static jit_word_t _boaddr_u(jit_state_t*,jit_word_t,jit_int32_t,jit_int32_t);
634# define boaddi_u(i0, r0, i1) _boaddi_u(_jit, i0, r0, i1)
635static jit_word_t _boaddi_u(jit_state_t*,jit_word_t,jit_int32_t,jit_word_t);
636# define bxaddr(i0, r0, r1) _bxaddr(_jit, i0, r0, r1)
637static jit_word_t _bxaddr(jit_state_t*,jit_word_t,jit_int32_t,jit_int32_t);
638# define bxaddi(i0, r0, i1) _bxaddi(_jit, i0, r0, i1)
639static jit_word_t _bxaddi(jit_state_t*,jit_word_t,jit_int32_t,jit_word_t);
640# define bxaddr_u(i0, r0, r1) _bxaddr_u(_jit, i0, r0, r1)
641static jit_word_t _bxaddr_u(jit_state_t*,jit_word_t,jit_int32_t,jit_int32_t);
642# define bxaddi_u(i0, r0, i1) _bxaddi_u(_jit, i0, r0, i1)
643static jit_word_t _bxaddi_u(jit_state_t*,jit_word_t,jit_int32_t,jit_word_t);
644# define bosubr(i0, r0, r1) _bosubr(_jit, i0, r0, r1)
645static jit_word_t _bosubr(jit_state_t*,jit_word_t,jit_int32_t,jit_int32_t);
646# define bosubi(i0, r0, i1) _bosubi(_jit, i0, r0, i1)
647static jit_word_t _bosubi(jit_state_t*,jit_word_t,jit_int32_t,jit_word_t);
648# define bosubr_u(i0, r0, r1) _bosubr_u(_jit, i0, r0, r1)
649static jit_word_t _bosubr_u(jit_state_t*,jit_word_t,jit_int32_t,jit_int32_t);
650# define bosubi_u(i0, r0, i1) _bosubi_u(_jit, i0, r0, i1)
651static jit_word_t _bosubi_u(jit_state_t*,jit_word_t,jit_int32_t,jit_word_t);
652# define bxsubr(i0, r0, r1) _bxsubr(_jit, i0, r0, r1)
653static jit_word_t _bxsubr(jit_state_t*,jit_word_t,jit_int32_t,jit_int32_t);
654# define bxsubi(i0, r0, i1) _bxsubi(_jit, i0, r0, i1)
655static jit_word_t _bxsubi(jit_state_t*,jit_word_t,jit_int32_t,jit_word_t);
656# define bxsubr_u(i0, r0, r1) _bxsubr_u(_jit, i0, r0, r1)
657static jit_word_t _bxsubr_u(jit_state_t*,jit_word_t,jit_int32_t,jit_int32_t);
658# define bxsubi_u(i0, r0, i1) _bxsubi_u(_jit, i0, r0, i1)
659static jit_word_t _bxsubi_u(jit_state_t*,jit_word_t,jit_int32_t,jit_word_t);
660# define callr(r0) _callr(_jit, r0)
661static void _callr(jit_state_t*, jit_int32_t);
662# define calli(i0) _calli(_jit, i0)
663static jit_word_t _calli(jit_state_t*, jit_word_t);
664# define jmpr(r0) _jmpr(_jit, r0)
665static void _jmpr(jit_state_t*, jit_int32_t);
666# define jmpi(i0) _jmpi(_jit, i0)
667static jit_word_t _jmpi(jit_state_t*, jit_word_t);
668# define jmpsi(i0) _jmpsi(_jit, i0)
669static void _jmpsi(jit_state_t*, jit_uint8_t);
670# define prolog(node) _prolog(_jit, node)
671static void _prolog(jit_state_t*, jit_node_t*);
672# define epilog(node) _epilog(_jit, node)
673static void _epilog(jit_state_t*, jit_node_t*);
674# define vastart(r0) _vastart(_jit, r0)
675static void _vastart(jit_state_t*, jit_int32_t);
676# define vaarg(r0, r1) _vaarg(_jit, r0, r1)
677static void _vaarg(jit_state_t*, jit_int32_t, jit_int32_t);
678# define vaarg_d(r0, r1, i0) _vaarg_d(_jit, r0, r1, i0)
679static void _vaarg_d(jit_state_t*, jit_int32_t, jit_int32_t, jit_bool_t);
680# define patch_at(node, instr, label) _patch_at(_jit, node, instr, label)
681static void _patch_at(jit_state_t*, jit_node_t*, jit_word_t, jit_word_t);
682# if !defined(HAVE_FFSL)
683# if __X32
684# define ffsl(i) __builtin_ffs(i)
685# else
686# define ffsl(l) __builtin_ffsl(l)
687# endif
688# endif
689#endif
690
691#if CODE
692static void
693_rex(jit_state_t *_jit, jit_int32_t l, jit_int32_t w,
694 jit_int32_t r, jit_int32_t x, jit_int32_t b)
695{
696#if __X64
697 jit_int32_t v = 0x40 | (w << 3);
698
699 if (r != _NOREG)
700 v |= (r & 8) >> 1;
701 if (x != _NOREG)
702 v |= (x & 8) >> 2;
703 if (b != _NOREG)
704 v |= (b & 8) >> 3;
705 if (l || v != 0x40)
706 ic(v);
707#endif
708}
709
710static void
711_rx(jit_state_t *_jit, jit_int32_t rd, jit_int32_t md,
712 jit_int32_t rb, jit_int32_t ri, jit_int32_t ms)
713{
714 if (ri == _NOREG) {
715 if (rb == _NOREG) {
716#if __X32
717 mrm(0x00, r7(rd), 0x05);
718#else
719 mrm(0x00, r7(rd), 0x04);
720 sib(_SCL1, 0x04, 0x05);
721#endif
722 ii(md);
723 }
724 else if (r7(rb) == _RSP_REGNO) {
725 if (md == 0) {
726 mrm(0x00, r7(rd), 0x04);
727 sib(ms, 0x04, 0x04);
728 }
729 else if ((jit_int8_t)md == md) {
730 mrm(0x01, r7(rd), 0x04);
731 sib(ms, 0x04, 0x04);
732 ic(md);
733 }
734 else {
735 mrm(0x02, r7(rd), 0x04);
736 sib(ms, 0x04, 0x04);
737 ii(md);
738 }
739 }
740 else {
741 if (md == 0 && r7(rb) != _RBP_REGNO)
742 mrm(0x00, r7(rd), r7(rb));
743 else if ((jit_int8_t)md == md) {
744 mrm(0x01, r7(rd), r7(rb));
745 ic(md);
746 }
747 else {
748 mrm(0x02, r7(rd), r7(rb));
749 ii(md);
750 }
751 }
752 }
753 else if (rb == _NOREG) {
754 mrm(0x00, r7(rd), 0x04);
755 sib(ms, r7(ri), 0x05);
756 ii(md);
757 }
758 else if (r8(ri) != _RSP_REGNO) {
759 if (md == 0 && r7(rb) != _RBP_REGNO) {
760 mrm(0x00, r7(rd), 0x04);
761 sib(ms, r7(ri), r7(rb));
762 }
763 else if ((jit_int8_t)md == md) {
764 mrm(0x01, r7(rd), 0x04);
765 sib(ms, r7(ri), r7(rb));
766 ic(md);
767 }
768 else {
769 mrm(0x02, r7(rd), 0x04);
770 sib(ms, r7(ri), r7(rb));
771 ic(md);
772 }
773 }
774 else {
775 fprintf(stderr, "illegal index register");
776 abort();
777 }
778}
779
780static void
781_nop(jit_state_t *_jit, jit_int32_t count)
782{
783 switch (count) {
784 case 0:
785 break;
786 case 1: /* NOP */
787 ic(0x90); break;
788 case 2: /* 66 NOP */
789 ic(0x66); ic(0x90);
790 break;
791 case 3: /* NOP DWORD ptr [EAX] */
792 ic(0x0f); ic(0x1f); ic(0x00);
793 break;
794 case 4: /* NOP DWORD ptr [EAX + 00H] */
795 ic(0x0f); ic(0x1f); ic(0x40); ic(0x00);
796 break;
797 case 5: /* NOP DWORD ptr [EAX + EAX*1 + 00H] */
798 ic(0x0f); ic(0x1f); ic(0x44); ic(0x00);
799 ic(0x00);
800 break;
801 case 6: /* 66 NOP DWORD ptr [EAX + EAX*1 + 00H] */
802 ic(0x66); ic(0x0f); ic(0x1f); ic(0x44);
803 ic(0x00); ic(0x00);
804 break;
805 case 7: /* NOP DWORD ptr [EAX + 00000000H] */
806 ic(0x0f); ic(0x1f); ic(0x80); ii(0x0000);
807 break;
808 case 8: /* NOP DWORD ptr [EAX + EAX*1 + 00000000H] */
809 ic(0x0f); ic(0x1f); ic(0x84); ic(0x00);
810 ii(0x0000);
811 break;
812 case 9: /* 66 NOP DWORD ptr [EAX + EAX*1 + 00000000H] */
813 ic(0x66); ic(0x0f); ic(0x1f); ic(0x84);
814 ic(0x00); ii(0x0000);
815 break;
816 default:
817 abort();
818 }
819}
820
821static void
822_lea(jit_state_t *_jit, jit_int32_t md, jit_int32_t rb,
823 jit_int32_t ri, jit_int32_t ms, jit_int32_t rd)
824{
825 rex(0, WIDE, rd, ri, rb);
826 ic(0x8d);
827 rx(rd, md, rb, ri, ms);
828}
829
830static void
831_pushr(jit_state_t *_jit, jit_int32_t r0)
832{
833 rex(0, WIDE, 0, 0, r0);
834 ic(0x50 | r7(r0));
835}
836
837static void
838_popr(jit_state_t *_jit, jit_int32_t r0)
839{
840 rex(0, WIDE, 0, 0, r0);
841 ic(0x58 | r7(r0));
842}
843
844static void
845_xchgr(jit_state_t *_jit, jit_int32_t r0, jit_int32_t r1)
846{
847 rex(0, WIDE, r1, _NOREG, r0);
848 ic(0x87);
849 mrm(0x03, r7(r1), r7(r0));
850}
851
852static void
853_testr(jit_state_t *_jit, jit_int32_t r0, jit_int32_t r1)
854{
855 rex(0, WIDE, r1, _NOREG, r0);
856 ic(0x85);
857 mrm(0x03, r7(r1), r7(r0));
858}
859
860static void
861_testi(jit_state_t *_jit, jit_int32_t r0, jit_word_t i0)
862{
863 rex(0, WIDE, _NOREG, _NOREG, r0);
864 if (r0 == _RAX_REGNO)
865 ic(0xa9);
866 else {
867 ic(0xf7);
868 mrm(0x03, 0x00, r7(r0));
869 }
870 ii(i0);
871}
872
873static void
874_cc(jit_state_t *_jit, jit_int32_t code, jit_int32_t r0)
875{
876 rex(0, 0, _NOREG, _NOREG, r0);
877 ic(0x0f);
878 ic(0x90 | code);
879 mrm(0x03, 0x00, r7(r0));
880}
881
882static void
883_alur(jit_state_t *_jit, jit_int32_t code, jit_int32_t r0, jit_int32_t r1)
884{
885 rex(0, WIDE, r1, _NOREG, r0);
886 ic(code | 0x01);
887 mrm(0x03, r7(r1), r7(r0));
888}
889
890static void
891_alui(jit_state_t *_jit, jit_int32_t code, jit_int32_t r0, jit_word_t i0)
892{
893 jit_int32_t reg;
894 if (can_sign_extend_int_p(i0)) {
895 rex(0, WIDE, _NOREG, _NOREG, r0);
896 if ((jit_int8_t)i0 == i0) {
897 ic(0x83);
898 ic(0xc0 | code | r7(r0));
899 ic(i0);
900 }
901 else {
902 if (r0 == _RAX_REGNO)
903 ic(code | 0x05);
904 else {
905 ic(0x81);
906 ic(0xc0 | code | r7(r0));
907 }
908 ii(i0);
909 }
910 }
911 else {
912 reg = jit_get_reg(jit_class_gpr);
913 movi(rn(reg), i0);
914 alur(code, r0, rn(reg));
915 jit_unget_reg(reg);
916 }
917}
918
919static void
920_save(jit_state_t *_jit, jit_int32_t r0)
921{
922 if (!_jitc->function->regoff[r0]) {
923 _jitc->function->regoff[r0] = jit_allocai(sizeof(jit_word_t));
924 _jitc->again = 1;
925 }
926 assert(!jit_regset_tstbit(&_jitc->regsav, r0));
927 jit_regset_setbit(&_jitc->regsav, r0);
928 stxi(_jitc->function->regoff[r0], _RBP_REGNO, r0);
929}
930
931static void
932_load(jit_state_t *_jit, jit_int32_t r0)
933{
934 assert(_jitc->function->regoff[r0]);
935 assert(jit_regset_tstbit(&_jitc->regsav, r0));
936 jit_regset_clrbit(&_jitc->regsav, r0);
937 ldxi(r0, _RBP_REGNO, _jitc->function->regoff[r0]);
938}
939
940static void
941_addr(jit_state_t *_jit, jit_int32_t r0, jit_int32_t r1, jit_int32_t r2)
942{
943 if (r0 == r1)
944 iaddr(r0, r2);
945 else if (r0 == r2)
946 iaddr(r0, r1);
947 else
948 lea(0, r1, r2, _SCL1, r0);
949}
950
951static void
952_addi(jit_state_t *_jit, jit_int32_t r0, jit_int32_t r1, jit_word_t i0)
953{
954 jit_int32_t reg;
955 if (i0 == 0)
956 movr(r0, r1);
957#if USE_INC_DEC
958 else if (i0 == 1)
959 incr(r0, r1);
960 else if (i0 == -1)
961 decr(r0, r1);
962#endif
963 else if (can_sign_extend_int_p(i0)) {
964 if (r0 == r1)
965 iaddi(r0, i0);
966 else
967 lea(i0, r1, _NOREG, _SCL1, r0);
968 }
969 else if (r0 != r1) {
970 movi(r0, i0);
971 iaddr(r0, r1);
972 }
973 else {
974 reg = jit_get_reg(jit_class_gpr);
975 movi(rn(reg), i0);
976 iaddr(r0, rn(reg));
977 jit_unget_reg(reg);
978 }
979}
980
981static void
982_addcr(jit_state_t *_jit, jit_int32_t r0, jit_int32_t r1, jit_int32_t r2)
983{
984 if (r0 == r2)
985 iaddr(r0, r1);
986 else {
987 movr(r0, r1);
988 iaddr(r0, r2);
989 }
990}
991
992static void
993_addci(jit_state_t *_jit, jit_int32_t r0, jit_int32_t r1, jit_word_t i0)
994{
995 jit_int32_t reg;
996 if (can_sign_extend_int_p(i0)) {
997 movr(r0, r1);
998 iaddi(r0, i0);
999 }
1000 else if (r0 == r1) {
1001 reg = jit_get_reg(jit_class_gpr);
1002 movi(rn(reg), i0);
1003 iaddr(r0, rn(reg));
1004 jit_unget_reg(reg);
1005 }
1006 else {
1007 movi(r0, i0);
1008 iaddr(r0, r1);
1009 }
1010}
1011
1012static void
1013_addxr(jit_state_t *_jit, jit_int32_t r0, jit_int32_t r1, jit_int32_t r2)
1014{
1015 if (r0 == r2)
1016 iaddxr(r0, r1);
1017 else {
1018 movr(r0, r1);
1019 iaddxr(r0, r2);
1020 }
1021}
1022
1023static void
1024_addxi(jit_state_t *_jit, jit_int32_t r0, jit_int32_t r1, jit_word_t i0)
1025{
1026 jit_int32_t reg;
1027 if (can_sign_extend_int_p(i0)) {
1028 movr(r0, r1);
1029 iaddxi(r0, i0);
1030 }
1031 else if (r0 == r1) {
1032 reg = jit_get_reg(jit_class_gpr);
1033 movi(rn(reg), i0);
1034 iaddxr(r0, rn(reg));
1035 jit_unget_reg(reg);
1036 }
1037 else {
1038 movi(r0, i0);
1039 iaddxr(r0, r1);
1040 }
1041}
1042
1043static void
1044_subr(jit_state_t *_jit, jit_int32_t r0, jit_int32_t r1, jit_int32_t r2)
1045{
1046 if (r1 == r2)
1047 ixorr(r0, r0);
1048 else if (r0 == r2) {
1049 isubr(r0, r1);
1050 inegr(r0);
1051 }
1052 else {
1053 movr(r0, r1);
1054 isubr(r0, r2);
1055 }
1056}
1057
1058static void
1059_subi(jit_state_t *_jit, jit_int32_t r0, jit_int32_t r1, jit_word_t i0)
1060{
1061 jit_int32_t reg;
1062 if (i0 == 0)
1063 movr(r0, r1);
1064#if USE_INC_DEC
1065 else if (i0 == 1)
1066 decr(r0, r1);
1067 else if (i0 == -1)
1068 incr(r0, r1);
1069#endif
1070 else if (can_sign_extend_int_p(i0)) {
1071 if (r0 == r1)
1072 isubi(r0, i0);
1073 else
1074 lea(-i0, r1, _NOREG, _SCL1, r0);
1075 }
1076 else if (r0 != r1) {
1077 movi(r0, -i0);
1078 iaddr(r0, r1);
1079 }
1080 else {
1081 reg = jit_get_reg(jit_class_gpr);
1082 movi(rn(reg), i0);
1083 isubr(r0, rn(reg));
1084 jit_unget_reg(reg);
1085 }
1086}
1087
1088static void
1089_subcr(jit_state_t *_jit, jit_int32_t r0, jit_int32_t r1, jit_int32_t r2)
1090{
1091 jit_int32_t reg;
1092 if (r0 == r2 && r0 != r1) {
1093 reg = jit_get_reg(jit_class_gpr);
1094 movr(rn(reg), r0);
1095 movr(r0, r1);
1096 isubr(r0, rn(reg));
1097 jit_unget_reg(reg);
1098 }
1099 else {
1100 movr(r0, r1);
1101 isubr(r0, r2);
1102 }
1103}
1104
1105static void
1106_subci(jit_state_t *_jit, jit_int32_t r0, jit_int32_t r1, jit_word_t i0)
1107{
1108 jit_int32_t reg;
1109 movr(r0, r1);
1110 if (can_sign_extend_int_p(i0))
1111 isubi(r0, i0);
1112 else {
1113 reg = jit_get_reg(jit_class_gpr);
1114 movi(rn(reg), i0);
1115 isubr(r0, rn(reg));
1116 jit_unget_reg(reg);
1117 }
1118}
1119
1120static void
1121_subxr(jit_state_t *_jit, jit_int32_t r0, jit_int32_t r1, jit_int32_t r2)
1122{
1123 jit_int32_t reg;
1124 if (r0 == r2 && r0 != r1) {
1125 reg = jit_get_reg(jit_class_gpr);
1126 movr(rn(reg), r0);
1127 movr(r0, r1);
1128 isubxr(r0, rn(reg));
1129 jit_unget_reg(reg);
1130 }
1131 else {
1132 movr(r0, r1);
1133 isubxr(r0, r2);
1134 }
1135}
1136
1137static void
1138_subxi(jit_state_t *_jit, jit_int32_t r0, jit_int32_t r1, jit_word_t i0)
1139{
1140 jit_int32_t reg;
1141 movr(r0, r1);
1142 if (can_sign_extend_int_p(i0))
1143 isubxi(r0, i0);
1144 else {
1145 reg = jit_get_reg(jit_class_gpr);
1146 imovi(rn(reg), i0);
1147 isubxr(r0, rn(reg));
1148 jit_unget_reg(reg);
1149 }
1150}
1151
1152static void
1153_rsbi(jit_state_t *_jit, jit_int32_t r0, jit_int32_t r1, jit_word_t i0)
1154{
1155 subi(r0, r1, i0);
1156 negr(r0, r0);
1157}
1158
1159static void
1160_imulr(jit_state_t *_jit, jit_int32_t r0, jit_int32_t r1)
1161{
1162 rex(0, WIDE, r0, _NOREG, r1);
1163 ic(0x0f);
1164 ic(0xaf);
1165 mrm(0x03, r7(r0), r7(r1));
1166}
1167
1168static void
1169_imuli(jit_state_t *_jit, jit_int32_t r0, jit_int32_t r1, jit_word_t i0)
1170{
1171 jit_int32_t reg;
1172 if (can_sign_extend_int_p(i0)) {
1173 rex(0, WIDE, r0, _NOREG, r1);
1174 if ((jit_int8_t)i0 == i0) {
1175 ic(0x6b);
1176 mrm(0x03, r7(r0), r7(r1));
1177 ic(i0);
1178 }
1179 else {
1180 ic(0x69);
1181 mrm(0x03, r7(r0), r7(r1));
1182 ii(i0);
1183 }
1184 }
1185 else {
1186 reg = jit_get_reg(jit_class_gpr);
1187 movi(rn(reg), i0);
1188 imulr(r0, rn(reg));
1189 jit_unget_reg(reg);
1190 }
1191}
1192
1193static void
1194_mulr(jit_state_t *_jit, jit_int32_t r0, jit_int32_t r1, jit_int32_t r2)
1195{
1196 if (r0 == r1)
1197 imulr(r0, r2);
1198 else if (r0 == r2)
1199 imulr(r0, r1);
1200 else {
1201 movr(r0, r1);
1202 imulr(r0, r2);
1203 }
1204}
1205
1206static void
1207_muli(jit_state_t *_jit, jit_int32_t r0, jit_int32_t r1, jit_word_t i0)
1208{
1209 switch (i0) {
1210 case 0:
1211 ixorr(r0, r0);
1212 break;
1213 case 1:
1214 movr(r0, r1);
1215 break;
1216 case -1:
1217 negr(r0, r1);
1218 break;
1219 case 2:
1220 lea(0, _NOREG, r1, _SCL2, r0);
1221 break;
1222 case 4:
1223 lea(0, _NOREG, r1, _SCL4, r0);
1224 break;
1225 case 8:
1226 lea(0, _NOREG, r1, _SCL8, r0);
1227 break;
1228 default:
1229 if (i0 > 0 && !(i0 & (i0 - 1)))
1230 lshi(r0, r1, ffsl(i0) - 1);
1231 else if (can_sign_extend_int_p(i0))
1232 imuli(r0, r1, i0);
1233 else if (r0 != r1) {
1234 movi(r0, i0);
1235 imulr(r0, r1);
1236 }
1237 else
1238 imuli(r0, r0, i0);
1239 break;
1240 }
1241}
1242
1243#define savset(rn) \
1244 if (r0 != rn) { \
1245 sav |= 1 << rn; \
1246 if (r1 != rn && r2 != rn) \
1247 set |= 1 << rn; \
1248 }
1249#define isavset(rn) \
1250 if (r0 != rn) { \
1251 sav |= 1 << rn; \
1252 if (r1 != rn) \
1253 set |= 1 << rn; \
1254 }
1255#define qsavset(rn) \
1256 if (r0 != rn && r1 != rn) { \
1257 sav |= 1 << rn; \
1258 if (r2 != rn && r3 != rn) \
1259 set |= 1 << rn; \
1260 }
1261#define allocr(rn, rv) \
1262 if (set & (1 << rn)) \
1263 (void)jit_get_reg(rv|jit_class_gpr|jit_class_named); \
1264 if (sav & (1 << rn)) { \
1265 if ( jit_regset_tstbit(&_jitc->regsav, rv) || \
1266 !jit_regset_tstbit(&_jitc->reglive, rv)) \
1267 sav &= ~(1 << rn); \
1268 else \
1269 save(rv); \
1270 }
1271#define clear(rn, rv) \
1272 if (set & (1 << rn)) \
1273 jit_unget_reg(rv); \
1274 if (sav & (1 << rn)) \
1275 load(rv);
1276static void
1277_iqmulr(jit_state_t *_jit, jit_int32_t r0, jit_int32_t r1,
1278 jit_int32_t r2, jit_int32_t r3, jit_bool_t sign)
1279{
1280 jit_int32_t mul;
1281 jit_int32_t sav;
1282 jit_int32_t set;
1283
1284 sav = set = 0;
1285 qsavset(_RDX_REGNO);
1286 qsavset(_RAX_REGNO);
1287 allocr(_RDX_REGNO, _RDX);
1288 allocr(_RAX_REGNO, _RAX);
1289
1290 if (r3 == _RAX_REGNO)
1291 mul = r2;
1292 else {
1293 mul = r3;
1294 movr(_RAX_REGNO, r2);
1295 }
1296 if (sign)
1297 umulr(mul);
1298 else
1299 umulr_u(mul);
1300
1301 if (r0 == _RDX_REGNO && r1 == _RAX_REGNO)
1302 xchgr(_RAX_REGNO, _RDX_REGNO);
1303 else {
1304 if (r0 != _RDX_REGNO)
1305 movr(r0, _RAX_REGNO);
1306 movr(r1, _RDX_REGNO);
1307 if (r0 == _RDX_REGNO)
1308 movr(r0, _RAX_REGNO);
1309 }
1310
1311 clear(_RDX_REGNO, _RDX);
1312 clear(_RAX_REGNO, _RAX);
1313}
1314
1315static void
1316_iqmuli(jit_state_t *_jit, jit_int32_t r0, jit_int32_t r1,
1317 jit_int32_t r2, jit_word_t i0, jit_bool_t sign)
1318{
1319 jit_int32_t reg;
1320
1321 if (i0 == 0) {
1322 ixorr(r0, r0);
1323 ixorr(r1, r1);
1324 }
1325 else {
1326 reg = jit_get_reg(jit_class_gpr);
1327 movi(rn(reg), i0);
1328 if (sign)
1329 qmulr(r0, r1, r2, rn(reg));
1330 else
1331 qmulr_u(r0, r1, r2, rn(reg));
1332 jit_unget_reg(reg);
1333 }
1334}
1335
1336static void
1337_sign_extend_rdx_rax(jit_state_t *_jit)
1338{
1339 rex(0, WIDE, 0, 0, 0);
1340 ic(0x99);
1341}
1342
1343static void
1344_divremr(jit_state_t *_jit, jit_int32_t r0, jit_int32_t r1, jit_int32_t r2,
1345 jit_bool_t sign, jit_bool_t divide)
1346{
1347 jit_int32_t div;
1348 jit_int32_t reg;
1349 jit_int32_t set;
1350 jit_int32_t sav;
1351 jit_int32_t use;
1352
1353 sav = set = use = 0;
1354 savset(_RDX_REGNO);
1355 savset(_RAX_REGNO);
1356 allocr(_RDX_REGNO, _RDX);
1357 allocr(_RAX_REGNO, _RAX);
1358
1359 if (r2 == _RAX_REGNO) {
1360 if (r0 == _RAX_REGNO || r0 == _RDX_REGNO) {
1361 if ((reg = jit_get_reg(jit_class_gpr|jit_class_chk)) == JIT_NOREG)
1362 reg = jit_get_reg((r1 == _RCX_REGNO ? _RBX : _RCX) |
1363 jit_class_gpr|jit_class_named);
1364 use = 1;
1365 div = rn(reg);
1366 movr(div, _RAX_REGNO);
1367 if (r1 != _RAX_REGNO)
1368 movr(_RAX_REGNO, r1);
1369 }
1370 else {
1371 if (r0 == r1)
1372 xchgr(r0, _RAX_REGNO);
1373 else {
1374 if (r0 != _RAX_REGNO)
1375 movr(r0, _RAX_REGNO);
1376 if (r1 != _RAX_REGNO)
1377 movr(_RAX_REGNO, r1);
1378 }
1379 div = r0;
1380 }
1381 }
1382 else if (r2 == _RDX_REGNO) {
1383 if (r0 == _RAX_REGNO || r0 == _RDX_REGNO) {
1384 if ((reg = jit_get_reg(jit_class_gpr|jit_class_chk)) == JIT_NOREG)
1385 reg = jit_get_reg((r1 == _RCX_REGNO ? _RBX : _RCX) |
1386 jit_class_gpr|jit_class_named);
1387 use = 1;
1388 div = rn(reg);
1389 movr(div, _RDX_REGNO);
1390 if (r1 != _RAX_REGNO)
1391 movr(_RAX_REGNO, r1);
1392 }
1393 else {
1394 if (r1 != _RAX_REGNO)
1395 movr(_RAX_REGNO, r1);
1396 movr(r0, _RDX_REGNO);
1397 div = r0;
1398 }
1399 }
1400 else {
1401 if (r1 != _RAX_REGNO)
1402 movr(_RAX_REGNO, r1);
1403 div = r2;
1404 }
1405
1406 if (sign) {
1407 sign_extend_rdx_rax();
1408 idivr(div);
1409 }
1410 else {
1411 ixorr(_RDX_REGNO, _RDX_REGNO);
1412 idivr_u(div);
1413 }
1414
1415 if (use)
1416 jit_unget_reg(reg);
1417
1418 if (divide)
1419 movr(r0, _RAX_REGNO);
1420 else
1421 movr(r0, _RDX_REGNO);
1422
1423 clear(_RDX_REGNO, _RDX);
1424 clear(_RAX_REGNO, _RAX);
1425}
1426
1427static void
1428_divremi(jit_state_t *_jit, jit_int32_t r0, jit_int32_t r1, jit_word_t i0,
1429 jit_bool_t sign, jit_bool_t divide)
1430{
1431 jit_int32_t reg;
1432 jit_int32_t div;
1433 jit_int32_t sav;
1434 jit_int32_t set;
1435 jit_int32_t use;
1436
1437 if (divide) {
1438 switch (i0) {
1439 case 1:
1440 movr(r0, r1);
1441 return;
1442 case -1:
1443 if (sign) {
1444 negr(r0, r1);
1445 return;
1446 }
1447 break;
1448 default:
1449 if (i0 > 0 && !(i0 & (i0 - 1))) {
1450 movr(r0, r1);
1451 if (sign)
1452 rshi(r0, r0, ffsl(i0) - 1);
1453 else
1454 rshi_u(r0, r0, ffsl(i0) - 1);
1455 return;
1456 }
1457 break;
1458 }
1459 }
1460 else if (i0 == 1 || (sign && i0 == -1)) {
1461 ixorr(r0, r0);
1462 return;
1463 }
1464 else if (!sign && i0 > 0 && !(i0 & (i0 - 1))) {
1465 if (can_sign_extend_int_p(i0)) {
1466 movr(r0, r1);
1467 iandi(r0, i0 - 1);
1468 }
1469 else if (r0 != r1) {
1470 movi(r0, i0 - 1);
1471 iandr(r0, r1);
1472 }
1473 else {
1474 reg = jit_get_reg(jit_class_gpr);
1475 movi(rn(reg), i0 - 1);
1476 iandr(r0, rn(reg));
1477 jit_unget_reg(reg);
1478 }
1479 return;
1480 }
1481
1482 sav = set = use = 0;
1483 isavset(_RDX_REGNO);
1484 isavset(_RAX_REGNO);
1485 allocr(_RDX_REGNO, _RDX);
1486 allocr(_RAX_REGNO, _RAX);
1487
1488 if (r0 == _RAX_REGNO || r0 == _RDX_REGNO || r0 == r1) {
1489 if ((reg = jit_get_reg(jit_class_gpr|jit_class_chk)) == JIT_NOREG)
1490 reg = jit_get_reg((r1 == _RCX_REGNO ? _RBX : _RCX) |
1491 jit_class_gpr|jit_class_named);
1492 use = 1;
1493 div = rn(reg);
1494 }
1495 else
1496 div = r0;
1497
1498 movi(div, i0);
1499 movr(_RAX_REGNO, r1);
1500
1501 if (sign) {
1502 sign_extend_rdx_rax();
1503 idivr(div);
1504 }
1505 else {
1506 ixorr(_RDX_REGNO, _RDX_REGNO);
1507 idivr_u(div);
1508 }
1509
1510 if (use)
1511 jit_unget_reg(reg);
1512
1513 if (divide)
1514 movr(r0, _RAX_REGNO);
1515 else
1516 movr(r0, _RDX_REGNO);
1517
1518 clear(_RDX_REGNO, _RDX);
1519 clear(_RAX_REGNO, _RAX);
1520}
1521
1522static void
1523_iqdivr(jit_state_t *_jit, jit_int32_t r0, jit_int32_t r1,
1524 jit_int32_t r2, jit_int32_t r3, jit_bool_t sign)
1525{
1526 jit_int32_t div;
1527 jit_int32_t reg;
1528 jit_int32_t sav;
1529 jit_int32_t set;
1530 jit_int32_t use;
1531
1532 sav = set = use = 0;
1533 qsavset(_RDX_REGNO);
1534 qsavset(_RAX_REGNO);
1535 allocr(_RDX_REGNO, _RDX);
1536 allocr(_RAX_REGNO, _RAX);
1537 if (r3 == _RAX_REGNO) {
1538 if (r0 == _RAX_REGNO || r0 == _RDX_REGNO) {
1539 if ((reg = jit_get_reg(jit_class_gpr|jit_class_chk)) == JIT_NOREG)
1540 reg = jit_get_reg((r1 == _RCX_REGNO ? _RBX : _RCX) |
1541 jit_class_gpr|jit_class_named);
1542 use = 1;
1543 div = rn(reg);
1544 movr(div, _RAX_REGNO);
1545 if (r2 != _RAX_REGNO)
1546 movr(_RAX_REGNO, r2);
1547 }
1548 else {
1549 if (r0 == r2)
1550 xchgr(r0, _RAX_REGNO);
1551 else {
1552 if (r0 != _RAX_REGNO)
1553 movr(r0, _RAX_REGNO);
1554 if (r2 != _RAX_REGNO)
1555 movr(_RAX_REGNO, r2);
1556 }
1557 div = r0;
1558 }
1559 }
1560 else if (r3 == _RDX_REGNO) {
1561 if (r0 == _RAX_REGNO || r0 == _RDX_REGNO) {
1562 if ((reg = jit_get_reg(jit_class_gpr|jit_class_chk)) == JIT_NOREG)
1563 reg = jit_get_reg((r1 == _RCX_REGNO ? _RBX : _RCX) |
1564 jit_class_gpr|jit_class_named);
1565 use = 1;
1566 div = rn(reg);
1567 movr(div, _RDX_REGNO);
1568 if (r2 != _RAX_REGNO)
1569 movr(_RAX_REGNO, r2);
1570 }
1571 else {
1572 if (r2 != _RAX_REGNO)
1573 movr(_RAX_REGNO, r2);
1574 movr(r0, _RDX_REGNO);
1575 div = r0;
1576 }
1577 }
1578 else {
1579 if (r2 != _RAX_REGNO)
1580 movr(_RAX_REGNO, r2);
1581 div = r3;
1582 }
1583 if (sign) {
1584 sign_extend_rdx_rax();
1585 idivr(div);
1586 }
1587 else {
1588 ixorr(_RDX_REGNO, _RDX_REGNO);
1589 idivr_u(div);
1590 }
1591 if (use)
1592 jit_unget_reg(reg);
1593
1594 if (r0 == _RDX_REGNO && r1 == _RAX_REGNO)
1595 xchgr(_RAX_REGNO, _RDX_REGNO);
1596 else {
1597 if (r0 != _RDX_REGNO)
1598 movr(r0, _RAX_REGNO);
1599 movr(r1, _RDX_REGNO);
1600 if (r0 == _RDX_REGNO)
1601 movr(r0, _RAX_REGNO);
1602 }
1603
1604 clear(_RDX_REGNO, _RDX);
1605 clear(_RAX_REGNO, _RAX);
1606}
1607
1608static void
1609_iqdivi(jit_state_t *_jit, jit_int32_t r0, jit_int32_t r1,
1610 jit_int32_t r2, jit_word_t i0, jit_bool_t sign)
1611{
1612 jit_int32_t reg;
1613
1614 reg = jit_get_reg(jit_class_gpr);
1615 movi(rn(reg), i0);
1616 if (sign)
1617 qdivr(r0, r1, r2, rn(reg));
1618 else
1619 qdivr_u(r0, r1, r2, rn(reg));
1620 jit_unget_reg(reg);
1621}
1622#undef clear
1623#undef allocr
1624#undef savset
1625
1626static void
1627_andr(jit_state_t *_jit, jit_int32_t r0, jit_int32_t r1, jit_int32_t r2)
1628{
1629 if (r1 == r2)
1630 movr(r0, r1);
1631 else if (r0 == r1)
1632 iandr(r0, r2);
1633 else if (r0 == r2)
1634 iandr(r0, r1);
1635 else {
1636 movr(r0, r1);
1637 iandr(r0, r2);
1638 }
1639}
1640
1641static void
1642_andi(jit_state_t *_jit, jit_int32_t r0, jit_int32_t r1, jit_word_t i0)
1643{
1644 jit_int32_t reg;
1645
1646 if (i0 == 0)
1647 ixorr(r0, r0);
1648 else if (i0 == -1)
1649 movr(r0, r1);
1650 else if (r0 == r1) {
1651 if (can_sign_extend_int_p(i0))
1652 iandi(r0, i0);
1653 else {
1654 reg = jit_get_reg(jit_class_gpr);
1655 movi(rn(reg), i0);
1656 iandr(r0, rn(reg));
1657 jit_unget_reg(reg);
1658 }
1659 }
1660 else {
1661 movi(r0, i0);
1662 iandr(r0, r1);
1663 }
1664}
1665
1666static void
1667_orr(jit_state_t *_jit, jit_int32_t r0, jit_int32_t r1, jit_int32_t r2)
1668{
1669 if (r1 == r2)
1670 movr(r0, r1);
1671 else if (r0 == r1)
1672 iorr(r0, r2);
1673 else if (r0 == r2)
1674 iorr(r0, r1);
1675 else {
1676 movr(r0, r1);
1677 iorr(r0, r2);
1678 }
1679}
1680
1681static void
1682_ori(jit_state_t *_jit, jit_int32_t r0, jit_int32_t r1, jit_word_t i0)
1683{
1684 jit_int32_t reg;
1685 if (i0 == 0)
1686 movr(r0, r1);
1687 else if (i0 == -1)
1688 movi(r0, -1);
1689 else if (can_sign_extend_int_p(i0)) {
1690 movr(r0, r1);
1691 iori(r0, i0);
1692 }
1693 else if (r0 != r1) {
1694 movi(r0, i0);
1695 iorr(r0, r1);
1696 }
1697 else {
1698 reg = jit_get_reg(jit_class_gpr);
1699 movi(rn(reg), i0);
1700 iorr(r0, rn(reg));
1701 jit_unget_reg(reg);
1702 }
1703}
1704
1705static void
1706_xorr(jit_state_t *_jit, jit_int32_t r0, jit_int32_t r1, jit_int32_t r2)
1707{
1708 if (r1 == r2)
1709 ixorr(r0, r0);
1710 else if (r0 == r1)
1711 ixorr(r0, r2);
1712 else if (r0 == r2)
1713 ixorr(r0, r1);
1714 else {
1715 movr(r0, r1);
1716 ixorr(r0, r2);
1717 }
1718}
1719
1720static void
1721_xori(jit_state_t *_jit, jit_int32_t r0, jit_int32_t r1, jit_word_t i0)
1722{
1723 jit_int32_t reg;
1724 if (i0 == 0)
1725 movr(r0, r1);
1726 else if (i0 == -1)
1727 comr(r0, r1);
1728 else if (can_sign_extend_int_p(i0)) {
1729 movr(r0, r1);
1730 ixori(r0, i0);
1731 }
1732 else if (r0 != r1) {
1733 movi(r0, i0);
1734 ixorr(r0, r1);
1735 }
1736 else {
1737 reg = jit_get_reg(jit_class_gpr);
1738 movi(rn(reg), i0);
1739 ixorr(r0, rn(reg));
1740 jit_unget_reg(reg);
1741 }
1742}
1743
1744static void
1745_irotshr(jit_state_t *_jit, jit_int32_t code, jit_int32_t r0)
1746{
1747 rex(0, WIDE, _RCX_REGNO, _NOREG, r0);
1748 ic(0xd3);
1749 mrm(0x03, code, r7(r0));
1750}
1751
1752static void
1753_rotshr(jit_state_t *_jit, jit_int32_t code,
1754 jit_int32_t r0, jit_int32_t r1, jit_int32_t r2)
1755{
1756 jit_int32_t reg;
1757 jit_int32_t use;
1758
1759 if (r0 == _RCX_REGNO) {
1760 reg = jit_get_reg(jit_class_gpr);
1761 movr(rn(reg), r1);
1762 if (r2 != _RCX_REGNO)
1763 movr(_RCX_REGNO, r2);
1764 irotshr(code, rn(reg));
1765 movr(_RCX_REGNO, rn(reg));
1766 jit_unget_reg(reg);
1767 }
1768 else if (r2 != _RCX_REGNO) {
1769 use = !jit_reg_free_p(_RCX);
1770 if (use) {
1771 reg = jit_get_reg(jit_class_gpr);
1772 movr(rn(reg), _RCX_REGNO);
1773 }
1774 else
1775 reg = 0;
1776 if (r1 == _RCX_REGNO) {
1777 if (r0 == r2)
1778 xchgr(r0, _RCX_REGNO);
1779 else {
1780 movr(r0, r1);
1781 movr(_RCX_REGNO, r2);
1782 }
1783 }
1784 else {
1785 movr(_RCX_REGNO, r2);
1786 movr(r0, r1);
1787 }
1788 irotshr(code, r0);
1789 if (use) {
1790 movr(_RCX_REGNO, rn(reg));
1791 jit_unget_reg(reg);
1792 }
1793 }
1794 else {
1795 movr(r0, r1);
1796 irotshr(code, r0);
1797 }
1798}
1799
1800static void
1801_irotshi(jit_state_t *_jit, jit_int32_t code, jit_int32_t r0, jit_word_t i0)
1802{
1803 rex(0, WIDE, _NOREG, _NOREG, r0);
1804 if (i0 == 1) {
1805 ic(0xd1);
1806 mrm(0x03, code, r7(r0));
1807 }
1808 else {
1809 ic(0xc1);
1810 mrm(0x03, code, r7(r0));
1811 ic(i0);
1812 }
1813}
1814
1815static void
1816_rotshi(jit_state_t *_jit, jit_int32_t code,
1817 jit_int32_t r0, jit_int32_t r1, jit_word_t i0)
1818{
1819 movr(r0, r1);
1820 if (i0)
1821 irotshi(code, r0, i0);
1822}
1823
1824static void
1825_lshi(jit_state_t *_jit, jit_int32_t r0, jit_int32_t r1, jit_word_t i0)
1826{
1827 if (i0 == 0)
1828 movr(r0, r1);
1829 else if (i0 <= 3)
1830 lea(0, _NOREG, r1, i0 == 1 ? _SCL2 : i0 == 2 ? _SCL4 : _SCL8, r0);
1831 else
1832 rotshi(X86_SHL, r0, r1, i0);
1833}
1834
1835static void
1836_unr(jit_state_t *_jit, jit_int32_t code, jit_int32_t r0)
1837{
1838 rex(0, WIDE, _NOREG, _NOREG, r0);
1839 ic(0xf7);
1840 mrm(0x03, code, r7(r0));
1841}
1842
1843static void
1844_negr(jit_state_t *_jit, jit_int32_t r0, jit_int32_t r1)
1845{
1846 if (r0 == r1)
1847 inegr(r0);
1848 else {
1849 ixorr(r0, r0);
1850 isubr(r0, r1);
1851 }
1852}
1853
1854static void
1855_comr(jit_state_t *_jit, jit_int32_t r0, jit_int32_t r1)
1856{
1857 movr(r0, r1);
1858 icomr(r0);
1859}
1860
1861#if USE_INC_DEC
1862static void
1863_incr(jit_state_t *_jit, jit_int32_t r0, jit_int32_t r1)
1864{
1865 movr(r0, r1);
1866# if __X64
1867 rex(0, WIDE, _NOREG, _NOREG, r0);
1868 ic(0xff);
1869 ic(0xc0 | r7(r0));
1870# else
1871 ic(0x40 | r7(r0));
1872# endif
1873}
1874
1875static void
1876_decr(jit_state_t *_jit, jit_int32_t r0, jit_int32_t r1)
1877{
1878 movr(r0, r1);
1879# if __X64
1880 rex(0, WIDE, _NOREG, _NOREG, r0);
1881 ic(0xff);
1882 ic(0xc8 | r7(r0));
1883# else
1884 ic(0x48 | r7(r0));
1885# endif
1886}
1887#endif
1888
1889static void
1890_cr(jit_state_t *_jit,
1891 jit_int32_t code, jit_int32_t r0, jit_int32_t r1, jit_int32_t r2)
1892{
1893 jit_int32_t reg;
1894 jit_bool_t same;
1895 if (reg8_p(r0)) {
1896 same = r0 == r1 || r0 == r2;
1897 if (!same)
1898 ixorr(r0, r0);
1899 icmpr(r1, r2);
1900 if (same)
1901 imovi(r0, 0);
1902 cc(code, r0);
1903 }
1904 else {
1905 reg = jit_get_reg(jit_class_gpr|jit_class_rg8);
1906 ixorr(rn(reg), rn(reg));
1907 icmpr(r1, r2);
1908 cc(code, rn(reg));
1909 movr(r0, rn(reg));
1910 jit_unget_reg(reg);
1911 }
1912}
1913
1914static void
1915_ci(jit_state_t *_jit,
1916 jit_int32_t code, jit_int32_t r0, jit_int32_t r1, jit_word_t i0)
1917{
1918 jit_int32_t reg;
1919 jit_bool_t same;
1920 if (reg8_p(r0)) {
1921 same = r0 == r1;
1922 if (!same)
1923 ixorr(r0, r0);
1924 icmpi(r1, i0);
1925 if (same)
1926 imovi(r0, 0);
1927 cc(code, r0);
1928 }
1929 else {
1930 reg = jit_get_reg(jit_class_gpr|jit_class_rg8);
1931 ixorr(rn(reg), rn(reg));
1932 icmpi(r1, i0);
1933 cc(code, rn(reg));
1934 movr(r0, rn(reg));
1935 jit_unget_reg(reg);
1936 }
1937}
1938
1939static void
1940_ci0(jit_state_t *_jit, jit_int32_t code, jit_int32_t r0, jit_int32_t r1)
1941{
1942 jit_int32_t reg;
1943 jit_bool_t same;
1944 if (reg8_p(r0)) {
1945 same = r0 == r1;
1946 if (!same)
1947 ixorr(r0, r0);
1948 testr(r1, r1);
1949 if (same)
1950 imovi(r0, 0);
1951 cc(code, r0);
1952 }
1953 else {
1954 reg = jit_get_reg(jit_class_gpr|jit_class_rg8);
1955 ixorr(rn(reg), rn(reg));
1956 testr(r1, r1);
1957 cc(code, rn(reg));
1958 movr(r0, rn(reg));
1959 jit_unget_reg(reg);
1960 }
1961}
1962
1963static void
1964_ltr(jit_state_t *_jit, jit_int32_t r0, jit_int32_t r1, jit_int32_t r2)
1965{
1966 if (r1 == r2)
1967 movi(r0, 0);
1968 else
1969 cr(X86_CC_L, r0, r1, r2);
1970}
1971
1972static void
1973_lti(jit_state_t *_jit, jit_int32_t r0, jit_int32_t r1, jit_word_t i0)
1974{
1975 if (i0)
1976 ci(X86_CC_L, r0, r1, i0);
1977 else
1978 ci0(X86_CC_S, r0, r1);
1979}
1980
1981static void
1982_ltr_u(jit_state_t *_jit, jit_int32_t r0, jit_int32_t r1, jit_int32_t r2)
1983{
1984 if (r1 == r2)
1985 movi(r0, 0);
1986 else
1987 cr(X86_CC_B, r0, r1, r2);
1988}
1989
1990static void
1991_ler(jit_state_t *_jit, jit_int32_t r0, jit_int32_t r1, jit_int32_t r2)
1992{
1993 if (r1 == r2)
1994 movi(r0, 1);
1995 else
1996 cr(X86_CC_LE, r0, r1, r2);
1997}
1998
1999static void
2000_ler_u(jit_state_t *_jit, jit_int32_t r0, jit_int32_t r1, jit_int32_t r2)
2001{
2002 if (r1 == r2)
2003 movi(r0, 1);
2004 else
2005 cr(X86_CC_BE, r0, r1, r2);
2006}
2007
2008static void
2009_lei_u(jit_state_t *_jit, jit_int32_t r0, jit_int32_t r1, jit_word_t i0)
2010{
2011 if (i0)
2012 ci(X86_CC_BE, r0, r1, i0);
2013 else
2014 ci0(X86_CC_E, r0, r1);
2015}
2016
2017static void
2018_eqr(jit_state_t *_jit, jit_int32_t r0, jit_int32_t r1, jit_int32_t r2)
2019{
2020 if (r1 == r2)
2021 movi(r0, 1);
2022 else
2023 cr(X86_CC_E, r0, r1, r2);
2024}
2025
2026static void
2027_eqi(jit_state_t *_jit, jit_int32_t r0, jit_int32_t r1, jit_word_t i0)
2028{
2029 if (i0)
2030 ci(X86_CC_E, r0, r1, i0);
2031 else
2032 ci0(X86_CC_E, r0, r1);
2033}
2034
2035static void
2036_ger(jit_state_t *_jit, jit_int32_t r0, jit_int32_t r1, jit_int32_t r2)
2037{
2038 if (r1 == r2)
2039 movi(r0, 1);
2040 else
2041 cr(X86_CC_GE, r0, r1, r2);
2042}
2043
2044static void
2045_gei(jit_state_t *_jit, jit_int32_t r0, jit_int32_t r1, jit_word_t i0)
2046{
2047 if (i0)
2048 ci(X86_CC_GE, r0, r1, i0);
2049 else
2050 ci0(X86_CC_NS, r0, r1);
2051}
2052
2053static void
2054_ger_u(jit_state_t *_jit, jit_int32_t r0, jit_int32_t r1, jit_int32_t r2)
2055{
2056 if (r1 == r2)
2057 movi(r0, 1);
2058 else
2059 cr(X86_CC_AE, r0, r1, r2);
2060}
2061
2062static void
2063_gei_u(jit_state_t *_jit, jit_int32_t r0, jit_int32_t r1, jit_word_t i0)
2064{
2065 if (i0)
2066 ci(X86_CC_AE, r0, r1, i0);
2067 else
2068 ci0(X86_CC_NB, r0, r1);
2069}
2070
2071static void
2072_gtr(jit_state_t *_jit, jit_int32_t r0, jit_int32_t r1, jit_int32_t r2)
2073{
2074 if (r1 == r2)
2075 movi(r0, 0);
2076 else
2077 cr(X86_CC_G, r0, r1, r2);
2078}
2079
2080static void
2081_gtr_u(jit_state_t *_jit, jit_int32_t r0, jit_int32_t r1, jit_int32_t r2)
2082{
2083 if (r1 == r2)
2084 movi(r0, 0);
2085 else
2086 cr(X86_CC_A, r0, r1, r2);
2087}
2088
2089static void
2090_gti_u(jit_state_t *_jit, jit_int32_t r0, jit_int32_t r1, jit_word_t i0)
2091{
2092 if (i0)
2093 ci(X86_CC_A, r0, r1, i0);
2094 else
2095 ci0(X86_CC_NE, r0, r1);
2096}
2097
2098static void
2099_ner(jit_state_t *_jit, jit_int32_t r0, jit_int32_t r1, jit_int32_t r2)
2100{
2101 if (r1 == r2)
2102 movi(r0, 0);
2103 else
2104 cr(X86_CC_NE, r0, r1, r2);
2105}
2106
2107static void
2108_nei(jit_state_t *_jit, jit_int32_t r0, jit_int32_t r1, jit_word_t i0)
2109{
2110 if (i0)
2111 ci(X86_CC_NE, r0, r1, i0);
2112 else
2113 ci0(X86_CC_NE, r0, r1);
2114}
2115
2116static void
2117_movr(jit_state_t *_jit, jit_int32_t r0, jit_int32_t r1)
2118{
2119 if (r0 != r1) {
2120 rex(0, 1, r1, _NOREG, r0);
2121 ic(0x89);
2122 ic(0xc0 | (r1 << 3) | r7(r0));
2123 }
2124}
2125
2126static void
2127_imovi(jit_state_t *_jit, jit_int32_t r0, jit_word_t i0)
2128{
2129#if __X64
2130# if !__X64_32
2131 if (fits_uint32_p(i0)) {
2132# endif
2133 rex(0, 0, _NOREG, _NOREG, r0);
2134 ic(0xb8 | r7(r0));
2135 ii(i0);
2136# if !__X64_32
2137 }
2138 else {
2139 rex(0, 1, _NOREG, _NOREG, r0);
2140 ic(0xb8 | r7(r0));
2141 il(i0);
2142 }
2143# endif
2144#else
2145 ic(0xb8 | r7(r0));
2146 ii(i0);
2147#endif
2148}
2149
2150static void
2151_movi(jit_state_t *_jit, jit_int32_t r0, jit_word_t i0)
2152{
2153 if (i0)
2154 imovi(r0, i0);
2155 else
2156 ixorr(r0, r0);
2157}
2158
2159static jit_word_t
2160_movi_p(jit_state_t *_jit, jit_int32_t r0, jit_word_t i0)
2161{
2162 rex(0, WIDE, _NOREG, _NOREG, r0);
2163 ic(0xb8 | r7(r0));
2164 il(i0);
2165 return (_jit->pc.w);
2166}
2167
2168static void
2169_movcr(jit_state_t *_jit, jit_int32_t r0, jit_int32_t r1)
2170{
2171 rex(0, WIDE, r0, _NOREG, r1);
2172 ic(0x0f);
2173 ic(0xbe);
2174 mrm(0x03, r7(r0), r7(r1));
2175}
2176
2177static void
2178_movcr_u(jit_state_t *_jit, jit_int32_t r0, jit_int32_t r1)
2179{
2180 rex(0, WIDE, r0, _NOREG, r1);
2181 ic(0x0f);
2182 ic(0xb6);
2183 mrm(0x03, r7(r0), r7(r1));
2184}
2185
2186static void
2187_movsr(jit_state_t *_jit, jit_int32_t r0, jit_int32_t r1)
2188{
2189 rex(0, WIDE, r0, _NOREG, r1);
2190 ic(0x0f);
2191 ic(0xbf);
2192 mrm(0x03, r7(r0), r7(r1));
2193}
2194
2195static void
2196_movsr_u(jit_state_t *_jit, jit_int32_t r0, jit_int32_t r1)
2197{
2198 rex(0, WIDE, r0, _NOREG, r1);
2199 ic(0x0f);
2200 ic(0xb7);
2201 mrm(0x03, r7(r0), r7(r1));
2202}
2203
2204#if __X64
2205static void
2206_movir(jit_state_t *_jit, jit_int32_t r0, jit_int32_t r1)
2207{
2208 rex(0, 1, r0, _NOREG, r1);
2209 ic(0x63);
2210 mrm(0x03, r7(r0), r7(r1));
2211}
2212
2213static void
2214_movir_u(jit_state_t *_jit, jit_int32_t r0, jit_int32_t r1)
2215{
2216 rex(0, 0, r1, _NOREG, r0);
2217 ic(0x89);
2218 ic(0xc0 | (r1 << 3) | r7(r0));
2219}
2220#endif
2221
2222static void
2223_htonr_us(jit_state_t *_jit, jit_int32_t r0, jit_int32_t r1)
2224{
2225 extr_us(r0, r1);
2226 ic(0x66);
2227 rex(0, 0, _NOREG, _NOREG, r0);
2228 ic(0xc1);
2229 mrm(0x03, X86_ROR, r7(r0));
2230 ic(8);
2231}
2232
2233static void
2234_htonr_ui(jit_state_t *_jit, jit_int32_t r0, jit_int32_t r1)
2235{
2236 movr(r0, r1);
2237 rex(0, 0, _NOREG, _NOREG, r0);
2238 ic(0x0f);
2239 ic(0xc8 | r7(r0));
2240}
2241
2242#if __X64 && !__X64_32
2243static void
2244_htonr_ul(jit_state_t *_jit, jit_int32_t r0, jit_int32_t r1)
2245{
2246 movr(r0, r1);
2247 rex(0, 1, _NOREG, _NOREG, r0);
2248 ic(0x0f);
2249 ic(0xc8 | r7(r0));
2250}
2251#endif
2252
2253static void
2254_extr_c(jit_state_t *_jit, jit_int32_t r0, jit_int32_t r1)
2255{
2256 jit_int32_t reg;
2257 if (reg8_p(r1))
2258 movcr(r0, r1);
2259 else {
2260 reg = jit_get_reg(jit_class_gpr|jit_class_rg8);
2261 movr(rn(reg), r1);
2262 movcr(r0, rn(reg));
2263 jit_unget_reg(reg);
2264 }
2265}
2266
2267static void
2268_extr_uc(jit_state_t *_jit, jit_int32_t r0, jit_int32_t r1)
2269{
2270 jit_int32_t reg;
2271 if (reg8_p(r1))
2272 movcr_u(r0, r1);
2273 else {
2274 reg = jit_get_reg(jit_class_gpr|jit_class_rg8);
2275 movr(rn(reg), r1);
2276 movcr_u(r0, rn(reg));
2277 jit_unget_reg(reg);
2278 }
2279}
2280
2281static void
2282_ldr_c(jit_state_t *_jit, jit_int32_t r0, jit_int32_t r1)
2283{
2284 rex(0, WIDE, r0, _NOREG, r1);
2285 ic(0x0f);
2286 ic(0xbe);
2287 rx(r0, 0, r1, _NOREG, _SCL1);
2288}
2289
2290static void
2291_ldi_c(jit_state_t *_jit, jit_int32_t r0, jit_word_t i0)
2292{
2293 jit_int32_t reg;
2294 if (can_sign_extend_int_p(i0)) {
2295 rex(0, WIDE, r0, _NOREG, _NOREG);
2296 ic(0x0f);
2297 ic(0xbe);
2298 rx(r0, i0, _NOREG, _NOREG, _SCL1);
2299 }
2300 else {
2301 reg = jit_get_reg(jit_class_gpr);
2302 movi(rn(reg), i0);
2303 ldr_c(r0, rn(reg));
2304 jit_unget_reg(reg);
2305 }
2306}
2307
2308static void
2309_ldr_uc(jit_state_t *_jit, jit_int32_t r0, jit_int32_t r1)
2310{
2311 rex(0, WIDE, r0, _NOREG, r1);
2312 ic(0x0f);
2313 ic(0xb6);
2314 rx(r0, 0, r1, _NOREG, _SCL1);
2315}
2316
2317static void
2318_ldi_uc(jit_state_t *_jit, jit_int32_t r0, jit_word_t i0)
2319{
2320 jit_int32_t reg;
2321 if (can_sign_extend_int_p(i0)) {
2322 rex(0, WIDE, r0, _NOREG, _NOREG);
2323 ic(0x0f);
2324 ic(0xb6);
2325 rx(r0, i0, _NOREG, _NOREG, _SCL1);
2326 }
2327 else {
2328 reg = jit_get_reg(jit_class_gpr);
2329 movi(rn(reg), i0);
2330 ldr_uc(r0, rn(reg));
2331 jit_unget_reg(reg);
2332 }
2333}
2334
2335static void
2336_ldr_s(jit_state_t *_jit, jit_int32_t r0, jit_int32_t r1)
2337{
2338 rex(0, WIDE, r0, _NOREG, r1);
2339 ic(0x0f);
2340 ic(0xbf);
2341 rx(r0, 0, r1, _NOREG, _SCL1);
2342}
2343
2344static void
2345_ldi_s(jit_state_t *_jit, jit_int32_t r0, jit_word_t i0)
2346{
2347 jit_int32_t reg;
2348 if (can_sign_extend_int_p(i0)) {
2349 rex(0, WIDE, r0, _NOREG, _NOREG);
2350 ic(0x0f);
2351 ic(0xbf);
2352 rx(r0, i0, _NOREG, _NOREG, _SCL1);
2353 }
2354 else {
2355 reg = jit_get_reg(jit_class_gpr);
2356 movi(rn(reg), i0);
2357 ldr_s(r0, rn(reg));
2358 jit_unget_reg(reg);
2359 }
2360}
2361
2362static void
2363_ldr_us(jit_state_t *_jit, jit_int32_t r0, jit_int32_t r1)
2364{
2365 rex(0, WIDE, r0, _NOREG, r1);
2366 ic(0x0f);
2367 ic(0xb7);
2368 rx(r0, 0, r1, _NOREG, _SCL1);
2369}
2370
2371static void
2372_ldi_us(jit_state_t *_jit, jit_int32_t r0, jit_word_t i0)
2373{
2374 jit_int32_t reg;
2375 if (can_sign_extend_int_p(i0)) {
2376 rex(0, WIDE, r0, _NOREG, _NOREG);
2377 ic(0x0f);
2378 ic(0xb7);
2379 rx(r0, i0, _NOREG, _NOREG, _SCL1);
2380 }
2381 else {
2382 reg = jit_get_reg(jit_class_gpr);
2383 movi(rn(reg), i0);
2384 ldr_us(r0, rn(reg));
2385 jit_unget_reg(reg);
2386 }
2387}
2388
2389#if __X32 || !__X64_32
2390static void
2391_ldr_i(jit_state_t *_jit, jit_int32_t r0, jit_int32_t r1)
2392{
2393#if __X64
2394 rex(0, WIDE, r0, _NOREG, r1);
2395 ic(0x63);
2396#else
2397 ic(0x8b);
2398#endif
2399 rx(r0, 0, r1, _NOREG, _SCL1);
2400}
2401
2402static void
2403_ldi_i(jit_state_t *_jit, jit_int32_t r0, jit_word_t i0)
2404{
2405 jit_int32_t reg;
2406 if (can_sign_extend_int_p(i0)) {
2407#if __X64
2408 rex(0, WIDE, r0, _NOREG, _NOREG);
2409 ic(0x63);
2410#else
2411 ic(0x8b);
2412#endif
2413 rx(r0, i0, _NOREG, _NOREG, _SCL1);
2414 }
2415 else {
2416 reg = jit_get_reg(jit_class_gpr);
2417 movi(rn(reg), i0);
2418 ldr_i(r0, rn(reg));
2419 jit_unget_reg(reg);
2420 }
2421}
2422#endif
2423
2424#if __X64
2425static void
2426_ldr_ui(jit_state_t *_jit, jit_int32_t r0, jit_int32_t r1)
2427{
2428 rex(0, 0, r0, _NOREG, r1);
2429 ic(0x63);
2430 rx(r0, 0, r1, _NOREG, _SCL1);
2431}
2432
2433static void
2434_ldi_ui(jit_state_t *_jit, jit_int32_t r0, jit_word_t i0)
2435{
2436 jit_int32_t reg;
2437 if (can_sign_extend_int_p(i0)) {
2438 rex(0, 0, r0, _NOREG, _NOREG);
2439 ic(0x63);
2440 rx(r0, i0, _NOREG, _NOREG, _SCL1);
2441 }
2442 else {
2443 reg = jit_get_reg(jit_class_gpr);
2444 movi(rn(reg), i0);
2445 ldr_ui(r0, rn(reg));
2446 jit_unget_reg(reg);
2447 }
2448}
2449
2450# if !__X64_32
2451static void
2452_ldr_l(jit_state_t *_jit, jit_int32_t r0, jit_int32_t r1)
2453{
2454 rex(0, 1, r0, _NOREG, r1);
2455 ic(0x8b);
2456 rx(r0, 0, r1, _NOREG, _SCL1);
2457}
2458
2459static void
2460_ldi_l(jit_state_t *_jit, jit_int32_t r0, jit_word_t i0)
2461{
2462 jit_int32_t reg;
2463 if (can_sign_extend_int_p(i0)) {
2464 rex(0, 1, r0, _NOREG, _NOREG);
2465 ic(0x8b);
2466 rx(r0, i0, _NOREG, _NOREG, _SCL1);
2467 }
2468 else {
2469 reg = jit_get_reg(jit_class_gpr);
2470 movi(rn(reg), i0);
2471 ldr_l(r0, rn(reg));
2472 jit_unget_reg(reg);
2473 }
2474}
2475# endif
2476#endif
2477
2478static void
2479_ldxr_c(jit_state_t *_jit, jit_int32_t r0, jit_int32_t r1, jit_int32_t r2)
2480{
2481#if __X64_32
2482 addr(r0, r1, r2);
2483 ldr_c(r0, r0);
2484#else
2485 rex(0, WIDE, r0, r1, r2);
2486 ic(0x0f);
2487 ic(0xbe);
2488 rx(r0, 0, r2, r1, _SCL1);
2489#endif
2490}
2491
2492static void
2493_ldxi_c(jit_state_t *_jit, jit_int32_t r0, jit_int32_t r1, jit_word_t i0)
2494{
2495 jit_int32_t reg;
2496 if (can_sign_extend_int_p(i0)) {
2497 rex(0, WIDE, r0, _NOREG, r1);
2498 ic(0x0f);
2499 ic(0xbe);
2500 rx(r0, i0, r1, _NOREG, _SCL1);
2501 }
2502 else {
2503 reg = jit_get_reg(jit_class_gpr);
2504 movi(rn(reg), i0);
2505 ldxr_c(r0, r1, rn(reg));
2506 jit_unget_reg(reg);
2507 }
2508}
2509
2510static void
2511_ldxr_uc(jit_state_t *_jit, jit_int32_t r0, jit_int32_t r1, jit_int32_t r2)
2512{
2513#if __X64_32
2514 addr(r0, r1, r2);
2515 ldr_uc(r0, r0);
2516#else
2517 rex(0, WIDE, r0, r1, r2);
2518 ic(0x0f);
2519 ic(0xb6);
2520 rx(r0, 0, r2, r1, _SCL1);
2521#endif
2522}
2523
2524static void
2525_ldxi_uc(jit_state_t *_jit, jit_int32_t r0, jit_int32_t r1, jit_word_t i0)
2526{
2527 jit_int32_t reg;
2528 if (can_sign_extend_int_p(i0)) {
2529 rex(0, WIDE, r0, _NOREG, r1);
2530 ic(0x0f);
2531 ic(0xb6);
2532 rx(r0, i0, r1, _NOREG, _SCL1);
2533 }
2534 else {
2535 reg = jit_get_reg(jit_class_gpr);
2536 movi(rn(reg), i0);
2537 ldxr_uc(r0, r1, rn(reg));
2538 jit_unget_reg(reg);
2539 }
2540}
2541
2542static void
2543_ldxr_s(jit_state_t *_jit, jit_int32_t r0, jit_int32_t r1, jit_int32_t r2)
2544{
2545#if __X64_32
2546 addr(r0, r1, r2);
2547 ldr_s(r0, r0);
2548#else
2549 rex(0, WIDE, r0, r1, r2);
2550 ic(0x0f);
2551 ic(0xbf);
2552 rx(r0, 0, r2, r1, _SCL1);
2553#endif
2554}
2555
2556static void
2557_ldxi_s(jit_state_t *_jit, jit_int32_t r0, jit_int32_t r1, jit_word_t i0)
2558{
2559 jit_int32_t reg;
2560 if (can_sign_extend_int_p(i0)) {
2561 rex(0, WIDE, r0, _NOREG, r1);
2562 ic(0x0f);
2563 ic(0xbf);
2564 rx(r0, i0, r1, _NOREG, _SCL1);
2565 }
2566 else {
2567 reg = jit_get_reg(jit_class_gpr);
2568 movi(rn(reg), i0);
2569 ldxr_s(r0, r1, rn(reg));
2570 jit_unget_reg(reg);
2571 }
2572}
2573
2574static void
2575_ldxr_us(jit_state_t *_jit, jit_int32_t r0, jit_int32_t r1, jit_int32_t r2)
2576{
2577#if __X64_32
2578 addr(r0, r1, r2);
2579 ldr_us(r0, r0);
2580#else
2581 rex(0, WIDE, r0, r1, r2);
2582 ic(0x0f);
2583 ic(0xb7);
2584 rx(r0, 0, r2, r1, _SCL1);
2585#endif
2586}
2587
2588static void
2589_ldxi_us(jit_state_t *_jit, jit_int32_t r0, jit_int32_t r1, jit_word_t i0)
2590{
2591 jit_int32_t reg;
2592 if (can_sign_extend_int_p(i0)) {
2593 rex(0, WIDE, r0, _NOREG, r1);
2594 ic(0x0f);
2595 ic(0xb7);
2596 rx(r0, i0, r1, _NOREG, _SCL1);
2597 }
2598 else {
2599 reg = jit_get_reg(jit_class_gpr);
2600 movi(rn(reg), i0);
2601 ldxr_us(r0, r1, rn(reg));
2602 jit_unget_reg(reg);
2603 }
2604}
2605
2606#if __X64 || !__X64_32
2607static void
2608_ldxr_i(jit_state_t *_jit, jit_int32_t r0, jit_int32_t r1, jit_int32_t r2)
2609{
2610#if __X64
2611 rex(0, WIDE, r0, r1, r2);
2612 ic(0x63);
2613#else
2614 ic(0x8b);
2615#endif
2616 rx(r0, 0, r2, r1, _SCL1);
2617}
2618
2619static void
2620_ldxi_i(jit_state_t *_jit, jit_int32_t r0, jit_int32_t r1, jit_word_t i0)
2621{
2622 jit_int32_t reg;
2623 if (can_sign_extend_int_p(i0)) {
2624#if __X64
2625 rex(0, WIDE, r0, _NOREG, r1);
2626 ic(0x63);
2627#else
2628 ic(0x8b);
2629#endif
2630 rx(r0, i0, r1, _NOREG, _SCL1);
2631 }
2632 else {
2633 reg = jit_get_reg(jit_class_gpr);
2634 movi(rn(reg), i0);
2635 ldxr_i(r0, r1, rn(reg));
2636 jit_unget_reg(reg);
2637 }
2638}
2639#endif
2640
2641#if __X64
2642static void
2643_ldxr_ui(jit_state_t *_jit, jit_int32_t r0, jit_int32_t r1, jit_int32_t r2)
2644{
2645#if __X64_32
2646 addr(r0, r1, r2);
2647 /* to avoid confusion with macro renames */
2648 _ldr_ui(_jit, r0, r0);
2649#else
2650 rex(0, 0, r0, r1, r2);
2651 ic(0x8b);
2652 rx(r0, 0, r2, r1, _SCL1);
2653#endif
2654}
2655
2656static void
2657_ldxi_ui(jit_state_t *_jit, jit_int32_t r0, jit_int32_t r1, jit_word_t i0)
2658{
2659 jit_int32_t reg;
2660 if (can_sign_extend_int_p(i0)) {
2661 rex(0, 0, r0, _NOREG, r1);
2662 ic(0x8b);
2663 rx(r0, i0, r1, _NOREG, _SCL1);
2664 }
2665 else {
2666 reg = jit_get_reg(jit_class_gpr);
2667 movi(rn(reg), i0);
2668 ldxr_ui(r0, r1, rn(reg));
2669 jit_unget_reg(reg);
2670 }
2671}
2672
2673# if !__X64_32
2674static void
2675_ldxr_l(jit_state_t *_jit, jit_int32_t r0, jit_int32_t r1, jit_int32_t r2)
2676{
2677 rex(0, 1, r0, r1, r2);
2678 ic(0x8b);
2679 rx(r0, 0, r2, r1, _SCL1);
2680}
2681
2682static void
2683_ldxi_l(jit_state_t *_jit, jit_int32_t r0, jit_int32_t r1, jit_word_t i0)
2684{
2685 jit_int32_t reg;
2686 if (can_sign_extend_int_p(i0)) {
2687 rex(0, 1, r0, _NOREG, r1);
2688 ic(0x8b);
2689 rx(r0, i0, r1, _NOREG, _SCL1);
2690 }
2691 else {
2692 reg = jit_get_reg(jit_class_gpr);
2693 movi(rn(reg), i0);
2694 ldxr_l(r0, r1, rn(reg));
2695 jit_unget_reg(reg);
2696 }
2697}
2698# endif
2699#endif
2700
2701static void
2702_str_c(jit_state_t *_jit, jit_int32_t r0, jit_int32_t r1)
2703{
2704 jit_int32_t reg;
2705 if (reg8_p(r1)) {
2706 rex(0, 0, r1, _NOREG, r0);
2707 ic(0x88);
2708 rx(r1, 0, r0, _NOREG, _SCL1);
2709 }
2710 else {
2711 reg = jit_get_reg(jit_class_gpr|jit_class_rg8);
2712 movr(rn(reg), r1);
2713 rex(0, 0, rn(reg), _NOREG, r0);
2714 ic(0x88);
2715 rx(rn(reg), 0, r0, _NOREG, _SCL1);
2716 jit_unget_reg(reg);
2717 }
2718}
2719
2720static void
2721_sti_c(jit_state_t *_jit, jit_word_t i0, jit_int32_t r0)
2722{
2723 jit_int32_t reg;
2724 if (can_sign_extend_int_p(i0)) {
2725 if (reg8_p(r0)) {
2726 rex(0, 0, r0, _NOREG, _NOREG);
2727 ic(0x88);
2728 rx(r0, i0, _NOREG, _NOREG, _SCL1);
2729 }
2730 else {
2731 reg = jit_get_reg(jit_class_gpr|jit_class_rg8);
2732 movr(rn(reg), r0);
2733 rex(0, 0, rn(reg), _NOREG, _NOREG);
2734 ic(0x88);
2735 rx(rn(reg), i0, _NOREG, _NOREG, _SCL1);
2736 jit_unget_reg(reg);
2737 }
2738 }
2739 else {
2740 reg = jit_get_reg(jit_class_gpr);
2741 movi(rn(reg), i0);
2742 str_c(rn(reg), r0);
2743 jit_unget_reg(reg);
2744 }
2745}
2746
2747static void
2748_str_s(jit_state_t *_jit, jit_int32_t r0, jit_int32_t r1)
2749{
2750 ic(0x66);
2751 rex(0, 0, r1, _NOREG, r0);
2752 ic(0x89);
2753 rx(r1, 0, r0, _NOREG, _SCL1);
2754}
2755
2756static void
2757_sti_s(jit_state_t *_jit, jit_word_t i0, jit_int32_t r0)
2758{
2759 jit_int32_t reg;
2760 if (can_sign_extend_int_p(i0)) {
2761 ic(0x66);
2762 rex(0, 0, r0, _NOREG, _NOREG);
2763 ic(0x89);
2764 rx(r0, i0, _NOREG, _NOREG, _SCL1);
2765 }
2766 else {
2767 reg = jit_get_reg(jit_class_gpr);
2768 movi(rn(reg), i0);
2769 str_s(rn(reg), r0);
2770 jit_unget_reg(reg);
2771 }
2772}
2773
2774static void
2775_str_i(jit_state_t *_jit, jit_int32_t r0, jit_int32_t r1)
2776{
2777 rex(0, 0, r1, _NOREG, r0);
2778 ic(0x89);
2779 rx(r1, 0, r0, _NOREG, _SCL1);
2780}
2781
2782static void
2783_sti_i(jit_state_t *_jit, jit_word_t i0, jit_int32_t r0)
2784{
2785 jit_int32_t reg;
2786 if (can_sign_extend_int_p(i0)) {
2787 rex(0, 0, r0, _NOREG, _NOREG);
2788 ic(0x89);
2789 rx(r0, i0, _NOREG, _NOREG, _SCL1);
2790 }
2791 else {
2792 reg = jit_get_reg(jit_class_gpr);
2793 movi(rn(reg), i0);
2794 str_i(rn(reg), r0);
2795 jit_unget_reg(reg);
2796 }
2797}
2798
2799#if __X64 && !__X64_32
2800static void
2801_str_l(jit_state_t *_jit, jit_int32_t r0, jit_int32_t r1)
2802{
2803 rex(0, 1, r1, _NOREG, r0);
2804 ic(0x89);
2805 rx(r1, 0, r0, _NOREG, _SCL1);
2806}
2807
2808static void
2809_sti_l(jit_state_t *_jit, jit_word_t i0, jit_int32_t r0)
2810{
2811 jit_int32_t reg;
2812 if (can_sign_extend_int_p(i0)) {
2813 rex(0, 1, r0, _NOREG, _NOREG);
2814 ic(0x89);
2815 rx(r0, i0, _NOREG, _NOREG, _SCL1);
2816 }
2817 else {
2818 reg = jit_get_reg(jit_class_gpr);
2819 movi(rn(reg), i0);
2820 str_l(rn(reg), r0);
2821 jit_unget_reg(reg);
2822 }
2823}
2824#endif
2825
2826static void
2827_stxr_c(jit_state_t *_jit, jit_int32_t r0, jit_int32_t r1, jit_int32_t r2)
2828{
2829 jit_int32_t reg;
2830#if __X64_32
2831 reg = jit_get_reg(jit_class_gpr);
2832 addr(rn(reg), r0, r1);
2833 str_c(rn(reg), r2);
2834 jit_unget_reg(reg);
2835#else
2836 if (reg8_p(r2)) {
2837 rex(0, 0, r2, r1, r0);
2838 ic(0x88);
2839 rx(r2, 0, r0, r1, _SCL1);
2840 }
2841 else {
2842 reg = jit_get_reg(jit_class_gpr|jit_class_rg8);
2843 movr(rn(reg), r2);
2844 rex(0, 0, rn(reg), r1, r0);
2845 ic(0x88);
2846 rx(rn(reg), 0, r0, r1, _SCL1);
2847 jit_unget_reg(reg);
2848 }
2849#endif
2850}
2851
2852static void
2853_stxi_c(jit_state_t *_jit, jit_word_t i0, jit_int32_t r0, jit_int32_t r1)
2854{
2855 jit_int32_t reg;
2856 if (can_sign_extend_int_p(i0)) {
2857 if (reg8_p(r1)) {
2858 rex(0, 0, r1, _NOREG, r0);
2859 ic(0x88);
2860 rx(r1, i0, r0, _NOREG, _SCL1);
2861 }
2862 else {
2863 reg = jit_get_reg(jit_class_gpr|jit_class_rg8);
2864 movr(rn(reg), r1);
2865 rex(0, 0, rn(reg), _NOREG, r0);
2866 ic(0x88);
2867 rx(rn(reg), i0, r0, _NOREG, _SCL1);
2868 jit_unget_reg(reg);
2869 }
2870 }
2871 else {
2872 reg = jit_get_reg(jit_class_gpr);
2873 movi(rn(reg), i0);
2874 stxr_c(rn(reg), r0, r1);
2875 jit_unget_reg(reg);
2876 }
2877}
2878
2879static void
2880_stxr_s(jit_state_t *_jit, jit_int32_t r0, jit_int32_t r1, jit_int32_t r2)
2881{
2882#if __X64_32
2883 jit_int32_t reg;
2884 reg = jit_get_reg(jit_class_gpr);
2885 addr(rn(reg), r0, r1);
2886 str_s(rn(reg), r2);
2887 jit_unget_reg(reg);
2888#else
2889 ic(0x66);
2890 rex(0, 0, r2, r1, r0);
2891 ic(0x89);
2892 rx(r2, 0, r0, r1, _SCL1);
2893#endif
2894}
2895
2896static void
2897_stxi_s(jit_state_t *_jit, jit_word_t i0, jit_int32_t r0, jit_int32_t r1)
2898{
2899 jit_int32_t reg;
2900 if (can_sign_extend_int_p(i0)) {
2901 ic(0x66);
2902 rex(0, 0, r1, _NOREG, r0);
2903 ic(0x89);
2904 rx(r1, i0, r0, _NOREG, _SCL1);
2905 }
2906 else {
2907 reg = jit_get_reg(jit_class_gpr);
2908 movi(rn(reg), i0);
2909 stxr_s(rn(reg), r0, r1);
2910 jit_unget_reg(reg);
2911 }
2912}
2913
2914static void
2915_stxr_i(jit_state_t *_jit, jit_int32_t r0, jit_int32_t r1, jit_int32_t r2)
2916{
2917#if __X64_32
2918 jit_int32_t reg;
2919 reg = jit_get_reg(jit_class_gpr);
2920 addr(rn(reg), r0, r1);
2921 str_i(rn(reg), r2);
2922 jit_unget_reg(reg);
2923#else
2924 rex(0, 0, r2, r1, r0);
2925 ic(0x89);
2926 rx(r2, 0, r0, r1, _SCL1);
2927#endif
2928}
2929
2930static void
2931_stxi_i(jit_state_t *_jit, jit_word_t i0, jit_int32_t r0, jit_int32_t r1)
2932{
2933 jit_int32_t reg;
2934 if (can_sign_extend_int_p(i0)) {
2935 rex(0, 0, r1, _NOREG, r0);
2936 ic(0x89);
2937 rx(r1, i0, r0, _NOREG, _SCL1);
2938 }
2939 else {
2940 reg = jit_get_reg(jit_class_gpr);
2941 movi(rn(reg), i0);
2942 stxr_i(rn(reg), r0, r1);
2943 jit_unget_reg(reg);
2944 }
2945}
2946
2947#if __X64 && !__X64_32
2948static void
2949_stxr_l(jit_state_t *_jit, jit_int32_t r0, jit_int32_t r1, jit_int32_t r2)
2950{
2951 rex(0, 1, r2, r1, r0);
2952 ic(0x89);
2953 rx(r2, 0, r0, r1, _SCL1);
2954}
2955
2956static void
2957_stxi_l(jit_state_t *_jit, jit_word_t i0, jit_int32_t r0, jit_int32_t r1)
2958{
2959 jit_int32_t reg;
2960 if (can_sign_extend_int_p(i0)) {
2961 rex(0, 1, r1, _NOREG, r0);
2962 ic(0x89);
2963 rx(r1, i0, r0, _NOREG, _SCL1);
2964 }
2965 else {
2966 reg = jit_get_reg(jit_class_gpr);
2967 movi(rn(reg), i0);
2968 stxr_l(rn(reg), r0, r1);
2969 jit_unget_reg(reg);
2970 }
2971}
2972#endif
2973
2974static void
2975_jccs(jit_state_t *_jit, jit_int32_t code, jit_word_t i0)
2976{
2977 jit_word_t w;
2978 ic(0x70 | code);
2979 w = i0 - (_jit->pc.w + 1);
2980 ic(w);
2981}
2982
2983static void
2984_jcc(jit_state_t *_jit, jit_int32_t code, jit_word_t i0)
2985{
2986 jit_word_t w;
2987 ic(0x0f);
2988 ic(0x80 | code);
2989 w = i0 - (_jit->pc.w + 4);
2990 ii(w);
2991}
2992
2993static void
2994_jcr(jit_state_t *_jit,
2995 jit_int32_t code, jit_word_t i0, jit_int32_t r0, jit_int32_t r1)
2996{
2997 alur(X86_CMP, r0, r1);
2998 jcc(code, i0);
2999}
3000
3001static void
3002_jci(jit_state_t *_jit,
3003 jit_int32_t code, jit_word_t i0, jit_int32_t r0, jit_word_t i1)
3004{
3005 alui(X86_CMP, r0, i1);
3006 jcc(code, i0);
3007}
3008
3009static void
3010_jci0(jit_state_t *_jit, jit_int32_t code, jit_word_t i0, jit_int32_t r0)
3011{
3012 testr(r0, r0);
3013 jcc(code, i0);
3014}
3015
3016static jit_word_t
3017_bltr(jit_state_t *_jit, jit_word_t i0, jit_int32_t r0, jit_int32_t r1)
3018{
3019 jcr(X86_CC_L, i0, r0, r1);
3020 return (_jit->pc.w);
3021}
3022
3023static jit_word_t
3024_blti(jit_state_t *_jit, jit_word_t i0, jit_int32_t r0, jit_word_t i1)
3025{
3026 if (i1) jci (X86_CC_L, i0, r0, i1);
3027 else jci0(X86_CC_S, i0, r0);
3028 return (_jit->pc.w);
3029}
3030
3031static jit_word_t
3032_bltr_u(jit_state_t *_jit, jit_word_t i0, jit_int32_t r0, jit_int32_t r1)
3033{
3034 jcr(X86_CC_B, i0, r0, r1);
3035 return (_jit->pc.w);
3036}
3037
3038static jit_word_t
3039_blti_u(jit_state_t *_jit, jit_word_t i0, jit_int32_t r0, jit_word_t i1)
3040{
3041 if (i1) jci (X86_CC_B, i0, r0, i1);
3042 else jci0(X86_CC_B, i0, r0);
3043 return (_jit->pc.w);
3044}
3045
3046static jit_word_t
3047_bler(jit_state_t *_jit, jit_word_t i0, jit_int32_t r0, jit_int32_t r1)
3048{
3049 if (r0 == r1) jmpi(i0);
3050 else jcr (X86_CC_LE, i0, r0, r1);
3051 return (_jit->pc.w);
3052}
3053
3054static jit_word_t
3055_blei(jit_state_t *_jit, jit_word_t i0, jit_int32_t r0, jit_word_t i1)
3056{
3057 if (i1) jci (X86_CC_LE, i0, r0, i1);
3058 else jci0(X86_CC_LE, i0, r0);
3059 return (_jit->pc.w);
3060}
3061
3062static jit_word_t
3063_bler_u(jit_state_t *_jit, jit_word_t i0, jit_int32_t r0, jit_int32_t r1)
3064{
3065 if (r0 == r1) jmpi(i0);
3066 else jcr (X86_CC_BE, i0, r0, r1);
3067 return (_jit->pc.w);
3068}
3069
3070static jit_word_t
3071_blei_u(jit_state_t *_jit, jit_word_t i0, jit_int32_t r0, jit_word_t i1)
3072{
3073 if (i1) jci (X86_CC_BE, i0, r0, i1);
3074 else jci0(X86_CC_BE, i0, r0);
3075 return (_jit->pc.w);
3076}
3077
3078static jit_word_t
3079_beqr(jit_state_t *_jit, jit_word_t i0, jit_int32_t r0, jit_int32_t r1)
3080{
3081 if (r0 == r1) jmpi(i0);
3082 else jcr (X86_CC_E, i0, r0, r1);
3083 return (_jit->pc.w);
3084}
3085
3086static jit_word_t
3087_beqi(jit_state_t *_jit, jit_word_t i0, jit_int32_t r0, jit_word_t i1)
3088{
3089 if (i1) jci (X86_CC_E, i0, r0, i1);
3090 else jci0(X86_CC_E, i0, r0);
3091 return (_jit->pc.w);
3092}
3093
3094static jit_word_t
3095_bger(jit_state_t *_jit, jit_word_t i0, jit_int32_t r0, jit_int32_t r1)
3096{
3097 if (r0 == r1) jmpi(i0);
3098 else jcr (X86_CC_GE, i0, r0, r1);
3099 return (_jit->pc.w);
3100}
3101
3102static jit_word_t
3103_bgei(jit_state_t *_jit, jit_word_t i0, jit_int32_t r0, jit_word_t i1)
3104{
3105 if (i1) jci (X86_CC_GE, i0, r0, i1);
3106 else jci0(X86_CC_NS, i0, r0);
3107 return (_jit->pc.w);
3108}
3109
3110static jit_word_t
3111_bger_u(jit_state_t *_jit, jit_word_t i0, jit_int32_t r0, jit_int32_t r1)
3112{
3113 if (r0 == r1) jmpi(i0);
3114 else jcr (X86_CC_AE, i0, r0, r1);
3115 return (_jit->pc.w);
3116}
3117
3118static jit_word_t
3119_bgei_u(jit_state_t *_jit, jit_word_t i0, jit_int32_t r0, jit_word_t i1)
3120{
3121 if (i1) jci (X86_CC_AE, i0, r0, i1);
3122 else jmpi(i0);
3123 return (_jit->pc.w);
3124}
3125
3126static jit_word_t
3127_bgtr(jit_state_t *_jit, jit_word_t i0, jit_int32_t r0, jit_int32_t r1)
3128{
3129 jcr(X86_CC_G, i0, r0, r1);
3130 return (_jit->pc.w);
3131}
3132
3133static jit_word_t
3134_bgti(jit_state_t *_jit, jit_word_t i0, jit_int32_t r0, jit_word_t i1)
3135{
3136 jci(X86_CC_G, i0, r0, i1);
3137 return (_jit->pc.w);
3138}
3139
3140static jit_word_t
3141_bgtr_u(jit_state_t *_jit, jit_word_t i0, jit_int32_t r0, jit_int32_t r1)
3142{
3143 jcr(X86_CC_A, i0, r0, r1);
3144 return (_jit->pc.w);
3145}
3146
3147static jit_word_t
3148_bgti_u(jit_state_t *_jit, jit_word_t i0, jit_int32_t r0, jit_word_t i1)
3149{
3150 if (i1) jci (X86_CC_A, i0, r0, i1);
3151 else jci0(X86_CC_NE, i0, r0);
3152 return (_jit->pc.w);
3153}
3154
3155static jit_word_t
3156_bner(jit_state_t *_jit, jit_word_t i0, jit_int32_t r0, jit_int32_t r1)
3157{
3158 jcr(X86_CC_NE, i0, r0, r1);
3159 return (_jit->pc.w);
3160}
3161
3162static jit_word_t
3163_bnei(jit_state_t *_jit, jit_word_t i0, jit_int32_t r0, jit_word_t i1)
3164{
3165 if (i1) jci (X86_CC_NE, i0, r0, i1);
3166 else jci0(X86_CC_NE, i0, r0);
3167 return (_jit->pc.w);
3168}
3169
3170static jit_word_t
3171_bmsr(jit_state_t *_jit, jit_word_t i0, jit_int32_t r0, jit_int32_t r1)
3172{
3173 testr(r0, r1);
3174 jnz(i0);
3175 return (_jit->pc.w);
3176}
3177
3178static jit_word_t
3179_bmsi(jit_state_t *_jit, jit_word_t i0, jit_int32_t r0, jit_word_t i1)
3180{
3181 jit_int32_t reg;
3182 if (can_zero_extend_int_p(i1))
3183 testi(r0, i1);
3184 else {
3185 reg = jit_get_reg(jit_class_gpr);
3186 movi(rn(reg), i1);
3187 testr(r0, rn(reg));
3188 jit_unget_reg(reg);
3189 }
3190 jnz(i0);
3191 return (_jit->pc.w);
3192}
3193
3194static jit_word_t
3195_bmcr(jit_state_t *_jit, jit_word_t i0, jit_int32_t r0, jit_int32_t r1)
3196{
3197 testr(r0, r1);
3198 jz(i0);
3199 return (_jit->pc.w);
3200}
3201
3202static jit_word_t
3203_bmci(jit_state_t *_jit, jit_word_t i0, jit_int32_t r0, jit_word_t i1)
3204{
3205 jit_int32_t reg;
3206 if (can_zero_extend_int_p(i1))
3207 testi(r0, i1);
3208 else {
3209 reg = jit_get_reg(jit_class_gpr);
3210 movi(rn(reg), i1);
3211 testr(r0, rn(reg));
3212 jit_unget_reg(reg);
3213 }
3214 jz(i0);
3215 return (_jit->pc.w);
3216}
3217
3218static jit_word_t
3219_boaddr(jit_state_t *_jit, jit_word_t i0, jit_int32_t r0, jit_int32_t r1)
3220{
3221 iaddr(r0, r1);
3222 jo(i0);
3223 return (_jit->pc.w);
3224}
3225
3226static jit_word_t
3227_boaddi(jit_state_t *_jit, jit_word_t i0, jit_int32_t r0, jit_word_t i1)
3228{
3229 jit_int32_t reg;
3230 if (can_sign_extend_int_p(i1)) {
3231 iaddi(r0, i1);
3232 jo(i0);
3233 return (_jit->pc.w);
3234 }
3235 reg = jit_get_reg(jit_class_gpr|jit_class_nospill);
3236 movi(rn(reg), i1);
3237 jit_unget_reg(reg);
3238 return (boaddr(i0, r0, rn(reg)));
3239}
3240
3241static jit_word_t
3242_boaddr_u(jit_state_t *_jit, jit_word_t i0, jit_int32_t r0, jit_int32_t r1)
3243{
3244 iaddr(r0, r1);
3245 jc(i0);
3246 return (_jit->pc.w);
3247}
3248
3249static jit_word_t
3250_boaddi_u(jit_state_t *_jit, jit_word_t i0, jit_int32_t r0, jit_word_t i1)
3251{
3252 jit_int32_t reg;
3253 if (can_sign_extend_int_p(i1)) {
3254 iaddi(r0, i1);
3255 jc(i0);
3256 return (_jit->pc.w);
3257 }
3258 reg = jit_get_reg(jit_class_gpr|jit_class_nospill);
3259 movi(rn(reg), i1);
3260 jit_unget_reg(reg);
3261 return (boaddr_u(i0, r0, rn(reg)));
3262}
3263
3264static jit_word_t
3265_bxaddr(jit_state_t *_jit, jit_word_t i0, jit_int32_t r0, jit_int32_t r1)
3266{
3267 iaddr(r0, r1);
3268 jno(i0);
3269 return (_jit->pc.w);
3270}
3271
3272static jit_word_t
3273_bxaddi(jit_state_t *_jit, jit_word_t i0, jit_int32_t r0, jit_word_t i1)
3274{
3275 jit_int32_t reg;
3276 if (can_sign_extend_int_p(i1)) {
3277 iaddi(r0, i1);
3278 jno(i0);
3279 return (_jit->pc.w);
3280 }
3281 reg = jit_get_reg(jit_class_gpr|jit_class_nospill);
3282 movi(rn(reg), i1);
3283 jit_unget_reg(reg);
3284 return (bxaddr(i0, r0, rn(reg)));
3285}
3286
3287static jit_word_t
3288_bxaddr_u(jit_state_t *_jit, jit_word_t i0, jit_int32_t r0, jit_int32_t r1)
3289{
3290 iaddr(r0, r1);
3291 jnc(i0);
3292 return (_jit->pc.w);
3293}
3294
3295static jit_word_t
3296_bxaddi_u(jit_state_t *_jit, jit_word_t i0, jit_int32_t r0, jit_word_t i1)
3297{
3298 jit_int32_t reg;
3299 if (can_sign_extend_int_p(i1)) {
3300 iaddi(r0, i1);
3301 jnc(i0);
3302 return (_jit->pc.w);
3303 }
3304 reg = jit_get_reg(jit_class_gpr|jit_class_nospill);
3305 movi(rn(reg), i1);
3306 jit_unget_reg(reg);
3307 return (bxaddr_u(i0, r0, rn(reg)));
3308}
3309
3310static jit_word_t
3311_bosubr(jit_state_t *_jit, jit_word_t i0, jit_int32_t r0, jit_int32_t r1)
3312{
3313 isubr(r0, r1);
3314 jo(i0);
3315 return (_jit->pc.w);
3316}
3317
3318static jit_word_t
3319_bosubi(jit_state_t *_jit, jit_word_t i0, jit_int32_t r0, jit_word_t i1)
3320{
3321 jit_int32_t reg;
3322 if (can_sign_extend_int_p(i1)) {
3323 isubi(r0, i1);
3324 jo(i0);
3325 return (_jit->pc.w);
3326 }
3327 reg = jit_get_reg(jit_class_gpr|jit_class_nospill);
3328 movi(rn(reg), i1);
3329 jit_unget_reg(reg);
3330 return (bosubr(i0, r0, rn(reg)));
3331}
3332
3333static jit_word_t
3334_bosubr_u(jit_state_t *_jit, jit_word_t i0, jit_int32_t r0, jit_int32_t r1)
3335{
3336 isubr(r0, r1);
3337 jc(i0);
3338 return (_jit->pc.w);
3339}
3340
3341static jit_word_t
3342_bosubi_u(jit_state_t *_jit, jit_word_t i0, jit_int32_t r0, jit_word_t i1)
3343{
3344 jit_int32_t reg;
3345 if (can_sign_extend_int_p(i1)) {
3346 isubi(r0, i1);
3347 jc(i0);
3348 return (_jit->pc.w);
3349 }
3350 reg = jit_get_reg(jit_class_gpr|jit_class_nospill);
3351 movi(rn(reg), i1);
3352 jit_unget_reg(reg);
3353 return (bosubr_u(i0, r0, rn(reg)));
3354}
3355
3356static jit_word_t
3357_bxsubr(jit_state_t *_jit, jit_word_t i0, jit_int32_t r0, jit_int32_t r1)
3358{
3359 isubr(r0, r1);
3360 jno(i0);
3361 return (_jit->pc.w);
3362}
3363
3364static jit_word_t
3365_bxsubi(jit_state_t *_jit, jit_word_t i0, jit_int32_t r0, jit_word_t i1)
3366{
3367 jit_int32_t reg;
3368 if (can_sign_extend_int_p(i1)) {
3369 isubi(r0, i1);
3370 jno(i0);
3371 return (_jit->pc.w);
3372 }
3373 reg = jit_get_reg(jit_class_gpr|jit_class_nospill);
3374 movi(rn(reg), i1);
3375 jit_unget_reg(reg);
3376 return (bxsubr(i0, r0, rn(reg)));
3377}
3378
3379static jit_word_t
3380_bxsubr_u(jit_state_t *_jit, jit_word_t i0, jit_int32_t r0, jit_int32_t r1)
3381{
3382 isubr(r0, r1);
3383 jnc(i0);
3384 return (_jit->pc.w);
3385}
3386
3387static jit_word_t
3388_bxsubi_u(jit_state_t *_jit, jit_word_t i0, jit_int32_t r0, jit_word_t i1)
3389{
3390 jit_int32_t reg;
3391 if (can_sign_extend_int_p(i1)) {
3392 isubi(r0, i1);
3393 jnc(i0);
3394 return (_jit->pc.w);
3395 }
3396 reg = jit_get_reg(jit_class_gpr|jit_class_nospill);
3397 movi(rn(reg), i1);
3398 jit_unget_reg(reg);
3399 return (bxsubr_u(i0, r0, rn(reg)));
3400}
3401
3402static void
3403_callr(jit_state_t *_jit, jit_int32_t r0)
3404{
3405 rex(0, 0, _NOREG, _NOREG, r0);
3406 ic(0xff);
3407 mrm(0x03, 0x02, r7(r0));
3408}
3409
3410static jit_word_t
3411_calli(jit_state_t *_jit, jit_word_t i0)
3412{
3413 jit_word_t word;
3414#if __X64
3415 jit_int32_t reg;
3416
3417 reg = jit_get_reg(jit_class_gpr);
3418 word = movi_p(rn(reg), i0);
3419 callr(rn(reg));
3420 jit_unget_reg(reg);
3421#else
3422 jit_word_t w;
3423 ic(0xe8);
3424 w = i0 - (_jit->pc.w + 4);
3425 ii(w);
3426 word = _jit->pc.w;
3427#endif
3428 return (word);
3429}
3430
3431static void
3432_jmpr(jit_state_t *_jit, jit_int32_t r0)
3433{
3434 rex(0, WIDE, _NOREG, _NOREG, r0);
3435 ic(0xff);
3436 mrm(0x03, 0x04, r7(r0));
3437}
3438
3439static jit_word_t
3440_jmpi(jit_state_t *_jit, jit_word_t i0)
3441{
3442 jit_word_t w;
3443 ic(0xe9);
3444 w = i0 - (_jit->pc.w + 4);
3445 ii(w);
3446 return (_jit->pc.w);
3447}
3448
3449static void
3450_jmpsi(jit_state_t *_jit, jit_uint8_t i0)
3451{
3452 ic(0xeb);
3453 ic(i0);
3454}
3455
3456static void
3457_prolog(jit_state_t *_jit, jit_node_t *node)
3458{
3459 jit_int32_t reg;
3460 if (_jitc->function->define_frame || _jitc->function->assume_frame) {
3461 jit_int32_t frame = -_jitc->function->frame;
3462 assert(_jitc->function->self.aoff >= frame);
3463 if (_jitc->function->assume_frame)
3464 return;
3465 _jitc->function->self.aoff = frame;
3466 }
3467 if (_jitc->function->allocar)
3468 _jitc->function->self.aoff &= -16;
3469#if __X64 && (__CYGWIN__ || _WIN32)
3470 _jitc->function->stack = (((/* first 32 bytes must be allocated */
3471 (_jitc->function->self.alen > 32 ?
3472 _jitc->function->self.alen : 32) -
3473 /* align stack at 16 bytes */
3474 _jitc->function->self.aoff) + 15) & -16) +
3475 stack_adjust;
3476#else
3477 _jitc->function->stack = (((_jitc->function->self.alen -
3478 _jitc->function->self.aoff) + 15) & -16) +
3479 stack_adjust;
3480#endif
3481 subi(_RSP_REGNO, _RSP_REGNO, stack_framesize - REAL_WORDSIZE);
3482 /* callee save registers */
3483#if __X32
3484 if (jit_regset_tstbit(&_jitc->function->regset, _RDI))
3485 stxi(12, _RSP_REGNO, _RDI_REGNO);
3486 if (jit_regset_tstbit(&_jitc->function->regset, _RSI))
3487 stxi( 8, _RSP_REGNO, _RSI_REGNO);
3488 if (jit_regset_tstbit(&_jitc->function->regset, _RBX))
3489 stxi( 4, _RSP_REGNO, _RBX_REGNO);
3490#else
3491# if __CYGWIN__ || _WIN32
3492 if (jit_regset_tstbit(&_jitc->function->regset, _XMM15))
3493 sse_stxi_d(136, _RSP_REGNO, _XMM15_REGNO);
3494 if (jit_regset_tstbit(&_jitc->function->regset, _XMM14))
3495 sse_stxi_d(128, _RSP_REGNO, _XMM14_REGNO);
3496 if (jit_regset_tstbit(&_jitc->function->regset, _XMM13))
3497 sse_stxi_d(120, _RSP_REGNO, _XMM13_REGNO);
3498 if (jit_regset_tstbit(&_jitc->function->regset, _XMM12))
3499 sse_stxi_d(112, _RSP_REGNO, _XMM12_REGNO);
3500 if (jit_regset_tstbit(&_jitc->function->regset, _XMM11))
3501 sse_stxi_d(104, _RSP_REGNO, _XMM11_REGNO);
3502 if (jit_regset_tstbit(&_jitc->function->regset, _XMM10))
3503 sse_stxi_d(96, _RSP_REGNO, _XMM10_REGNO);
3504 if (jit_regset_tstbit(&_jitc->function->regset, _XMM9))
3505 sse_stxi_d(88, _RSP_REGNO, _XMM9_REGNO);
3506 if (jit_regset_tstbit(&_jitc->function->regset, _XMM8))
3507 sse_stxi_d(80, _RSP_REGNO, _XMM8_REGNO);
3508 if (jit_regset_tstbit(&_jitc->function->regset, _XMM7))
3509 sse_stxi_d(72, _RSP_REGNO, _XMM7_REGNO);
3510 if (jit_regset_tstbit(&_jitc->function->regset, _XMM6))
3511 sse_stxi_d(64, _RSP_REGNO, _XMM6_REGNO);
3512 if (jit_regset_tstbit(&_jitc->function->regset, _R15))
3513 stxi(56, _RSP_REGNO, _R15_REGNO);
3514 if (jit_regset_tstbit(&_jitc->function->regset, _R14))
3515 stxi(48, _RSP_REGNO, _R14_REGNO);
3516 if (jit_regset_tstbit(&_jitc->function->regset, _R13))
3517 stxi(40, _RSP_REGNO, _R13_REGNO);
3518 if (jit_regset_tstbit(&_jitc->function->regset, _R12))
3519 stxi(32, _RSP_REGNO, _R12_REGNO);
3520 if (jit_regset_tstbit(&_jitc->function->regset, _RSI))
3521 stxi(24, _RSP_REGNO, _RSI_REGNO);
3522 if (jit_regset_tstbit(&_jitc->function->regset, _RDI))
3523 stxi(16, _RSP_REGNO, _RDI_REGNO);
3524 if (jit_regset_tstbit(&_jitc->function->regset, _RBX))
3525 stxi( 8, _RSP_REGNO, _RBX_REGNO);
3526# else
3527 if (jit_regset_tstbit(&_jitc->function->regset, _RBX))
3528 stxi(40, _RSP_REGNO, _RBX_REGNO);
3529 if (jit_regset_tstbit(&_jitc->function->regset, _R12))
3530 stxi(32, _RSP_REGNO, _R12_REGNO);
3531 if (jit_regset_tstbit(&_jitc->function->regset, _R13))
3532 stxi(24, _RSP_REGNO, _R13_REGNO);
3533 if (jit_regset_tstbit(&_jitc->function->regset, _R14))
3534 stxi(16, _RSP_REGNO, _R14_REGNO);
3535 if (jit_regset_tstbit(&_jitc->function->regset, _R15))
3536 stxi( 8, _RSP_REGNO, _R15_REGNO);
3537# endif
3538#endif
3539 stxi(0, _RSP_REGNO, _RBP_REGNO);
3540 movr(_RBP_REGNO, _RSP_REGNO);
3541
3542 /* alloca */
3543 subi(_RSP_REGNO, _RSP_REGNO, _jitc->function->stack);
3544 if (_jitc->function->allocar) {
3545 reg = jit_get_reg(jit_class_gpr);
3546 movi(rn(reg), _jitc->function->self.aoff);
3547 stxi_i(_jitc->function->aoffoff, _RBP_REGNO, rn(reg));
3548 jit_unget_reg(reg);
3549 }
3550
3551#if __X64 && !(__CYGWIN__ || _WIN32)
3552 if (_jitc->function->self.call & jit_call_varargs) {
3553 jit_word_t nofp_code;
3554
3555 /* Save gp registers in the save area, if any is a vararg */
3556 for (reg = first_gp_from_offset(_jitc->function->vagp);
3557 jit_arg_reg_p(reg); ++reg)
3558 stxi(_jitc->function->vaoff + first_gp_offset +
3559 reg * 8, _RBP_REGNO, rn(JIT_RA0 - reg));
3560
3561 reg = first_fp_from_offset(_jitc->function->vafp);
3562 if (jit_arg_f_reg_p(reg)) {
3563 /* Skip over if no float registers were passed as argument */
3564 /* test %al, %al */
3565 ic(0x84);
3566 ic(0xc0);
3567 jes(0);
3568 nofp_code = _jit->pc.w;
3569
3570 /* Save fp registers in the save area, if any is a vararg */
3571 /* Note that the full 16 byte xmm is not saved, because
3572 * lightning only handles float and double, and, while
3573 * attempting to provide a va_list compatible pointer as
3574 * jit_va_start return, does not guarantee it (on all ports). */
3575 for (; jit_arg_f_reg_p(reg); ++reg)
3576 sse_stxi_d(_jitc->function->vaoff + first_fp_offset +
3577 reg * va_fp_increment, _RBP_REGNO, rn(_XMM0 - reg));
3578
3579 patch_rel_char(nofp_code, _jit->pc.w);
3580 }
3581 }
3582#endif
3583}
3584
3585static void
3586_epilog(jit_state_t *_jit, jit_node_t *node)
3587{
3588 if (_jitc->function->assume_frame)
3589 return;
3590 /* callee save registers */
3591 movr(_RSP_REGNO, _RBP_REGNO);
3592#if __X32
3593 if (jit_regset_tstbit(&_jitc->function->regset, _RDI))
3594 ldxi(_RDI_REGNO, _RSP_REGNO, 12);
3595 if (jit_regset_tstbit(&_jitc->function->regset, _RSI))
3596 ldxi(_RSI_REGNO, _RSP_REGNO, 8);
3597 if (jit_regset_tstbit(&_jitc->function->regset, _RBX))
3598 ldxi(_RBX_REGNO, _RSP_REGNO, 4);
3599#else
3600# if __CYGWIN__ || _WIN32
3601 if (jit_regset_tstbit(&_jitc->function->regset, _XMM15))
3602 sse_ldxi_d(_XMM15_REGNO, _RSP_REGNO, 136);
3603 if (jit_regset_tstbit(&_jitc->function->regset, _XMM14))
3604 sse_ldxi_d(_XMM14_REGNO, _RSP_REGNO, 128);
3605 if (jit_regset_tstbit(&_jitc->function->regset, _XMM13))
3606 sse_ldxi_d(_XMM13_REGNO, _RSP_REGNO, 120);
3607 if (jit_regset_tstbit(&_jitc->function->regset, _XMM12))
3608 sse_ldxi_d(_XMM12_REGNO, _RSP_REGNO, 112);
3609 if (jit_regset_tstbit(&_jitc->function->regset, _XMM11))
3610 sse_ldxi_d(_XMM11_REGNO, _RSP_REGNO, 104);
3611 if (jit_regset_tstbit(&_jitc->function->regset, _XMM10))
3612 sse_ldxi_d(_XMM10_REGNO, _RSP_REGNO, 96);
3613 if (jit_regset_tstbit(&_jitc->function->regset, _XMM9))
3614 sse_ldxi_d(_XMM9_REGNO, _RSP_REGNO, 88);
3615 if (jit_regset_tstbit(&_jitc->function->regset, _XMM8))
3616 sse_ldxi_d(_XMM8_REGNO, _RSP_REGNO, 80);
3617 if (jit_regset_tstbit(&_jitc->function->regset, _XMM7))
3618 sse_ldxi_d(_XMM7_REGNO, _RSP_REGNO, 72);
3619 if (jit_regset_tstbit(&_jitc->function->regset, _XMM6))
3620 sse_ldxi_d(_XMM6_REGNO, _RSP_REGNO, 64);
3621 if (jit_regset_tstbit(&_jitc->function->regset, _R15))
3622 ldxi(_R15_REGNO, _RSP_REGNO, 56);
3623 if (jit_regset_tstbit(&_jitc->function->regset, _R14))
3624 ldxi(_R14_REGNO, _RSP_REGNO, 48);
3625 if (jit_regset_tstbit(&_jitc->function->regset, _R13))
3626 ldxi(_R13_REGNO, _RSP_REGNO, 40);
3627 if (jit_regset_tstbit(&_jitc->function->regset, _R12))
3628 ldxi(_R12_REGNO, _RSP_REGNO, 32);
3629 if (jit_regset_tstbit(&_jitc->function->regset, _RSI))
3630 ldxi(_RSI_REGNO, _RSP_REGNO, 24);
3631 if (jit_regset_tstbit(&_jitc->function->regset, _RDI))
3632 ldxi(_RDI_REGNO, _RSP_REGNO, 16);
3633 if (jit_regset_tstbit(&_jitc->function->regset, _RBX))
3634 ldxi(_RBX_REGNO, _RSP_REGNO, 8);
3635# else
3636 if (jit_regset_tstbit(&_jitc->function->regset, _RBX))
3637 ldxi(_RBX_REGNO, _RSP_REGNO, 40);
3638 if (jit_regset_tstbit(&_jitc->function->regset, _R12))
3639 ldxi(_R12_REGNO, _RSP_REGNO, 32);
3640 if (jit_regset_tstbit(&_jitc->function->regset, _R13))
3641 ldxi(_R13_REGNO, _RSP_REGNO, 24);
3642 if (jit_regset_tstbit(&_jitc->function->regset, _R14))
3643 ldxi(_R14_REGNO, _RSP_REGNO, 16);
3644 if (jit_regset_tstbit(&_jitc->function->regset, _R15))
3645 ldxi(_R15_REGNO, _RSP_REGNO, 8);
3646# endif
3647#endif
3648 ldxi(_RBP_REGNO, _RSP_REGNO, 0);
3649 addi(_RSP_REGNO, _RSP_REGNO, stack_framesize - REAL_WORDSIZE);
3650
3651 ic(0xc3);
3652}
3653
3654static void
3655_vastart(jit_state_t *_jit, jit_int32_t r0)
3656{
3657#if __X32 || __CYGWIN__ || _WIN32
3658 assert(_jitc->function->self.call & jit_call_varargs);
3659 addi(r0, _RBP_REGNO, _jitc->function->self.size);
3660#else
3661 jit_int32_t reg;
3662
3663 assert(_jitc->function->self.call & jit_call_varargs);
3664
3665 /* Return jit_va_list_t in the register argument */
3666 addi(r0, _RBP_REGNO, _jitc->function->vaoff);
3667 reg = jit_get_reg(jit_class_gpr);
3668
3669 /* Initialize gp offset in the save area. */
3670 movi(rn(reg), _jitc->function->vagp);
3671 stxi_i(offsetof(jit_va_list_t, gpoff), r0, rn(reg));
3672
3673 /* Initialize fp offset in the save area. */
3674 movi(rn(reg), _jitc->function->vafp);
3675 stxi_i(offsetof(jit_va_list_t, fpoff), r0, rn(reg));
3676
3677 /* Initialize overflow pointer to the first stack argument. */
3678 addi(rn(reg), _RBP_REGNO, _jitc->function->self.size);
3679 stxi(offsetof(jit_va_list_t, over), r0, rn(reg));
3680
3681 /* Initialize register save area pointer. */
3682 addi(rn(reg), r0, first_gp_offset);
3683 stxi(offsetof(jit_va_list_t, save), r0, rn(reg));
3684
3685 jit_unget_reg(reg);
3686#endif
3687}
3688
3689static void
3690_vaarg(jit_state_t *_jit, jit_int32_t r0, jit_int32_t r1)
3691{
3692#if __X32 || __CYGWIN__ || _WIN32
3693 assert(_jitc->function->self.call & jit_call_varargs);
3694 ldr(r0, r1);
3695 addi(r1, r1, va_gp_increment);
3696#else
3697 jit_int32_t rg0;
3698 jit_int32_t rg1;
3699 jit_word_t ge_code;
3700 jit_word_t lt_code;
3701
3702 assert(_jitc->function->self.call & jit_call_varargs);
3703
3704 rg0 = jit_get_reg(jit_class_gpr);
3705 rg1 = jit_get_reg(jit_class_gpr);
3706
3707 /* Load the gp offset in save area in the first temporary. */
3708 ldxi_i(rn(rg0), r1, offsetof(jit_va_list_t, gpoff));
3709
3710 /* Jump over if there are no remaining arguments in the save area. */
3711 icmpi(rn(rg0), va_gp_max_offset);
3712 jaes(0);
3713 ge_code = _jit->pc.w;
3714
3715 /* Load the save area pointer in the second temporary. */
3716 ldxi(rn(rg1), r1, offsetof(jit_va_list_t, save));
3717
3718 /* Load the vararg argument in the first argument. */
3719 ldxr(r0, rn(rg1), rn(rg0));
3720
3721 /* Update the gp offset. */
3722 addi(rn(rg0), rn(rg0), 8);
3723 stxi_i(offsetof(jit_va_list_t, gpoff), r1, rn(rg0));
3724
3725 /* Will only need one temporary register below. */
3726 jit_unget_reg(rg1);
3727
3728 /* Jump over overflow code. */
3729 jmpsi(0);
3730 lt_code = _jit->pc.w;
3731
3732 /* Where to land if argument is in overflow area. */
3733 patch_rel_char(ge_code, _jit->pc.w);
3734
3735 /* Load overflow pointer. */
3736 ldxi(rn(rg0), r1, offsetof(jit_va_list_t, over));
3737
3738 /* Load argument. */
3739 ldr(r0, rn(rg0));
3740
3741 /* Update overflow pointer. */
3742 addi(rn(rg0), rn(rg0), va_gp_increment);
3743 stxi(offsetof(jit_va_list_t, over), r1, rn(rg0));
3744
3745 /* Where to land if argument is in save area. */
3746 patch_rel_char(lt_code, _jit->pc.w);
3747
3748 jit_unget_reg(rg0);
3749#endif
3750}
3751
3752/* The x87 boolean argument tells if will put the result in a x87
3753 * register if non false, in a sse register otherwise. */
3754static void
3755_vaarg_d(jit_state_t *_jit, jit_int32_t r0, jit_int32_t r1, jit_bool_t x87)
3756{
3757#if __X32 || __CYGWIN__ || _WIN32
3758 assert(_jitc->function->self.call & jit_call_varargs);
3759 if (x87)
3760 x87_ldr_d(r0, r1);
3761 else
3762 sse_ldr_d(r0, r1);
3763 addi(r1, r1, 8);
3764#else
3765 jit_int32_t rg0;
3766 jit_int32_t rg1;
3767 jit_word_t ge_code;
3768 jit_word_t lt_code;
3769
3770 assert(_jitc->function->self.call & jit_call_varargs);
3771
3772 rg0 = jit_get_reg(jit_class_gpr);
3773 rg1 = jit_get_reg(jit_class_gpr);
3774
3775 /* Load the fp offset in save area in the first temporary. */
3776 ldxi_i(rn(rg0), r1, offsetof(jit_va_list_t, fpoff));
3777
3778 /* Jump over if there are no remaining arguments in the save area. */
3779 icmpi(rn(rg0), va_fp_max_offset);
3780 jaes(0);
3781 ge_code = _jit->pc.w;
3782
3783 /* Load the save area pointer in the second temporary. */
3784 ldxi(rn(rg1), r1, offsetof(jit_va_list_t, save));
3785
3786 /* Load the vararg argument in the first argument. */
3787 if (x87)
3788 x87_ldxr_d(r0, rn(rg1), rn(rg0));
3789 else
3790 sse_ldxr_d(r0, rn(rg1), rn(rg0));
3791
3792 /* Update the fp offset. */
3793 addi(rn(rg0), rn(rg0), va_fp_increment);
3794 stxi_i(offsetof(jit_va_list_t, fpoff), r1, rn(rg0));
3795
3796 /* Will only need one temporary register below. */
3797 jit_unget_reg(rg1);
3798
3799 /* Jump over overflow code. */
3800 jmpsi(0);
3801 lt_code = _jit->pc.w;
3802
3803 /* Where to land if argument is in overflow area. */
3804 patch_rel_char(ge_code, _jit->pc.w);
3805
3806 /* Load overflow pointer. */
3807 ldxi(rn(rg0), r1, offsetof(jit_va_list_t, over));
3808
3809 /* Load argument. */
3810 if (x87)
3811 x87_ldr_d(r0, rn(rg0));
3812 else
3813 sse_ldr_d(r0, rn(rg0));
3814
3815 /* Update overflow pointer. */
3816 addi(rn(rg0), rn(rg0), 8);
3817 stxi(offsetof(jit_va_list_t, over), r1, rn(rg0));
3818
3819 /* Where to land if argument is in save area. */
3820 patch_rel_char(lt_code, _jit->pc.w);
3821
3822 jit_unget_reg(rg0);
3823#endif
3824}
3825
3826static void
3827_patch_at(jit_state_t *_jit, jit_node_t *node,
3828 jit_word_t instr, jit_word_t label)
3829{
3830 switch (node->code) {
3831# if __X64
3832 case jit_code_calli:
3833# endif
3834 case jit_code_movi:
3835 patch_abs(instr, label);
3836 break;
3837 default:
3838 patch_rel(instr, label);
3839 break;
3840 }
3841}
3842#endif
ARM optimized PCSX fork