notaz.gp2x.de / pcsx_rearmed.git / blame
? search:
summary | shortlog | log | commit | commitdiff | tree
blob | blame (incremental) | history | HEAD
git subrepo pull (merge) --force deps/lightning
[pcsx_rearmed.git] / deps / lightning / lib / jit_x86.c
CommitLineData
4a71579b 1/*
79bfeef6 2 * Copyright (C) 2012-2023 Free Software Foundation, Inc.
4a71579b
PC		 3 *
4 * This file is part of GNU lightning.
5 *
6 * GNU lightning is free software; you can redistribute it and/or modify it
7 * under the terms of the GNU Lesser General Public License as published
8 * by the Free Software Foundation; either version 3, or (at your option)
9 * any later version.
10 *
11 * GNU lightning is distributed in the hope that it will be useful, but
12 * WITHOUT ANY WARRANTY; without even the implied warranty of MERCHANTABILITY
13 * or FITNESS FOR A PARTICULAR PURPOSE. See the GNU Lesser General Public
14 * License for more details.
15 *
16 * Authors:
17 * Paulo Cesar Pereira de Andrade
18 */
19
20#include <lightning.h>
21#include <lightning/jit_private.h>
22
23#if __X32
79bfeef6
PC		 24# define CAN_RIP_ADDRESS 0
25# define address_p(i0) 1
4a71579b
PC		 26# define jit_arg_reg_p(i) 0
27# define jit_arg_f_reg_p(i) 0
79bfeef6
PC		 28/* callee save + 16 byte align
29 * align16(%ebp + %rbx + %rsi + %rdi) + (16 - 4) */
30# define stack_framesize 28
4a71579b
PC		 31# define REAL_WORDSIZE 4
32# define va_gp_increment 4
33# define va_fp_increment 8
34#else
79bfeef6
PC		 35# if _WIN32 || __X64_32
36# define CAN_RIP_ADDRESS 0
37# else
38# define CAN_RIP_ADDRESS 1
39# endif
40# if __X64_32
41# define address_p(i0) ((jit_word_t)(i0) >= 0)
42# else
43# define address_p(i0) can_sign_extend_int_p(i0)
44# endif
4a71579b
PC		 45# if __CYGWIN__ || _WIN32
46# define jit_arg_reg_p(i) ((i) >= 0 && (i) < 4)
47# define jit_arg_f_reg_p(i) jit_arg_reg_p(i)
79bfeef6
PC		 48/* callee save + 16 byte align
49 * align16(%rbp+%rbx+%rdi+%rsi+%r1[2-5]+%xmm[6-9]+%xmm1[0-5]) + (16 - 8) */
4a71579b
PC		 50# define stack_framesize 152
51# define va_fp_increment 8
52# else
53# define jit_arg_reg_p(i) ((i) >= 0 && (i) < 6)
54# define jit_arg_f_reg_p(i) ((i) >= 0 && (i) < 8)
79bfeef6
PC		 55/* callee save + 16 byte align
56 * align16(%rbp + %r15 + %r14 + %r13 + %r12 + %rbx) + (16 - 8) */
4a71579b
PC		 57# define stack_framesize 56
58# define first_gp_argument rdi
59# define first_gp_offset offsetof(jit_va_list_t, rdi)
60# define first_gp_from_offset(gp) ((gp) / 8)
61# define last_gp_argument r9
62# define va_gp_max_offset \
63 (offsetof(jit_va_list_t, r9) - offsetof(jit_va_list_t, rdi) + 8)
64# define first_fp_argument xmm0
65# define first_fp_offset offsetof(jit_va_list_t, xmm0)
66# define last_fp_argument xmm7
67# define va_fp_max_offset \
68 (offsetof(jit_va_list_t, xmm7) - offsetof(jit_va_list_t, rdi) + 16)
69# define va_fp_increment 16
70# define first_fp_from_offset(fp) (((fp) - va_gp_max_offset) / 16)
71# endif
72# define va_gp_increment 8
4a71579b
PC		 73# define REAL_WORDSIZE 8
74#endif
79bfeef6
PC		 75#define CVT_OFFSET _jitc->function->cvt_offset
76
77#define CHECK_CVT_OFFSET() \
78 do { \
79 if (!_jitc->function->cvt_offset) { \
80 _jitc->again = 1; \
81 _jitc->function->cvt_offset = \
82 jit_allocai(sizeof(jit_float64_t)); \
83 } \
84 } while (0)
4a71579b
PC		 85
86/*
87 * Types
88 */
89#if __X32 || __CYGWIN__ || _WIN32
90typedef jit_pointer_t jit_va_list_t;
91#else
92typedef struct jit_va_list {
93 jit_int32_t gpoff;
94 jit_int32_t fpoff;
95 jit_pointer_t over;
96 jit_pointer_t save;
97 /* Declared explicitly as int64 for the x32 abi */
98 jit_int64_t rdi;
99 jit_int64_t rsi;
100 jit_int64_t rdx;
101 jit_int64_t rcx;
102 jit_int64_t r8;
103 jit_int64_t r9;
104 jit_float64_t xmm0;
105 jit_float64_t _up0;
106 jit_float64_t xmm1;
107 jit_float64_t _up1;
108 jit_float64_t xmm2;
109 jit_float64_t _up2;
110 jit_float64_t xmm3;
111 jit_float64_t _up3;
112 jit_float64_t xmm4;
113 jit_float64_t _up4;
114 jit_float64_t xmm5;
115 jit_float64_t _up5;
116 jit_float64_t xmm6;
117 jit_float64_t _up6;
118 jit_float64_t xmm7;
119 jit_float64_t _up7;
120} jit_va_list_t;
121#endif
122
123/*
124 * Prototypes
125 */
79bfeef6
PC		 126#define compute_framesize() _compute_framesize(_jit)
127static void _compute_framesize(jit_state_t*);
4a71579b
PC		 128#define patch(instr, node) _patch(_jit, instr, node)
129static void _patch(jit_state_t*,jit_word_t,jit_node_t*);
130#define sse_from_x87_f(r0, r1) _sse_from_x87_f(_jit, r0, r1)
131static void _sse_from_x87_f(jit_state_t*,jit_int32_t,jit_int32_t);
132#define sse_from_x87_d(r0, r1) _sse_from_x87_d(_jit, r0, r1)
133static void _sse_from_x87_d(jit_state_t*,jit_int32_t,jit_int32_t);
134#define x87_from_sse_f(r0, r1) _x87_from_sse_f(_jit, r0, r1)
135static void _x87_from_sse_f(jit_state_t*,jit_int32_t,jit_int32_t);
136#define x87_from_sse_d(r0, r1) _x87_from_sse_d(_jit, r0, r1)
137static void _x87_from_sse_d(jit_state_t*,jit_int32_t,jit_int32_t);
138
139#define PROTO 1
140# include "jit_x86-cpu.c"
141# include "jit_x86-sse.c"
142# include "jit_x86-x87.c"
143#undef PROTO
144
145/*
146 * Initialization
147 */
148jit_cpu_t jit_cpu;
149jit_register_t _rvs[] = {
150#if __X32
151 { rc(gpr) | rc(rg8) | 0, "%eax" },
152 { rc(gpr) | rc(rg8) | 1, "%ecx" },
153 { rc(gpr) | rc(rg8) | 2, "%edx" },
154 { rc(sav) | rc(rg8) | rc(gpr) | 3, "%ebx" },
155 { rc(sav) | rc(gpr) | 6, "%esi" },
156 { rc(sav) | rc(gpr) | 7, "%edi" },
157 { rc(sav) | 4, "%esp" },
158 { rc(sav) | 5, "%ebp" },
159 { rc(xpr) | rc(fpr) | 0, "%xmm0" },
160 { rc(xpr) | rc(fpr) | 1, "%xmm1" },
161 { rc(xpr) | rc(fpr) | 2, "%xmm2" },
162 { rc(xpr) | rc(fpr) | 3, "%xmm3" },
163 { rc(xpr) | rc(fpr) | 4, "%xmm4" },
164 { rc(xpr) | rc(fpr) | 5, "%xmm5" },
165 { rc(xpr) | rc(fpr) | 6, "%xmm6" },
166 { rc(xpr) | rc(fpr) | 7, "%xmm7" },
167 { rc(fpr) | 0, "st(0)" },
168 { rc(fpr) | 1, "st(1)" },
169 { rc(fpr) | 2, "st(2)" },
170 { rc(fpr) | 3, "st(3)" },
171 { rc(fpr) | 4, "st(4)" },
172 { rc(fpr) | 5, "st(5)" },
173 { rc(fpr) | 6, "st(6)" },
174 { rc(fpr) | 7, "st(7)" },
175#else
176# if __CYGWIN__ || _WIN32
177 { rc(gpr) | rc(rg8) | 0, "%rax" },
178 { rc(gpr) | rc(rg8) | rc(rg8) | 10, "%r10" },
179 { rc(gpr) | rc(rg8) | rc(rg8) | 11, "%r11" },
180 { rc(sav) | rc(rg8) | rc(gpr) | 3, "%rbx" },
181 { rc(sav) | rc(gpr) | 7, "%rdi" },
182 { rc(sav) | rc(gpr) | 6, "%rsi" },
183 { rc(sav) | rc(gpr) | 12, "%r12" },
184 { rc(sav) | rc(gpr) | 13, "%r13" },
185 { rc(sav) | rc(gpr) | 14, "%r14" },
186 { rc(sav) | rc(gpr) | 15, "%r15" },
187 { rc(arg) | rc(rg8) | rc(gpr) | 9, "%r9" },
188 { rc(arg) | rc(rg8) | rc(gpr) | 8, "%r8" },
189 { rc(arg) | rc(rg8) | rc(gpr) | 2, "%rdx" },
190 { rc(arg) | rc(rg8) | rc(gpr) | 1, "%rcx" },
191 { rc(sav) | 4, "%rsp" },
192 { rc(sav) | 5, "%rbp" },
193 { rc(xpr) | rc(fpr) | 4, "%xmm4" },
194 { rc(xpr) | rc(fpr) | 5, "%xmm5" },
195 { rc(sav) | rc(xpr) | rc(fpr) | 6, "%xmm6" },
196 { rc(sav) | rc(xpr) | rc(fpr) | 7, "%xmm7" },
197 { rc(sav) | rc(xpr) | rc(fpr) | 8, "%xmm8" },
198 { rc(sav) | rc(xpr) | rc(fpr) | 9, "%xmm9" },
199 { rc(sav) | rc(xpr) | rc(fpr) | 10, "%xmm10" },
200 { rc(sav) | rc(xpr) | rc(fpr) | 11, "%xmm11" },
201 { rc(sav) | rc(xpr) | rc(fpr) | 12, "%xmm12" },
202 { rc(sav) | rc(xpr) | rc(fpr) | 13, "%xmm13" },
203 { rc(sav) | rc(xpr) | rc(fpr) | 14, "%xmm14" },
204 { rc(sav) | rc(xpr) | rc(fpr) | 15, "%xmm15" },
205 { rc(xpr) | rc(arg) | rc(fpr) | 3, "%xmm3" },
206 { rc(xpr) | rc(arg) | rc(fpr) | 2, "%xmm2" },
207 { rc(xpr) | rc(arg) | rc(fpr) | 1, "%xmm1" },
208 { rc(xpr) | rc(arg) | rc(fpr) | 0, "%xmm0" },
209#else
210 /* %rax is a pseudo flag argument for varargs functions */
211 { rc(arg) | rc(gpr) | rc(rg8) | 0, "%rax" },
212 { rc(gpr) | rc(rg8) | 10, "%r10" },
213 { rc(gpr) | rc(rg8) | 11, "%r11" },
214 { rc(sav) | rc(rg8) | rc(gpr) | 3, "%rbx" },
215 { rc(sav) | rc(rg8) | rc(gpr) | 13, "%r13" },
216 { rc(sav) | rc(rg8) | rc(gpr) | 14, "%r14" },
217 { rc(sav) | rc(rg8) | rc(gpr) | 15, "%r15" },
218 { rc(sav) | rc(gpr) | rc(rg8) | 12, "%r12" },
219 { rc(arg) | rc(rg8) | rc(gpr) | 9, "%r9" },
220 { rc(arg) | rc(rg8) | rc(gpr) | 8, "%r8" },
221 { rc(arg) | rc(rg8) | rc(gpr) | 1, "%rcx" },
222 { rc(arg) | rc(rg8) | rc(gpr) | 2, "%rdx" },
223 { rc(arg) | rc(rg8) | rc(gpr) | 6, "%rsi" },
224 { rc(arg) | rc(rg8) | rc(gpr) | 7, "%rdi" },
225 { rc(sav) | 4, "%rsp" },
226 { rc(sav) | 5, "%rbp" },
227 { rc(xpr) | rc(fpr) | 8, "%xmm8" },
228 { rc(xpr) | rc(fpr) | 9, "%xmm9" },
229 { rc(xpr) | rc(fpr) | 10, "%xmm10" },
230 { rc(xpr) | rc(fpr) | 11, "%xmm11" },
231 { rc(xpr) | rc(fpr) | 12, "%xmm12" },
232 { rc(xpr) | rc(fpr) | 13, "%xmm13" },
233 { rc(xpr) | rc(fpr) | 14, "%xmm14" },
234 { rc(xpr) | rc(fpr) | 15, "%xmm15" },
235 { rc(xpr) | rc(arg) | rc(fpr) | 7, "%xmm7" },
236 { rc(xpr) | rc(arg) | rc(fpr) | 6, "%xmm6" },
237 { rc(xpr) | rc(arg) | rc(fpr) | 5, "%xmm5" },
238 { rc(xpr) | rc(arg) | rc(fpr) | 4, "%xmm4" },
239 { rc(xpr) | rc(arg) | rc(fpr) | 3, "%xmm3" },
240 { rc(xpr) | rc(arg) | rc(fpr) | 2, "%xmm2" },
241 { rc(xpr) | rc(arg) | rc(fpr) | 1, "%xmm1" },
242 { rc(xpr) | rc(arg) | rc(fpr) | 0, "%xmm0" },
243# endif
244 { rc(fpr) | 0, "st(0)" },
245 { rc(fpr) | 1, "st(1)" },
246 { rc(fpr) | 2, "st(2)" },
247 { rc(fpr) | 3, "st(3)" },
248 { rc(fpr) | 4, "st(4)" },
249 { rc(fpr) | 5, "st(5)" },
250 { rc(fpr) | 6, "st(6)" },
251 { rc(fpr) | 7, "st(7)" },
252#endif
253 { _NOREG, "<none>" },
254};
255
79bfeef6
PC		 256static jit_int32_t iregs[] = {
257#if __X32
258 _RBX, _RSI, _RDI,
259#elif (__CYGWIN__ || _WIN32)
260 _RBX, _RDI, _RSI, _R12, _R13, _R14, _R15,
261#else
262 _R15, _R14, _R13, _R12, _RBX,
263#endif
264};
265
266#if __X64 && (__CYGWIN__ || _WIN32)
267static jit_int32_t fregs[] = {
268 _XMM6, _XMM7, _XMM8, _XMM9, _XMM10, _XMM11, _XMM12, _XMM13, _XMM14, _XMM15,
269};
270#endif
271
4a71579b
PC		 272/*
273 * Implementation
274 */
275void
276jit_get_cpu(void)
277{
278 union {
79bfeef6
PC		 279 /* eax=7 and ecx=0 */
280 struct {
281 jit_uword_t fsgsbase : 1;
282 jit_uword_t IA32_TSC_ADJUST : 1;
283 jit_uword_t sgx : 1;
284 jit_uword_t bmi1 : 1;
285 jit_uword_t hle : 1;
286 jit_uword_t avx2 : 1;
287 jit_uword_t FDP_EXCPTN_ONLY : 1;
288 jit_uword_t smep : 1;
289 jit_uword_t bmi2 : 1;
290 jit_uword_t erms : 1;
291 jit_uword_t invpcid : 1;
292 jit_uword_t rtm : 1;
293 jit_uword_t rdt_m_pqm : 1;
294 jit_uword_t dep_FPU_CS_DS : 1;
295 jit_uword_t mpx : 1;
296 jit_uword_t rdt_a_pqe : 1;
297 jit_uword_t avx512_f : 1;
298 jit_uword_t avx512_dq : 1;
299 jit_uword_t rdseed : 1;
300 jit_uword_t adx : 1;
301 jit_uword_t smap : 1;
302 jit_uword_t avx512_ifma : 1;
303 jit_uword_t __reserved0 : 1;
304 jit_uword_t clflushopt : 1;
305 jit_uword_t clwb : 1;
306 jit_uword_t pt : 1;
307 jit_uword_t avx512_pf : 1;
308 jit_uword_t avx512_er : 1;
309 jit_uword_t avx512_cd : 1;
310 jit_uword_t sha : 1;
311 jit_uword_t avx512_bw : 1;
312 jit_uword_t avx512_vl : 1;
313 } bits;
314 jit_uword_t cpuid;
315 } ebx;
316 union {
317 /* eax=0 */
4a71579b
PC		 318 struct {
319 jit_uint32_t sse3 : 1;
320 jit_uint32_t pclmulqdq : 1;
321 jit_uint32_t dtes64 : 1; /* amd reserved */
322 jit_uint32_t monitor : 1;
323 jit_uint32_t ds_cpl : 1; /* amd reserved */
324 jit_uint32_t vmx : 1; /* amd reserved */
325 jit_uint32_t smx : 1; /* amd reserved */
326 jit_uint32_t est : 1; /* amd reserved */
327 jit_uint32_t tm2 : 1; /* amd reserved */
328 jit_uint32_t ssse3 : 1;
329 jit_uint32_t cntx_id : 1; /* amd reserved */
330 jit_uint32_t __reserved0 : 1;
331 jit_uint32_t fma : 1;
332 jit_uint32_t cmpxchg16b : 1;
333 jit_uint32_t xtpr : 1; /* amd reserved */
334 jit_uint32_t pdcm : 1; /* amd reserved */
335 jit_uint32_t __reserved1 : 1;
336 jit_uint32_t pcid : 1; /* amd reserved */
337 jit_uint32_t dca : 1; /* amd reserved */
338 jit_uint32_t sse4_1 : 1;
339 jit_uint32_t sse4_2 : 1;
340 jit_uint32_t x2apic : 1; /* amd reserved */
341 jit_uint32_t movbe : 1; /* amd reserved */
342 jit_uint32_t popcnt : 1;
343 jit_uint32_t tsc : 1; /* amd reserved */
344 jit_uint32_t aes : 1;
345 jit_uint32_t xsave : 1;
346 jit_uint32_t osxsave : 1;
347 jit_uint32_t avx : 1;
348 jit_uint32_t __reserved2 : 1; /* amd F16C */
349 jit_uint32_t __reserved3 : 1;
350 jit_uint32_t __alwayszero : 1; /* amd RAZ */
351 } bits;
352 jit_uword_t cpuid;
353 } ecx;
354 union {
79bfeef6 355 /* eax=0 */
4a71579b
PC		 356 struct {
357 jit_uint32_t fpu : 1;
358 jit_uint32_t vme : 1;
359 jit_uint32_t de : 1;
360 jit_uint32_t pse : 1;
361 jit_uint32_t tsc : 1;
362 jit_uint32_t msr : 1;
363 jit_uint32_t pae : 1;
364 jit_uint32_t mce : 1;
365 jit_uint32_t cmpxchg8b : 1;
366 jit_uint32_t apic : 1;
367 jit_uint32_t __reserved0 : 1;
368 jit_uint32_t sep : 1;
369 jit_uint32_t mtrr : 1;
370 jit_uint32_t pge : 1;
371 jit_uint32_t mca : 1;
372 jit_uint32_t cmov : 1;
373 jit_uint32_t pat : 1;
374 jit_uint32_t pse36 : 1;
375 jit_uint32_t psn : 1; /* amd reserved */
376 jit_uint32_t clfsh : 1;
377 jit_uint32_t __reserved1 : 1;
378 jit_uint32_t ds : 1; /* amd reserved */
379 jit_uint32_t acpi : 1; /* amd reserved */
380 jit_uint32_t mmx : 1;
381 jit_uint32_t fxsr : 1;
382 jit_uint32_t sse : 1;
383 jit_uint32_t sse2 : 1;
384 jit_uint32_t ss : 1; /* amd reserved */
385 jit_uint32_t htt : 1;
386 jit_uint32_t tm : 1; /* amd reserved */
387 jit_uint32_t __reserved2 : 1;
388 jit_uint32_t pbe : 1; /* amd reserved */
389 } bits;
390 jit_uword_t cpuid;
391 } edx;
392#if __X32
393 int ac, flags;
394#endif
79bfeef6 395 jit_uword_t eax;
4a71579b
PC		 396
397#if __X32
398 /* adapted from glibc __sysconf */
399 __asm__ volatile ("pushfl;\n\t"
400 "popl %0;\n\t"
401 "movl $0x240000, %1;\n\t"
402 "xorl %0, %1;\n\t"
403 "pushl %1;\n\t"
404 "popfl;\n\t"
405 "pushfl;\n\t"
406 "popl %1;\n\t"
407 "xorl %0, %1;\n\t"
408 "pushl %0;\n\t"
409 "popfl"
410 : "=r" (flags), "=r" (ac));
411
412 /* i386 or i486 without cpuid */
413 if ((ac & (1 << 21)) == 0)
414 /* probably without x87 as well */
415 return;
416#endif
417
418 /* query %eax = 1 function */
419#if __X32 || __X64_32
420 __asm__ volatile ("xchgl %%ebx, %1; cpuid; xchgl %%ebx, %1"
421#else
422 __asm__ volatile ("xchgq %%rbx, %1; cpuid; xchgq %%rbx, %1"
423#endif
79bfeef6 424 : "=a" (eax), "=r" (ebx.cpuid),
4a71579b
PC		 425 "=c" (ecx.cpuid), "=d" (edx.cpuid)
426 : "0" (1));
427
428 jit_cpu.fpu = edx.bits.fpu;
429 jit_cpu.cmpxchg8b = edx.bits.cmpxchg8b;
430 jit_cpu.cmov = edx.bits.cmov;
431 jit_cpu.mmx = edx.bits.mmx;
432 jit_cpu.sse = edx.bits.sse;
433 jit_cpu.sse2 = edx.bits.sse2;
434 jit_cpu.sse3 = ecx.bits.sse3;
435 jit_cpu.pclmulqdq = ecx.bits.pclmulqdq;
436 jit_cpu.ssse3 = ecx.bits.ssse3;
437 jit_cpu.fma = ecx.bits.fma;
438 jit_cpu.cmpxchg16b = ecx.bits.cmpxchg16b;
439 jit_cpu.sse4_1 = ecx.bits.sse4_1;
440 jit_cpu.sse4_2 = ecx.bits.sse4_2;
441 jit_cpu.movbe = ecx.bits.movbe;
442 jit_cpu.popcnt = ecx.bits.popcnt;
443 jit_cpu.aes = ecx.bits.aes;
444 jit_cpu.avx = ecx.bits.avx;
445
79bfeef6
PC		 446 /* query %eax = 7 and ecx = 0 function */
447#if __X64
448 __asm__ volatile ("cpuid"
449 : "=a" (eax), "=b" (ebx.cpuid), "=c" (ecx), "=d" (edx)
450 : "a" (7), "c" (0));
451#endif
ba86ff93
PC		 452 jit_cpu.adx = ebx.bits.adx;
453 jit_cpu.bmi2 = ebx.bits.bmi2;
79bfeef6
PC		 454
455
4a71579b
PC		 456 /* query %eax = 0x80000001 function */
457#if __X64
458# if __X64_32
459 __asm__ volatile ("xchgl %%ebx, %1; cpuid; xchgl %%ebx, %1"
460# else
461 __asm__ volatile ("xchgq %%rbx, %1; cpuid; xchgq %%rbx, %1"
462# endif
79bfeef6 463 : "=a" (eax), "=r" (ebx.cpuid),
4a71579b
PC		 464 "=c" (ecx.cpuid), "=d" (edx.cpuid)
465 : "0" (0x80000001));
79bfeef6
PC		 466 jit_cpu.lahf = !!(ecx.cpuid & 1);
467 jit_cpu.abm = !!(ecx.cpuid & 32);
ba86ff93 468 jit_cpu.fma4 = !!(ecx.cpuid & (1 << 16));
4a71579b
PC		 469#endif
470}
471
472void
473_jit_init(jit_state_t *_jit)
474{
475#if __X32
476 jit_int32_t regno;
477 static jit_bool_t first = 1;
478#endif
479
480 _jitc->reglen = jit_size(_rvs) - 1;
481#if __X32
482 if (first) {
483 if (!jit_cpu.sse2) {
484 for (regno = _jitc->reglen; regno >= 0; regno--) {
485 if (_rvs[regno].spec & jit_class_xpr)
486 _rvs[regno].spec = 0;
487 }
488 }
489 first = 0;
490 }
491#endif
492}
493
494void
495_jit_prolog(jit_state_t *_jit)
496{
497 jit_int32_t offset;
498
499 if (_jitc->function)
500 jit_epilog();
501 assert(jit_regset_cmp_ui(&_jitc->regarg, 0) == 0);
502 jit_regset_set_ui(&_jitc->regsav, 0);
503 offset = _jitc->functions.offset;
504 if (offset >= _jitc->functions.length) {
505 jit_realloc((jit_pointer_t *)&_jitc->functions.ptr,
506 _jitc->functions.length * sizeof(jit_function_t),
507 (_jitc->functions.length + 16) * sizeof(jit_function_t));
508 _jitc->functions.length += 16;
509 }
510 _jitc->function = _jitc->functions.ptr + _jitc->functions.offset++;
79bfeef6
PC		 511 /* One extra stack slot for implicit saved returned address */
512 _jitc->function->self.size = stack_framesize + REAL_WORDSIZE;
4a71579b
PC		 513 _jitc->function->self.argi = _jitc->function->self.argf =
514 _jitc->function->self.aoff = _jitc->function->self.alen = 0;
79bfeef6
PC		 515 _jitc->function->cvt_offset = 0;
516#if __X64 && (__CYGWIN__ || _WIN32)
517 /* force framepointer */
518 jit_check_frame();
519#endif
4a71579b
PC		 520 _jitc->function->self.call = jit_call_default;
521 jit_alloc((jit_pointer_t *)&_jitc->function->regoff,
522 _jitc->reglen * sizeof(jit_int32_t));
523
524 /* _no_link here does not mean the jit_link() call can be removed
525 * by rewriting as:
526 * _jitc->function->prolog = jit_new_node(jit_code_prolog);
527 */
528 _jitc->function->prolog = jit_new_node_no_link(jit_code_prolog);
529 jit_link(_jitc->function->prolog);
530 _jitc->function->prolog->w.w = offset;
531 _jitc->function->epilog = jit_new_node_no_link(jit_code_epilog);
532 /* u: label value
533 * v: offset in blocks vector
534 * w: offset in functions vector
535 */
536 _jitc->function->epilog->w.w = offset;
537
538 jit_regset_new(&_jitc->function->regset);
539}
540
541jit_int32_t
542_jit_allocai(jit_state_t *_jit, jit_int32_t length)
543{
544 assert(_jitc->function);
79bfeef6
PC		 545 jit_check_frame();
546#if __X32
547 /* Stack is 4 bytes aligned but jit functions keep it 8 bytes aligned.
548 * Called functions have 16 byte aligned stack. */
549 if (!_jitc->function->self.aoff)
550 _jitc->function->self.aoff = -4;
551#endif
4a71579b
PC		 552 switch (length) {
553 case 0: case 1: break;
554 case 2: _jitc->function->self.aoff &= -2; break;
555 case 3: case 4: _jitc->function->self.aoff &= -4; break;
556 default: _jitc->function->self.aoff &= -8; break;
557 }
558 _jitc->function->self.aoff -= length;
559
560 /* jit_allocai() may be called from jit_x86-cpu.c, and force a function
561 * generation restart on some conditions: div/rem and qmul/qdiv, due
562 * to registers constraints.
563 * The check is to prevent an assertion of a jit_xyz() being called
564 * during code generation, and attempting to add a node to the tail
565 * of the current IR generation. */
566 if (!_jitc->realize) {
567 jit_inc_synth_ww(allocai, _jitc->function->self.aoff, length);
568 jit_dec_synth();
569 }
570
571 return (_jitc->function->self.aoff);
572}
573
574void
575_jit_allocar(jit_state_t *_jit, jit_int32_t u, jit_int32_t v)
576{
577 jit_int32_t reg;
578 assert(_jitc->function);
579 jit_inc_synth_ww(allocar, u, v);
580 if (!_jitc->function->allocar) {
581 _jitc->function->aoffoff = jit_allocai(sizeof(jit_int32_t));
582 _jitc->function->allocar = 1;
583 }
584 reg = jit_get_reg(jit_class_gpr);
585 jit_negr(reg, v);
586 jit_andi(reg, reg, -16);
587 jit_ldxi_i(u, JIT_FP, _jitc->function->aoffoff);
588 jit_addr(u, u, reg);
589 jit_addr(JIT_SP, JIT_SP, reg);
590 jit_stxi_i(_jitc->function->aoffoff, JIT_FP, u);
591 jit_unget_reg(reg);
592 jit_dec_synth();
593}
594
595void
596_jit_ret(jit_state_t *_jit)
597{
598 jit_node_t *instr;
599 assert(_jitc->function);
600 jit_inc_synth(ret);
601 /* jump to epilog */
602 instr = jit_jmpi();
603 jit_patch_at(instr, _jitc->function->epilog);
604 jit_dec_synth();
605}
606
607void
79bfeef6
PC		 608_jit_retr(jit_state_t *_jit, jit_int32_t u, jit_code_t code)
609{
610 jit_code_inc_synth_w(code, u);
611 jit_movr(JIT_RET, u);
4a71579b
PC		 612 jit_ret();
613 jit_dec_synth();
614}
615
616void
79bfeef6 617_jit_reti(jit_state_t *_jit, jit_word_t u, jit_code_t code)
4a71579b 618{
79bfeef6 619 jit_code_inc_synth_w(code, u);
4a71579b
PC		 620 jit_movi(JIT_RET, u);
621 jit_ret();
622 jit_dec_synth();
623}
624
625void
626_jit_retr_f(jit_state_t *_jit, jit_int32_t u)
627{
628 jit_inc_synth_w(retr_f, u);
629 if (JIT_FRET != u)
630 jit_movr_f(JIT_FRET, u);
631 else
632 jit_live(JIT_FRET);
633 jit_ret();
634 jit_dec_synth();
635}
636
637void
638_jit_reti_f(jit_state_t *_jit, jit_float32_t u)
639{
640 jit_inc_synth_f(reti_f, u);
641 jit_movi_f(JIT_FRET, u);
642 jit_ret();
643 jit_dec_synth();
644}
645
646void
647_jit_retr_d(jit_state_t *_jit, jit_int32_t u)
648{
649 jit_inc_synth_w(retr_d, u);
650 if (JIT_FRET != u)
651 jit_movr_d(JIT_FRET, u);
652 else
653 jit_live(JIT_FRET);
654 jit_ret();
655 jit_dec_synth();
656}
657
658void
659_jit_reti_d(jit_state_t *_jit, jit_float64_t u)
660{
661 jit_inc_synth_d(reti_d, u);
662 jit_movi_d(JIT_FRET, u);
663 jit_ret();
664 jit_dec_synth();
665}
666
667void
668_jit_epilog(jit_state_t *_jit)
669{
670 assert(_jitc->function);
671 assert(_jitc->function->epilog->next == NULL);
672 jit_link(_jitc->function->epilog);
673 _jitc->function = NULL;
674}
675
676jit_bool_t
677_jit_arg_register_p(jit_state_t *_jit, jit_node_t *u)
678{
79bfeef6 679 if (u->code >= jit_code_arg_c && u->code <= jit_code_arg)
4a71579b
PC		 680 return (jit_arg_reg_p(u->u.w));
681 assert(u->code == jit_code_arg_f || u->code == jit_code_arg_d);
682 return (jit_arg_f_reg_p(u->u.w));
683}
684
685void
686_jit_ellipsis(jit_state_t *_jit)
687{
688 jit_inc_synth(ellipsis);
79bfeef6 689 jit_check_frame();
4a71579b
PC		 690 if (_jitc->prepare) {
691 jit_link_prepare();
692 /* Remember that a varargs function call is being constructed. */
693 assert(!(_jitc->function->call.call & jit_call_varargs));
694 _jitc->function->call.call |= jit_call_varargs;
695 }
696 else {
697 jit_link_prolog();
698 /* Remember the current function is varargs. */
699 assert(!(_jitc->function->self.call & jit_call_varargs));
700 _jitc->function->self.call |= jit_call_varargs;
701
702#if __X64 && !(__CYGWIN__ || _WIN32)
703 /* Allocate va_list like object in the stack.
704 * If applicable, with enough space to save all argument
705 * registers, and use fixed offsets for them. */
706 _jitc->function->vaoff = jit_allocai(sizeof(jit_va_list_t));
707
708 /* Initialize gp offset in save area. */
709 if (jit_arg_reg_p(_jitc->function->self.argi))
710 _jitc->function->vagp = _jitc->function->self.argi * 8;
711 else
712 _jitc->function->vagp = va_gp_max_offset;
713
714 /* Initialize fp offset in save area. */
715 if (jit_arg_f_reg_p(_jitc->function->self.argf))
716 _jitc->function->vafp = _jitc->function->self.argf * 16 +
717 va_gp_max_offset;
718 else
719 _jitc->function->vafp = va_fp_max_offset;
720#endif
721 }
722 jit_dec_synth();
723}
724
725void
726_jit_va_push(jit_state_t *_jit, jit_int32_t u)
727{
728 jit_inc_synth_w(va_push, u);
729 jit_pushargr(u);
730 jit_dec_synth();
731}
732
733jit_node_t *
79bfeef6 734_jit_arg(jit_state_t *_jit, jit_code_t code)
4a71579b
PC		 735{
736 jit_node_t *node;
737 jit_int32_t offset;
738 assert(_jitc->function);
739 assert(!(_jitc->function->self.call & jit_call_varargs));
79bfeef6
PC		 740#if STRONG_TYPE_CHECKING
741 assert(code >= jit_code_arg_c && code <= jit_code_arg);
742#endif
4a71579b
PC		 743#if __X64
744 if (jit_arg_reg_p(_jitc->function->self.argi)) {
745 offset = _jitc->function->self.argi++;
746# if __CYGWIN__ || _WIN32
747 _jitc->function->self.size += sizeof(jit_word_t);
748# endif
749 }
750 else
751#endif
752 {
753 offset = _jitc->function->self.size;
754 _jitc->function->self.size += REAL_WORDSIZE;
79bfeef6 755 jit_check_frame();
4a71579b 756 }
79bfeef6 757 node = jit_new_node_ww(code, offset,
4a71579b
PC		 758 ++_jitc->function->self.argn);
759 jit_link_prolog();
760 return (node);
761}
762
763jit_node_t *
764_jit_arg_f(jit_state_t *_jit)
765{
766 jit_node_t *node;
767 jit_int32_t offset;
768 assert(_jitc->function);
769 assert(!(_jitc->function->self.call & jit_call_varargs));
770#if __X64
771# if __CYGWIN__ || _WIN32
772 if (jit_arg_reg_p(_jitc->function->self.argi)) {
773 offset = _jitc->function->self.argi++;
774 _jitc->function->self.size += sizeof(jit_word_t);
775 }
776# else
777 if (jit_arg_f_reg_p(_jitc->function->self.argf))
778 offset = _jitc->function->self.argf++;
779# endif
780 else
781#endif
782 {
783 offset = _jitc->function->self.size;
784 _jitc->function->self.size += REAL_WORDSIZE;
79bfeef6 785 jit_check_frame();
4a71579b
PC		 786 }
787 node = jit_new_node_ww(jit_code_arg_f, offset,
788 ++_jitc->function->self.argn);
789 jit_link_prolog();
790 return (node);
791}
792
793jit_node_t *
794_jit_arg_d(jit_state_t *_jit)
795{
796 jit_node_t *node;
797 jit_int32_t offset;
798 assert(_jitc->function);
799 assert(!(_jitc->function->self.call & jit_call_varargs));
800#if __X64
801# if __CYGWIN__ || _WIN32
802 if (jit_arg_reg_p(_jitc->function->self.argi)) {
803 offset = _jitc->function->self.argi++;
804 _jitc->function->self.size += sizeof(jit_word_t);
805 }
806# else
807 if (jit_arg_f_reg_p(_jitc->function->self.argf))
808 offset = _jitc->function->self.argf++;
809# endif
810 else
811#endif
812 {
813 offset = _jitc->function->self.size;
814 _jitc->function->self.size += sizeof(jit_float64_t);
79bfeef6 815 jit_check_frame();
4a71579b
PC		 816 }
817 node = jit_new_node_ww(jit_code_arg_d, offset,
818 ++_jitc->function->self.argn);
819 jit_link_prolog();
820 return (node);
821}
822
823void
824_jit_getarg_c(jit_state_t *_jit, jit_int32_t u, jit_node_t *v)
825{
79bfeef6 826 assert_arg_type(v->code, jit_code_arg_c);
4a71579b
PC		 827 jit_inc_synth_wp(getarg_c, u, v);
828#if __X64
829 if (jit_arg_reg_p(v->u.w))
830 jit_extr_c(u, JIT_RA0 - v->u.w);
831 else
832#endif
79bfeef6
PC		 833 {
834 jit_node_t *node = jit_ldxi_c(u, _RBP, v->u.w);
835 jit_link_alist(node);
836 }
4a71579b
PC		 837 jit_dec_synth();
838}
839
840void
841_jit_getarg_uc(jit_state_t *_jit, jit_int32_t u, jit_node_t *v)
842{
79bfeef6 843 assert_arg_type(v->code, jit_code_arg_c);
4a71579b
PC		 844 jit_inc_synth_wp(getarg_uc, u, v);
845#if __X64
846 if (jit_arg_reg_p(v->u.w))
847 jit_extr_uc(u, JIT_RA0 - v->u.w);
848 else
849#endif
79bfeef6
PC		 850 {
851 jit_node_t *node = jit_ldxi_uc(u, _RBP, v->u.w);
852 jit_link_alist(node);
853 }
4a71579b
PC		 854 jit_dec_synth();
855}
856
857void
858_jit_getarg_s(jit_state_t *_jit, jit_int32_t u, jit_node_t *v)
859{
79bfeef6 860 assert_arg_type(v->code, jit_code_arg_s);
4a71579b
PC		 861 jit_inc_synth_wp(getarg_s, u, v);
862#if __X64
863 if (jit_arg_reg_p(v->u.w))
864 jit_extr_s(u, JIT_RA0 - v->u.w);
865 else
866#endif
79bfeef6
PC		 867 {
868 jit_node_t *node = jit_ldxi_s(u, _RBP, v->u.w);
869 jit_link_alist(node);
870 }
4a71579b
PC		 871 jit_dec_synth();
872}
873
874void
875_jit_getarg_us(jit_state_t *_jit, jit_int32_t u, jit_node_t *v)
876{
79bfeef6 877 assert_arg_type(v->code, jit_code_arg_s);
4a71579b
PC		 878 jit_inc_synth_wp(getarg_us, u, v);
879#if __X64
880 if (jit_arg_reg_p(v->u.w))
881 jit_extr_us(u, JIT_RA0 - v->u.w);
882 else
883#endif
79bfeef6
PC		 884 {
885 jit_node_t *node = jit_ldxi_us(u, _RBP, v->u.w);
886 jit_link_alist(node);
887 }
4a71579b
PC		 888 jit_dec_synth();
889}
890
891void
892_jit_getarg_i(jit_state_t *_jit, jit_int32_t u, jit_node_t *v)
893{
79bfeef6 894 assert_arg_type(v->code, jit_code_arg_i);
4a71579b
PC		 895 jit_inc_synth_wp(getarg_i, u, v);
896#if __X64
897 if (jit_arg_reg_p(v->u.w)) {
898# if __X64_32
899 jit_movr(u, JIT_RA0 - v->u.w);
900# else
901 jit_extr_i(u, JIT_RA0 - v->u.w);
902# endif
903 }
904 else
905#endif
79bfeef6
PC		 906 {
907 jit_node_t *node = jit_ldxi_i(u, _RBP, v->u.w);
908 jit_link_alist(node);
909 }
4a71579b
PC		 910 jit_dec_synth();
911}
912
913#if __X64 && !__X64_32
914void
915_jit_getarg_ui(jit_state_t *_jit, jit_int32_t u, jit_node_t *v)
916{
79bfeef6 917 assert_arg_type(v->code, jit_code_arg_i);
4a71579b
PC		 918 jit_inc_synth_wp(getarg_ui, u, v);
919 if (jit_arg_reg_p(v->u.w))
920 jit_extr_ui(u, JIT_RA0 - v->u.w);
79bfeef6
PC		 921 else {
922 jit_node_t *node = jit_ldxi_ui(u, _RBP, v->u.w);
923 jit_link_alist(node);
924 }
4a71579b
PC		 925 jit_dec_synth();
926}
927
928void
929_jit_getarg_l(jit_state_t *_jit, jit_int32_t u, jit_node_t *v)
930{
79bfeef6 931 assert_arg_type(v->code, jit_code_arg_l);
4a71579b
PC		 932 jit_inc_synth_wp(getarg_l, u, v);
933 if (jit_arg_reg_p(v->u.w))
934 jit_movr(u, JIT_RA0 - v->u.w);
79bfeef6
PC		 935 else {
936 jit_node_t *node = jit_ldxi_l(u, _RBP, v->u.w);
937 jit_link_alist(node);
938 }
4a71579b
PC		 939 jit_dec_synth();
940}
941#endif
942
943void
79bfeef6 944_jit_putargr(jit_state_t *_jit, jit_int32_t u, jit_node_t *v, jit_code_t code)
4a71579b 945{
79bfeef6
PC		 946 assert_putarg_type(code, v->code);
947 jit_code_inc_synth_wp(code, u, v);
4a71579b
PC		 948#if __X64
949 if (jit_arg_reg_p(v->u.w))
950 jit_movr(JIT_RA0 - v->u.w, u);
951 else
952#endif
79bfeef6
PC		 953 {
954 jit_node_t *node = jit_stxi(v->u.w, _RBP, u);
955 jit_link_alist(node);
956 }
4a71579b
PC		 957 jit_dec_synth();
958}
959
960void
79bfeef6 961_jit_putargi(jit_state_t *_jit, jit_word_t u, jit_node_t *v, jit_code_t code)
4a71579b
PC		 962{
963 jit_int32_t regno;
79bfeef6
PC		 964 assert_putarg_type(code, v->code);
965 jit_code_inc_synth_wp(code, u, v);
4a71579b
PC		 966#if __X64
967 if (jit_arg_reg_p(v->u.w))
968 jit_movi(JIT_RA0 - v->u.w, u);
969 else
970#endif
971 {
79bfeef6 972 jit_node_t *node;
4a71579b
PC		 973 regno = jit_get_reg(jit_class_gpr);
974 jit_movi(regno, u);
79bfeef6
PC		 975 node = jit_stxi(v->u.w, _RBP, regno);
976 jit_link_alist(node);
4a71579b
PC		 977 jit_unget_reg(regno);
978 }
979 jit_dec_synth();
980}
981
982void
983_jit_getarg_f(jit_state_t *_jit, jit_int32_t u, jit_node_t *v)
984{
985 assert(v->code == jit_code_arg_f);
986 jit_inc_synth_wp(getarg_f, u, v);
987#if __X64
988 if (jit_arg_f_reg_p(v->u.w))
989 jit_movr_f(u, _XMM0 - v->u.w);
990 else
991#endif
79bfeef6
PC		 992 {
993 jit_node_t *node = jit_ldxi_f(u, _RBP, v->u.w);
994 jit_link_alist(node);
995 }
4a71579b
PC		 996 jit_dec_synth();
997}
998
999void
1000_jit_putargr_f(jit_state_t *_jit, jit_int32_t u, jit_node_t *v)
1001{
1002 assert(v->code == jit_code_arg_f);
1003 jit_inc_synth_wp(putargr_f, u, v);
1004#if __X64
79bfeef6 1005 if (jit_arg_f_reg_p(v->u.w))
4a71579b
PC		 1006 jit_movr_f(_XMM0 - v->u.w, u);
1007 else
1008#endif
79bfeef6
PC		 1009 {
1010 jit_node_t *node = jit_stxi_f(v->u.w, _RBP, u);
1011 jit_link_alist(node);
1012 }
4a71579b
PC		 1013 jit_dec_synth();
1014}
1015
1016void
1017_jit_putargi_f(jit_state_t *_jit, jit_float32_t u, jit_node_t *v)
1018{
1019 jit_int32_t regno;
1020 assert(v->code == jit_code_arg_f);
1021 jit_inc_synth_fp(putargi_f, u, v);
1022#if __X64
79bfeef6 1023 if (jit_arg_f_reg_p(v->u.w))
4a71579b
PC		 1024 jit_movi_f(_XMM0 - v->u.w, u);
1025 else
1026#endif
1027 {
79bfeef6
PC		 1028 jit_node_t *node;
1029 regno = jit_get_reg(jit_class_fpr);
4a71579b 1030 jit_movi_f(regno, u);
79bfeef6
PC		 1031 node = jit_stxi_f(v->u.w, _RBP, regno);
1032 jit_link_alist(node);
4a71579b
PC		 1033 jit_unget_reg(regno);
1034 }
1035 jit_dec_synth();
1036}
1037
1038void
1039_jit_getarg_d(jit_state_t *_jit, jit_int32_t u, jit_node_t *v)
1040{
1041 assert(v->code == jit_code_arg_d);
1042 jit_inc_synth_wp(getarg_d, u, v);
1043#if __X64
1044 if (jit_arg_f_reg_p(v->u.w))
1045 jit_movr_d(u, _XMM0 - v->u.w);
1046 else
1047#endif
79bfeef6
PC		 1048 {
1049 jit_node_t *node = jit_ldxi_d(u, _RBP, v->u.w);
1050 jit_link_alist(node);
1051 }
4a71579b
PC		 1052 jit_dec_synth();
1053}
1054
1055void
1056_jit_putargr_d(jit_state_t *_jit, jit_int32_t u, jit_node_t *v)
1057{
1058 assert(v->code == jit_code_arg_d);
1059 jit_inc_synth_wp(putargr_d, u, v);
1060#if __X64
79bfeef6 1061 if (jit_arg_f_reg_p(v->u.w))
4a71579b
PC		 1062 jit_movr_d(_XMM0 - v->u.w, u);
1063 else
1064#endif
79bfeef6
PC		 1065 {
1066 jit_node_t *node = jit_stxi_d(v->u.w, _RBP, u);
1067 jit_link_alist(node);
1068 }
4a71579b
PC		 1069 jit_dec_synth();
1070}
1071
1072void
1073_jit_putargi_d(jit_state_t *_jit, jit_float64_t u, jit_node_t *v)
1074{
1075 jit_int32_t regno;
1076 assert(v->code == jit_code_arg_d);
1077 jit_inc_synth_dp(putargi_d, u, v);
1078#if __X64
79bfeef6 1079 if (jit_arg_f_reg_p(v->u.w))
4a71579b
PC		 1080 jit_movi_d(_XMM0 - v->u.w, u);
1081 else
1082#endif
1083 {
79bfeef6
PC		 1084 jit_node_t *node;
1085 regno = jit_get_reg(jit_class_fpr);
4a71579b 1086 jit_movi_d(regno, u);
79bfeef6
PC		 1087 node = jit_stxi_d(v->u.w, _RBP, regno);
1088 jit_link_alist(node);
4a71579b
PC		 1089 jit_unget_reg(regno);
1090 }
1091 jit_dec_synth();
1092}
1093
1094void
79bfeef6 1095_jit_pushargr(jit_state_t *_jit, jit_int32_t u, jit_code_t code)
4a71579b
PC		 1096{
1097 assert(_jitc->function);
79bfeef6 1098 jit_code_inc_synth_w(code, u);
4a71579b
PC		 1099 jit_link_prepare();
1100#if __X64
1101 if (jit_arg_reg_p(_jitc->function->call.argi)) {
1102 jit_movr(JIT_RA0 - _jitc->function->call.argi, u);
1103 ++_jitc->function->call.argi;
1104# if __CYGWIN__ || _WIN32
1105 if (_jitc->function->call.call & jit_call_varargs)
1106 jit_stxi(_jitc->function->call.size, _RSP, u);
1107 _jitc->function->call.size += sizeof(jit_word_t);
1108# endif
1109 }
1110 else
1111#endif
1112 {
1113 jit_stxi(_jitc->function->call.size, _RSP, u);
1114 _jitc->function->call.size += REAL_WORDSIZE;
79bfeef6 1115 jit_check_frame();
4a71579b
PC		 1116 }
1117 jit_dec_synth();
1118}
1119
1120void
79bfeef6 1121_jit_pushargi(jit_state_t *_jit, jit_word_t u, jit_code_t code)
4a71579b
PC		 1122{
1123 jit_int32_t regno;
1124 assert(_jitc->function);
79bfeef6 1125 jit_code_inc_synth_w(code, u);
4a71579b
PC		 1126 jit_link_prepare();
1127#if __X64
1128 if (jit_arg_reg_p(_jitc->function->call.argi)) {
1129 jit_movi(JIT_RA0 - _jitc->function->call.argi, u);
1130# if __CYGWIN__ || _WIN32
1131 if (_jitc->function->call.call & jit_call_varargs)
1132 jit_stxi(_jitc->function->call.size, _RSP,
1133 JIT_RA0 - _jitc->function->call.argi);
1134 _jitc->function->call.size += sizeof(jit_word_t);
1135# endif
1136 ++_jitc->function->call.argi;
1137 }
1138 else
1139#endif
1140 {
1141 regno = jit_get_reg(jit_class_gpr);
1142 jit_movi(regno, u);
1143 jit_stxi(_jitc->function->call.size, _RSP, regno);
1144 _jitc->function->call.size += REAL_WORDSIZE;
1145 jit_unget_reg(regno);
79bfeef6 1146 jit_check_frame();
4a71579b
PC		 1147 }
1148 jit_dec_synth();
1149}
1150
1151void
1152_jit_pushargr_f(jit_state_t *_jit, jit_int32_t u)
1153{
1154 assert(_jitc->function);
1155 jit_inc_synth_w(pushargr_f, u);
1156 jit_link_prepare();
1157#if __X64
1158# if __CYGWIN__ || _WIN32
1159 if (jit_arg_reg_p(_jitc->function->call.argi)) {
1160 jit_movr_f(_XMM0 - _jitc->function->call.argi, u);
1161 if (_jitc->function->call.call & jit_call_varargs) {
1162 jit_stxi_f(_jitc->function->call.size, _RSP,
1163 _XMM0 - _jitc->function->call.argi);
1164 jit_ldxi_i(JIT_RA0 - _jitc->function->call.argi, _RSP,
1165 _jitc->function->call.size);
1166 }
1167 ++_jitc->function->call.argi;
1168 _jitc->function->call.size += sizeof(jit_word_t);
1169 }
1170# else
1171 if (jit_arg_f_reg_p(_jitc->function->self.argf)) {
1172 jit_movr_f(_XMM0 - _jitc->function->call.argf, u);
1173 ++_jitc->function->call.argf;
1174 }
1175# endif
1176 else
1177#endif
1178 {
1179 jit_stxi_f(_jitc->function->call.size, _RSP, u);
1180 _jitc->function->call.size += REAL_WORDSIZE;
79bfeef6 1181 jit_check_frame();
4a71579b
PC		 1182 }
1183 jit_dec_synth();
1184}
1185
1186void
1187_jit_pushargi_f(jit_state_t *_jit, jit_float32_t u)
1188{
1189 jit_int32_t regno;
1190 assert(_jitc->function);
1191 jit_inc_synth_f(pushargi_f, u);
1192 jit_link_prepare();
1193#if __X64
1194# if __CYGWIN__ || _WIN32
1195 if (jit_arg_reg_p(_jitc->function->call.argi)) {
1196 jit_movi_f(_XMM0 - _jitc->function->call.argi, u);
1197 if (_jitc->function->call.call & jit_call_varargs) {
1198 jit_stxi_f(_jitc->function->call.size, _RSP,
1199 _XMM0 - _jitc->function->call.argi);
1200 jit_ldxi_i(JIT_RA0 - _jitc->function->call.argi, _RSP,
1201 _jitc->function->call.size);
1202 }
1203 ++_jitc->function->call.argi;
1204 _jitc->function->call.size += sizeof(jit_word_t);
1205 }
1206# else
1207 if (jit_arg_f_reg_p(_jitc->function->call.argf)) {
1208 jit_movi_f(_XMM0 - _jitc->function->call.argf, u);
1209 ++_jitc->function->call.argf;
1210 }
1211# endif
1212 else
1213#endif
1214 {
1215 regno = jit_get_reg(jit_class_fpr);
1216 jit_movi_f(regno, u);
1217 jit_stxi_f(_jitc->function->call.size, _RSP, regno);
1218 _jitc->function->call.size += REAL_WORDSIZE;
1219 jit_unget_reg(regno);
79bfeef6 1220 jit_check_frame();
4a71579b
PC		 1221 }
1222 jit_dec_synth();
1223}
1224
1225void
1226_jit_pushargr_d(jit_state_t *_jit, jit_int32_t u)
1227{
1228 assert(_jitc->function);
1229 jit_inc_synth_w(pushargr_d, u);
1230 jit_link_prepare();
1231#if __X64
1232# if __CYGWIN__ || _WIN32
1233 if (jit_arg_reg_p(_jitc->function->call.argi)) {
1234 jit_movr_d(_XMM0 - _jitc->function->call.argi, u);
1235 if (_jitc->function->call.call & jit_call_varargs) {
1236 jit_stxi_d(_jitc->function->call.size, _RSP,
1237 _XMM0 - _jitc->function->call.argi);
1238 jit_ldxi_l(JIT_RA0 - _jitc->function->call.argi, _RSP,
1239 _jitc->function->call.size);
1240 }
1241 ++_jitc->function->call.argi;
1242 _jitc->function->call.size += sizeof(jit_word_t);
1243 }
1244# else
1245 if (jit_arg_f_reg_p(_jitc->function->call.argf)) {
1246 jit_movr_d(_XMM0 - _jitc->function->call.argf, u);
1247 ++_jitc->function->call.argf;
1248 }
1249# endif
1250 else
1251#endif
1252 {
1253 jit_stxi_d(_jitc->function->call.size, _RSP, u);
1254 _jitc->function->call.size += sizeof(jit_float64_t);
79bfeef6 1255 jit_check_frame();
4a71579b
PC		 1256 }
1257 jit_dec_synth();
1258}
1259
1260void
1261_jit_pushargi_d(jit_state_t *_jit, jit_float64_t u)
1262{
1263 jit_int32_t regno;
1264 assert(_jitc->function);
1265 jit_inc_synth_d(pushargi_d, u);
1266 jit_link_prepare();
1267#if __X64
1268# if __CYGWIN__ || _WIN32
1269 if (jit_arg_reg_p(_jitc->function->call.argi)) {
1270 jit_movi_d(_XMM0 - _jitc->function->call.argi, u);
1271 if (_jitc->function->call.call & jit_call_varargs) {
1272 jit_stxi_d(_jitc->function->call.size, _RSP,
1273 _XMM0 - _jitc->function->call.argi);
1274 jit_ldxi_l(JIT_RA0 - _jitc->function->call.argi, _RSP,
1275 _jitc->function->call.size);
1276 }
1277 ++_jitc->function->call.argi;
1278 _jitc->function->call.size += sizeof(jit_word_t);
1279 }
1280# else
1281 if (jit_arg_f_reg_p(_jitc->function->call.argf)) {
1282 jit_movi_d(_XMM0 - _jitc->function->call.argf, u);
1283 ++_jitc->function->call.argf;
1284 }
1285# endif
1286 else
1287#endif
1288 {
1289 regno = jit_get_reg(jit_class_fpr);
1290 jit_movi_d(regno, u);
1291 jit_stxi_d(_jitc->function->call.size, _RSP, regno);
1292 _jitc->function->call.size += sizeof(jit_float64_t);
1293 jit_unget_reg(regno);
79bfeef6 1294 jit_check_frame();
4a71579b
PC		 1295 }
1296 jit_dec_synth();
1297}
1298
1299jit_bool_t
1300_jit_regarg_p(jit_state_t *_jit, jit_node_t *node, jit_int32_t regno)
1301{
1302#if __X64
1303 jit_int32_t spec;
1304
1305 spec = jit_class(_rvs[regno].spec);
1306 if (spec & jit_class_arg) {
1307 if (spec & jit_class_gpr) {
1308 regno = JIT_RA0 - regno;
1309 if (regno >= 0 && regno < node->v.w)
1310 return (1);
1311 }
1312 else if (spec & jit_class_fpr) {
1313 regno = _XMM0 - regno;
1314 if (regno >= 0 && regno < node->w.w)
1315 return (1);
1316 }
1317 }
1318#endif
1319 return (0);
1320}
1321
1322void
1323_jit_finishr(jit_state_t *_jit, jit_int32_t r0)
1324{
1325 jit_int32_t reg;
1326 jit_node_t *call;
1327 assert(_jitc->function);
79bfeef6 1328 jit_check_frame();
4a71579b
PC		 1329 reg = r0;
1330 jit_inc_synth_w(finishr, r0);
1331 if (_jitc->function->self.alen < _jitc->function->call.size)
1332 _jitc->function->self.alen = _jitc->function->call.size;
1333#if __X64
1334# if !(__CYGWIN__ || _WIN32)
1335 if (_jitc->function->call.call & jit_call_varargs) {
1336 if (jit_regno(reg) == _RAX) {
1337 reg = jit_get_reg(jit_class_gpr);
1338 jit_movr(reg, _RAX);
1339 }
1340 if (_jitc->function->call.argf)
1341 jit_movi(_RAX, _jitc->function->call.argf);
1342 else
1343 jit_movi(_RAX, 0);
1344 if (reg != r0)
1345 jit_unget_reg(reg);
1346 }
1347# endif
1348#endif
1349 call = jit_callr(reg);
1350 call->v.w = _jitc->function->call.argi;
1351 call->w.w = _jitc->function->call.argf;
1352 _jitc->function->call.argi = _jitc->function->call.argf =
1353 _jitc->function->call.size = 0;
1354 _jitc->prepare = 0;
1355 jit_dec_synth();
1356}
1357
1358jit_node_t *
1359_jit_finishi(jit_state_t *_jit, jit_pointer_t i0)
1360{
4a71579b
PC		 1361 jit_node_t *node;
1362 assert(_jitc->function);
79bfeef6 1363 jit_check_frame();
4a71579b
PC		 1364 jit_inc_synth_w(finishi, (jit_word_t)i0);
1365 if (_jitc->function->self.alen < _jitc->function->call.size)
1366 _jitc->function->self.alen = _jitc->function->call.size;
1367#if __X64
79bfeef6
PC		 1368# if !(__CYGWIN__ || _WIN32)
1369 if (_jitc->function->call.call & jit_call_varargs) {
1370 if (_jitc->function->call.argf)
1371 jit_movi(_RAX, _jitc->function->call.argf);
1372 else
1373 jit_movi(_RAX, 0);
1374 jit_live(_RAX);
1375 }
1376# endif
1377#endif
4a71579b
PC		 1378 node = jit_calli(i0);
1379 node->v.w = _jitc->function->call.argi;
1380 node->w.w = _jitc->function->call.argf;
4a71579b
PC		 1381 _jitc->function->call.argi = _jitc->function->call.argf =
1382 _jitc->function->call.size = 0;
1383 _jitc->prepare = 0;
1384 jit_dec_synth();
1385 return (node);
1386}
1387
1388void
1389_jit_retval_c(jit_state_t *_jit, jit_int32_t r0)
1390{
1391 jit_inc_synth_w(retval_c, r0);
1392 jit_extr_c(r0, JIT_RET);
1393 jit_dec_synth();
1394}
1395
1396void
1397_jit_retval_uc(jit_state_t *_jit, jit_int32_t r0)
1398{
1399 jit_inc_synth_w(retval_uc, r0);
1400 jit_extr_uc(r0, JIT_RET);
1401 jit_dec_synth();
1402}
1403
1404void
1405_jit_retval_s(jit_state_t *_jit, jit_int32_t r0)
1406{
1407 jit_inc_synth_w(retval_s, r0);
1408 jit_extr_s(r0, JIT_RET);
1409 jit_dec_synth();
1410}
1411
1412void
1413_jit_retval_us(jit_state_t *_jit, jit_int32_t r0)
1414{
1415 jit_inc_synth_w(retval_us, r0);
1416 jit_extr_us(r0, JIT_RET);
1417 jit_dec_synth();
1418}
1419
1420void
1421_jit_retval_i(jit_state_t *_jit, jit_int32_t r0)
1422{
1423 jit_inc_synth_w(retval_i, r0);
1424#if __X32 || __X64_32
1425 if (r0 != JIT_RET)
1426 jit_movr(r0, JIT_RET);
1427#else
1428 jit_extr_i(r0, JIT_RET);
1429#endif
1430 jit_dec_synth();
1431}
1432
1433#if __X64 && !__X64_32
1434void
1435_jit_retval_ui(jit_state_t *_jit, jit_int32_t r0)
1436{
1437 jit_inc_synth_w(retval_ui, r0);
1438 jit_extr_ui(r0, JIT_RET);
1439 jit_dec_synth();
1440}
1441
1442void
1443_jit_retval_l(jit_state_t *_jit, jit_int32_t r0)
1444{
1445 jit_inc_synth_w(retval_l, r0);
1446 if (r0 != JIT_RET)
1447 jit_movr(r0, JIT_RET);
1448 jit_dec_synth();
1449}
1450#endif
1451
1452void
1453_jit_retval_f(jit_state_t *_jit, jit_int32_t r0)
1454{
1455 jit_inc_synth_w(retval_f, r0);
1456#if __X64
1457 if (r0 != JIT_FRET)
1458 jit_movr_f(r0, JIT_FRET);
1459#endif
1460 jit_dec_synth();
1461}
1462
1463void
1464_jit_retval_d(jit_state_t *_jit, jit_int32_t r0)
1465{
1466 jit_inc_synth_w(retval_d, r0);
1467#if __X64
1468 if (r0 != JIT_FRET)
1469 jit_movr_d(r0, JIT_FRET);
1470#endif
1471 jit_dec_synth();
1472}
1473
1474jit_pointer_t
1475_emit_code(jit_state_t *_jit)
1476{
1477 jit_node_t *node;
1478 jit_node_t *temp;
1479 jit_word_t word;
1480 jit_int32_t value;
1481 jit_int32_t offset;
1482 struct {
1483 jit_node_t *node;
1484 jit_word_t word;
79bfeef6 1485 jit_function_t func;
4a71579b
PC		 1486#if DEVEL_DISASSEMBLER
1487 jit_word_t prevw;
1488#endif
1489 jit_int32_t patch_offset;
1490 } undo;
1491#if DEVEL_DISASSEMBLER
1492 jit_word_t prevw;
1493#endif
1494
1495 _jitc->function = NULL;
1496
1497 jit_reglive_setup();
1498
1499 undo.word = 0;
1500 undo.node = NULL;
1501 undo.patch_offset = 0;
1502#define case_rr(name, type) \
1503 case jit_code_##name##r##type: \
1504 name##r##type(rn(node->u.w), rn(node->v.w)); \
1505 break
1506#define case_rw(name, type) \
1507 case jit_code_##name##i##type: \
1508 name##i##type(rn(node->u.w), node->v.w); \
1509 break
1510#define case_rf(name, type) \
1511 case jit_code_##name##r##type: \
1512 if (jit_x87_reg_p(node->v.w)) \
1513 x87_##name##r##type(rn(node->u.w), rn(node->v.w)); \
1514 else \
1515 sse_##name##r##type(rn(node->u.w), rn(node->v.w)); \
1516 break
1517#define case_fr(name, type) \
1518 case jit_code_##name##r##type: \
1519 if (jit_x87_reg_p(node->u.w)) \
1520 x87_##name##r##type(rn(node->u.w), rn(node->v.w)); \
1521 else \
1522 sse_##name##r##type(rn(node->u.w), rn(node->v.w)); \
1523 break
1524#define case_fw(name, type) \
1525 case jit_code_##name##i##type: \
1526 if (jit_x87_reg_p(node->u.w)) \
1527 x87_##name##i##type(rn(node->u.w), node->v.w); \
1528 else \
1529 sse_##name##i##type(rn(node->u.w), node->v.w); \
1530 break
1531#define case_wr(name, type) \
1532 case jit_code_##name##i##type: \
1533 name##i##type(node->u.w, rn(node->v.w)); \
1534 break
1535#define case_wf(name, type) \
1536 case jit_code_##name##i##type: \
1537 if (jit_x87_reg_p(node->v.w)) \
1538 x87_##name##i##type(node->u.w, rn(node->v.w)); \
1539 else \
1540 sse_##name##i##type(node->u.w, rn(node->v.w)); \
1541 break
1542#define case_ff(name, type) \
1543 case jit_code_##name##r##type: \
1544 if (jit_x87_reg_p(node->u.w) && \
1545 jit_x87_reg_p(node->v.w)) \
1546 x87_##name##r##type(rn(node->u.w), rn(node->v.w)); \
1547 else \
1548 sse_##name##r##type(rn(node->u.w), rn(node->v.w)); \
1549 break;
1550#define case_rrr(name, type) \
1551 case jit_code_##name##r##type: \
1552 name##r##type(rn(node->u.w), \
1553 rn(node->v.w), rn(node->w.w)); \
1554 break
1555#define case_rrrr(name, type) \
1556 case jit_code_##name##r##type: \
1557 name##r##type(rn(node->u.q.l), rn(node->u.q.h), \
1558 rn(node->v.w), rn(node->w.w)); \
1559 break
ba86ff93
PC		 1560#define case_rqr(name, type) \
1561 case jit_code_##name##r##type: \
1562 if (jit_x87_reg_p(node->u.w) && \
1563 jit_x87_reg_p(node->v.q.l) && \
1564 jit_x87_reg_p(node->v.q.h) && \
1565 jit_x87_reg_p(node->w.w)) \
1566 x87_##name##r##type(rn(node->u.w), \
1567 rn(node->v.q.l), \
1568 rn(node->v.q.h), \
1569 rn(node->w.w)); \
1570 else { \
1571 assert(jit_sse_reg_p(node->u.w) && \
1572 jit_sse_reg_p(node->v.q.l) && \
1573 jit_sse_reg_p(node->v.q.h) && \
1574 jit_sse_reg_p(node->w.w)); \
1575 sse_##name##r##type(rn(node->u.w), \
1576 rn(node->v.q.l), \
1577 rn(node->v.q.h), \
1578 rn(node->w.w)); \
1579 } \
1580 break;
4a71579b
PC		 1581#define case_frr(name, type) \
1582 case jit_code_##name##r##type: \
1583 if (jit_x87_reg_p(node->u.w)) \
1584 x87_##name##r##type(rn(node->u.w), \
1585 rn(node->v.w), rn(node->w.w)); \
1586 else \
1587 sse_##name##r##type(rn(node->u.w), \
1588 rn(node->v.w), rn(node->w.w)); \
1589 break
1590#define case_rrf(name, type) \
1591 case jit_code_##name##r##type: \
1592 if (jit_x87_reg_p(node->w.w)) \
1593 x87_##name##r##type(rn(node->u.w), \
1594 rn(node->v.w), rn(node->w.w)); \
1595 else \
1596 sse_##name##r##type(rn(node->u.w), \
1597 rn(node->v.w), rn(node->w.w)); \
1598 break
1599#define case_rrw(name, type) \
1600 case jit_code_##name##i##type: \
1601 name##i##type(rn(node->u.w), rn(node->v.w), node->w.w); \
1602 break
1603#define case_rrrw(name, type) \
1604 case jit_code_##name##i##type: \
1605 name##i##type(rn(node->u.q.l), rn(node->u.q.h), \
1606 rn(node->v.w), node->w.w); \
1607 break
1608#define case_frw(name, type) \
1609 case jit_code_##name##i##type: \
1610 if (jit_x87_reg_p(node->u.w)) \
1611 x87_##name##i##type(rn(node->u.w), \
1612 rn(node->v.w), node->w.w); \
1613 else \
1614 sse_##name##i##type(rn(node->u.w), \
1615 rn(node->v.w), node->w.w); \
1616 break
1617#define case_wrr(name, type) \
1618 case jit_code_##name##i##type: \
1619 name##i##type(node->u.w, rn(node->v.w), rn(node->w.w)); \
1620 break
1621#define case_wrf(name, type) \
1622 case jit_code_##name##i##type: \
1623 if (jit_x87_reg_p(node->w.w)) \
1624 x87_##name##i##type(node->u.w, \
1625 rn(node->v.w), rn(node->w.w)); \
1626 else \
1627 sse_##name##i##type(node->u.w, \
1628 rn(node->v.w), rn(node->w.w)); \
1629 break
1630#define case_brr(name, type) \
1631 case jit_code_##name##r##type: \
1632 temp = node->u.n; \
1633 assert(temp->code == jit_code_label || \
1634 temp->code == jit_code_epilog); \
1635 if (temp->flag & jit_flag_patch) \
1636 name##r##type(temp->u.w, rn(node->v.w), \
1637 rn(node->w.w)); \
1638 else { \
1639 word = name##r##type(_jit->pc.w, \
1640 rn(node->v.w), rn(node->w.w)); \
1641 patch(word, node); \
1642 } \
1643 break
1644#define case_brw(name, type) \
1645 case jit_code_##name##i##type: \
1646 temp = node->u.n; \
1647 assert(temp->code == jit_code_label || \
1648 temp->code == jit_code_epilog); \
1649 if (temp->flag & jit_flag_patch) \
1650 name##i##type(temp->u.w, \
1651 rn(node->v.w), node->w.w); \
1652 else { \
1653 word = name##i##type(_jit->pc.w, \
1654 rn(node->v.w), node->w.w); \
1655 patch(word, node); \
1656 } \
1657 break
1658#define case_rff(name, type) \
1659 case jit_code_##name##r##type: \
1660 if (jit_x87_reg_p(node->v.w) && \
1661 jit_x87_reg_p(node->w.w)) \
1662 x87_##name##r##type(rn(node->u.w), rn(node->v.w), \
1663 rn(node->w.w)); \
1664 else \
1665 sse_##name##r##type(rn(node->u.w), rn(node->v.w), \
1666 rn(node->w.w)); \
1667 break;
1668#define case_rfw(name, type, size) \
1669 case jit_code_##name##i##type: \
1670 assert(node->flag & jit_flag_data); \
1671 if (jit_x87_reg_p(node->v.w)) \
1672 x87_##name##i##type(rn(node->u.w), rn(node->v.w), \
1673 (jit_float##size##_t *)node->w.n->u.w); \
1674 else \
1675 sse_##name##i##type(rn(node->u.w), rn(node->v.w), \
1676 (jit_float##size##_t *)node->w.n->u.w); \
1677 break
1678#define case_fff(name, type) \
1679 case jit_code_##name##r##type: \
1680 if (jit_x87_reg_p(node->u.w) && \
1681 jit_x87_reg_p(node->v.w) && \
1682 jit_x87_reg_p(node->w.w)) \
1683 x87_##name##r##type(rn(node->u.w), \
1684 rn(node->v.w), rn(node->w.w)); \
1685 else \
1686 sse_##name##r##type(rn(node->u.w), \
1687 rn(node->v.w), rn(node->w.w)); \
1688 break
1689#define case_ffw(name, type, size) \
1690 case jit_code_##name##i##type: \
1691 assert(node->flag & jit_flag_data); \
1692 if (jit_x87_reg_p(node->u.w) && \
1693 jit_x87_reg_p(node->v.w)) \
1694 x87_##name##i##type(rn(node->u.w), rn(node->v.w), \
1695 (jit_float##size##_t *)node->w.n->u.w); \
1696 else \
1697 sse_##name##i##type(rn(node->u.w), rn(node->v.w), \
1698 (jit_float##size##_t *)node->w.n->u.w); \
1699 break
1700#define case_bff(name, type) \
1701 case jit_code_b##name##r##type: \
1702 temp = node->u.n; \
1703 assert(temp->code == jit_code_label || \
1704 temp->code == jit_code_epilog); \
1705 if (temp->flag & jit_flag_patch) { \
1706 if (jit_x87_reg_p(node->v.w) && \
1707 jit_x87_reg_p(node->w.w)) \
1708 x87_b##name##r##type(temp->u.w, \
1709 rn(node->v.w), rn(node->w.w)); \
1710 else \
1711 sse_b##name##r##type(temp->u.w, \
1712 rn(node->v.w), rn(node->w.w)); \
1713 } \
1714 else { \
1715 if (jit_x87_reg_p(node->v.w) && \
1716 jit_x87_reg_p(node->w.w)) \
1717 word = x87_b##name##r##type(_jit->pc.w, \
1718 rn(node->v.w), rn(node->w.w)); \
1719 else \
1720 word = sse_b##name##r##type(_jit->pc.w, \
1721 rn(node->v.w), rn(node->w.w)); \
1722 patch(word, node); \
1723 } \
1724 break
1725#define case_bfw(name, type, size) \
1726 case jit_code_b##name##i##type: \
1727 temp = node->u.n; \
1728 assert(temp->code == jit_code_label || \
1729 temp->code == jit_code_epilog); \
1730 if (temp->flag & jit_flag_patch) { \
1731 if (jit_x87_reg_p(node->v.w)) \
1732 x87_b##name##i##type(temp->u.w, \
1733 rn(node->v.w), \
1734 (jit_float##size##_t *)node->w.n->u.w); \
1735 else \
1736 sse_b##name##i##type(temp->u.w, \
1737 rn(node->v.w), \
1738 (jit_float##size##_t *)node->w.n->u.w); \
1739 } \
1740 else { \
1741 if (jit_x87_reg_p(node->v.w)) \
1742 word = x87_b##name##i##type(_jit->pc.w, \
1743 rn(node->v.w), \
1744 (jit_float##size##_t *)node->w.n->u.w); \
1745 else \
1746 word = sse_b##name##i##type(_jit->pc.w, \
1747 rn(node->v.w), \
1748 (jit_float##size##_t *)node->w.n->u.w); \
1749 patch(word, node); \
1750 } \
1751 break
1752#if DEVEL_DISASSEMBLER
1753 prevw = _jit->pc.w;
1754#endif
1755 for (node = _jitc->head; node; node = node->next) {
1756 if (_jit->pc.uc >= _jitc->code.end)
1757 return (NULL);
1758
1759#if DEVEL_DISASSEMBLER
1760 node->offset = (jit_uword_t)_jit->pc.w - (jit_uword_t)prevw;
1761 prevw = _jit->pc.w;
1762#endif
1763 value = jit_classify(node->code);
1764 jit_regarg_set(node, value);
1765 switch (node->code) {
1766 case jit_code_align:
c0c16242
PC		 1767 /* Must align to a power of two */
1768 assert(!(node->u.w & (node->u.w - 1)));
4a71579b
PC		 1769 if ((word = _jit->pc.w & (node->u.w - 1)))
1770 nop(node->u.w - word);
1771 break;
79bfeef6
PC		 1772 case jit_code_skip:
1773 nop(node->u.w);
1774 break;
1775 case jit_code_note: case jit_code_name:
4a71579b
PC		 1776 node->u.w = _jit->pc.w;
1777 break;
1778 case jit_code_label:
1779 if ((node->link || (node->flag & jit_flag_use)) &&
1780 (word = _jit->pc.w & (sizeof(jit_word_t) - 1)))
1781 nop(sizeof(jit_word_t) - word);
1782 /* remember label is defined */
1783 node->flag |= jit_flag_patch;
1784 node->u.w = _jit->pc.w;
1785 break;
1786 case_rrr(add,);
1787 case_rrw(add,);
1788 case_rrr(addx,);
1789 case_rrw(addx,);
1790 case_rrr(addc,);
1791 case_rrw(addc,);
1792 case_rrr(sub,);
1793 case_rrw(sub,);
1794 case_rrr(subx,);
1795 case_rrw(subx,);
1796 case_rrr(subc,);
1797 case_rrw(subc,);
1798 case_rrw(rsb,);
1799 case_rrr(mul,);
1800 case_rrw(mul,);
ba86ff93
PC		 1801 case_rrr(hmul,);
1802 case_rrw(hmul,);
1803 case_rrr(hmul, _u);
1804 case_rrw(hmul, _u);
4a71579b
PC		 1805 case_rrrr(qmul,);
1806 case_rrrw(qmul,);
1807 case_rrrr(qmul, _u);
1808 case_rrrw(qmul, _u);
1809 case_rrr(div,);
1810 case_rrw(div,);
1811 case_rrr(div, _u);
1812 case_rrw(div, _u);
1813 case_rrrr(qdiv,);
1814 case_rrrw(qdiv,);
1815 case_rrrr(qdiv, _u);
1816 case_rrrw(qdiv, _u);
1817 case_rrr(rem,);
1818 case_rrw(rem,);
1819 case_rrr(rem, _u);
1820 case_rrw(rem, _u);
1821 case_rrr(and,);
1822 case_rrw(and,);
1823 case_rrr(or,);
1824 case_rrw(or,);
1825 case_rrr(xor,);
1826 case_rrw(xor,);
1827 case_rrr(lsh,);
1828 case_rrw(lsh,);
ba86ff93
PC		 1829 case_rrrr(qlsh,);
1830 case_rrrw(qlsh,);
1831 case_rrrr(qlsh, _u);
1832 case_rrrw(qlsh, _u);
4a71579b
PC		 1833 case_rrr(rsh,);
1834 case_rrw(rsh,);
ba86ff93
PC		 1835 case_rrrr(qrsh,);
1836 case_rrrw(qrsh,);
4a71579b
PC		 1837 case_rrr(rsh, _u);
1838 case_rrw(rsh, _u);
ba86ff93
PC		 1839 case_rrrr(qrsh, _u);
1840 case_rrrw(qrsh, _u);
1841 case_rrr(lrot,);
1842 case_rrw(lrot,);
1843 case_rrr(rrot,);
1844 case_rrw(rrot,);
4a71579b
PC		 1845 case_rr(neg,);
1846 case_rr(com,);
79bfeef6
PC		 1847 case_rr(clo,);
1848 case_rr(clz,);
1849 case_rr(cto,);
1850 case_rr(ctz,);
ba86ff93
PC		 1851 case_rr(rbit,);
1852 case_rr(popcnt,);
4a71579b
PC		 1853 case_rrr(lt,);
1854 case_rrw(lt,);
1855 case_rrr(lt, _u);
1856 case_rrw(lt, _u);
1857 case_rrr(le,);
1858 case_rrw(le,);
1859 case_rrr(le, _u);
1860 case_rrw(le, _u);
1861 case_rrr(eq,);
1862 case_rrw(eq,);
1863 case_rrr(ge,);
1864 case_rrw(ge,);
1865 case_rrr(ge, _u);
1866 case_rrw(ge, _u);
1867 case_rrr(gt,);
1868 case_rrw(gt,);
1869 case_rrr(gt, _u);
1870 case_rrw(gt, _u);
1871 case_rrr(ne,);
1872 case_rrw(ne,);
ba3814c1
PC		 1873 case jit_code_casr:
1874 casr(rn(node->u.w), rn(node->v.w),
1875 rn(node->w.q.l), rn(node->w.q.h));
1876 break;
1877 case jit_code_casi:
1878 casi(rn(node->u.w), node->v.w,
1879 rn(node->w.q.l), rn(node->w.q.h));
1880 break;
1f22b268
PC		 1881 case_rrr(movn,);
1882 case_rrr(movz,);
4a71579b
PC		 1883 case_rr(mov,);
1884 case jit_code_movi:
1885 if (node->flag & jit_flag_node) {
1886 temp = node->v.n;
1887 if (temp->code == jit_code_data ||
1888 (temp->code == jit_code_label &&
1889 (temp->flag & jit_flag_patch)))
1890 movi(rn(node->u.w), temp->u.w);
1891 else {
1892 assert(temp->code == jit_code_label ||
1893 temp->code == jit_code_epilog);
79bfeef6
PC		 1894#if CAN_RIP_ADDRESS
1895 word = _jit->code.length -
1896 (_jit->pc.uc - _jit->code.ptr);
1897 if ((jit_int32_t)word == word)
1898 word = movi(rn(node->u.w), _jit->pc.w);
1899 else
1900#endif
1901 word = movi_p(rn(node->u.w), node->v.w);
4a71579b
PC		 1902 patch(word, node);
1903 }
1904 }
1905 else
1906 movi(rn(node->u.w), node->v.w);
1907 break;
1908 case_rr(hton, _us);
1909 case_rr(hton, _ui);
1910#if __X64 && !__X64_32
1911 case_rr(hton, _ul);
40a44dcb
PC		 1912#endif
1913 case_rr(bswap, _us);
1914 case_rr(bswap, _ui);
1915#if __X64 && !__X64_32
1916 case_rr(bswap, _ul);
4a71579b 1917#endif
ba86ff93
PC		 1918 case jit_code_extr:
1919 extr(rn(node->u.w), rn(node->v.w), node->w.q.l, node->w.q.h);
1920 break;
1921 case jit_code_extr_u:
1922 extr_u(rn(node->u.w), rn(node->v.w), node->w.q.l, node->w.q.h);
1923 break;
1924 case jit_code_depr:
1925 depr(rn(node->u.w), rn(node->v.w), node->w.q.l, node->w.q.h);
1926 break;
1927 case jit_code_depi:
1928 depi(rn(node->u.w), node->v.w, node->w.q.l, node->w.q.h);
1929 break;
4a71579b
PC		 1930 case_rr(ext, _c);
1931 case_rr(ext, _uc);
1932 case_rr(ext, _s);
1933 case_rr(ext, _us);
1934#if __X64 && !__X64_32
1935 case_rr(ext, _i);
1936 case_rr(ext, _ui);
1937#endif
1938 case_rf(trunc, _f_i);
1939 case_rf(trunc, _d_i);
1940#if __X64
1941 case_rf(trunc, _f_l);
1942 case_rf(trunc, _d_l);
1943#endif
1944 case_rr(ld, _c);
1945 case_rw(ld, _c);
1946 case_rr(ld, _uc);
1947 case_rw(ld, _uc);
1948 case_rr(ld, _s);
1949 case_rw(ld, _s);
1950 case_rr(ld, _us);
1951 case_rw(ld, _us);
1952 case_rr(ld, _i);
1953 case_rw(ld, _i);
1954#if __X64 && !__X64_32
1955 case_rr(ld, _ui);
1956 case_rw(ld, _ui);
1957 case_rr(ld, _l);
1958 case_rw(ld, _l);
1959#endif
1960 case_rrr(ldx, _c);
1961 case_rrw(ldx, _c);
1962 case_rrr(ldx, _uc);
1963 case_rrw(ldx, _uc);
1964 case_rrr(ldx, _s);
1965 case_rrw(ldx, _s);
1966 case_rrr(ldx, _us);
1967 case_rrw(ldx, _us);
1968 case_rrr(ldx, _i);
1969 case_rrw(ldx, _i);
1970#if __X64 && !__X64_32
1971 case_rrr(ldx, _ui);
1972 case_rrw(ldx, _ui);
1973 case_rrr(ldx, _l);
1974 case_rrw(ldx, _l);
1975#endif
ba86ff93
PC		 1976 case jit_code_unldr:
1977 unldr(rn(node->u.w), rn(node->v.w), node->w.w);
1978 break;
1979 case jit_code_unldi:
1980 unldi(rn(node->u.w), node->v.w, node->w.w);
1981 break;
1982 case jit_code_unldr_u:
1983 unldr_u(rn(node->u.w), rn(node->v.w), node->w.w);
1984 break;
1985 case jit_code_unldi_u:
1986 unldi_u(rn(node->u.w), node->v.w, node->w.w);
1987 break;
4a71579b
PC		 1988 case_rr(st, _c);
1989 case_wr(st, _c);
1990 case_rr(st, _s);
1991 case_wr(st, _s);
1992 case_rr(st, _i);
1993 case_wr(st, _i);
1994#if __X64 && !__X64_32
1995 case_rr(st, _l);
1996 case_wr(st, _l);
1997#endif
1998 case_rrr(stx, _c);
1999 case_wrr(stx, _c);
2000 case_rrr(stx, _s);
2001 case_wrr(stx, _s);
2002 case_rrr(stx, _i);
2003 case_wrr(stx, _i);
2004#if __X64 && !__X64_32
2005 case_rrr(stx, _l);
2006 case_wrr(stx, _l);
2007#endif
ba86ff93
PC		 2008 case jit_code_unstr:
2009 unstr(rn(node->u.w), rn(node->v.w), node->w.w);
2010 break;
2011 case jit_code_unsti:
2012 unsti(node->u.w, rn(node->v.w), node->w.w);
2013 break;
4a71579b
PC		 2014 case_brr(blt,);
2015 case_brw(blt,);
2016 case_brr(blt, _u);
2017 case_brw(blt, _u);
2018 case_brr(ble,);
2019 case_brw(ble,);
2020 case_brr(ble, _u);
2021 case_brw(ble, _u);
2022 case_brr(beq,);
2023 case_brw(beq,);
2024 case_brr(bge,);
2025 case_brw(bge,);
2026 case_brr(bge, _u);
2027 case_brw(bge, _u);
2028 case_brr(bgt,);
2029 case_brw(bgt,);
2030 case_brr(bgt, _u);
2031 case_brw(bgt, _u);
2032 case_brr(bne,);
2033 case_brw(bne,);
2034 case_brr(bms,);
2035 case_brw(bms,);
2036 case_brr(bmc,);
2037 case_brw(bmc,);
2038 case_brr(boadd,);
2039 case_brw(boadd,);
2040 case_brr(boadd, _u);
2041 case_brw(boadd, _u);
2042 case_brr(bxadd,);
2043 case_brw(bxadd,);
2044 case_brr(bxadd, _u);
2045 case_brw(bxadd, _u);
2046 case_brr(bosub,);
2047 case_brw(bosub,);
2048 case_brr(bosub, _u);
2049 case_brw(bosub, _u);
2050 case_brr(bxsub,);
2051 case_brw(bxsub,);
2052 case_brr(bxsub, _u);
2053 case_brw(bxsub, _u);
2054 case_fff(add, _f);
2055 case_ffw(add, _f, 32);
2056 case_fff(sub, _f);
2057 case_ffw(sub, _f, 32);
2058 case_ffw(rsb, _f, 32);
2059 case_fff(mul, _f);
2060 case_ffw(mul, _f, 32);
2061 case_fff(div, _f);
2062 case_ffw(div, _f, 32);
2063 case_ff(abs, _f);
2064 case_ff(neg, _f);
2065 case_ff(sqrt, _f);
ba86ff93
PC		 2066 case_rqr(fma, _f);
2067 case_rqr(fms, _f);
2068 case_rqr(fnma, _f);
2069 case_rqr(fnms, _f);
4a71579b
PC		 2070 case_fr(ext, _f);
2071 case_fr(ext, _d_f);
2072 case_rff(lt, _f);
2073 case_rfw(lt, _f, 32);
2074 case_rff(le, _f);
2075 case_rfw(le, _f, 32);
2076 case_rff(eq, _f);
2077 case_rfw(eq, _f, 32);
2078 case_rff(ge, _f);
2079 case_rfw(ge, _f, 32);
2080 case_rff(gt, _f);
2081 case_rfw(gt, _f, 32);
2082 case_rff(ne, _f);
2083 case_rfw(ne, _f, 32);
2084 case_rff(unlt, _f);
2085 case_rfw(unlt, _f, 32);
2086 case_rff(unle, _f);
2087 case_rfw(unle, _f, 32);
2088 case_rff(uneq, _f);
2089 case_rfw(uneq, _f, 32);
2090 case_rff(unge, _f);
2091 case_rfw(unge, _f, 32);
2092 case_rff(ungt, _f);
2093 case_rfw(ungt, _f, 32);
2094 case_rff(ltgt, _f);
2095 case_rfw(ltgt, _f, 32);
2096 case_rff(ord, _f);
2097 case_rfw(ord, _f, 32);
2098 case_rff(unord, _f);
2099 case_rfw(unord, _f, 32);
2100 case jit_code_movr_f:
2101 if (jit_x87_reg_p(node->u.w)) {
2102 if (jit_x87_reg_p(node->v.w))
2103 x87_movr_f(rn(node->u.w), rn(node->v.w));
2104 else
2105 x87_from_sse_f(rn(node->u.w), rn(node->v.w));
2106 }
2107 else {
2108 if (jit_sse_reg_p(node->v.w))
2109 sse_movr_f(rn(node->u.w), rn(node->v.w));
2110 else
2111 sse_from_x87_f(rn(node->u.w), rn(node->v.w));
2112 }
2113 break;
2114 case jit_code_movi_f:
2115 assert(node->flag & jit_flag_data);
2116 if (jit_x87_reg_p(node->u.w))
2117 x87_movi_f(rn(node->u.w), (jit_float32_t *)node->v.n->u.w);
2118 else
2119 sse_movi_f(rn(node->u.w), (jit_float32_t *)node->v.n->u.w);
2120 break;
2121 case_fr(ld, _f);
2122 case_fw(ld, _f);
2123 case_frr(ldx, _f);
2124 case_frw(ldx, _f);
ba86ff93
PC		 2125 case jit_code_unldr_x:
2126 if (jit_x87_reg_p(node->u.w))
2127 x87_unldr_x(rn(node->u.w), rn(node->v.w), node->w.w);
2128 else
2129 sse_unldr_x(rn(node->u.w), rn(node->v.w), node->w.w);
2130 break;
2131 case jit_code_unldi_x:
2132 if (jit_x87_reg_p(node->u.w))
2133 x87_unldi_x(rn(node->u.w), node->v.w, node->w.w);
2134 else
2135 sse_unldi_x(rn(node->u.w), node->v.w, node->w.w);
2136 break;
4a71579b
PC		 2137 case_rf(st, _f);
2138 case_wf(st, _f);
2139 case_rrf(stx, _f);
2140 case_wrf(stx, _f);
ba86ff93
PC		 2141 case jit_code_unstr_x:
2142 if (jit_x87_reg_p(node->v.w))
2143 x87_unstr_x(rn(node->u.w), rn(node->v.w), node->w.w);
2144 else
2145 sse_unstr_x(rn(node->u.w), rn(node->v.w), node->w.w);
2146 break;
2147 case jit_code_unsti_x:
2148 if (jit_x87_reg_p(node->v.w))
2149 x87_unsti_x(node->u.w, rn(node->v.w), node->w.w);
2150 else
2151 sse_unsti_x(node->u.w, rn(node->v.w), node->w.w);
2152 break;
4a71579b
PC		 2153 case_bff(lt, _f);
2154 case_bfw(lt, _f, 32);
2155 case_bff(le, _f);
2156 case_bfw(le, _f, 32);
2157 case_bff(eq, _f);
2158 case_bfw(eq, _f, 32);
2159 case_bff(ge, _f);
2160 case_bfw(ge, _f, 32);
2161 case_bff(gt, _f);
2162 case_bfw(gt, _f, 32);
2163 case_bff(ne, _f);
2164 case_bfw(ne, _f, 32);
2165 case_bff(unlt, _f);
2166 case_bfw(unlt, _f, 32);
2167 case_bff(unle, _f);
2168 case_bfw(unle, _f, 32);
2169 case_bff(uneq, _f);
2170 case_bfw(uneq, _f, 32);
2171 case_bff(unge, _f);
2172 case_bfw(unge, _f, 32);
2173 case_bff(ungt, _f);
2174 case_bfw(ungt, _f, 32);
2175 case_bff(ltgt, _f);
2176 case_bfw(ltgt, _f, 32);
2177 case_bff(ord, _f);
2178 case_bfw(ord, _f, 32);
2179 case_bff(unord, _f);
2180 case_bfw(unord, _f, 32);
2181 case_fff(add, _d);
2182 case_ffw(add, _d, 64);
2183 case_fff(sub, _d);
2184 case_ffw(sub, _d, 64);
2185 case_ffw(rsb, _d, 64);
2186 case_fff(mul, _d);
2187 case_ffw(mul, _d, 64);
2188 case_fff(div, _d);
2189 case_ffw(div, _d, 64);
2190 case_ff(abs, _d);
2191 case_ff(neg, _d);
2192 case_ff(sqrt, _d);
ba86ff93
PC		 2193 case_rqr(fma, _d);
2194 case_rqr(fms, _d);
2195 case_rqr(fnma, _d);
2196 case_rqr(fnms, _d);
4a71579b
PC		 2197 case_fr(ext, _d);
2198 case_fr(ext, _f_d);
2199 case_rff(lt, _d);
2200 case_rfw(lt, _d, 64);
2201 case_rff(le, _d);
2202 case_rfw(le, _d, 64);
2203 case_rff(eq, _d);
2204 case_rfw(eq, _d, 64);
2205 case_rff(ge, _d);
2206 case_rfw(ge, _d, 64);
2207 case_rff(gt, _d);
2208 case_rfw(gt, _d, 64);
2209 case_rff(ne, _d);
2210 case_rfw(ne, _d, 64);
2211 case_rff(unlt, _d);
2212 case_rfw(unlt, _d, 64);
2213 case_rff(unle, _d);
2214 case_rfw(unle, _d, 64);
2215 case_rff(uneq, _d);
2216 case_rfw(uneq, _d, 64);
2217 case_rff(unge, _d);
2218 case_rfw(unge, _d, 64);
2219 case_rff(ungt, _d);
2220 case_rfw(ungt, _d, 64);
2221 case_rff(ltgt, _d);
2222 case_rfw(ltgt, _d, 64);
2223 case_rff(ord, _d);
2224 case_rfw(ord, _d, 64);
2225 case_rff(unord, _d);
2226 case_rfw(unord, _d, 64);
2227 case jit_code_movr_d:
2228 if (jit_x87_reg_p(node->u.w)) {
2229 if (jit_x87_reg_p(node->v.w))
2230 x87_movr_d(rn(node->u.w), rn(node->v.w));
2231 else
2232 x87_from_sse_d(rn(node->u.w), rn(node->v.w));
2233 }
2234 else {
2235 if (jit_sse_reg_p(node->v.w))
2236 sse_movr_d(rn(node->u.w), rn(node->v.w));
2237 else
2238 sse_from_x87_d(rn(node->u.w), rn(node->v.w));
2239 }
2240 break;
2241 case jit_code_movi_d:
2242 assert(node->flag & jit_flag_data);
2243 if (jit_x87_reg_p(node->u.w))
2244 x87_movi_d(rn(node->u.w), (jit_float64_t *)node->v.n->u.w);
2245 else
2246 sse_movi_d(rn(node->u.w), (jit_float64_t *)node->v.n->u.w);
2247 break;
2248 case_fr(ld, _d);
2249 case_fw(ld, _d);
2250 case_frr(ldx, _d);
2251 case_frw(ldx, _d);
2252 case_rf(st, _d);
2253 case_wf(st, _d);
2254 case_rrf(stx, _d);
2255 case_wrf(stx, _d);
2256 case_bff(lt, _d);
2257 case_bfw(lt, _d, 64);
2258 case_bff(le, _d);
2259 case_bfw(le, _d, 64);
2260 case_bff(eq, _d);
2261 case_bfw(eq, _d, 64);
2262 case_bff(ge, _d);
2263 case_bfw(ge, _d, 64);
2264 case_bff(gt, _d);
2265 case_bfw(gt, _d, 64);
2266 case_bff(ne, _d);
2267 case_bfw(ne, _d, 64);
2268 case_bff(unlt, _d);
2269 case_bfw(unlt, _d, 64);
2270 case_bff(unle, _d);
2271 case_bfw(unle, _d, 64);
2272 case_bff(uneq, _d);
2273 case_bfw(uneq, _d, 64);
2274 case_bff(unge, _d);
2275 case_bfw(unge, _d, 64);
2276 case_bff(ungt, _d);
2277 case_bfw(ungt, _d, 64);
2278 case_bff(ltgt, _d);
2279 case_bfw(ltgt, _d, 64);
2280 case_bff(ord, _d);
2281 case_bfw(ord, _d, 64);
2282 case_bff(unord, _d);
2283 case_bfw(unord, _d, 64);
2284 case jit_code_jmpr:
79bfeef6 2285 jit_check_frame();
4a71579b
PC		 2286 jmpr(rn(node->u.w));
2287 break;
2288 case jit_code_jmpi:
2289 if (node->flag & jit_flag_node) {
2290 temp = node->u.n;
2291 assert(temp->code == jit_code_label ||
2292 temp->code == jit_code_epilog);
2293 if (temp->flag & jit_flag_patch)
2294 jmpi(temp->u.w);
2295 else {
79bfeef6
PC		 2296#if __X64
2297 word = _jit->code.length -
2298 (_jit->pc.uc - _jit->code.ptr);
2299 if ((jit_int32_t)word == word)
2300 word = jmpi(_jit->pc.w);
2301 else
2302#endif
2303 word = jmpi_p(_jit->pc.w);
4a71579b
PC		 2304 patch(word, node);
2305 }
2306 }
79bfeef6
PC		 2307 else {
2308 jit_check_frame();
4a71579b 2309 jmpi(node->u.w);
79bfeef6 2310 }
4a71579b
PC		 2311 break;
2312 case jit_code_callr:
79bfeef6 2313 jit_check_frame();
4a71579b
PC		 2314 callr(rn(node->u.w));
2315 break;
2316 case jit_code_calli:
2317 if (node->flag & jit_flag_node) {
2318 temp = node->u.n;
2319 assert(temp->code == jit_code_label ||
2320 temp->code == jit_code_epilog);
519a9ea1
PC		 2321 if (temp->flag & jit_flag_patch)
2322 calli(temp->u.w);
2323 else {
79bfeef6
PC		 2324#if __X64
2325 word = _jit->code.length -
2326 (_jit->pc.uc - _jit->code.ptr);
2327 if ((jit_int32_t)word == word)
2328 word = calli(_jit->pc.w);
2329 else
2330#endif
2331 word = calli_p(_jit->pc.w);
4a71579b 2332 patch(word, node);
519a9ea1 2333 }
4a71579b 2334 }
79bfeef6
PC		 2335 else {
2336 jit_check_frame();
4a71579b 2337 calli(node->u.w);
79bfeef6 2338 }
4a71579b
PC		 2339 break;
2340 case jit_code_prolog:
2341 _jitc->function = _jitc->functions.ptr + node->w.w;
2342 undo.node = node;
2343 undo.word = _jit->pc.w;
79bfeef6 2344 memcpy(&undo.func, _jitc->function, sizeof(undo.func));
4a71579b
PC		 2345#if DEVEL_DISASSEMBLER
2346 undo.prevw = prevw;
2347#endif
2348 undo.patch_offset = _jitc->patches.offset;
2349 restart_function:
79bfeef6
PC		 2350 compute_framesize();
2351 patch_alist(0);
4a71579b
PC		 2352 _jitc->again = 0;
2353 prolog(node);
2354 break;
2355 case jit_code_epilog:
2356 assert(_jitc->function == _jitc->functions.ptr + node->w.w);
2357 if (_jitc->again) {
2358 for (temp = undo.node->next;
2359 temp != node; temp = temp->next) {
2360 if (temp->code == jit_code_label ||
2361 temp->code == jit_code_epilog)
2362 temp->flag &= ~jit_flag_patch;
2363 }
2364 temp->flag &= ~jit_flag_patch;
2365 node = undo.node;
2366 _jit->pc.w = undo.word;
79bfeef6
PC		 2367 /* undo.func.self.aoff and undo.func.regset should not
2368 * be undone, as they will be further updated, and are
2369 * the reason of the undo. */
2370 undo.func.self.aoff = _jitc->function->frame +
2371 _jitc->function->self.aoff;
2372 undo.func.need_frame = _jitc->function->need_frame;
2373 jit_regset_set(&undo.func.regset, &_jitc->function->regset);
2374 /* allocar information also does not need to be undone */
2375 undo.func.aoffoff = _jitc->function->aoffoff;
2376 undo.func.allocar = _jitc->function->allocar;
2377 /* real stack framesize is not in the jit_function_t,
2378 * if it were, would need to not be undone */
2379 /* cvt_offset must also not be undone */
2380 undo.func.cvt_offset = _jitc->function->cvt_offset;
2381 /* this will be recomputed but undo anyway to have it
2382 * better self documented.*/
2383 undo.func.need_stack = _jitc->function->need_stack;
2384 memcpy(_jitc->function, &undo.func, sizeof(undo.func));
4a71579b
PC		 2385#if DEVEL_DISASSEMBLER
2386 prevw = undo.prevw;
2387#endif
2388 _jitc->patches.offset = undo.patch_offset;
79bfeef6 2389 patch_alist(1);
4a71579b
PC		 2390 goto restart_function;
2391 }
2392 if (node->link &&
2393 (word = _jit->pc.w & (sizeof(jit_word_t) - 1)))
2394 nop(sizeof(jit_word_t) - word);
2395 /* remember label is defined */
2396 node->flag |= jit_flag_patch;
2397 node->u.w = _jit->pc.w;
2398 epilog(node);
2399 _jitc->function = NULL;
2400 break;
ba86ff93
PC		 2401 case jit_code_movr_w_f:
2402 if (jit_sse_reg_p(node->u.w))
2403 sse_movr_w_f(rn(node->u.w), rn(node->v.w));
2404 else
2405 x87_movr_w_f(rn(node->u.w), rn(node->v.w));
2406 break;
2407 case jit_code_movr_f_w:
2408 if (jit_sse_reg_p(node->v.w))
2409 sse_movr_f_w(rn(node->u.w), rn(node->v.w));
2410 else
2411 x87_movr_f_w(rn(node->u.w), rn(node->v.w));
2412 break;
2413 case jit_code_movi_f_w:
2414 assert(node->flag & jit_flag_data);
2415 movi_f_w(rn(node->u.w), *(jit_float32_t *)node->v.n->u.w);
2416 break;
2417 case jit_code_movi_w_f:
2418 if (jit_sse_reg_p(node->u.w))
2419 sse_movi_w_f(rn(node->u.w), node->v.w);
2420 else
2421 x87_movi_w_f(rn(node->u.w), node->v.w);
2422 break;
2423# if __X32 || __X64_32
2424 case jit_code_movr_ww_d:
2425 if (jit_sse_reg_p(node->u.w))
2426 sse_movr_ww_d(rn(node->u.w), rn(node->v.w), rn(node->w.w));
2427 else
2428 x87_movr_ww_d(rn(node->u.w), rn(node->v.w), rn(node->w.w));
2429 break;
2430 case jit_code_movr_d_ww:
2431 if (jit_sse_reg_p(node->w.w))
2432 sse_movr_d_ww(rn(node->u.w), rn(node->v.w), rn(node->w.w));
2433 else
2434 x87_movr_d_ww(rn(node->u.w), rn(node->v.w), rn(node->w.w));
2435 break;
2436 case jit_code_movi_d_ww:
2437 assert(node->flag & jit_flag_data);
2438 movi_d_ww(rn(node->u.w), rn(node->v.w),
2439 *(jit_float64_t *)node->w.n->u.w);
2440 break;
2441 case jit_code_movi_ww_d:
2442 if (jit_sse_reg_p(node->u.w))
2443 sse_movi_ww_d(rn(node->u.w), node->v.w, node->w.w);
2444 else
2445 x87_movi_ww_d(rn(node->u.w), node->v.w, node->w.w);
2446 break;
2447# else
2448 case jit_code_movr_w_d:
2449 if (jit_sse_reg_p(node->u.w))
2450 sse_movr_w_d(rn(node->u.w), rn(node->v.w));
2451 else
2452 x87_movr_w_d(rn(node->u.w), rn(node->v.w));
2453 break;
2454 case jit_code_movr_d_w:
2455 if (jit_sse_reg_p(node->v.w))
2456 sse_movr_d_w(rn(node->u.w), rn(node->v.w));
2457 else
2458 x87_movr_d_w(rn(node->u.w), rn(node->v.w));
2459 break;
2460 case jit_code_movi_d_w:
2461 assert(node->flag & jit_flag_data);
2462 movi_d_w(rn(node->u.w), *(jit_float64_t *)node->v.n->u.w);
2463 break;
2464 case jit_code_movi_w_d:
2465 if (jit_sse_reg_p(node->u.w))
2466 sse_movi_w_d(rn(node->u.w), node->v.w);
2467 else
2468 x87_movi_w_d(rn(node->u.w), node->v.w);
2469 break;
2470# endif
4a71579b
PC		 2471 case jit_code_va_start:
2472 vastart(rn(node->u.w));
2473 break;
2474 case jit_code_va_arg:
2475 vaarg(rn(node->u.w), rn(node->v.w));
2476 break;
2477 case jit_code_va_arg_d:
2478 vaarg_d(rn(node->u.w), rn(node->v.w), jit_x87_reg_p(node->u.w));
2479 break;
2480 case jit_code_live: case jit_code_ellipsis:
2481 case jit_code_va_push:
2482 case jit_code_allocai: case jit_code_allocar:
79bfeef6
PC		 2483 case jit_code_arg_c: case jit_code_arg_s:
2484 case jit_code_arg_i:
2485# if __WORDSIZE == 64
2486 case jit_code_arg_l:
2487# endif
4a71579b
PC		 2488 case jit_code_arg_f: case jit_code_arg_d:
2489 case jit_code_va_end:
2490 case jit_code_ret:
79bfeef6
PC		 2491 case jit_code_retr_c: case jit_code_reti_c:
2492 case jit_code_retr_uc: case jit_code_reti_uc:
2493 case jit_code_retr_s: case jit_code_reti_s:
2494 case jit_code_retr_us: case jit_code_reti_us:
2495 case jit_code_retr_i: case jit_code_reti_i:
2496#if __WORDSIZE == 64
2497 case jit_code_retr_ui: case jit_code_reti_ui:
2498 case jit_code_retr_l: case jit_code_reti_l:
2499#endif
4a71579b
PC		 2500 case jit_code_retr_f: case jit_code_reti_f:
2501 case jit_code_retr_d: case jit_code_reti_d:
2502 case jit_code_getarg_c: case jit_code_getarg_uc:
2503 case jit_code_getarg_s: case jit_code_getarg_us:
2504 case jit_code_getarg_i:
2505#if __X64 && !__X64_32
2506 case jit_code_getarg_ui: case jit_code_getarg_l:
2507#endif
2508 case jit_code_getarg_f: case jit_code_getarg_d:
79bfeef6
PC		 2509 case jit_code_putargr_c: case jit_code_putargi_c:
2510 case jit_code_putargr_uc: case jit_code_putargi_uc:
2511 case jit_code_putargr_s: case jit_code_putargi_s:
2512 case jit_code_putargr_us: case jit_code_putargi_us:
2513 case jit_code_putargr_i: case jit_code_putargi_i:
2514#if __WORDSIZE == 64
2515 case jit_code_putargr_ui: case jit_code_putargi_ui:
2516 case jit_code_putargr_l: case jit_code_putargi_l:
2517#endif
4a71579b
PC		 2518 case jit_code_putargr_f: case jit_code_putargi_f:
2519 case jit_code_putargr_d: case jit_code_putargi_d:
79bfeef6
PC		 2520 case jit_code_pushargr_c: case jit_code_pushargi_c:
2521 case jit_code_pushargr_uc: case jit_code_pushargi_uc:
2522 case jit_code_pushargr_s: case jit_code_pushargi_s:
2523 case jit_code_pushargr_us: case jit_code_pushargi_us:
2524 case jit_code_pushargr_i: case jit_code_pushargi_i:
2525#if __WORDSIZE == 64
2526 case jit_code_pushargr_ui: case jit_code_pushargi_ui:
2527 case jit_code_pushargr_l: case jit_code_pushargi_l:
2528#endif
4a71579b
PC		 2529 case jit_code_pushargr_f: case jit_code_pushargi_f:
2530 case jit_code_pushargr_d: case jit_code_pushargi_d:
2531 case jit_code_retval_c: case jit_code_retval_uc:
2532 case jit_code_retval_s: case jit_code_retval_us:
2533 case jit_code_retval_i:
2534#if __X64 && !__X32
2535 case jit_code_retval_ui: case jit_code_retval_l:
2536#endif
2537 case jit_code_prepare:
2538 case jit_code_finishr: case jit_code_finishi:
ba86ff93
PC		 2539 case jit_code_negi_f: case jit_code_absi_f:
2540 case jit_code_sqrti_f: case jit_code_negi_d:
2541 case jit_code_absi_d: case jit_code_sqrti_d:
2542 case jit_code_fmai_f: case jit_code_fmsi_f:
2543 case jit_code_fmai_d: case jit_code_fmsi_d:
2544 case jit_code_fnmai_f: case jit_code_fnmsi_f:
2545 case jit_code_fnmai_d: case jit_code_fnmsi_d:
4a71579b
PC		 2546 break;
2547 case jit_code_retval_f:
2548#if __X32
2549 if (jit_sse_reg_p(node->u.w)) {
2550 fstpr(_ST1_REGNO);
2551 sse_from_x87_f(rn(node->u.w), _ST0_REGNO);
2552 }
2553 else
2554 fstpr(rn(node->u.w) + 1);
2555#endif
2556 break;
2557 case jit_code_retval_d:
2558#if __X32
2559 if (jit_sse_reg_p(node->u.w)) {
2560 fstpr(_ST1_REGNO);
2561 sse_from_x87_d(rn(node->u.w), _ST0_REGNO);
2562 }
2563 else
2564 fstpr(rn(node->u.w) + 1);
2565#endif
2566 break;
ba86ff93
PC		 2567 case jit_code_negi:
2568 negi(rn(node->u.w), node->v.w);
2569 break;
2570 case jit_code_comi:
2571 comi(rn(node->u.w), node->v.w);
2572 break;
2573 case jit_code_exti_c:
2574 exti_c(rn(node->u.w), node->v.w);
2575 break;
2576 case jit_code_exti_uc:
2577 exti_uc(rn(node->u.w), node->v.w);
2578 break;
2579 case jit_code_exti_s:
2580 exti_s(rn(node->u.w), node->v.w);
2581 break;
2582 case jit_code_exti_us:
2583 exti_us(rn(node->u.w), node->v.w);
2584 break;
2585 case jit_code_bswapi_us:
2586 bswapi_us(rn(node->u.w), node->v.w);
2587 break;
2588 case jit_code_bswapi_ui:
2589 bswapi_ui(rn(node->u.w), node->v.w);
2590 break;
2591 case jit_code_htoni_us:
2592 htoni_us(rn(node->u.w), node->v.w);
2593 break;
2594 case jit_code_htoni_ui:
2595 htoni_ui(rn(node->u.w), node->v.w);
2596 break;
2597#if __X64 && !__X64_32
2598 case jit_code_exti_i:
2599 exti_i(rn(node->u.w), node->v.w);
2600 break;
2601 case jit_code_exti_ui:
2602 exti_ui(rn(node->u.w), node->v.w);
2603 break;
2604 case jit_code_bswapi_ul:
2605 bswapi_ul(rn(node->u.w), node->v.w);
2606 break;
2607 case jit_code_htoni_ul:
2608 htoni_ul(rn(node->u.w), node->v.w);
2609 break;
2610#endif
2611 case jit_code_cloi:
2612 cloi(rn(node->u.w), node->v.w);
2613 break;
2614 case jit_code_clzi:
2615 clzi(rn(node->u.w), node->v.w);
2616 break;
2617 case jit_code_ctoi:
2618 ctoi(rn(node->u.w), node->v.w);
2619 break;
2620 case jit_code_ctzi:
2621 ctzi(rn(node->u.w), node->v.w);
2622 break;
2623 case jit_code_rbiti:
2624 rbiti(rn(node->u.w), node->v.w);
2625 break;
2626 case jit_code_popcnti:
2627 popcnti(rn(node->u.w), node->v.w);
2628 break;
2629 case jit_code_exti:
2630 exti(rn(node->u.w), node->v.w, node->w.q.l, node->w.q.h);
2631 break;
2632 case jit_code_exti_u:
2633 exti_u(rn(node->u.w), node->v.w, node->w.q.l, node->w.q.h);
2634 break;
4a71579b
PC		 2635 default:
2636 abort();
2637 }
2638 jit_regarg_clr(node, value);
2639 assert(_jitc->regarg == 0 && _jitc->synth == 0);
2640 /* update register live state */
2641 jit_reglive(node);
2642 }
2643#undef case_bfw
2644#undef case_bff
2645#undef case_ffw
2646#undef case_rfw
2647#undef case_rff
2648#undef case_brw
2649#undef case_brr
2650#undef case_wrf
2651#undef case_wrr
2652#undef case_frw
2653#undef case_rrf
2654#undef case_rrw
2655#undef case_frr
2656#undef case_rrr
2657#undef case_wf
2658#undef case_fw
2659#undef case_fr
2660#undef case_rr
2661
2662 for (offset = 0; offset < _jitc->patches.offset; offset++) {
2663 node = _jitc->patches.ptr[offset].node;
2664 word = node->code == jit_code_movi ? node->v.n->u.w : node->u.n->u.w;
79bfeef6 2665 patch_at(_jitc->patches.ptr[offset].inst, word);
4a71579b
PC		 2666 }
2667
2668 jit_flush(_jit->code.ptr, _jit->pc.uc);
2669
2670 return (_jit->code.ptr);
2671}
2672
2673#define CODE 1
2674# include "jit_x86-cpu.c"
2675# include "jit_x86-sse.c"
2676# include "jit_x86-x87.c"
2677#undef CODE
2678
2679void
2680jit_flush(void *fptr, void *tptr)
2681{
2682}
2683
2684void
2685_emit_ldxi(jit_state_t *_jit, jit_gpr_t r0, jit_gpr_t r1, jit_word_t i0)
2686{
2687 ldxi(rn(r0), rn(r1), i0);
2688}
2689
2690void
2691_emit_stxi(jit_state_t *_jit, jit_word_t i0, jit_gpr_t r0, jit_gpr_t r1)
2692{
2693 stxi(i0, rn(r0), rn(r1));
2694}
2695
2696void
2697_emit_ldxi_d(jit_state_t *_jit, jit_fpr_t r0, jit_gpr_t r1, jit_word_t i0)
2698{
2699 if (jit_x87_reg_p(r0))
2700 x87_ldxi_d(rn(r0), rn(r1), i0);
2701 else
2702 sse_ldxi_d(rn(r0), rn(r1), i0);
2703}
2704
2705void
2706_emit_stxi_d(jit_state_t *_jit, jit_word_t i0, jit_gpr_t r0, jit_fpr_t r1)
2707{
2708 if (jit_x87_reg_p(r1))
2709 x87_stxi_d(i0, rn(r0), rn(r1));
2710 else
2711 sse_stxi_d(i0, rn(r0), rn(r1));
2712}
2713
79bfeef6
PC		 2714static void
2715_compute_framesize(jit_state_t *_jit)
2716{
2717 jit_int32_t reg;
2718 /* Save stack pointer in first slot */
2719 _jitc->framesize = REAL_WORDSIZE;
2720 for (reg = 0; reg < jit_size(iregs); reg++)
2721 if (jit_regset_tstbit(&_jitc->function->regset, iregs[reg]))
2722 _jitc->framesize += REAL_WORDSIZE;
2723
2724#if __X64 && (__CYGWIN__ || _WIN32)
2725 for (reg = 0; reg < jit_size(fregs); reg++)
2726 if (jit_regset_tstbit(&_jitc->function->regset, fregs[reg]))
2727 _jitc->framesize += sizeof(jit_float64_t);
2728#endif
2729 /* Make sure functions called have a 16 byte aligned stack */
2730 _jitc->framesize = (_jitc->framesize + 15) & -16;
2731 _jitc->framesize += 16 - REAL_WORDSIZE;
2732}
2733
4a71579b
PC		 2734static void
2735_patch(jit_state_t *_jit, jit_word_t instr, jit_node_t *node)
2736{
2737 jit_int32_t flag;
2738
2739 assert(node->flag & jit_flag_node);
2740 if (node->code == jit_code_movi)
2741 flag = node->v.n->flag;
2742 else
2743 flag = node->u.n->flag;
2744 assert(!(flag & jit_flag_patch));
2745 if (_jitc->patches.offset >= _jitc->patches.length) {
2746 jit_realloc((jit_pointer_t *)&_jitc->patches.ptr,
2747 _jitc->patches.length * sizeof(jit_patch_t),
2748 (_jitc->patches.length + 1024) * sizeof(jit_patch_t));
2749 _jitc->patches.length += 1024;
2750 }
2751 _jitc->patches.ptr[_jitc->patches.offset].inst = instr;
2752 _jitc->patches.ptr[_jitc->patches.offset].node = node;
2753 ++_jitc->patches.offset;
2754}
2755
2756static void
2757_sse_from_x87_f(jit_state_t *_jit, jit_int32_t r0, jit_int32_t r1)
2758{
79bfeef6 2759 CHECK_CVT_OFFSET();
4a71579b
PC		 2760 x87_stxi_f(CVT_OFFSET, _RBP_REGNO, r1);
2761 sse_ldxi_f(r0, _RBP_REGNO, CVT_OFFSET);
2762}
2763
2764static void
2765_sse_from_x87_d(jit_state_t *_jit, jit_int32_t r0, jit_int32_t r1)
2766{
79bfeef6 2767 CHECK_CVT_OFFSET();
4a71579b
PC		 2768 x87_stxi_d(CVT_OFFSET, _RBP_REGNO, r1);
2769 sse_ldxi_d(r0, _RBP_REGNO, CVT_OFFSET);
2770}
2771
2772static void
2773_x87_from_sse_f(jit_state_t *_jit, jit_int32_t r0, jit_int32_t r1)
2774{
79bfeef6 2775 CHECK_CVT_OFFSET();
4a71579b
PC		 2776 sse_stxi_f(CVT_OFFSET, _RBP_REGNO, r1);
2777 x87_ldxi_f(r0, _RBP_REGNO, CVT_OFFSET);
2778}
2779
2780static void
2781_x87_from_sse_d(jit_state_t *_jit, jit_int32_t r0, jit_int32_t r1)
2782{
79bfeef6 2783 CHECK_CVT_OFFSET();
4a71579b
PC		 2784 sse_stxi_d(CVT_OFFSET, _RBP_REGNO, r1);
2785 x87_ldxi_d(r0, _RBP_REGNO, CVT_OFFSET);
2786}
ARM optimized PCSX fork