2 * Copyright (C) 2012-2019 Free Software Foundation, Inc.
4 * This file is part of GNU lightning.
6 * GNU lightning is free software; you can redistribute it and/or modify it
7 * under the terms of the GNU Lesser General Public License as published
8 * by the Free Software Foundation; either version 3, or (at your option)
11 * GNU lightning is distributed in the hope that it will be useful, but
12 * WITHOUT ANY WARRANTY; without even the implied warranty of MERCHANTABILITY
13 * or FITNESS FOR A PARTICULAR PURPOSE. See the GNU Lesser General Public
14 * License for more details.
17 * Paulo Cesar Pereira de Andrade
20 #include <lightning.h>
21 #include <lightning/jit_private.h>
24 # define jit_arg_reg_p(i) 0
25 # define jit_arg_f_reg_p(i) 0
26 # define stack_framesize 20
27 # define stack_adjust 12
28 # define CVT_OFFSET -12
29 # define REAL_WORDSIZE 4
30 # define va_gp_increment 4
31 # define va_fp_increment 8
33 # if __CYGWIN__ || _WIN32
34 # define jit_arg_reg_p(i) ((i) >= 0 && (i) < 4)
35 # define jit_arg_f_reg_p(i) jit_arg_reg_p(i)
36 # define stack_framesize 152
37 # define va_fp_increment 8
39 # define jit_arg_reg_p(i) ((i) >= 0 && (i) < 6)
40 # define jit_arg_f_reg_p(i) ((i) >= 0 && (i) < 8)
41 # define stack_framesize 56
42 # define first_gp_argument rdi
43 # define first_gp_offset offsetof(jit_va_list_t, rdi)
44 # define first_gp_from_offset(gp) ((gp) / 8)
45 # define last_gp_argument r9
46 # define va_gp_max_offset \
47 (offsetof(jit_va_list_t, r9) - offsetof(jit_va_list_t, rdi) + 8)
48 # define first_fp_argument xmm0
49 # define first_fp_offset offsetof(jit_va_list_t, xmm0)
50 # define last_fp_argument xmm7
51 # define va_fp_max_offset \
52 (offsetof(jit_va_list_t, xmm7) - offsetof(jit_va_list_t, rdi) + 16)
53 # define va_fp_increment 16
54 # define first_fp_from_offset(fp) (((fp) - va_gp_max_offset) / 16)
56 # define va_gp_increment 8
57 # define stack_adjust 8
58 # define CVT_OFFSET -8
59 # define REAL_WORDSIZE 8
65 #if __X32 || __CYGWIN__ || _WIN32
66 typedef jit_pointer_t jit_va_list_t;
68 typedef struct jit_va_list {
73 /* Declared explicitly as int64 for the x32 abi */
102 #define patch(instr, node) _patch(_jit, instr, node)
103 static void _patch(jit_state_t*,jit_word_t,jit_node_t*);
104 #define sse_from_x87_f(r0, r1) _sse_from_x87_f(_jit, r0, r1)
105 static void _sse_from_x87_f(jit_state_t*,jit_int32_t,jit_int32_t);
106 #define sse_from_x87_d(r0, r1) _sse_from_x87_d(_jit, r0, r1)
107 static void _sse_from_x87_d(jit_state_t*,jit_int32_t,jit_int32_t);
108 #define x87_from_sse_f(r0, r1) _x87_from_sse_f(_jit, r0, r1)
109 static void _x87_from_sse_f(jit_state_t*,jit_int32_t,jit_int32_t);
110 #define x87_from_sse_d(r0, r1) _x87_from_sse_d(_jit, r0, r1)
111 static void _x87_from_sse_d(jit_state_t*,jit_int32_t,jit_int32_t);
114 # include "jit_x86-cpu.c"
115 # include "jit_x86-sse.c"
116 # include "jit_x86-x87.c"
123 jit_register_t _rvs[] = {
125 { rc(gpr) | rc(rg8) | 0, "%eax" },
126 { rc(gpr) | rc(rg8) | 1, "%ecx" },
127 { rc(gpr) | rc(rg8) | 2, "%edx" },
128 { rc(sav) | rc(rg8) | rc(gpr) | 3, "%ebx" },
129 { rc(sav) | rc(gpr) | 6, "%esi" },
130 { rc(sav) | rc(gpr) | 7, "%edi" },
131 { rc(sav) | 4, "%esp" },
132 { rc(sav) | 5, "%ebp" },
133 { rc(xpr) | rc(fpr) | 0, "%xmm0" },
134 { rc(xpr) | rc(fpr) | 1, "%xmm1" },
135 { rc(xpr) | rc(fpr) | 2, "%xmm2" },
136 { rc(xpr) | rc(fpr) | 3, "%xmm3" },
137 { rc(xpr) | rc(fpr) | 4, "%xmm4" },
138 { rc(xpr) | rc(fpr) | 5, "%xmm5" },
139 { rc(xpr) | rc(fpr) | 6, "%xmm6" },
140 { rc(xpr) | rc(fpr) | 7, "%xmm7" },
141 { rc(fpr) | 0, "st(0)" },
142 { rc(fpr) | 1, "st(1)" },
143 { rc(fpr) | 2, "st(2)" },
144 { rc(fpr) | 3, "st(3)" },
145 { rc(fpr) | 4, "st(4)" },
146 { rc(fpr) | 5, "st(5)" },
147 { rc(fpr) | 6, "st(6)" },
148 { rc(fpr) | 7, "st(7)" },
150 # if __CYGWIN__ || _WIN32
151 { rc(gpr) | rc(rg8) | 0, "%rax" },
152 { rc(gpr) | rc(rg8) | rc(rg8) | 10, "%r10" },
153 { rc(gpr) | rc(rg8) | rc(rg8) | 11, "%r11" },
154 { rc(sav) | rc(rg8) | rc(gpr) | 3, "%rbx" },
155 { rc(sav) | rc(gpr) | 7, "%rdi" },
156 { rc(sav) | rc(gpr) | 6, "%rsi" },
157 { rc(sav) | rc(gpr) | 12, "%r12" },
158 { rc(sav) | rc(gpr) | 13, "%r13" },
159 { rc(sav) | rc(gpr) | 14, "%r14" },
160 { rc(sav) | rc(gpr) | 15, "%r15" },
161 { rc(arg) | rc(rg8) | rc(gpr) | 9, "%r9" },
162 { rc(arg) | rc(rg8) | rc(gpr) | 8, "%r8" },
163 { rc(arg) | rc(rg8) | rc(gpr) | 2, "%rdx" },
164 { rc(arg) | rc(rg8) | rc(gpr) | 1, "%rcx" },
165 { rc(sav) | 4, "%rsp" },
166 { rc(sav) | 5, "%rbp" },
167 { rc(xpr) | rc(fpr) | 4, "%xmm4" },
168 { rc(xpr) | rc(fpr) | 5, "%xmm5" },
169 { rc(sav) | rc(xpr) | rc(fpr) | 6, "%xmm6" },
170 { rc(sav) | rc(xpr) | rc(fpr) | 7, "%xmm7" },
171 { rc(sav) | rc(xpr) | rc(fpr) | 8, "%xmm8" },
172 { rc(sav) | rc(xpr) | rc(fpr) | 9, "%xmm9" },
173 { rc(sav) | rc(xpr) | rc(fpr) | 10, "%xmm10" },
174 { rc(sav) | rc(xpr) | rc(fpr) | 11, "%xmm11" },
175 { rc(sav) | rc(xpr) | rc(fpr) | 12, "%xmm12" },
176 { rc(sav) | rc(xpr) | rc(fpr) | 13, "%xmm13" },
177 { rc(sav) | rc(xpr) | rc(fpr) | 14, "%xmm14" },
178 { rc(sav) | rc(xpr) | rc(fpr) | 15, "%xmm15" },
179 { rc(xpr) | rc(arg) | rc(fpr) | 3, "%xmm3" },
180 { rc(xpr) | rc(arg) | rc(fpr) | 2, "%xmm2" },
181 { rc(xpr) | rc(arg) | rc(fpr) | 1, "%xmm1" },
182 { rc(xpr) | rc(arg) | rc(fpr) | 0, "%xmm0" },
184 /* %rax is a pseudo flag argument for varargs functions */
185 { rc(arg) | rc(gpr) | rc(rg8) | 0, "%rax" },
186 { rc(gpr) | rc(rg8) | 10, "%r10" },
187 { rc(gpr) | rc(rg8) | 11, "%r11" },
188 { rc(sav) | rc(rg8) | rc(gpr) | 3, "%rbx" },
189 { rc(sav) | rc(rg8) | rc(gpr) | 13, "%r13" },
190 { rc(sav) | rc(rg8) | rc(gpr) | 14, "%r14" },
191 { rc(sav) | rc(rg8) | rc(gpr) | 15, "%r15" },
192 { rc(sav) | rc(gpr) | rc(rg8) | 12, "%r12" },
193 { rc(arg) | rc(rg8) | rc(gpr) | 9, "%r9" },
194 { rc(arg) | rc(rg8) | rc(gpr) | 8, "%r8" },
195 { rc(arg) | rc(rg8) | rc(gpr) | 1, "%rcx" },
196 { rc(arg) | rc(rg8) | rc(gpr) | 2, "%rdx" },
197 { rc(arg) | rc(rg8) | rc(gpr) | 6, "%rsi" },
198 { rc(arg) | rc(rg8) | rc(gpr) | 7, "%rdi" },
199 { rc(sav) | 4, "%rsp" },
200 { rc(sav) | 5, "%rbp" },
201 { rc(xpr) | rc(fpr) | 8, "%xmm8" },
202 { rc(xpr) | rc(fpr) | 9, "%xmm9" },
203 { rc(xpr) | rc(fpr) | 10, "%xmm10" },
204 { rc(xpr) | rc(fpr) | 11, "%xmm11" },
205 { rc(xpr) | rc(fpr) | 12, "%xmm12" },
206 { rc(xpr) | rc(fpr) | 13, "%xmm13" },
207 { rc(xpr) | rc(fpr) | 14, "%xmm14" },
208 { rc(xpr) | rc(fpr) | 15, "%xmm15" },
209 { rc(xpr) | rc(arg) | rc(fpr) | 7, "%xmm7" },
210 { rc(xpr) | rc(arg) | rc(fpr) | 6, "%xmm6" },
211 { rc(xpr) | rc(arg) | rc(fpr) | 5, "%xmm5" },
212 { rc(xpr) | rc(arg) | rc(fpr) | 4, "%xmm4" },
213 { rc(xpr) | rc(arg) | rc(fpr) | 3, "%xmm3" },
214 { rc(xpr) | rc(arg) | rc(fpr) | 2, "%xmm2" },
215 { rc(xpr) | rc(arg) | rc(fpr) | 1, "%xmm1" },
216 { rc(xpr) | rc(arg) | rc(fpr) | 0, "%xmm0" },
218 { rc(fpr) | 0, "st(0)" },
219 { rc(fpr) | 1, "st(1)" },
220 { rc(fpr) | 2, "st(2)" },
221 { rc(fpr) | 3, "st(3)" },
222 { rc(fpr) | 4, "st(4)" },
223 { rc(fpr) | 5, "st(5)" },
224 { rc(fpr) | 6, "st(6)" },
225 { rc(fpr) | 7, "st(7)" },
227 { _NOREG, "<none>" },
238 jit_uint32_t sse3 : 1;
239 jit_uint32_t pclmulqdq : 1;
240 jit_uint32_t dtes64 : 1; /* amd reserved */
241 jit_uint32_t monitor : 1;
242 jit_uint32_t ds_cpl : 1; /* amd reserved */
243 jit_uint32_t vmx : 1; /* amd reserved */
244 jit_uint32_t smx : 1; /* amd reserved */
245 jit_uint32_t est : 1; /* amd reserved */
246 jit_uint32_t tm2 : 1; /* amd reserved */
247 jit_uint32_t ssse3 : 1;
248 jit_uint32_t cntx_id : 1; /* amd reserved */
249 jit_uint32_t __reserved0 : 1;
250 jit_uint32_t fma : 1;
251 jit_uint32_t cmpxchg16b : 1;
252 jit_uint32_t xtpr : 1; /* amd reserved */
253 jit_uint32_t pdcm : 1; /* amd reserved */
254 jit_uint32_t __reserved1 : 1;
255 jit_uint32_t pcid : 1; /* amd reserved */
256 jit_uint32_t dca : 1; /* amd reserved */
257 jit_uint32_t sse4_1 : 1;
258 jit_uint32_t sse4_2 : 1;
259 jit_uint32_t x2apic : 1; /* amd reserved */
260 jit_uint32_t movbe : 1; /* amd reserved */
261 jit_uint32_t popcnt : 1;
262 jit_uint32_t tsc : 1; /* amd reserved */
263 jit_uint32_t aes : 1;
264 jit_uint32_t xsave : 1;
265 jit_uint32_t osxsave : 1;
266 jit_uint32_t avx : 1;
267 jit_uint32_t __reserved2 : 1; /* amd F16C */
268 jit_uint32_t __reserved3 : 1;
269 jit_uint32_t __alwayszero : 1; /* amd RAZ */
275 jit_uint32_t fpu : 1;
276 jit_uint32_t vme : 1;
278 jit_uint32_t pse : 1;
279 jit_uint32_t tsc : 1;
280 jit_uint32_t msr : 1;
281 jit_uint32_t pae : 1;
282 jit_uint32_t mce : 1;
283 jit_uint32_t cmpxchg8b : 1;
284 jit_uint32_t apic : 1;
285 jit_uint32_t __reserved0 : 1;
286 jit_uint32_t sep : 1;
287 jit_uint32_t mtrr : 1;
288 jit_uint32_t pge : 1;
289 jit_uint32_t mca : 1;
290 jit_uint32_t cmov : 1;
291 jit_uint32_t pat : 1;
292 jit_uint32_t pse36 : 1;
293 jit_uint32_t psn : 1; /* amd reserved */
294 jit_uint32_t clfsh : 1;
295 jit_uint32_t __reserved1 : 1;
296 jit_uint32_t ds : 1; /* amd reserved */
297 jit_uint32_t acpi : 1; /* amd reserved */
298 jit_uint32_t mmx : 1;
299 jit_uint32_t fxsr : 1;
300 jit_uint32_t sse : 1;
301 jit_uint32_t sse2 : 1;
302 jit_uint32_t ss : 1; /* amd reserved */
303 jit_uint32_t htt : 1;
304 jit_uint32_t tm : 1; /* amd reserved */
305 jit_uint32_t __reserved2 : 1;
306 jit_uint32_t pbe : 1; /* amd reserved */
313 jit_uword_t eax, ebx;
316 /* adapted from glibc __sysconf */
317 __asm__ volatile ("pushfl;\n\t"
319 "movl $0x240000, %1;\n\t"
328 : "=r" (flags), "=r" (ac));
330 /* i386 or i486 without cpuid */
331 if ((ac & (1 << 21)) == 0)
332 /* probably without x87 as well */
336 /* query %eax = 1 function */
337 #if __X32 || __X64_32
338 __asm__ volatile ("xchgl %%ebx, %1; cpuid; xchgl %%ebx, %1"
340 __asm__ volatile ("xchgq %%rbx, %1; cpuid; xchgq %%rbx, %1"
342 : "=a" (eax), "=r" (ebx),
343 "=c" (ecx.cpuid), "=d" (edx.cpuid)
346 jit_cpu.fpu = edx.bits.fpu;
347 jit_cpu.cmpxchg8b = edx.bits.cmpxchg8b;
348 jit_cpu.cmov = edx.bits.cmov;
349 jit_cpu.mmx = edx.bits.mmx;
350 jit_cpu.sse = edx.bits.sse;
351 jit_cpu.sse2 = edx.bits.sse2;
352 jit_cpu.sse3 = ecx.bits.sse3;
353 jit_cpu.pclmulqdq = ecx.bits.pclmulqdq;
354 jit_cpu.ssse3 = ecx.bits.ssse3;
355 jit_cpu.fma = ecx.bits.fma;
356 jit_cpu.cmpxchg16b = ecx.bits.cmpxchg16b;
357 jit_cpu.sse4_1 = ecx.bits.sse4_1;
358 jit_cpu.sse4_2 = ecx.bits.sse4_2;
359 jit_cpu.movbe = ecx.bits.movbe;
360 jit_cpu.popcnt = ecx.bits.popcnt;
361 jit_cpu.aes = ecx.bits.aes;
362 jit_cpu.avx = ecx.bits.avx;
364 /* query %eax = 0x80000001 function */
367 __asm__ volatile ("xchgl %%ebx, %1; cpuid; xchgl %%ebx, %1"
369 __asm__ volatile ("xchgq %%rbx, %1; cpuid; xchgq %%rbx, %1"
371 : "=a" (eax), "=r" (ebx),
372 "=c" (ecx.cpuid), "=d" (edx.cpuid)
374 jit_cpu.lahf = ecx.cpuid & 1;
379 _jit_init(jit_state_t *_jit)
383 static jit_bool_t first = 1;
386 _jitc->reglen = jit_size(_rvs) - 1;
390 for (regno = _jitc->reglen; regno >= 0; regno--) {
391 if (_rvs[regno].spec & jit_class_xpr)
392 _rvs[regno].spec = 0;
401 _jit_prolog(jit_state_t *_jit)
407 assert(jit_regset_cmp_ui(&_jitc->regarg, 0) == 0);
408 jit_regset_set_ui(&_jitc->regsav, 0);
409 offset = _jitc->functions.offset;
410 if (offset >= _jitc->functions.length) {
411 jit_realloc((jit_pointer_t *)&_jitc->functions.ptr,
412 _jitc->functions.length * sizeof(jit_function_t),
413 (_jitc->functions.length + 16) * sizeof(jit_function_t));
414 _jitc->functions.length += 16;
416 _jitc->function = _jitc->functions.ptr + _jitc->functions.offset++;
417 _jitc->function->self.size = stack_framesize;
418 _jitc->function->self.argi = _jitc->function->self.argf =
419 _jitc->function->self.aoff = _jitc->function->self.alen = 0;
420 /* sse/x87 conversion */
421 _jitc->function->self.aoff = CVT_OFFSET;
422 _jitc->function->self.call = jit_call_default;
423 jit_alloc((jit_pointer_t *)&_jitc->function->regoff,
424 _jitc->reglen * sizeof(jit_int32_t));
426 /* _no_link here does not mean the jit_link() call can be removed
428 * _jitc->function->prolog = jit_new_node(jit_code_prolog);
430 _jitc->function->prolog = jit_new_node_no_link(jit_code_prolog);
431 jit_link(_jitc->function->prolog);
432 _jitc->function->prolog->w.w = offset;
433 _jitc->function->epilog = jit_new_node_no_link(jit_code_epilog);
435 * v: offset in blocks vector
436 * w: offset in functions vector
438 _jitc->function->epilog->w.w = offset;
440 jit_regset_new(&_jitc->function->regset);
444 _jit_allocai(jit_state_t *_jit, jit_int32_t length)
446 assert(_jitc->function);
448 case 0: case 1: break;
449 case 2: _jitc->function->self.aoff &= -2; break;
450 case 3: case 4: _jitc->function->self.aoff &= -4; break;
451 default: _jitc->function->self.aoff &= -8; break;
453 _jitc->function->self.aoff -= length;
455 /* jit_allocai() may be called from jit_x86-cpu.c, and force a function
456 * generation restart on some conditions: div/rem and qmul/qdiv, due
457 * to registers constraints.
458 * The check is to prevent an assertion of a jit_xyz() being called
459 * during code generation, and attempting to add a node to the tail
460 * of the current IR generation. */
461 if (!_jitc->realize) {
462 jit_inc_synth_ww(allocai, _jitc->function->self.aoff, length);
466 return (_jitc->function->self.aoff);
470 _jit_allocar(jit_state_t *_jit, jit_int32_t u, jit_int32_t v)
473 assert(_jitc->function);
474 jit_inc_synth_ww(allocar, u, v);
475 if (!_jitc->function->allocar) {
476 _jitc->function->aoffoff = jit_allocai(sizeof(jit_int32_t));
477 _jitc->function->allocar = 1;
479 reg = jit_get_reg(jit_class_gpr);
481 jit_andi(reg, reg, -16);
482 jit_ldxi_i(u, JIT_FP, _jitc->function->aoffoff);
484 jit_addr(JIT_SP, JIT_SP, reg);
485 jit_stxi_i(_jitc->function->aoffoff, JIT_FP, u);
491 _jit_ret(jit_state_t *_jit)
494 assert(_jitc->function);
498 jit_patch_at(instr, _jitc->function->epilog);
503 _jit_retr(jit_state_t *_jit, jit_int32_t u)
505 jit_inc_synth_w(retr, u);
506 /* movr(%ret, %ret) would be optimized out */
508 jit_movr(JIT_RET, u);
509 /* explicitly tell it is live */
516 _jit_reti(jit_state_t *_jit, jit_word_t u)
518 jit_inc_synth_w(reti, u);
519 jit_movi(JIT_RET, u);
525 _jit_retr_f(jit_state_t *_jit, jit_int32_t u)
527 jit_inc_synth_w(retr_f, u);
529 jit_movr_f(JIT_FRET, u);
537 _jit_reti_f(jit_state_t *_jit, jit_float32_t u)
539 jit_inc_synth_f(reti_f, u);
540 jit_movi_f(JIT_FRET, u);
546 _jit_retr_d(jit_state_t *_jit, jit_int32_t u)
548 jit_inc_synth_w(retr_d, u);
550 jit_movr_d(JIT_FRET, u);
558 _jit_reti_d(jit_state_t *_jit, jit_float64_t u)
560 jit_inc_synth_d(reti_d, u);
561 jit_movi_d(JIT_FRET, u);
567 _jit_epilog(jit_state_t *_jit)
569 assert(_jitc->function);
570 assert(_jitc->function->epilog->next == NULL);
571 jit_link(_jitc->function->epilog);
572 _jitc->function = NULL;
576 _jit_arg_register_p(jit_state_t *_jit, jit_node_t *u)
578 if (u->code == jit_code_arg)
579 return (jit_arg_reg_p(u->u.w));
580 assert(u->code == jit_code_arg_f || u->code == jit_code_arg_d);
581 return (jit_arg_f_reg_p(u->u.w));
585 _jit_ellipsis(jit_state_t *_jit)
587 jit_inc_synth(ellipsis);
588 if (_jitc->prepare) {
590 /* Remember that a varargs function call is being constructed. */
591 assert(!(_jitc->function->call.call & jit_call_varargs));
592 _jitc->function->call.call |= jit_call_varargs;
596 /* Remember the current function is varargs. */
597 assert(!(_jitc->function->self.call & jit_call_varargs));
598 _jitc->function->self.call |= jit_call_varargs;
600 #if __X64 && !(__CYGWIN__ || _WIN32)
601 /* Allocate va_list like object in the stack.
602 * If applicable, with enough space to save all argument
603 * registers, and use fixed offsets for them. */
604 _jitc->function->vaoff = jit_allocai(sizeof(jit_va_list_t));
606 /* Initialize gp offset in save area. */
607 if (jit_arg_reg_p(_jitc->function->self.argi))
608 _jitc->function->vagp = _jitc->function->self.argi * 8;
610 _jitc->function->vagp = va_gp_max_offset;
612 /* Initialize fp offset in save area. */
613 if (jit_arg_f_reg_p(_jitc->function->self.argf))
614 _jitc->function->vafp = _jitc->function->self.argf * 16 +
617 _jitc->function->vafp = va_fp_max_offset;
624 _jit_va_push(jit_state_t *_jit, jit_int32_t u)
626 jit_inc_synth_w(va_push, u);
632 _jit_arg(jit_state_t *_jit)
636 assert(_jitc->function);
637 assert(!(_jitc->function->self.call & jit_call_varargs));
639 if (jit_arg_reg_p(_jitc->function->self.argi)) {
640 offset = _jitc->function->self.argi++;
641 # if __CYGWIN__ || _WIN32
642 _jitc->function->self.size += sizeof(jit_word_t);
648 offset = _jitc->function->self.size;
649 _jitc->function->self.size += REAL_WORDSIZE;
651 node = jit_new_node_ww(jit_code_arg, offset,
652 ++_jitc->function->self.argn);
658 _jit_arg_f(jit_state_t *_jit)
662 assert(_jitc->function);
663 assert(!(_jitc->function->self.call & jit_call_varargs));
665 # if __CYGWIN__ || _WIN32
666 if (jit_arg_reg_p(_jitc->function->self.argi)) {
667 offset = _jitc->function->self.argi++;
668 _jitc->function->self.size += sizeof(jit_word_t);
671 if (jit_arg_f_reg_p(_jitc->function->self.argf))
672 offset = _jitc->function->self.argf++;
677 offset = _jitc->function->self.size;
678 _jitc->function->self.size += REAL_WORDSIZE;
680 node = jit_new_node_ww(jit_code_arg_f, offset,
681 ++_jitc->function->self.argn);
687 _jit_arg_d(jit_state_t *_jit)
691 assert(_jitc->function);
692 assert(!(_jitc->function->self.call & jit_call_varargs));
694 # if __CYGWIN__ || _WIN32
695 if (jit_arg_reg_p(_jitc->function->self.argi)) {
696 offset = _jitc->function->self.argi++;
697 _jitc->function->self.size += sizeof(jit_word_t);
700 if (jit_arg_f_reg_p(_jitc->function->self.argf))
701 offset = _jitc->function->self.argf++;
706 offset = _jitc->function->self.size;
707 _jitc->function->self.size += sizeof(jit_float64_t);
709 node = jit_new_node_ww(jit_code_arg_d, offset,
710 ++_jitc->function->self.argn);
716 _jit_getarg_c(jit_state_t *_jit, jit_int32_t u, jit_node_t *v)
718 assert(v->code == jit_code_arg);
719 jit_inc_synth_wp(getarg_c, u, v);
721 if (jit_arg_reg_p(v->u.w))
722 jit_extr_c(u, JIT_RA0 - v->u.w);
725 jit_ldxi_c(u, _RBP, v->u.w);
730 _jit_getarg_uc(jit_state_t *_jit, jit_int32_t u, jit_node_t *v)
732 assert(v->code == jit_code_arg);
733 jit_inc_synth_wp(getarg_uc, u, v);
735 if (jit_arg_reg_p(v->u.w))
736 jit_extr_uc(u, JIT_RA0 - v->u.w);
739 jit_ldxi_uc(u, _RBP, v->u.w);
744 _jit_getarg_s(jit_state_t *_jit, jit_int32_t u, jit_node_t *v)
746 assert(v->code == jit_code_arg);
747 jit_inc_synth_wp(getarg_s, u, v);
749 if (jit_arg_reg_p(v->u.w))
750 jit_extr_s(u, JIT_RA0 - v->u.w);
753 jit_ldxi_s(u, _RBP, v->u.w);
758 _jit_getarg_us(jit_state_t *_jit, jit_int32_t u, jit_node_t *v)
760 assert(v->code == jit_code_arg);
761 jit_inc_synth_wp(getarg_us, u, v);
763 if (jit_arg_reg_p(v->u.w))
764 jit_extr_us(u, JIT_RA0 - v->u.w);
767 jit_ldxi_us(u, _RBP, v->u.w);
772 _jit_getarg_i(jit_state_t *_jit, jit_int32_t u, jit_node_t *v)
774 assert(v->code == jit_code_arg);
775 jit_inc_synth_wp(getarg_i, u, v);
777 if (jit_arg_reg_p(v->u.w)) {
779 jit_movr(u, JIT_RA0 - v->u.w);
781 jit_extr_i(u, JIT_RA0 - v->u.w);
786 jit_ldxi_i(u, _RBP, v->u.w);
790 #if __X64 && !__X64_32
792 _jit_getarg_ui(jit_state_t *_jit, jit_int32_t u, jit_node_t *v)
794 assert(v->code == jit_code_arg);
795 jit_inc_synth_wp(getarg_ui, u, v);
796 if (jit_arg_reg_p(v->u.w))
797 jit_extr_ui(u, JIT_RA0 - v->u.w);
799 jit_ldxi_ui(u, _RBP, v->u.w);
804 _jit_getarg_l(jit_state_t *_jit, jit_int32_t u, jit_node_t *v)
806 assert(v->code == jit_code_arg);
807 jit_inc_synth_wp(getarg_l, u, v);
808 if (jit_arg_reg_p(v->u.w))
809 jit_movr(u, JIT_RA0 - v->u.w);
811 jit_ldxi_l(u, _RBP, v->u.w);
817 _jit_putargr(jit_state_t *_jit, jit_int32_t u, jit_node_t *v)
819 assert(v->code == jit_code_arg);
820 jit_inc_synth_wp(putargr, u, v);
822 if (jit_arg_reg_p(v->u.w))
823 jit_movr(JIT_RA0 - v->u.w, u);
826 jit_stxi(v->u.w, _RBP, u);
831 _jit_putargi(jit_state_t *_jit, jit_word_t u, jit_node_t *v)
834 assert(v->code == jit_code_arg);
835 jit_inc_synth_wp(putargi, u, v);
837 if (jit_arg_reg_p(v->u.w))
838 jit_movi(JIT_RA0 - v->u.w, u);
842 regno = jit_get_reg(jit_class_gpr);
844 jit_stxi(v->u.w, _RBP, regno);
845 jit_unget_reg(regno);
851 _jit_getarg_f(jit_state_t *_jit, jit_int32_t u, jit_node_t *v)
853 assert(v->code == jit_code_arg_f);
854 jit_inc_synth_wp(getarg_f, u, v);
856 if (jit_arg_f_reg_p(v->u.w))
857 jit_movr_f(u, _XMM0 - v->u.w);
860 jit_ldxi_f(u, _RBP, v->u.w);
865 _jit_putargr_f(jit_state_t *_jit, jit_int32_t u, jit_node_t *v)
867 assert(v->code == jit_code_arg_f);
868 jit_inc_synth_wp(putargr_f, u, v);
870 if (jit_arg_reg_p(v->u.w))
871 jit_movr_f(_XMM0 - v->u.w, u);
874 jit_stxi_f(v->u.w, _RBP, u);
879 _jit_putargi_f(jit_state_t *_jit, jit_float32_t u, jit_node_t *v)
882 assert(v->code == jit_code_arg_f);
883 jit_inc_synth_fp(putargi_f, u, v);
885 if (jit_arg_reg_p(v->u.w))
886 jit_movi_f(_XMM0 - v->u.w, u);
890 regno = jit_get_reg(jit_class_gpr);
891 jit_movi_f(regno, u);
892 jit_stxi_f(v->u.w, _RBP, regno);
893 jit_unget_reg(regno);
899 _jit_getarg_d(jit_state_t *_jit, jit_int32_t u, jit_node_t *v)
901 assert(v->code == jit_code_arg_d);
902 jit_inc_synth_wp(getarg_d, u, v);
904 if (jit_arg_f_reg_p(v->u.w))
905 jit_movr_d(u, _XMM0 - v->u.w);
908 jit_ldxi_d(u, _RBP, v->u.w);
913 _jit_putargr_d(jit_state_t *_jit, jit_int32_t u, jit_node_t *v)
915 assert(v->code == jit_code_arg_d);
916 jit_inc_synth_wp(putargr_d, u, v);
918 if (jit_arg_reg_p(v->u.w))
919 jit_movr_d(_XMM0 - v->u.w, u);
922 jit_stxi_d(v->u.w, _RBP, u);
927 _jit_putargi_d(jit_state_t *_jit, jit_float64_t u, jit_node_t *v)
930 assert(v->code == jit_code_arg_d);
931 jit_inc_synth_dp(putargi_d, u, v);
933 if (jit_arg_reg_p(v->u.w))
934 jit_movi_d(_XMM0 - v->u.w, u);
938 regno = jit_get_reg(jit_class_gpr);
939 jit_movi_d(regno, u);
940 jit_stxi_d(v->u.w, _RBP, regno);
941 jit_unget_reg(regno);
947 _jit_pushargr(jit_state_t *_jit, jit_int32_t u)
949 assert(_jitc->function);
950 jit_inc_synth_w(pushargr, u);
953 if (jit_arg_reg_p(_jitc->function->call.argi)) {
954 jit_movr(JIT_RA0 - _jitc->function->call.argi, u);
955 ++_jitc->function->call.argi;
956 # if __CYGWIN__ || _WIN32
957 if (_jitc->function->call.call & jit_call_varargs)
958 jit_stxi(_jitc->function->call.size, _RSP, u);
959 _jitc->function->call.size += sizeof(jit_word_t);
965 jit_stxi(_jitc->function->call.size, _RSP, u);
966 _jitc->function->call.size += REAL_WORDSIZE;
972 _jit_pushargi(jit_state_t *_jit, jit_word_t u)
975 assert(_jitc->function);
976 jit_inc_synth_w(pushargi, u);
979 if (jit_arg_reg_p(_jitc->function->call.argi)) {
980 jit_movi(JIT_RA0 - _jitc->function->call.argi, u);
981 # if __CYGWIN__ || _WIN32
982 if (_jitc->function->call.call & jit_call_varargs)
983 jit_stxi(_jitc->function->call.size, _RSP,
984 JIT_RA0 - _jitc->function->call.argi);
985 _jitc->function->call.size += sizeof(jit_word_t);
987 ++_jitc->function->call.argi;
992 regno = jit_get_reg(jit_class_gpr);
994 jit_stxi(_jitc->function->call.size, _RSP, regno);
995 _jitc->function->call.size += REAL_WORDSIZE;
996 jit_unget_reg(regno);
1002 _jit_pushargr_f(jit_state_t *_jit, jit_int32_t u)
1004 assert(_jitc->function);
1005 jit_inc_synth_w(pushargr_f, u);
1008 # if __CYGWIN__ || _WIN32
1009 if (jit_arg_reg_p(_jitc->function->call.argi)) {
1010 jit_movr_f(_XMM0 - _jitc->function->call.argi, u);
1011 if (_jitc->function->call.call & jit_call_varargs) {
1012 jit_stxi_f(_jitc->function->call.size, _RSP,
1013 _XMM0 - _jitc->function->call.argi);
1014 jit_ldxi_i(JIT_RA0 - _jitc->function->call.argi, _RSP,
1015 _jitc->function->call.size);
1017 ++_jitc->function->call.argi;
1018 _jitc->function->call.size += sizeof(jit_word_t);
1021 if (jit_arg_f_reg_p(_jitc->function->self.argf)) {
1022 jit_movr_f(_XMM0 - _jitc->function->call.argf, u);
1023 ++_jitc->function->call.argf;
1029 jit_stxi_f(_jitc->function->call.size, _RSP, u);
1030 _jitc->function->call.size += REAL_WORDSIZE;
1036 _jit_pushargi_f(jit_state_t *_jit, jit_float32_t u)
1039 assert(_jitc->function);
1040 jit_inc_synth_f(pushargi_f, u);
1043 # if __CYGWIN__ || _WIN32
1044 if (jit_arg_reg_p(_jitc->function->call.argi)) {
1045 jit_movi_f(_XMM0 - _jitc->function->call.argi, u);
1046 if (_jitc->function->call.call & jit_call_varargs) {
1047 jit_stxi_f(_jitc->function->call.size, _RSP,
1048 _XMM0 - _jitc->function->call.argi);
1049 jit_ldxi_i(JIT_RA0 - _jitc->function->call.argi, _RSP,
1050 _jitc->function->call.size);
1052 ++_jitc->function->call.argi;
1053 _jitc->function->call.size += sizeof(jit_word_t);
1056 if (jit_arg_f_reg_p(_jitc->function->call.argf)) {
1057 jit_movi_f(_XMM0 - _jitc->function->call.argf, u);
1058 ++_jitc->function->call.argf;
1064 regno = jit_get_reg(jit_class_fpr);
1065 jit_movi_f(regno, u);
1066 jit_stxi_f(_jitc->function->call.size, _RSP, regno);
1067 _jitc->function->call.size += REAL_WORDSIZE;
1068 jit_unget_reg(regno);
1074 _jit_pushargr_d(jit_state_t *_jit, jit_int32_t u)
1076 assert(_jitc->function);
1077 jit_inc_synth_w(pushargr_d, u);
1080 # if __CYGWIN__ || _WIN32
1081 if (jit_arg_reg_p(_jitc->function->call.argi)) {
1082 jit_movr_d(_XMM0 - _jitc->function->call.argi, u);
1083 if (_jitc->function->call.call & jit_call_varargs) {
1084 jit_stxi_d(_jitc->function->call.size, _RSP,
1085 _XMM0 - _jitc->function->call.argi);
1086 jit_ldxi_l(JIT_RA0 - _jitc->function->call.argi, _RSP,
1087 _jitc->function->call.size);
1089 ++_jitc->function->call.argi;
1090 _jitc->function->call.size += sizeof(jit_word_t);
1093 if (jit_arg_f_reg_p(_jitc->function->call.argf)) {
1094 jit_movr_d(_XMM0 - _jitc->function->call.argf, u);
1095 ++_jitc->function->call.argf;
1101 jit_stxi_d(_jitc->function->call.size, _RSP, u);
1102 _jitc->function->call.size += sizeof(jit_float64_t);
1108 _jit_pushargi_d(jit_state_t *_jit, jit_float64_t u)
1111 assert(_jitc->function);
1112 jit_inc_synth_d(pushargi_d, u);
1115 # if __CYGWIN__ || _WIN32
1116 if (jit_arg_reg_p(_jitc->function->call.argi)) {
1117 jit_movi_d(_XMM0 - _jitc->function->call.argi, u);
1118 if (_jitc->function->call.call & jit_call_varargs) {
1119 jit_stxi_d(_jitc->function->call.size, _RSP,
1120 _XMM0 - _jitc->function->call.argi);
1121 jit_ldxi_l(JIT_RA0 - _jitc->function->call.argi, _RSP,
1122 _jitc->function->call.size);
1124 ++_jitc->function->call.argi;
1125 _jitc->function->call.size += sizeof(jit_word_t);
1128 if (jit_arg_f_reg_p(_jitc->function->call.argf)) {
1129 jit_movi_d(_XMM0 - _jitc->function->call.argf, u);
1130 ++_jitc->function->call.argf;
1136 regno = jit_get_reg(jit_class_fpr);
1137 jit_movi_d(regno, u);
1138 jit_stxi_d(_jitc->function->call.size, _RSP, regno);
1139 _jitc->function->call.size += sizeof(jit_float64_t);
1140 jit_unget_reg(regno);
1146 _jit_regarg_p(jit_state_t *_jit, jit_node_t *node, jit_int32_t regno)
1151 spec = jit_class(_rvs[regno].spec);
1152 if (spec & jit_class_arg) {
1153 if (spec & jit_class_gpr) {
1154 regno = JIT_RA0 - regno;
1155 if (regno >= 0 && regno < node->v.w)
1158 else if (spec & jit_class_fpr) {
1159 regno = _XMM0 - regno;
1160 if (regno >= 0 && regno < node->w.w)
1169 _jit_finishr(jit_state_t *_jit, jit_int32_t r0)
1173 assert(_jitc->function);
1175 jit_inc_synth_w(finishr, r0);
1176 if (_jitc->function->self.alen < _jitc->function->call.size)
1177 _jitc->function->self.alen = _jitc->function->call.size;
1179 # if !(__CYGWIN__ || _WIN32)
1180 if (_jitc->function->call.call & jit_call_varargs) {
1181 if (jit_regno(reg) == _RAX) {
1182 reg = jit_get_reg(jit_class_gpr);
1183 jit_movr(reg, _RAX);
1185 if (_jitc->function->call.argf)
1186 jit_movi(_RAX, _jitc->function->call.argf);
1194 call = jit_callr(reg);
1195 call->v.w = _jitc->function->call.argi;
1196 call->w.w = _jitc->function->call.argf;
1197 _jitc->function->call.argi = _jitc->function->call.argf =
1198 _jitc->function->call.size = 0;
1204 _jit_finishi(jit_state_t *_jit, jit_pointer_t i0)
1210 assert(_jitc->function);
1211 jit_inc_synth_w(finishi, (jit_word_t)i0);
1212 if (_jitc->function->self.alen < _jitc->function->call.size)
1213 _jitc->function->self.alen = _jitc->function->call.size;
1215 /* FIXME preventing %rax allocation is good enough, but for consistency
1216 * it should automatically detect %rax is dead, in case it has run out
1217 * registers, and not save/restore it, what would be wrong if using the
1218 * the return value, otherwise, just a needless noop */
1219 /* >> prevent %rax from being allocated as the function pointer */
1220 jit_regset_setbit(&_jitc->regarg, _RAX);
1221 reg = jit_get_reg(jit_class_gpr);
1222 node = jit_movi(reg, (jit_word_t)i0);
1225 /* << prevent %rax from being allocated as the function pointer */
1226 jit_regset_clrbit(&_jitc->regarg, _RAX);
1228 node = jit_calli(i0);
1229 node->v.w = _jitc->function->call.argi;
1230 node->w.w = _jitc->function->call.argf;
1232 _jitc->function->call.argi = _jitc->function->call.argf =
1233 _jitc->function->call.size = 0;
1240 _jit_retval_c(jit_state_t *_jit, jit_int32_t r0)
1242 jit_inc_synth_w(retval_c, r0);
1243 jit_extr_c(r0, JIT_RET);
1248 _jit_retval_uc(jit_state_t *_jit, jit_int32_t r0)
1250 jit_inc_synth_w(retval_uc, r0);
1251 jit_extr_uc(r0, JIT_RET);
1256 _jit_retval_s(jit_state_t *_jit, jit_int32_t r0)
1258 jit_inc_synth_w(retval_s, r0);
1259 jit_extr_s(r0, JIT_RET);
1264 _jit_retval_us(jit_state_t *_jit, jit_int32_t r0)
1266 jit_inc_synth_w(retval_us, r0);
1267 jit_extr_us(r0, JIT_RET);
1272 _jit_retval_i(jit_state_t *_jit, jit_int32_t r0)
1274 jit_inc_synth_w(retval_i, r0);
1275 #if __X32 || __X64_32
1277 jit_movr(r0, JIT_RET);
1279 jit_extr_i(r0, JIT_RET);
1284 #if __X64 && !__X64_32
1286 _jit_retval_ui(jit_state_t *_jit, jit_int32_t r0)
1288 jit_inc_synth_w(retval_ui, r0);
1289 jit_extr_ui(r0, JIT_RET);
1294 _jit_retval_l(jit_state_t *_jit, jit_int32_t r0)
1296 jit_inc_synth_w(retval_l, r0);
1298 jit_movr(r0, JIT_RET);
1304 _jit_retval_f(jit_state_t *_jit, jit_int32_t r0)
1306 jit_inc_synth_w(retval_f, r0);
1309 jit_movr_f(r0, JIT_FRET);
1315 _jit_retval_d(jit_state_t *_jit, jit_int32_t r0)
1317 jit_inc_synth_w(retval_d, r0);
1320 jit_movr_d(r0, JIT_FRET);
1326 _emit_code(jit_state_t *_jit)
1336 #if DEVEL_DISASSEMBLER
1339 jit_int32_t patch_offset;
1341 #if DEVEL_DISASSEMBLER
1345 _jitc->function = NULL;
1347 jit_reglive_setup();
1351 undo.patch_offset = 0;
1352 #define case_rr(name, type) \
1353 case jit_code_##name##r##type: \
1354 name##r##type(rn(node->u.w), rn(node->v.w)); \
1356 #define case_rw(name, type) \
1357 case jit_code_##name##i##type: \
1358 name##i##type(rn(node->u.w), node->v.w); \
1360 #define case_rf(name, type) \
1361 case jit_code_##name##r##type: \
1362 if (jit_x87_reg_p(node->v.w)) \
1363 x87_##name##r##type(rn(node->u.w), rn(node->v.w)); \
1365 sse_##name##r##type(rn(node->u.w), rn(node->v.w)); \
1367 #define case_fr(name, type) \
1368 case jit_code_##name##r##type: \
1369 if (jit_x87_reg_p(node->u.w)) \
1370 x87_##name##r##type(rn(node->u.w), rn(node->v.w)); \
1372 sse_##name##r##type(rn(node->u.w), rn(node->v.w)); \
1374 #define case_fw(name, type) \
1375 case jit_code_##name##i##type: \
1376 if (jit_x87_reg_p(node->u.w)) \
1377 x87_##name##i##type(rn(node->u.w), node->v.w); \
1379 sse_##name##i##type(rn(node->u.w), node->v.w); \
1381 #define case_wr(name, type) \
1382 case jit_code_##name##i##type: \
1383 name##i##type(node->u.w, rn(node->v.w)); \
1385 #define case_wf(name, type) \
1386 case jit_code_##name##i##type: \
1387 if (jit_x87_reg_p(node->v.w)) \
1388 x87_##name##i##type(node->u.w, rn(node->v.w)); \
1390 sse_##name##i##type(node->u.w, rn(node->v.w)); \
1392 #define case_ff(name, type) \
1393 case jit_code_##name##r##type: \
1394 if (jit_x87_reg_p(node->u.w) && \
1395 jit_x87_reg_p(node->v.w)) \
1396 x87_##name##r##type(rn(node->u.w), rn(node->v.w)); \
1398 sse_##name##r##type(rn(node->u.w), rn(node->v.w)); \
1400 #define case_rrr(name, type) \
1401 case jit_code_##name##r##type: \
1402 name##r##type(rn(node->u.w), \
1403 rn(node->v.w), rn(node->w.w)); \
1405 #define case_rrrr(name, type) \
1406 case jit_code_##name##r##type: \
1407 name##r##type(rn(node->u.q.l), rn(node->u.q.h), \
1408 rn(node->v.w), rn(node->w.w)); \
1410 #define case_frr(name, type) \
1411 case jit_code_##name##r##type: \
1412 if (jit_x87_reg_p(node->u.w)) \
1413 x87_##name##r##type(rn(node->u.w), \
1414 rn(node->v.w), rn(node->w.w)); \
1416 sse_##name##r##type(rn(node->u.w), \
1417 rn(node->v.w), rn(node->w.w)); \
1419 #define case_rrf(name, type) \
1420 case jit_code_##name##r##type: \
1421 if (jit_x87_reg_p(node->w.w)) \
1422 x87_##name##r##type(rn(node->u.w), \
1423 rn(node->v.w), rn(node->w.w)); \
1425 sse_##name##r##type(rn(node->u.w), \
1426 rn(node->v.w), rn(node->w.w)); \
1428 #define case_rrw(name, type) \
1429 case jit_code_##name##i##type: \
1430 name##i##type(rn(node->u.w), rn(node->v.w), node->w.w); \
1432 #define case_rrrw(name, type) \
1433 case jit_code_##name##i##type: \
1434 name##i##type(rn(node->u.q.l), rn(node->u.q.h), \
1435 rn(node->v.w), node->w.w); \
1437 #define case_frw(name, type) \
1438 case jit_code_##name##i##type: \
1439 if (jit_x87_reg_p(node->u.w)) \
1440 x87_##name##i##type(rn(node->u.w), \
1441 rn(node->v.w), node->w.w); \
1443 sse_##name##i##type(rn(node->u.w), \
1444 rn(node->v.w), node->w.w); \
1446 #define case_wrr(name, type) \
1447 case jit_code_##name##i##type: \
1448 name##i##type(node->u.w, rn(node->v.w), rn(node->w.w)); \
1450 #define case_wrf(name, type) \
1451 case jit_code_##name##i##type: \
1452 if (jit_x87_reg_p(node->w.w)) \
1453 x87_##name##i##type(node->u.w, \
1454 rn(node->v.w), rn(node->w.w)); \
1456 sse_##name##i##type(node->u.w, \
1457 rn(node->v.w), rn(node->w.w)); \
1459 #define case_brr(name, type) \
1460 case jit_code_##name##r##type: \
1462 assert(temp->code == jit_code_label || \
1463 temp->code == jit_code_epilog); \
1464 if (temp->flag & jit_flag_patch) \
1465 name##r##type(temp->u.w, rn(node->v.w), \
1468 word = name##r##type(_jit->pc.w, \
1469 rn(node->v.w), rn(node->w.w)); \
1470 patch(word, node); \
1473 #define case_brw(name, type) \
1474 case jit_code_##name##i##type: \
1476 assert(temp->code == jit_code_label || \
1477 temp->code == jit_code_epilog); \
1478 if (temp->flag & jit_flag_patch) \
1479 name##i##type(temp->u.w, \
1480 rn(node->v.w), node->w.w); \
1482 word = name##i##type(_jit->pc.w, \
1483 rn(node->v.w), node->w.w); \
1484 patch(word, node); \
1487 #define case_rff(name, type) \
1488 case jit_code_##name##r##type: \
1489 if (jit_x87_reg_p(node->v.w) && \
1490 jit_x87_reg_p(node->w.w)) \
1491 x87_##name##r##type(rn(node->u.w), rn(node->v.w), \
1494 sse_##name##r##type(rn(node->u.w), rn(node->v.w), \
1497 #define case_rfw(name, type, size) \
1498 case jit_code_##name##i##type: \
1499 assert(node->flag & jit_flag_data); \
1500 if (jit_x87_reg_p(node->v.w)) \
1501 x87_##name##i##type(rn(node->u.w), rn(node->v.w), \
1502 (jit_float##size##_t *)node->w.n->u.w); \
1504 sse_##name##i##type(rn(node->u.w), rn(node->v.w), \
1505 (jit_float##size##_t *)node->w.n->u.w); \
1507 #define case_fff(name, type) \
1508 case jit_code_##name##r##type: \
1509 if (jit_x87_reg_p(node->u.w) && \
1510 jit_x87_reg_p(node->v.w) && \
1511 jit_x87_reg_p(node->w.w)) \
1512 x87_##name##r##type(rn(node->u.w), \
1513 rn(node->v.w), rn(node->w.w)); \
1515 sse_##name##r##type(rn(node->u.w), \
1516 rn(node->v.w), rn(node->w.w)); \
1518 #define case_ffw(name, type, size) \
1519 case jit_code_##name##i##type: \
1520 assert(node->flag & jit_flag_data); \
1521 if (jit_x87_reg_p(node->u.w) && \
1522 jit_x87_reg_p(node->v.w)) \
1523 x87_##name##i##type(rn(node->u.w), rn(node->v.w), \
1524 (jit_float##size##_t *)node->w.n->u.w); \
1526 sse_##name##i##type(rn(node->u.w), rn(node->v.w), \
1527 (jit_float##size##_t *)node->w.n->u.w); \
1529 #define case_bff(name, type) \
1530 case jit_code_b##name##r##type: \
1532 assert(temp->code == jit_code_label || \
1533 temp->code == jit_code_epilog); \
1534 if (temp->flag & jit_flag_patch) { \
1535 if (jit_x87_reg_p(node->v.w) && \
1536 jit_x87_reg_p(node->w.w)) \
1537 x87_b##name##r##type(temp->u.w, \
1538 rn(node->v.w), rn(node->w.w)); \
1540 sse_b##name##r##type(temp->u.w, \
1541 rn(node->v.w), rn(node->w.w)); \
1544 if (jit_x87_reg_p(node->v.w) && \
1545 jit_x87_reg_p(node->w.w)) \
1546 word = x87_b##name##r##type(_jit->pc.w, \
1547 rn(node->v.w), rn(node->w.w)); \
1549 word = sse_b##name##r##type(_jit->pc.w, \
1550 rn(node->v.w), rn(node->w.w)); \
1551 patch(word, node); \
1554 #define case_bfw(name, type, size) \
1555 case jit_code_b##name##i##type: \
1557 assert(temp->code == jit_code_label || \
1558 temp->code == jit_code_epilog); \
1559 if (temp->flag & jit_flag_patch) { \
1560 if (jit_x87_reg_p(node->v.w)) \
1561 x87_b##name##i##type(temp->u.w, \
1563 (jit_float##size##_t *)node->w.n->u.w); \
1565 sse_b##name##i##type(temp->u.w, \
1567 (jit_float##size##_t *)node->w.n->u.w); \
1570 if (jit_x87_reg_p(node->v.w)) \
1571 word = x87_b##name##i##type(_jit->pc.w, \
1573 (jit_float##size##_t *)node->w.n->u.w); \
1575 word = sse_b##name##i##type(_jit->pc.w, \
1577 (jit_float##size##_t *)node->w.n->u.w); \
1578 patch(word, node); \
1581 #if DEVEL_DISASSEMBLER
1584 for (node = _jitc->head; node; node = node->next) {
1585 if (_jit->pc.uc >= _jitc->code.end)
1588 #if DEVEL_DISASSEMBLER
1589 node->offset = (jit_uword_t)_jit->pc.w - (jit_uword_t)prevw;
1592 value = jit_classify(node->code);
1593 jit_regarg_set(node, value);
1594 switch (node->code) {
1595 case jit_code_align:
1596 assert(!(node->u.w & (node->u.w - 1)) &&
1597 node->u.w <= sizeof(jit_word_t));
1598 if ((word = _jit->pc.w & (node->u.w - 1)))
1599 nop(node->u.w - word);
1601 case jit_code_note: case jit_code_name:
1602 node->u.w = _jit->pc.w;
1604 case jit_code_label:
1605 if ((node->link || (node->flag & jit_flag_use)) &&
1606 (word = _jit->pc.w & (sizeof(jit_word_t) - 1)))
1607 nop(sizeof(jit_word_t) - word);
1608 /* remember label is defined */
1609 node->flag |= jit_flag_patch;
1610 node->u.w = _jit->pc.w;
1629 case_rrrr(qmul, _u);
1630 case_rrrw(qmul, _u);
1637 case_rrrr(qdiv, _u);
1638 case_rrrw(qdiv, _u);
1679 if (node->flag & jit_flag_node) {
1681 if (temp->code == jit_code_data ||
1682 (temp->code == jit_code_label &&
1683 (temp->flag & jit_flag_patch)))
1684 movi(rn(node->u.w), temp->u.w);
1686 assert(temp->code == jit_code_label ||
1687 temp->code == jit_code_epilog);
1688 word = movi_p(rn(node->u.w), node->v.w);
1693 movi(rn(node->u.w), node->v.w);
1697 #if __X64 && !__X64_32
1704 #if __X64 && !__X64_32
1708 case_rf(trunc, _f_i);
1709 case_rf(trunc, _d_i);
1711 case_rf(trunc, _f_l);
1712 case_rf(trunc, _d_l);
1724 #if __X64 && !__X64_32
1740 #if __X64 && !__X64_32
1752 #if __X64 && !__X64_32
1762 #if __X64 && !__X64_32
1792 case_brr(boadd, _u);
1793 case_brw(boadd, _u);
1796 case_brr(bxadd, _u);
1797 case_brw(bxadd, _u);
1800 case_brr(bosub, _u);
1801 case_brw(bosub, _u);
1804 case_brr(bxsub, _u);
1805 case_brw(bxsub, _u);
1807 case_ffw(add, _f, 32);
1809 case_ffw(sub, _f, 32);
1810 case_ffw(rsb, _f, 32);
1812 case_ffw(mul, _f, 32);
1814 case_ffw(div, _f, 32);
1821 case_rfw(lt, _f, 32);
1823 case_rfw(le, _f, 32);
1825 case_rfw(eq, _f, 32);
1827 case_rfw(ge, _f, 32);
1829 case_rfw(gt, _f, 32);
1831 case_rfw(ne, _f, 32);
1833 case_rfw(unlt, _f, 32);
1835 case_rfw(unle, _f, 32);
1837 case_rfw(uneq, _f, 32);
1839 case_rfw(unge, _f, 32);
1841 case_rfw(ungt, _f, 32);
1843 case_rfw(ltgt, _f, 32);
1845 case_rfw(ord, _f, 32);
1846 case_rff(unord, _f);
1847 case_rfw(unord, _f, 32);
1848 case jit_code_movr_f:
1849 if (jit_x87_reg_p(node->u.w)) {
1850 if (jit_x87_reg_p(node->v.w))
1851 x87_movr_f(rn(node->u.w), rn(node->v.w));
1853 x87_from_sse_f(rn(node->u.w), rn(node->v.w));
1856 if (jit_sse_reg_p(node->v.w))
1857 sse_movr_f(rn(node->u.w), rn(node->v.w));
1859 sse_from_x87_f(rn(node->u.w), rn(node->v.w));
1862 case jit_code_movi_f:
1863 assert(node->flag & jit_flag_data);
1864 if (jit_x87_reg_p(node->u.w))
1865 x87_movi_f(rn(node->u.w), (jit_float32_t *)node->v.n->u.w);
1867 sse_movi_f(rn(node->u.w), (jit_float32_t *)node->v.n->u.w);
1878 case_bfw(lt, _f, 32);
1880 case_bfw(le, _f, 32);
1882 case_bfw(eq, _f, 32);
1884 case_bfw(ge, _f, 32);
1886 case_bfw(gt, _f, 32);
1888 case_bfw(ne, _f, 32);
1890 case_bfw(unlt, _f, 32);
1892 case_bfw(unle, _f, 32);
1894 case_bfw(uneq, _f, 32);
1896 case_bfw(unge, _f, 32);
1898 case_bfw(ungt, _f, 32);
1900 case_bfw(ltgt, _f, 32);
1902 case_bfw(ord, _f, 32);
1903 case_bff(unord, _f);
1904 case_bfw(unord, _f, 32);
1906 case_ffw(add, _d, 64);
1908 case_ffw(sub, _d, 64);
1909 case_ffw(rsb, _d, 64);
1911 case_ffw(mul, _d, 64);
1913 case_ffw(div, _d, 64);
1920 case_rfw(lt, _d, 64);
1922 case_rfw(le, _d, 64);
1924 case_rfw(eq, _d, 64);
1926 case_rfw(ge, _d, 64);
1928 case_rfw(gt, _d, 64);
1930 case_rfw(ne, _d, 64);
1932 case_rfw(unlt, _d, 64);
1934 case_rfw(unle, _d, 64);
1936 case_rfw(uneq, _d, 64);
1938 case_rfw(unge, _d, 64);
1940 case_rfw(ungt, _d, 64);
1942 case_rfw(ltgt, _d, 64);
1944 case_rfw(ord, _d, 64);
1945 case_rff(unord, _d);
1946 case_rfw(unord, _d, 64);
1947 case jit_code_movr_d:
1948 if (jit_x87_reg_p(node->u.w)) {
1949 if (jit_x87_reg_p(node->v.w))
1950 x87_movr_d(rn(node->u.w), rn(node->v.w));
1952 x87_from_sse_d(rn(node->u.w), rn(node->v.w));
1955 if (jit_sse_reg_p(node->v.w))
1956 sse_movr_d(rn(node->u.w), rn(node->v.w));
1958 sse_from_x87_d(rn(node->u.w), rn(node->v.w));
1961 case jit_code_movi_d:
1962 assert(node->flag & jit_flag_data);
1963 if (jit_x87_reg_p(node->u.w))
1964 x87_movi_d(rn(node->u.w), (jit_float64_t *)node->v.n->u.w);
1966 sse_movi_d(rn(node->u.w), (jit_float64_t *)node->v.n->u.w);
1977 case_bfw(lt, _d, 64);
1979 case_bfw(le, _d, 64);
1981 case_bfw(eq, _d, 64);
1983 case_bfw(ge, _d, 64);
1985 case_bfw(gt, _d, 64);
1987 case_bfw(ne, _d, 64);
1989 case_bfw(unlt, _d, 64);
1991 case_bfw(unle, _d, 64);
1993 case_bfw(uneq, _d, 64);
1995 case_bfw(unge, _d, 64);
1997 case_bfw(ungt, _d, 64);
1999 case_bfw(ltgt, _d, 64);
2001 case_bfw(ord, _d, 64);
2002 case_bff(unord, _d);
2003 case_bfw(unord, _d, 64);
2005 jmpr(rn(node->u.w));
2008 if (node->flag & jit_flag_node) {
2010 assert(temp->code == jit_code_label ||
2011 temp->code == jit_code_epilog);
2012 if (temp->flag & jit_flag_patch)
2015 word = jmpi(_jit->pc.w);
2022 case jit_code_callr:
2023 callr(rn(node->u.w));
2025 case jit_code_calli:
2026 if (node->flag & jit_flag_node) {
2028 assert(temp->code == jit_code_label ||
2029 temp->code == jit_code_epilog);
2030 word = calli(temp->u.w);
2031 if (!(temp->flag & jit_flag_patch))
2037 case jit_code_prolog:
2038 _jitc->function = _jitc->functions.ptr + node->w.w;
2040 undo.word = _jit->pc.w;
2041 #if DEVEL_DISASSEMBLER
2044 undo.patch_offset = _jitc->patches.offset;
2049 case jit_code_epilog:
2050 assert(_jitc->function == _jitc->functions.ptr + node->w.w);
2052 for (temp = undo.node->next;
2053 temp != node; temp = temp->next) {
2054 if (temp->code == jit_code_label ||
2055 temp->code == jit_code_epilog)
2056 temp->flag &= ~jit_flag_patch;
2058 temp->flag &= ~jit_flag_patch;
2060 _jit->pc.w = undo.word;
2061 #if DEVEL_DISASSEMBLER
2064 _jitc->patches.offset = undo.patch_offset;
2065 goto restart_function;
2068 (word = _jit->pc.w & (sizeof(jit_word_t) - 1)))
2069 nop(sizeof(jit_word_t) - word);
2070 /* remember label is defined */
2071 node->flag |= jit_flag_patch;
2072 node->u.w = _jit->pc.w;
2074 _jitc->function = NULL;
2076 case jit_code_va_start:
2077 vastart(rn(node->u.w));
2079 case jit_code_va_arg:
2080 vaarg(rn(node->u.w), rn(node->v.w));
2082 case jit_code_va_arg_d:
2083 vaarg_d(rn(node->u.w), rn(node->v.w), jit_x87_reg_p(node->u.w));
2085 case jit_code_live: case jit_code_ellipsis:
2086 case jit_code_va_push:
2087 case jit_code_allocai: case jit_code_allocar:
2089 case jit_code_arg_f: case jit_code_arg_d:
2090 case jit_code_va_end:
2092 case jit_code_retr: case jit_code_reti:
2093 case jit_code_retr_f: case jit_code_reti_f:
2094 case jit_code_retr_d: case jit_code_reti_d:
2095 case jit_code_getarg_c: case jit_code_getarg_uc:
2096 case jit_code_getarg_s: case jit_code_getarg_us:
2097 case jit_code_getarg_i:
2098 #if __X64 && !__X64_32
2099 case jit_code_getarg_ui: case jit_code_getarg_l:
2101 case jit_code_getarg_f: case jit_code_getarg_d:
2102 case jit_code_putargr: case jit_code_putargi:
2103 case jit_code_putargr_f: case jit_code_putargi_f:
2104 case jit_code_putargr_d: case jit_code_putargi_d:
2105 case jit_code_pushargr: case jit_code_pushargi:
2106 case jit_code_pushargr_f: case jit_code_pushargi_f:
2107 case jit_code_pushargr_d: case jit_code_pushargi_d:
2108 case jit_code_retval_c: case jit_code_retval_uc:
2109 case jit_code_retval_s: case jit_code_retval_us:
2110 case jit_code_retval_i:
2112 case jit_code_retval_ui: case jit_code_retval_l:
2114 case jit_code_prepare:
2115 case jit_code_finishr: case jit_code_finishi:
2117 case jit_code_retval_f:
2119 if (jit_sse_reg_p(node->u.w)) {
2121 sse_from_x87_f(rn(node->u.w), _ST0_REGNO);
2124 fstpr(rn(node->u.w) + 1);
2127 case jit_code_retval_d:
2129 if (jit_sse_reg_p(node->u.w)) {
2131 sse_from_x87_d(rn(node->u.w), _ST0_REGNO);
2134 fstpr(rn(node->u.w) + 1);
2140 jit_regarg_clr(node, value);
2141 assert(_jitc->regarg == 0 && _jitc->synth == 0);
2142 /* update register live state */
2164 for (offset = 0; offset < _jitc->patches.offset; offset++) {
2165 node = _jitc->patches.ptr[offset].node;
2166 word = node->code == jit_code_movi ? node->v.n->u.w : node->u.n->u.w;
2167 patch_at(node, _jitc->patches.ptr[offset].inst, word);
2170 jit_flush(_jit->code.ptr, _jit->pc.uc);
2172 return (_jit->code.ptr);
2176 # include "jit_x86-cpu.c"
2177 # include "jit_x86-sse.c"
2178 # include "jit_x86-x87.c"
2182 jit_flush(void *fptr, void *tptr)
2187 _emit_ldxi(jit_state_t *_jit, jit_gpr_t r0, jit_gpr_t r1, jit_word_t i0)
2189 ldxi(rn(r0), rn(r1), i0);
2193 _emit_stxi(jit_state_t *_jit, jit_word_t i0, jit_gpr_t r0, jit_gpr_t r1)
2195 stxi(i0, rn(r0), rn(r1));
2199 _emit_ldxi_d(jit_state_t *_jit, jit_fpr_t r0, jit_gpr_t r1, jit_word_t i0)
2201 if (jit_x87_reg_p(r0))
2202 x87_ldxi_d(rn(r0), rn(r1), i0);
2204 sse_ldxi_d(rn(r0), rn(r1), i0);
2208 _emit_stxi_d(jit_state_t *_jit, jit_word_t i0, jit_gpr_t r0, jit_fpr_t r1)
2210 if (jit_x87_reg_p(r1))
2211 x87_stxi_d(i0, rn(r0), rn(r1));
2213 sse_stxi_d(i0, rn(r0), rn(r1));
2217 _patch(jit_state_t *_jit, jit_word_t instr, jit_node_t *node)
2221 assert(node->flag & jit_flag_node);
2222 if (node->code == jit_code_movi)
2223 flag = node->v.n->flag;
2225 flag = node->u.n->flag;
2226 assert(!(flag & jit_flag_patch));
2227 if (_jitc->patches.offset >= _jitc->patches.length) {
2228 jit_realloc((jit_pointer_t *)&_jitc->patches.ptr,
2229 _jitc->patches.length * sizeof(jit_patch_t),
2230 (_jitc->patches.length + 1024) * sizeof(jit_patch_t));
2231 _jitc->patches.length += 1024;
2233 _jitc->patches.ptr[_jitc->patches.offset].inst = instr;
2234 _jitc->patches.ptr[_jitc->patches.offset].node = node;
2235 ++_jitc->patches.offset;
2239 _sse_from_x87_f(jit_state_t *_jit, jit_int32_t r0, jit_int32_t r1)
2241 x87_stxi_f(CVT_OFFSET, _RBP_REGNO, r1);
2242 sse_ldxi_f(r0, _RBP_REGNO, CVT_OFFSET);
2246 _sse_from_x87_d(jit_state_t *_jit, jit_int32_t r0, jit_int32_t r1)
2248 x87_stxi_d(CVT_OFFSET, _RBP_REGNO, r1);
2249 sse_ldxi_d(r0, _RBP_REGNO, CVT_OFFSET);
2253 _x87_from_sse_f(jit_state_t *_jit, jit_int32_t r0, jit_int32_t r1)
2255 sse_stxi_f(CVT_OFFSET, _RBP_REGNO, r1);
2256 x87_ldxi_f(r0, _RBP_REGNO, CVT_OFFSET);
2260 _x87_from_sse_d(jit_state_t *_jit, jit_int32_t r0, jit_int32_t r1)
2262 sse_stxi_d(CVT_OFFSET, _RBP_REGNO, r1);
2263 x87_ldxi_d(r0, _RBP_REGNO, CVT_OFFSET);