add a thp-based huge page alloc fallback
[pcsx_rearmed.git] / deps / lightning / lib / jit_sparc.c
1 /*
2  * Copyright (C) 2013-2023  Free Software Foundation, Inc.
3  *
4  * This file is part of GNU lightning.
5  *
6  * GNU lightning is free software; you can redistribute it and/or modify it
7  * under the terms of the GNU Lesser General Public License as published
8  * by the Free Software Foundation; either version 3, or (at your option)
9  * any later version.
10  *
11  * GNU lightning is distributed in the hope that it will be useful, but
12  * WITHOUT ANY WARRANTY; without even the implied warranty of MERCHANTABILITY
13  * or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU Lesser General Public
14  * License for more details.
15  *
16  * Authors:
17  *      Paulo Cesar Pereira de Andrade
18  */
19
20 /* Handling SIGILL should not be done by Lightning, but can either use
21  * sample, or use another approach to set jit_cpu.lzcnt
22  */
23 #define CHECK_LZCNT     0
24
25 #if CHECK_LZCNT
26 #include <signal.h>
27 #include <setjmp.h>
28 #endif
29
30 #define jit_arg_reg_p(i)                ((i) >= 0 && (i) < 6)
31 #if __WORDSIZE == 32
32 #  define jit_arg_d_reg_p(i)            ((i) >= 0 && (i) < 5)
33 #  define BIAS(n)                       (n)
34 #else
35 #  define jit_arg_d_reg_p(i)            ((i) >= 0 && (i) < 16)
36 #  define BIAS(n)                       ((n) + 2047)
37 #endif
38
39 /*
40  * Types
41  */
42 typedef jit_pointer_t   jit_va_list_t;
43
44 /*
45  * Prototypes
46  */
47 #define patch(instr, node)              _patch(_jit, instr, node)
48 static void _patch(jit_state_t*,jit_word_t,jit_node_t*);
49
50 #define PROTO                           1
51 #  include "jit_sparc-cpu.c"
52 #  include "jit_sparc-fpu.c"
53 #  include "jit_fallback.c"
54 #undef PROTO
55
56 /*
57  * Initialization
58  */
59 jit_cpu_t               jit_cpu;
60 jit_register_t          _rvs[] = {
61     { 0x00,                             "%g0" },
62     { 0x01,                             "%g1" },
63     { rc(gpr) | 0x02,                   "%g2" },
64     { rc(gpr) | 0x03,                   "%g3" },
65     { rc(gpr) | 0x04,                   "%g4" },
66     { 0x05,                             "%g5" },
67     { 0x06,                             "%g6" },
68     { 0x07,                             "%g7" },
69     { rc(arg) | rc(gpr) | 0x08,         "%o0" },
70     { rc(arg) | rc(gpr) | 0x09,         "%o1" },
71     { rc(arg) | rc(gpr) | 0x0a,         "%o2" },
72     { rc(arg) | rc(gpr) | 0x0b,         "%o3" },
73     { rc(arg) | rc(gpr) | 0x0c,         "%o4" },
74     { rc(arg) | rc(gpr) | 0x0d,         "%o5" },
75     { rc(sav) | 0x0e,                   "%sp" },
76     { 0x0f,                             "%o7" },
77     { rc(sav) | rc(gpr) | 0x10,         "%l0" },
78     { rc(sav) | rc(gpr) | 0x11,         "%l1" },
79     { rc(sav) | rc(gpr) | 0x12,         "%l2" },
80     { rc(sav) | rc(gpr) | 0x13,         "%l3" },
81     { rc(sav) | rc(gpr) | 0x14,         "%l4" },
82     { rc(sav) | rc(gpr) | 0x15,         "%l5" },
83     { rc(sav) | rc(gpr) | 0x16,         "%l6" },
84     { rc(sav) | rc(gpr) | 0x17,         "%l7" },
85     { 0x18,                             "%i0" },
86     { 0x19,                             "%i1" },
87     { 0x1a,                             "%i2" },
88     { 0x1b,                             "%i3" },
89     { 0x1c,                             "%i4" },
90     { 0x1d,                             "%i5" },
91     { rc(sav) | 0x1e,                   "%fp" },
92     { 0x1f,                             "%i7" },
93 #  if __WORDSIZE == 32
94     { rc(fpr) | 0x00,                   "%f0" },
95     { 0x01,                             "%f1" },
96     { rc(fpr) | 0x02,                   "%f2" },
97     { 0x03,                             "%f3" },
98     { rc(fpr) | 0x04,                   "%f4" },
99     { 0x05,                             "%f5" },
100     { rc(fpr) | 0x06,                   "%f6" },
101     { 0x07,                             "%f7" },
102     { rc(fpr) | 0x08,                   "%f8" },
103     { 0x09,                             "%f9" },
104     { rc(fpr) | 0x0a,                   "%f10" },
105     { 0x0b,                             "%f11" },
106     { rc(fpr) | 0x0c,                   "%f12" },
107     { 0x0d,                             "%f13" },
108     { rc(fpr) | 0x0e,                   "%f14" },
109     { 0x0f,                             "%f15" },
110 #  else
111     { rc(fpr) | rc(dbl) | 0x3e,         "%f62" },
112     { rc(fpr) | rc(dbl) | 0x3c,         "%f60" },
113     { rc(fpr) | rc(dbl) | 0x3a,         "%f58" },
114     { rc(fpr) | rc(dbl) | 0x38,         "%f56" },
115     { rc(fpr) | rc(dbl) | 0x36,         "%f54" },
116     { rc(fpr) | rc(dbl) | 0x34,         "%f52" },
117     { rc(fpr) | rc(dbl) | 0x32,         "%f50" },
118     { rc(fpr) | rc(dbl) | 0x30,         "%f48" },
119     { rc(fpr) | rc(dbl) | 0x2e,         "%f46" },
120     { rc(fpr) | rc(dbl) | 0x2c,         "%f44" },
121     { rc(fpr) | rc(dbl) | 0x2a,         "%f42" },
122     { rc(fpr) | rc(dbl) | 0x28,         "%f40" },
123     { rc(fpr) | rc(dbl) | 0x26,         "%f38" },
124     { rc(fpr) | rc(dbl) | 0x24,         "%f36" },
125     { rc(fpr) | rc(dbl) | 0x22,         "%f34" },
126     { rc(fpr) | rc(dbl) | 0x20,         "%f32" },
127     { 0x1f,                             "%f31" },
128     { rc(arg)|rc(fpr)|rc(sng)|0x1e,     "%f30" },
129     { 0x1d,                             "%f29" },
130     { rc(arg)|rc(fpr)|rc(sng)|0x1c,     "%f28" },
131     { 0x1b,                             "%f27" },
132     { rc(arg)|rc(fpr)|rc(sng)|0x1a,     "%f26" },
133     { 0x19,                             "%f25" },
134     { rc(arg)|rc(fpr)|rc(sng)|0x18,     "%f24" },
135     { 0x17,                             "%f23" },
136     { rc(arg)|rc(fpr)|rc(sng)|0x16,     "%f22" },
137     { 0x15,                             "%f21" },
138     { rc(arg)|rc(fpr)|rc(sng)|0x14,     "%f20" },
139     { 0x13,                             "%f19" },
140     { rc(arg)|rc(fpr)|rc(sng)|0x12,     "%f18" },
141     { 0x11,                             "%f17" },
142     { rc(arg)|rc(fpr)|rc(sng)|0x10,     "%f16" },
143     { 0x0f,                             "%f15" },
144     { rc(arg)|rc(fpr)|rc(sng)|0x0e,     "%f14" },
145     { 0x0d,                             "%f13" },
146     { rc(arg)|rc(fpr)|rc(sng)|0x0c,     "%f12" },
147     { 0x0b,                             "%f11" },
148     { rc(arg)|rc(fpr)|rc(sng)|0x0a,     "%f10" },
149     { 0x09,                             "%f9" },
150     { rc(arg)|rc(fpr)|rc(sng)|0x08,     "%f8" },
151     { 0x07,                             "%f7" },
152     { rc(arg)|rc(fpr)|rc(sng)|0x06,     "%f6" },
153     { 0x05,                             "%f5" },
154     { rc(arg)|rc(fpr)|rc(sng)|0x04,     "%f4" },
155     { 0x03,                             "%f3" },
156     { rc(arg)|rc(fpr)|rc(sng)|0x02,     "%f2" },
157     { 0x01,                             "%f1" },
158     { rc(arg)|rc(fpr)|rc(sng)|0x00,     "%f0" },
159 #  endif
160     { _NOREG,                           "<none>" },
161 };
162 #if CHECK_LZCNT
163 sigjmp_buf              jit_env;
164 #endif
165
166 /*
167  * Implementation
168  */
169 #if CHECK_LZCNT
170 static void
171 sigill_handler(int signum)
172 {
173     jit_cpu.lzcnt = 0;
174     siglongjmp(jit_env, 1);
175 }
176 #endif
177
178 void
179 jit_get_cpu(void)
180 {
181 #if CHECK_LZCNT
182     int                 g2;
183     struct              sigaction new_action, old_action;
184     new_action.sa_handler = sigill_handler;
185     sigemptyset(&new_action.sa_mask);
186     new_action.sa_flags = 0;
187     sigaction(SIGILL, NULL, &old_action);
188     if (old_action.sa_handler != SIG_IGN) {
189         sigaction(SIGILL, &new_action, NULL);
190         if (!sigsetjmp(jit_env, 1)) {
191             jit_cpu.lzcnt = 1;
192             /* lzcnt %g2, %g2 */
193             __asm__ volatile("mov %%g2, %0; .long 0xa3b0021; mov %0, %%g2"
194                              : "=r" (g2));
195             sigaction(SIGILL, &old_action, NULL);
196         }
197     }
198 #else
199     jit_cpu.lzcnt = 0;
200 #endif
201 }
202
203 void
204 _jit_init(jit_state_t *_jit)
205 {
206     _jitc->reglen = jit_size(_rvs) - 1;
207 #  if __WORDSIZE == 64
208     jit_carry = _NOREG;
209 #  endif
210 }
211
212 void
213 _jit_prolog(jit_state_t *_jit)
214 {
215     jit_int32_t          offset;
216
217     if (_jitc->function)
218         jit_epilog();
219     assert(jit_regset_cmp_ui(&_jitc->regarg, 0) == 0);
220     jit_regset_set_ui(&_jitc->regsav, 0);
221     offset = _jitc->functions.offset;
222     if (offset >= _jitc->functions.length) {
223         jit_realloc((jit_pointer_t *)&_jitc->functions.ptr,
224                     _jitc->functions.length * sizeof(jit_function_t),
225                     (_jitc->functions.length + 16) * sizeof(jit_function_t));
226         _jitc->functions.length += 16;
227     }
228     _jitc->function = _jitc->functions.ptr + _jitc->functions.offset++;
229     _jitc->function->self.size = stack_framesize;
230     _jitc->function->self.argi = _jitc->function->self.argf =
231         _jitc->function->self.alen = 0;
232     /* float conversion */
233 #  if __WORDSIZE == 32
234     _jitc->function->self.aoff = -8;
235 #  else
236     /* extra slots in case qmul is called */
237     _jitc->function->self.aoff = -24;
238 #  endif
239      _jitc->function->self.call = jit_call_default;
240     jit_alloc((jit_pointer_t *)&_jitc->function->regoff,
241               _jitc->reglen * sizeof(jit_int32_t));
242
243     /* _no_link here does not mean the jit_link() call can be removed
244      * by rewriting as:
245      * _jitc->function->prolog = jit_new_node(jit_code_prolog);
246      */
247     _jitc->function->prolog = jit_new_node_no_link(jit_code_prolog);
248     jit_link(_jitc->function->prolog);
249     _jitc->function->prolog->w.w = offset;
250     _jitc->function->epilog = jit_new_node_no_link(jit_code_epilog);
251     /*  u:      label value
252      *  v:      offset in blocks vector
253      *  w:      offset in functions vector
254      */
255     _jitc->function->epilog->w.w = offset;
256
257     jit_regset_new(&_jitc->function->regset);
258 }
259
260 jit_int32_t
261 _jit_allocai(jit_state_t *_jit, jit_int32_t length)
262 {
263     assert(_jitc->function);
264     switch (length) {
265         case 0: case 1:                                         break;
266         case 2:         _jitc->function->self.aoff &= -2;       break;
267         case 3: case 4: _jitc->function->self.aoff &= -4;       break;
268         default:        _jitc->function->self.aoff &= -8;       break;
269     }
270     _jitc->function->self.aoff -= length;
271     if (!_jitc->realize) {
272         jit_inc_synth_ww(allocai, _jitc->function->self.aoff, length);
273         jit_dec_synth();
274     }
275     return (BIAS(_jitc->function->self.aoff));
276 }
277
278 void
279 _jit_allocar(jit_state_t *_jit, jit_int32_t u, jit_int32_t v)
280 {
281     jit_int32_t          reg;
282     assert(_jitc->function);
283     jit_inc_synth_ww(allocar, u, v);
284     if (!_jitc->function->allocar) {
285         _jitc->function->aoffoff = jit_allocai(sizeof(jit_int32_t));
286         _jitc->function->allocar = 1;
287     }
288     reg = jit_get_reg(jit_class_gpr);
289     jit_negr(reg, v);
290     jit_andi(reg, reg, -16);
291     jit_ldxi_i(u, JIT_FP, _jitc->function->aoffoff);
292     jit_addr(u, u, reg);
293     jit_addr(_SP, _SP, reg);
294     jit_stxi_i(_jitc->function->aoffoff, JIT_FP, u);
295     jit_unget_reg(reg);
296     jit_dec_synth();
297 }
298
299 void
300 _jit_ret(jit_state_t *_jit)
301 {
302     jit_node_t          *instr;
303     assert(_jitc->function);
304     jit_inc_synth(ret);
305     /* jump to epilog */
306     instr = jit_jmpi();
307     jit_patch_at(instr, _jitc->function->epilog);
308     jit_dec_synth();
309 }
310
311 void
312 _jit_retr(jit_state_t *_jit, jit_int32_t u, jit_code_t code)
313 {
314     jit_code_inc_synth_w(code, u);
315     jit_movr(JIT_RET, u);
316     jit_ret();
317     jit_dec_synth();
318 }
319
320 void
321 _jit_reti(jit_state_t *_jit, jit_word_t u, jit_code_t code)
322 {
323     jit_code_inc_synth_w(code, u);
324     jit_movi(JIT_RET, u);
325     jit_ret();
326     jit_dec_synth();
327 }
328
329 void
330 _jit_retr_f(jit_state_t *_jit, jit_int32_t u)
331 {
332     jit_inc_synth_w(retr_f, u);
333     if (JIT_FRET != u)
334         jit_movr_f(JIT_FRET, u);
335     else
336         jit_live(JIT_FRET);
337     jit_ret();
338     jit_dec_synth();
339 }
340
341 void
342 _jit_reti_f(jit_state_t *_jit, jit_float32_t u)
343 {
344     jit_inc_synth_f(reti_f, u);
345     jit_movi_f(JIT_FRET, u);
346     jit_ret();
347     jit_dec_synth();
348 }
349
350 void
351 _jit_retr_d(jit_state_t *_jit, jit_int32_t u)
352 {
353     jit_inc_synth_w(retr_d, u);
354     if (JIT_FRET != u)
355         jit_movr_d(JIT_FRET, u);
356     else
357         jit_live(JIT_FRET);
358     jit_ret();
359     jit_dec_synth();
360 }
361
362 void
363 _jit_reti_d(jit_state_t *_jit, jit_float64_t u)
364 {
365     jit_inc_synth_d(reti_d, u);
366     jit_movi_d(JIT_FRET, u);
367     jit_ret();
368     jit_dec_synth();
369 }
370
371 void
372 _jit_epilog(jit_state_t *_jit)
373 {
374     assert(_jitc->function);
375     assert(_jitc->function->epilog->next == NULL);
376     jit_link(_jitc->function->epilog);
377     _jitc->function = NULL;
378 }
379
380 jit_bool_t
381 _jit_arg_register_p(jit_state_t *_jit, jit_node_t *u)
382 {
383 #  if __WORDSIZE == 32
384     if ((u->code >= jit_code_arg_c && u->code <= jit_code_arg) ||
385         u->code == jit_code_arg_f)
386         return (jit_arg_reg_p(u->u.w));
387     assert(u->code == jit_code_arg_d);
388     return (jit_arg_d_reg_p(u->u.w));
389 #  else
390     if (u->code >= jit_code_arg_c && u->code <= jit_code_arg)
391         return (jit_arg_reg_p(u->u.w));
392     assert(u->code == jit_code_arg_d || u->code == jit_code_arg_f);
393     return (jit_arg_d_reg_p(u->u.w));
394 #  endif
395 }
396
397 void
398 _jit_ellipsis(jit_state_t *_jit)
399 {
400     jit_inc_synth(ellipsis);
401     if (_jitc->prepare) {
402         jit_link_prepare();
403         assert(!(_jitc->function->call.call & jit_call_varargs));
404         _jitc->function->call.call |= jit_call_varargs;
405     }
406     else {
407         jit_link_prolog();
408         assert(!(_jitc->function->self.call & jit_call_varargs));
409         _jitc->function->self.call |= jit_call_varargs;
410
411         _jitc->function->vagp = _jitc->function->self.argi;
412     }
413     jit_dec_synth();
414 }
415
416 void
417 _jit_va_push(jit_state_t *_jit, jit_int32_t u)
418 {
419     jit_inc_synth_w(va_push, u);
420     jit_pushargr(u);
421     jit_dec_synth();
422 }
423
424 jit_node_t *
425 _jit_arg(jit_state_t *_jit, jit_code_t code)
426 {
427     jit_node_t          *node;
428     jit_int32_t          offset;
429     assert(_jitc->function);
430     assert(!(_jitc->function->self.call & jit_call_varargs));
431 #if STRONG_TYPE_CHECKING
432     assert(code >= jit_code_arg_c && code <= jit_code_arg);
433 #endif
434     if (jit_arg_reg_p(_jitc->function->self.argi))
435         offset = _jitc->function->self.argi++;
436     else {
437 #  if __WORDSIZE == 64
438         if (jit_arg_d_reg_p(_jitc->function->self.argi))
439             ++_jitc->function->self.argi;
440 #   endif
441         offset = BIAS(_jitc->function->self.size);
442         _jitc->function->self.size += sizeof(jit_word_t);
443     }
444     node = jit_new_node_ww(code, offset,
445                            ++_jitc->function->self.argn);
446     jit_link_prolog();
447     return (node);
448 }
449
450 jit_node_t *
451 _jit_arg_f(jit_state_t *_jit)
452 {
453     jit_node_t          *node;
454     jit_int32_t          offset;
455 #  if __WORDSIZE == 64
456     jit_bool_t           inc;
457 #  endif
458     assert(_jitc->function);
459 #  if __WORDSIZE == 32
460     if (jit_arg_reg_p(_jitc->function->self.argi))
461         offset = _jitc->function->self.argi++;
462     else {
463         offset = _jitc->function->self.size;
464         _jitc->function->self.size += sizeof(jit_word_t);
465     }
466 #  else
467     inc = !jit_arg_reg_p(_jitc->function->self.argi);
468     if (jit_arg_d_reg_p(_jitc->function->self.argi))
469         offset = _jitc->function->self.argi++;
470     else
471         offset = BIAS(_jitc->function->self.size);
472     if (inc)
473         _jitc->function->self.size += sizeof(jit_word_t);
474 #  endif
475     node = jit_new_node_ww(jit_code_arg_f, offset,
476                            ++_jitc->function->self.argn);
477     jit_link_prolog();
478     return (node);
479 }
480
481 jit_node_t *
482 _jit_arg_d(jit_state_t *_jit)
483 {
484     jit_node_t          *node;
485     jit_int32_t          offset;
486 #  if __WORDSIZE == 64
487     jit_bool_t           inc;
488 #  endif
489     assert(_jitc->function);
490 #  if __WORDSIZE == 32
491     if (jit_arg_d_reg_p(_jitc->function->self.argi)) {
492         offset = _jitc->function->self.argi;
493         _jitc->function->self.argi += 2;
494     }
495     else if (jit_arg_reg_p(_jitc->function->self.argi)) {
496         offset = _jitc->function->self.argi++;
497         _jitc->function->self.size += sizeof(jit_float32_t);
498     }
499     else {
500         offset = _jitc->function->self.size;
501         _jitc->function->self.size += sizeof(jit_float64_t);
502     }
503 #  else
504     inc = !jit_arg_reg_p(_jitc->function->self.argi);
505     if (jit_arg_d_reg_p(_jitc->function->self.argi))
506         offset = _jitc->function->self.argi++;
507     else
508         offset = BIAS(_jitc->function->self.size);
509     if (inc)
510         _jitc->function->self.size += sizeof(jit_word_t);
511 #  endif
512     node = jit_new_node_ww(jit_code_arg_d, offset,
513                            ++_jitc->function->self.argn);
514     jit_link_prolog();
515     return (node);
516 }
517
518 void
519 _jit_getarg_c(jit_state_t *_jit, jit_int32_t u, jit_node_t *v)
520 {
521     assert_arg_type(v->code, jit_code_arg_c);
522     jit_inc_synth_wp(getarg_c, u, v);
523     if (jit_arg_reg_p(v->u.w))
524         jit_extr_c(u, _I0 + v->u.w);
525     else
526         jit_ldxi_c(u, JIT_FP,
527                    v->u.w + (__WORDSIZE >> 3) - sizeof(jit_int8_t));
528     jit_dec_synth();
529 }
530
531 void
532 _jit_getarg_uc(jit_state_t *_jit, jit_int32_t u, jit_node_t *v)
533 {
534     assert_arg_type(v->code, jit_code_arg_c);
535     jit_inc_synth_wp(getarg_uc, u, v);
536     if (jit_arg_reg_p(v->u.w))
537         jit_extr_uc(u, _I0 + v->u.w);
538     else
539         jit_ldxi_uc(u, JIT_FP,
540                     v->u.w + (__WORDSIZE >> 3) - sizeof(jit_uint8_t));
541     jit_dec_synth();
542 }
543
544 void
545 _jit_getarg_s(jit_state_t *_jit, jit_int32_t u, jit_node_t *v)
546 {
547     assert_arg_type(v->code, jit_code_arg_s);
548     jit_inc_synth_wp(getarg_s, u, v);
549     if (jit_arg_reg_p(v->u.w))
550         jit_extr_s(u, _I0 + v->u.w);
551     else
552         jit_ldxi_s(u, JIT_FP,
553                    v->u.w + (__WORDSIZE >> 3) - sizeof(jit_int16_t));
554     jit_dec_synth();
555 }
556
557 void
558 _jit_getarg_us(jit_state_t *_jit, jit_int32_t u, jit_node_t *v)
559 {
560     assert_arg_type(v->code, jit_code_arg_s);
561     jit_inc_synth_wp(getarg_us, u, v);
562     if (jit_arg_reg_p(v->u.w))
563         jit_extr_us(u, _I0 + v->u.w);
564     else
565         jit_ldxi_us(u, JIT_FP,
566                     v->u.w + (__WORDSIZE >> 3) - sizeof(jit_uint16_t));
567     jit_dec_synth();
568 }
569
570 void
571 _jit_getarg_i(jit_state_t *_jit, jit_int32_t u, jit_node_t *v)
572 {
573     assert_arg_type(v->code, jit_code_arg_i);
574     jit_inc_synth_wp(getarg_i, u, v);
575     if (jit_arg_reg_p(v->u.w)) {
576 #  if __WORDSIZE == 64
577         jit_extr_i(u, _I0 + v->u.w);
578 #  else
579         jit_movr(u, _I0 + v->u.w);
580 #  endif
581     }
582     else
583         jit_ldxi_i(u, JIT_FP,
584                    v->u.w + (__WORDSIZE >> 3) - sizeof(jit_int32_t));
585     jit_dec_synth();
586 }
587
588 #  if __WORDSIZE == 64
589 void
590 _jit_getarg_ui(jit_state_t *_jit, jit_int32_t u, jit_node_t *v)
591 {
592     assert_arg_type(v->code, jit_code_arg_i);
593     jit_inc_synth_wp(getarg_i, u, v);
594     if (jit_arg_reg_p(v->u.w))
595         jit_extr_ui(u, _I0 + v->u.w);
596     else
597         jit_ldxi_ui(u, JIT_FP,
598                     v->u.w + (__WORDSIZE >> 3) - sizeof(jit_int32_t));
599     jit_dec_synth();
600 }
601
602 void
603 _jit_getarg_l(jit_state_t *_jit, jit_int32_t u, jit_node_t *v)
604 {
605     assert_arg_type(v->code, jit_code_arg_l);
606     jit_inc_synth_wp(getarg_i, u, v);
607     if (jit_arg_reg_p(v->u.w))
608         jit_movr(u, _I0 + v->u.w);
609     else
610         jit_ldxi_l(u, JIT_FP, v->u.w);
611     jit_dec_synth();
612 }
613 #  endif
614
615 void
616 _jit_putargr(jit_state_t *_jit, jit_int32_t u, jit_node_t *v, jit_code_t code)
617 {
618     assert_putarg_type(code, v->code);
619     jit_code_inc_synth_wp(code, u, v);
620     if (jit_arg_reg_p(v->u.w))
621         jit_movr(_I0 + v->u.w, u);
622     else
623         jit_stxi(v->u.w, JIT_FP, u);
624     jit_dec_synth();
625 }
626
627 void
628 _jit_putargi(jit_state_t *_jit, jit_word_t u, jit_node_t *v, jit_code_t code)
629 {
630     jit_int32_t         regno;
631     assert_putarg_type(code, v->code);
632     jit_code_inc_synth_wp(code, u, v);
633     if (jit_arg_reg_p(v->u.w))
634         jit_movi(_I0 + v->u.w, u);
635     else {
636         regno = jit_get_reg(jit_class_gpr);
637         jit_movi(regno, u);
638         jit_stxi(v->u.w, JIT_FP, regno);
639         jit_unget_reg(regno);
640     }
641     jit_dec_synth();
642 }
643
644 void
645 _jit_getarg_f(jit_state_t *_jit, jit_int32_t u, jit_node_t *v)
646 {
647     assert(v->code == jit_code_arg_f);
648     assert(_jitc->function);
649     jit_inc_synth_wp(getarg_f, u, v);
650 #  if __WORDSIZE == 32
651     if (jit_arg_reg_p(v->u.w)) {
652         jit_stxi_i(-4, JIT_FP, _I0 + v->u.w);
653         jit_ldxi_f(u, JIT_FP, -4);
654     }
655 #  else
656     if (jit_arg_d_reg_p(v->u.w)) {
657         jit_live(_F0 - (v->u.w << 1));  /* pair of registers is live */
658         jit_movr_f(u, (_F0 - (v->u.w << 1)) - 1);
659     }
660 #  endif
661     else
662         jit_ldxi_f(u, JIT_FP, v->u.w + (__WORDSIZE >> 3) -
663                    sizeof(jit_float32_t));
664     jit_dec_synth();
665 }
666
667 void
668 _jit_putargr_f(jit_state_t *_jit, jit_int32_t u, jit_node_t *v)
669 {
670     assert(v->code == jit_code_arg_f);
671     jit_inc_synth_wp(putargr_f, u, v);
672 #  if __WORDSIZE == 32
673     if (jit_arg_reg_p(v->u.w)) {
674         jit_stxi_f(-4, JIT_FP, u);
675         jit_ldxi_i(_I0 + v->u.w, JIT_FP, -4);
676     }
677 #  else
678     if (jit_arg_d_reg_p(v->u.w)) {
679         jit_live(_F0 - (v->u.w << 1));  /* pair of registers is live */
680         jit_movr_f((_F0 - (v->u.w << 1)) - 1, u);
681     }
682 #  endif
683     else
684         jit_stxi_f(v->u.w + (__WORDSIZE >> 3) -
685                    sizeof(jit_float32_t), JIT_FP, u);
686     jit_dec_synth();
687 }
688
689 void
690 _jit_putargi_f(jit_state_t *_jit, jit_float32_t u, jit_node_t *v)
691 {
692     jit_int32_t         regno;
693     assert(v->code == jit_code_arg_f);
694     jit_inc_synth_fp(putargi_f, u, v);
695 #  if __WORDSIZE == 32
696     regno = jit_get_reg(jit_class_fpr);
697     jit_movi_f(regno, u);
698     if (jit_arg_reg_p(v->u.w)) {
699         jit_stxi_f(-4, JIT_FP, regno);
700         jit_ldxi_i(_I0 + v->u.w, JIT_FP, -4);
701     }
702     else
703         jit_stxi_f(v->u.w, JIT_FP, regno);
704     jit_unget_reg(regno);
705 #  else
706     if (jit_arg_d_reg_p(v->u.w)) {
707         jit_live(_F0 - (v->u.w << 1));  /* pair of registers is live */
708         jit_movi_f((_F0 - (v->u.w << 1)) - 1, u);
709     }
710     else {
711         regno = jit_get_reg(jit_class_fpr | jit_class_sng);
712         jit_movi_f(regno, u);
713         jit_stxi_f(v->u.w + (__WORDSIZE >> 3) -
714                    sizeof(jit_float32_t), JIT_FP, regno);
715         jit_unget_reg(regno);
716     }
717 #  endif
718     jit_dec_synth();
719 }
720
721 void
722 _jit_getarg_d(jit_state_t *_jit, jit_int32_t u, jit_node_t *v)
723 {
724     assert(v->code == jit_code_arg_d);
725     assert(_jitc->function);
726     jit_inc_synth_wp(getarg_d, u, v);
727     if (jit_arg_d_reg_p(v->u.w)) {
728 #  if __WORDSIZE == 32
729         jit_stxi(-8, JIT_FP, _I0 + v->u.w);
730         jit_stxi(-4, JIT_FP, _I0 + v->u.w + 1);
731         jit_ldxi_d(u, JIT_FP, -8);
732 #  else
733         jit_movr_d(u, _F0 - (v->u.w << 1));
734 #  endif
735     }
736 #  if __WORDSIZE == 32
737     else if (jit_arg_reg_p(v->u.w)) {
738         jit_stxi(-8, JIT_FP, _I0 + v->u.w);
739         jit_ldxi_f(u, JIT_FP, -8);
740         jit_ldxi_f(u + 1, JIT_FP, stack_framesize);
741     }
742 #  endif
743     else {
744 #  if __WORDSIZE == 32
745         jit_ldxi_f(u, JIT_FP, v->u.w);
746         jit_ldxi_f(u + 1, JIT_FP, v->u.w + 4);
747 #  else
748         jit_ldxi_d(u, JIT_FP, v->u.w);
749 #  endif
750     }
751     jit_dec_synth();
752 }
753
754 void
755 _jit_putargr_d(jit_state_t *_jit, jit_int32_t u, jit_node_t *v)
756 {
757    jit_int32_t          regno;
758     assert(v->code == jit_code_arg_d);
759     jit_inc_synth_wp(putargr_d, u, v);
760 #  if __WORDSIZE == 32
761     if (jit_arg_d_reg_p(v->u.w)) {
762         jit_stxi_d(-8, JIT_FP, u);
763         jit_ldxi(_I0 + v->u.w, JIT_FP, -8);
764         jit_ldxi(_I0 + v->u.w + 1, JIT_FP, -4);
765     }
766     else if (jit_arg_reg_p(v->u.w)) {
767         regno = jit_get_reg(jit_class_gpr);
768         jit_stxi_d(-8, JIT_FP, u);
769         jit_ldxi(_I0 + v->u.w, JIT_FP, -8);
770         jit_ldxi(regno, JIT_FP, -4);
771         jit_stxi(stack_framesize, JIT_FP, regno);
772         jit_unget_reg(regno);
773     }
774     else if ((v->u.w & 7) == 0)
775         jit_stxi_d(v->u.w, JIT_FP, u);
776     else {
777         jit_stxi_d(-8, JIT_FP, u);
778         regno = jit_get_reg(jit_class_gpr);
779         jit_ldxi(regno, JIT_FP, -8);
780         jit_stxi(v->u.w, JIT_FP, regno);
781         jit_ldxi(regno, JIT_FP, -4);
782         jit_stxi(v->u.w + 4, JIT_FP, regno);
783         jit_unget_reg(regno);
784     }
785 #  else
786     if (jit_arg_d_reg_p(v->u.w))
787         jit_movr_d(_F0 - (v->u.w << 1), u);
788     else
789         jit_stxi_d(v->u.w, JIT_FP, u);
790 #  endif
791     jit_dec_synth();
792 }
793
794 void
795 _jit_putargi_d(jit_state_t *_jit, jit_float64_t u, jit_node_t *v)
796 {
797 #  if __WORDSIZE == 32
798     jit_int32_t         gpr;
799 #  endif
800    jit_int32_t          regno;
801     assert(v->code == jit_code_arg_d);
802     jit_inc_synth_dp(putargi_d, u, v);
803 #  if __WORDSIZE == 32
804     regno = jit_get_reg(jit_class_fpr);
805     jit_movi_d(regno, u);
806     if (jit_arg_d_reg_p(v->u.w)) {
807         jit_stxi_d(-8, JIT_FP, regno);
808         jit_ldxi(_I0 + v->u.w, JIT_FP, -8);
809         jit_ldxi(_I0 + v->u.w + 1, JIT_FP, -4);
810     }
811     else if (jit_arg_reg_p(v->u.w)) {
812         gpr = jit_get_reg(jit_class_gpr);
813         jit_stxi_d(-8, JIT_FP, regno);
814         jit_ldxi(_I0 + v->u.w, JIT_FP, -8);
815         jit_ldxi(gpr, JIT_FP, -4);
816         jit_stxi(stack_framesize, JIT_FP, gpr);
817         jit_unget_reg(gpr);
818     }
819     else if ((v->u.w & 7) == 0)
820         jit_stxi_d(v->u.w, JIT_FP, regno);
821     else {
822         jit_stxi_d(-8, JIT_FP, regno);
823         gpr = jit_get_reg(jit_class_gpr);
824         jit_ldxi(gpr, JIT_FP, -8);
825         jit_stxi(v->u.w, JIT_FP, gpr);
826         jit_ldxi(gpr, JIT_FP, -4);
827         jit_stxi(v->u.w + 4, JIT_FP, gpr);
828         jit_unget_reg(gpr);
829     }
830     jit_unget_reg(regno);
831 #  else
832     if (jit_arg_d_reg_p(v->u.w))
833         jit_movi_d(_F0 - (v->u.w << 1), u);
834     else {
835         regno = jit_get_reg(jit_class_fpr | jit_class_dbl);
836         jit_movi_d(regno, u);
837         jit_stxi_d(v->u.w, JIT_FP, regno);
838         jit_unget_reg(regno);
839     }
840 #  endif
841     jit_dec_synth();
842 }
843
844 void
845 _jit_pushargr(jit_state_t *_jit, jit_int32_t u, jit_code_t code)
846 {
847     jit_code_inc_synth_w(code, u);
848     jit_link_prepare();
849     if (jit_arg_reg_p(_jitc->function->call.argi)) {
850         jit_movr(_O0 + _jitc->function->call.argi, u);
851         ++_jitc->function->call.argi;
852     }
853     else {
854 #  if __WORDSIZE == 64
855         if (jit_arg_d_reg_p(_jitc->function->call.argi))
856             ++_jitc->function->call.argi;
857 #  endif
858         jit_stxi(BIAS(_jitc->function->call.size + stack_framesize),
859                  JIT_SP, u);
860         _jitc->function->call.size += sizeof(jit_word_t);
861     }
862     jit_dec_synth();
863 }
864
865 void
866 _jit_pushargi(jit_state_t *_jit, jit_word_t u, jit_code_t code)
867 {
868     jit_int32_t         regno;
869     jit_code_inc_synth_w(code, u);
870     jit_link_prepare();
871     if (jit_arg_reg_p(_jitc->function->call.argi)) {
872         jit_movi(_O0 + _jitc->function->call.argi, u);
873         ++_jitc->function->call.argi;
874     }
875     else {
876 #  if __WORDSIZE == 64
877         if (jit_arg_d_reg_p(_jitc->function->call.argi))
878             ++_jitc->function->call.argi;
879 #  endif
880         regno = jit_get_reg(jit_class_gpr);
881         jit_movi(regno, u);
882         jit_stxi(BIAS(_jitc->function->call.size + stack_framesize),
883                  JIT_SP, regno);
884         jit_unget_reg(regno);
885         _jitc->function->call.size += sizeof(jit_word_t);
886     }
887     jit_dec_synth();
888 }
889
890 void
891 _jit_pushargr_f(jit_state_t *_jit, jit_int32_t u)
892 {
893     jit_inc_synth_w(pushargr_f, u);
894     jit_link_prepare();
895 #  if __WORDSIZE == 32
896     if (jit_arg_reg_p(_jitc->function->call.argi)) {
897         jit_stxi_f(-8, JIT_FP, u);
898         jit_ldxi(_O0 + _jitc->function->call.argi, JIT_FP, -8);
899         ++_jitc->function->call.argi;
900     }
901     else {
902         jit_stxi_f(_jitc->function->call.size + stack_framesize,
903                    JIT_SP, u);
904         _jitc->function->call.size += sizeof(jit_float32_t);
905     }
906 #  else
907     if ((_jitc->function->call.call & jit_call_varargs) &&
908         jit_arg_reg_p(_jitc->function->call.argi)) {
909         jit_stxi_f(BIAS(-8), JIT_FP, u);
910         jit_ldxi_i(_O0 + _jitc->function->call.argi, JIT_FP, BIAS(-8));
911         ++_jitc->function->call.argi;
912     }
913     else if (!(_jitc->function->call.call & jit_call_varargs) &&
914              jit_arg_d_reg_p(_jitc->function->call.argi)) {
915         /* pair of registers is live */
916         jit_live(_F0 - (_jitc->function->call.argi << 1));
917         jit_movr_f((_F0 - (_jitc->function->call.argi << 1)) - 1, u);
918         if (!jit_arg_reg_p(_jitc->function->call.argi))
919             _jitc->function->call.size += sizeof(jit_float64_t);
920         ++_jitc->function->call.argi;
921     }
922     else {
923         jit_stxi_f(BIAS(_jitc->function->call.size + stack_framesize + 4),
924                    JIT_SP, u);
925         _jitc->function->call.size += sizeof(jit_float64_t);
926     }
927 #  endif
928     jit_dec_synth();
929 }
930
931 void
932 _jit_pushargi_f(jit_state_t *_jit, jit_float32_t u)
933 {
934     jit_int32_t         regno;
935     jit_inc_synth_f(pushargi_f, u);
936     jit_link_prepare();
937 #  if __WORDSIZE == 32
938     regno = jit_get_reg(jit_class_fpr);
939     jit_movi_f(regno, u);
940     if (jit_arg_reg_p(_jitc->function->call.argi)) {
941         jit_stxi_f(-8, JIT_FP, regno);
942         jit_ldxi(_O0 + _jitc->function->call.argi, JIT_FP, -8);
943         _jitc->function->call.argi++;
944     }
945     else {
946         jit_stxi_f(_jitc->function->call.size + stack_framesize,
947                    JIT_SP, regno);
948         _jitc->function->call.size += sizeof(jit_float32_t);
949     }
950     jit_unget_reg(regno);
951 #  else
952     if ((_jitc->function->call.call & jit_call_varargs) &&
953         jit_arg_reg_p(_jitc->function->call.argi)) {
954         regno = jit_get_reg(jit_class_fpr | jit_class_sng);
955         jit_movi_f(regno, u);
956         jit_stxi_f(BIAS(-8), JIT_FP, regno);
957         jit_ldxi_i(_O0 + _jitc->function->call.argi, JIT_FP, BIAS(-8));
958         ++_jitc->function->call.argi;
959         jit_unget_reg(regno);
960     }
961     else if (!(_jitc->function->call.call & jit_call_varargs) &&
962              jit_arg_d_reg_p(_jitc->function->call.argi)) {
963         /* pair of registers is live */
964         jit_live(_F0 - (_jitc->function->call.argi << 1));
965         jit_movi_f((_F0 - (_jitc->function->call.argi << 1)) - 1, u);
966         if (!jit_arg_reg_p(_jitc->function->call.argi))
967             _jitc->function->call.size += sizeof(jit_float64_t);
968         ++_jitc->function->call.argi;
969     }
970     else {
971         regno = jit_get_reg(jit_class_fpr | jit_class_sng);
972         jit_movi_f(regno, u);
973         jit_stxi_f(BIAS(_jitc->function->call.size + stack_framesize + 4),
974                    JIT_SP, regno);
975         jit_unget_reg(regno);
976         _jitc->function->call.size += sizeof(jit_float64_t);
977     }
978 #  endif
979     jit_dec_synth();
980 }
981
982 void
983 _jit_pushargr_d(jit_state_t *_jit, jit_int32_t u)
984 {
985     jit_inc_synth_w(pushargr_d, u);
986     jit_link_prepare();
987 #  if __WORDSIZE == 32
988     if (jit_arg_d_reg_p(_jitc->function->call.argi)) {
989         jit_stxi_d(BIAS(-8), JIT_FP, u);
990         jit_ldxi(_O0 + _jitc->function->call.argi, JIT_FP, BIAS(-8));
991         jit_ldxi(_O0 + _jitc->function->call.argi + 1, JIT_FP, -4);
992         _jitc->function->call.argi += 2;
993     }
994     else if (jit_arg_reg_p(_jitc->function->call.argi)) {
995         jit_stxi_f(-8, JIT_FP, u);
996         jit_ldxi(_O0 + _jitc->function->call.argi, JIT_FP, -8);
997         ++_jitc->function->call.argi;
998         jit_stxi_f(stack_framesize, JIT_SP, u + 1);
999         _jitc->function->call.size += sizeof(jit_float32_t);
1000     }
1001     else {
1002         jit_stxi_f(_jitc->function->call.size + stack_framesize,
1003                    JIT_SP, u);
1004         jit_stxi_f(_jitc->function->call.size + stack_framesize + 4,
1005                    JIT_SP, u + 1);
1006         _jitc->function->call.size += sizeof(jit_float64_t);
1007     }
1008 #  else
1009     if ((_jitc->function->call.call & jit_call_varargs) &&
1010         jit_arg_reg_p(_jitc->function->call.argi)) {
1011         jit_stxi_d(BIAS(-8), JIT_FP, u);
1012         jit_ldxi(_O0 + _jitc->function->call.argi, JIT_FP, BIAS(-8));
1013         ++_jitc->function->call.argi;
1014     }
1015     else if (!(_jitc->function->call.call & jit_call_varargs) &&
1016              jit_arg_d_reg_p(_jitc->function->call.argi)) {
1017         jit_movr_d(_F0 - (_jitc->function->call.argi << 1), u);
1018         if (!jit_arg_reg_p(_jitc->function->call.argi))
1019             _jitc->function->call.size += sizeof(jit_float64_t);
1020         ++_jitc->function->call.argi;
1021     }
1022     else {
1023         jit_stxi_d(BIAS(_jitc->function->call.size + stack_framesize),
1024                    JIT_SP, u);
1025         _jitc->function->call.size += sizeof(jit_float64_t);
1026     }
1027 #  endif
1028     jit_dec_synth();
1029 }
1030
1031 void
1032 _jit_pushargi_d(jit_state_t *_jit, jit_float64_t u)
1033 {
1034     jit_int32_t         regno;
1035     jit_inc_synth_d(pushargi_d, u);
1036     jit_link_prepare();
1037 #  if __WORDSIZE == 32
1038     regno = jit_get_reg(jit_class_fpr);
1039     jit_movi_d(regno, u);
1040     if (jit_arg_d_reg_p(_jitc->function->call.argi)) {
1041         jit_stxi_d(BIAS(-8), JIT_FP, regno);
1042         jit_ldxi(_O0 + _jitc->function->call.argi, JIT_FP, BIAS(-8));
1043         jit_ldxi(_O0 + _jitc->function->call.argi + 1, JIT_FP, -4);
1044         _jitc->function->call.argi += 2;
1045     }
1046     else if (jit_arg_reg_p(_jitc->function->call.argi)) {
1047         jit_stxi_f(-8, JIT_FP, regno);
1048         jit_ldxi(_O0 + _jitc->function->call.argi, JIT_FP, -8);
1049         ++_jitc->function->call.argi;
1050         jit_stxi_f(stack_framesize, JIT_SP, u + 1);
1051         _jitc->function->call.size += sizeof(jit_float32_t);
1052     }
1053     else {
1054         jit_stxi_f(_jitc->function->call.size + stack_framesize,
1055                    JIT_SP, regno);
1056         jit_stxi_f(_jitc->function->call.size + stack_framesize + 4,
1057                    JIT_SP, regno + 1);
1058         _jitc->function->call.size += sizeof(jit_float64_t);
1059     }
1060     jit_unget_reg(regno);
1061 #  else
1062     if ((_jitc->function->call.call & jit_call_varargs) &&
1063         jit_arg_reg_p(_jitc->function->call.argi)) {
1064         regno = jit_get_reg(jit_class_fpr | jit_class_dbl);
1065         jit_movi_d(regno, u);
1066         jit_stxi_d(BIAS(-8), JIT_FP, regno);
1067         jit_ldxi(_O0 + _jitc->function->call.argi, JIT_FP, BIAS(-8));
1068         ++_jitc->function->call.argi;
1069         jit_unget_reg(regno);
1070     }
1071     else if (!(_jitc->function->call.call & jit_call_varargs) &&
1072              jit_arg_d_reg_p(_jitc->function->call.argi)) {
1073         jit_movi_d(_F0 - (_jitc->function->call.argi << 1), u);
1074         if (!jit_arg_reg_p(_jitc->function->call.argi))
1075             _jitc->function->call.size += sizeof(jit_float64_t);
1076         ++_jitc->function->call.argi;
1077     }
1078     else {
1079         regno = jit_get_reg(jit_class_fpr | jit_class_dbl);
1080         jit_movi_d(regno, u);
1081         jit_stxi_d(BIAS(_jitc->function->call.size + stack_framesize),
1082                    JIT_SP, regno);
1083         jit_unget_reg(regno);
1084         _jitc->function->call.size += sizeof(jit_float64_t);
1085     }
1086 #  endif
1087     jit_dec_synth();
1088 }
1089
1090 jit_bool_t
1091 _jit_regarg_p(jit_state_t *_jit, jit_node_t *node, jit_int32_t regno)
1092 {
1093     jit_int32_t         spec;
1094
1095     spec = jit_class(_rvs[regno].spec);
1096     if ((spec & (jit_class_arg|jit_class_gpr)) ==
1097         (jit_class_arg|jit_class_gpr)) {
1098         regno -= _O0;
1099         if (regno >= 0 && regno < node->v.w)
1100             return (1);
1101     }
1102 #  if __WORDSIZE == 64
1103     if ((spec & (jit_class_arg|jit_class_fpr)) ==
1104         (jit_class_arg|jit_class_fpr)) {
1105         regno = _F0 - (regno >> 1);
1106         if (regno >= 0 && regno < node->v.w)
1107             return (1);
1108     }
1109 #  endif
1110
1111     return (0);
1112 }
1113
1114 void
1115 _jit_finishr(jit_state_t *_jit, jit_int32_t r0)
1116 {
1117     jit_node_t          *call;
1118     assert(_jitc->function);
1119     jit_inc_synth_w(finishr, r0);
1120     if (_jitc->function->self.alen < _jitc->function->call.size)
1121         _jitc->function->self.alen = _jitc->function->call.size;
1122     call = jit_callr(r0);
1123     call->v.w = _jitc->function->self.argi;
1124     call->w.w = _jitc->function->self.argf;
1125     _jitc->function->call.argi = _jitc->function->call.argf =
1126         _jitc->function->call.size = 0;
1127     _jitc->prepare = 0;
1128     jit_dec_synth();
1129 }
1130
1131 jit_node_t *
1132 _jit_finishi(jit_state_t *_jit, jit_pointer_t i0)
1133 {
1134     jit_node_t          *node;
1135     assert(_jitc->function);
1136     jit_inc_synth_w(finishi, (jit_word_t)i0);
1137     if (_jitc->function->self.alen < _jitc->function->call.size)
1138         _jitc->function->self.alen = _jitc->function->call.size;
1139     node = jit_calli(i0);
1140     node->v.w = _jitc->function->call.argi;
1141     node->w.w = _jitc->function->call.argf;
1142     _jitc->function->call.argi = _jitc->function->call.argf =
1143         _jitc->function->call.size = 0;
1144     _jitc->prepare = 0;
1145     jit_dec_synth();
1146     return (node);
1147 }
1148
1149 void
1150 _jit_retval_c(jit_state_t *_jit, jit_int32_t r0)
1151 {
1152     jit_inc_synth_w(retval_c, r0);
1153     jit_extr_c(r0, _O0);
1154     jit_dec_synth();
1155 }
1156
1157 void
1158 _jit_retval_uc(jit_state_t *_jit, jit_int32_t r0)
1159 {
1160     jit_inc_synth_w(retval_uc, r0);
1161     jit_extr_uc(r0, _O0);
1162     jit_dec_synth();
1163 }
1164
1165 void
1166 _jit_retval_s(jit_state_t *_jit, jit_int32_t r0)
1167 {
1168     jit_inc_synth_w(retval_s, r0);
1169     jit_extr_s(r0, _O0);
1170     jit_dec_synth();
1171 }
1172
1173 void
1174 _jit_retval_us(jit_state_t *_jit, jit_int32_t r0)
1175 {
1176     jit_inc_synth_w(retval_us, r0);
1177     jit_extr_us(r0, _O0);
1178     jit_dec_synth();
1179 }
1180
1181 void
1182 _jit_retval_i(jit_state_t *_jit, jit_int32_t r0)
1183 {
1184     jit_inc_synth_w(retval_i, r0);
1185 #  if __WORDSIZE == 32
1186     if (r0 != _O0)
1187         jit_movr(r0, _O0);
1188 #  else
1189     jit_extr_i(r0, _O0);
1190 #  endif
1191     jit_dec_synth();
1192 }
1193
1194 #  if __WORDSIZE == 64
1195 void
1196 _jit_retval_ui(jit_state_t *_jit, jit_int32_t r0)
1197 {
1198     jit_inc_synth_w(retval_i, r0);
1199     if (r0 != _O0)
1200         jit_extr_ui(r0, _O0);
1201     jit_dec_synth();
1202 }
1203
1204 void
1205 _jit_retval_l(jit_state_t *_jit, jit_int32_t r0)
1206 {
1207     jit_inc_synth_w(retval_i, r0);
1208     if (r0 != _O0)
1209         jit_movr(r0, _O0);
1210     jit_dec_synth();
1211 }
1212 #  endif
1213
1214 void
1215 _jit_retval_f(jit_state_t *_jit, jit_int32_t r0)
1216 {
1217     jit_inc_synth_w(retval_f, r0);
1218     if (r0 != JIT_FRET)
1219         jit_movr_f(r0, JIT_FRET);
1220     jit_dec_synth();
1221 }
1222
1223 void
1224 _jit_retval_d(jit_state_t *_jit, jit_int32_t r0)
1225 {
1226     jit_inc_synth_w(retval_d, r0);
1227     if (r0 != JIT_FRET)
1228         jit_movr_d(r0, JIT_FRET);
1229     jit_dec_synth();
1230 }
1231
1232 jit_pointer_t
1233 _emit_code(jit_state_t *_jit)
1234 {
1235     jit_node_t          *node;
1236     jit_node_t          *temp;
1237     jit_word_t           word;
1238     jit_int32_t          value;
1239     jit_int32_t          offset;
1240     struct {
1241         jit_node_t      *node;
1242         jit_word_t       word;
1243         jit_function_t   func;
1244 #if DEVEL_DISASSEMBLER
1245         jit_word_t       prevw;
1246 #endif
1247         jit_int32_t      patch_offset;
1248     } undo;
1249 #if DEVEL_DISASSEMBLER
1250     jit_word_t           prevw;
1251 #endif
1252
1253     _jitc->function = NULL;
1254
1255     jit_reglive_setup();
1256
1257     undo.word = 0;
1258     undo.node = NULL;
1259     undo.patch_offset = 0;
1260
1261 #define case_rr(name, type)                                             \
1262             case jit_code_##name##r##type:                              \
1263                 name##r##type(rn(node->u.w), rn(node->v.w));            \
1264                 break
1265 #define case_rw(name, type)                                             \
1266             case jit_code_##name##i##type:                              \
1267                 name##i##type(rn(node->u.w), node->v.w);                \
1268                 break
1269 #define case_wr(name, type)                                             \
1270             case jit_code_##name##i##type:                              \
1271                 name##i##type(node->u.w, rn(node->v.w));                \
1272                 break
1273 #define case_rf(name)                                                   \
1274             case jit_code_##name##i##type:                              \
1275                 assert(node->flag & jit_flag_data);                     \
1276                 name##_f(rn(node->u.w),                                 \
1277                 (jit_float32_t *)node->v.n->u.w);                       \
1278                 break
1279 #define case_rd(name)                                                   \
1280             case jit_code_##name##i_d:                                  \
1281                 assert(node->flag & jit_flag_data);                     \
1282                 name##_d(rn(node->u.w),                                 \
1283                          (jit_float64_t *)node->v.n->u.w);              \
1284                 break
1285 #define case_rrr(name, type)                                            \
1286             case jit_code_##name##r##type:                              \
1287                 name##r##type(rn(node->u.w),                            \
1288                               rn(node->v.w), rn(node->w.w));            \
1289                 break
1290 #define case_rrrr(name, type)                                           \
1291             case jit_code_##name##r##type:                              \
1292                 name##r##type(rn(node->u.q.l), rn(node->u.q.h),         \
1293                               rn(node->v.w), rn(node->w.w));            \
1294                 break
1295 #define case_rqr(name, type)                                            \
1296             case jit_code_##name##r##type:                              \
1297                 name##r##type(rn(node->u.w), rn(node->v.q.l),           \
1298                               rn(node->v.q.h), rn(node->w.w));          \
1299             case jit_code_##name##i##type:                              \
1300                 break;
1301 #define case_rrw(name, type)                                            \
1302             case jit_code_##name##i##type:                              \
1303                 name##i##type(rn(node->u.w),                            \
1304                               rn(node->v.w), node->w.w);                \
1305                 break
1306 #define case_rrrw(name, type)                                           \
1307             case jit_code_##name##i##type:                              \
1308                 name##i##type(rn(node->u.q.l), rn(node->u.q.h),         \
1309                               rn(node->v.w), node->w.w);                \
1310                 break
1311 #define case_rrf(name, type, size)                                      \
1312             case jit_code_##name##i##type:                              \
1313                 assert(node->flag & jit_flag_data);                     \
1314                 name##i##type(rn(node->u.w), rn(node->v.w),             \
1315                               (jit_float##size##_t *)node->w.n->u.w);   \
1316                 break
1317 #define case_wrr(name, type)                                            \
1318             case jit_code_##name##i##type:                              \
1319                 name##i##type(node->u.w, rn(node->v.w), rn(node->w.w)); \
1320                 break
1321 #define case_brr(name, type)                                            \
1322             case jit_code_##name##r##type:                              \
1323                 temp = node->u.n;                                       \
1324                 assert(temp->code == jit_code_label ||                  \
1325                        temp->code == jit_code_epilog);                  \
1326                 if (temp->flag & jit_flag_patch)                        \
1327                     name##r##type(temp->u.w, rn(node->v.w),             \
1328                                   rn(node->w.w));                       \
1329                 else {                                                  \
1330                     word = name##r##type(_jit->pc.w,                    \
1331                                          rn(node->v.w), rn(node->w.w)); \
1332                     patch(word, node);                                  \
1333                 }                                                       \
1334                 break
1335 #define case_brw(name, type)                                            \
1336             case jit_code_##name##i##type:                              \
1337                 temp = node->u.n;                                       \
1338                 assert(temp->code == jit_code_label ||                  \
1339                        temp->code == jit_code_epilog);                  \
1340                 if (temp->flag & jit_flag_patch)                        \
1341                     name##i##type(temp->u.w,                            \
1342                                   rn(node->v.w), node->w.w);            \
1343                 else {                                                  \
1344                     word = name##i##type(_jit->pc.w,                    \
1345                                          rn(node->v.w), node->w.w);     \
1346                     patch(word, node);                                  \
1347                 }                                                       \
1348                 break
1349 #define case_brf(name, type, size)                                      \
1350             case jit_code_##name##i##type:                              \
1351                 temp = node->u.n;                                       \
1352                 assert(temp->code == jit_code_label ||                  \
1353                        temp->code == jit_code_epilog);                  \
1354                 if (temp->flag & jit_flag_patch)                        \
1355                     name##i##type(temp->u.w, rn(node->v.w),             \
1356                                 (jit_float##size##_t *)node->w.n->u.w); \
1357                 else {                                                  \
1358                     word = name##i##type(_jit->pc.w, rn(node->v.w),     \
1359                                 (jit_float##size##_t *)node->w.n->u.w); \
1360                     patch(word, node);                                  \
1361                 }                                                       \
1362                 break
1363 #if DEVEL_DISASSEMBLER
1364     prevw = _jit->pc.w;
1365 #endif
1366     for (node = _jitc->head; node; node = node->next) {
1367         if (_jit->pc.uc >= _jitc->code.end)
1368             return (NULL);
1369
1370 #if DEVEL_DISASSEMBLER
1371         node->offset = (jit_uword_t)_jit->pc.w - (jit_uword_t)prevw;
1372         prevw = _jit->pc.w;
1373 #endif
1374         value = jit_classify(node->code);
1375         jit_regarg_set(node, value);
1376         switch (node->code) {
1377             case jit_code_align:
1378                 /* Must align to a power of two */
1379                 assert(!(node->u.w & (node->u.w - 1)));
1380                 if ((word = _jit->pc.w & (node->u.w - 1)))
1381                     nop(node->u.w - word);
1382                 break;
1383             case jit_code_skip:
1384                 nop((node->u.w + 3) & ~3);
1385                 break;
1386             case jit_code_note:         case jit_code_name:
1387                 node->u.w = _jit->pc.w;
1388                 break;
1389             case jit_code_label:
1390                 if ((node->link || (node->flag & jit_flag_use)) &&
1391                     (word = _jit->pc.w & (sizeof(jit_word_t) - 1)))
1392                     nop(sizeof(jit_word_t) - word);
1393                 /* remember label is defined */
1394                 node->flag |= jit_flag_patch;
1395                 node->u.w = _jit->pc.w;
1396                 break;
1397                 case_rrr(add,);
1398                 case_rrw(add,);
1399                 case_rrr(addc,);
1400                 case_rrw(addc,);
1401                 case_rrr(addx,);
1402                 case_rrw(addx,);
1403                 case_rrr(sub,);
1404                 case_rrw(sub,);
1405                 case_rrr(subc,);
1406                 case_rrw(subc,);
1407                 case_rrr(subx,);
1408                 case_rrw(subx,);
1409                 case_rrw(rsb,);
1410                 case_rrr(mul,);
1411                 case_rrw(mul,);
1412                 case_rrr(hmul,);
1413                 case_rrw(hmul,);
1414                 case_rrr(hmul, _u);
1415                 case_rrw(hmul, _u);
1416                 case_rrrr(qmul,);
1417                 case_rrrw(qmul,);
1418                 case_rrrr(qmul, _u);
1419                 case_rrrw(qmul, _u);
1420                 case_rrr(div,);
1421                 case_rrw(div,);
1422                 case_rrr(div, _u);
1423                 case_rrw(div, _u);
1424                 case_rrrr(qdiv,);
1425                 case_rrrw(qdiv,);
1426                 case_rrrr(qdiv, _u);
1427                 case_rrrw(qdiv, _u);
1428                 case_rrr(rem,);
1429                 case_rrw(rem,);
1430                 case_rrr(rem, _u);
1431                 case_rrw(rem, _u);
1432                 case_rrr(and,);
1433                 case_rrw(and,);
1434                 case_rrr(or,);
1435                 case_rrw(or,);
1436                 case_rrr(xor,);
1437                 case_rrw(xor,);
1438                 case_rrr(lsh,);
1439                 case_rrw(lsh,);
1440 #define qlshr(r0, r1, r2, r3)   fallback_qlshr(r0, r1, r2, r3)
1441 #define qlshi(r0, r1, r2, i0)   fallback_qlshi(r0, r1, r2, i0)
1442 #define qlshr_u(r0, r1, r2, r3) fallback_qlshr_u(r0, r1, r2, r3)
1443 #define qlshi_u(r0, r1, r2, i0) fallback_qlshi_u(r0, r1, r2, i0)
1444                 case_rrrr(qlsh,);
1445                 case_rrrw(qlsh,);
1446                 case_rrrr(qlsh, _u);
1447                 case_rrrw(qlsh, _u);
1448                 case_rrr(rsh,);
1449                 case_rrw(rsh,);
1450                 case_rrr(rsh, _u);
1451                 case_rrw(rsh, _u);
1452 #define qrshr(r0, r1, r2, r3)   fallback_qrshr(r0, r1, r2, r3)
1453 #define qrshi(r0, r1, r2, i0)   fallback_qrshi(r0, r1, r2, i0)
1454 #define qrshr_u(r0, r1, r2, r3) fallback_qrshr_u(r0, r1, r2, r3)
1455 #define qrshi_u(r0, r1, r2, i0) fallback_qrshi_u(r0, r1, r2, i0)
1456                 case_rrrr(qrsh,);
1457                 case_rrrw(qrsh,);
1458                 case_rrrr(qrsh, _u);
1459                 case_rrrw(qrsh, _u);
1460 #define lrotr(r0,r1,r2) fallback_lrotr(r0,r1,r2)
1461 #define lroti(r0,r1,i0) fallback_lroti(r0,r1,i0)
1462 #define rrotr(r0,r1,r2) fallback_rrotr(r0,r1,r2)
1463 #define rroti(r0,r1,i0) fallback_rroti(r0,r1,i0)
1464                 case_rrr(lrot,);
1465                 case_rrw(lrot,);
1466                 case_rrr(rrot,);
1467                 case_rrw(rrot,);
1468                 case_rr(trunc, _f_i);
1469                 case_rr(trunc, _d_i);
1470 #if __WORDSIZE == 64
1471                 case_rr(trunc, _f_l);
1472                 case_rr(trunc, _d_l);
1473 #endif
1474                 case_rrr(lt,);
1475                 case_rrw(lt,);
1476                 case_rrr(lt, _u);
1477                 case_rrw(lt, _u);
1478                 case_rrr(le,);
1479                 case_rrw(le,);
1480                 case_rrr(le, _u);
1481                 case_rrw(le, _u);
1482                 case_rrr(eq,);
1483                 case_rrw(eq,);
1484                 case_rrr(ge,);
1485                 case_rrw(ge,);
1486                 case_rrr(ge, _u);
1487                 case_rrw(ge, _u);
1488                 case_rrr(gt,);
1489                 case_rrw(gt,);
1490                 case_rrr(gt, _u);
1491                 case_rrw(gt, _u);
1492                 case_rrr(ne,);
1493                 case_rrw(ne,);
1494                 case_rr(ld, _c);
1495                 case_rw(ld, _c);
1496                 case_rr(ld, _uc);
1497                 case_rw(ld, _uc);
1498                 case_rr(ld, _s);
1499                 case_rw(ld, _s);
1500                 case_rr(ld, _us);
1501                 case_rw(ld, _us);
1502                 case_rr(ld, _i);
1503                 case_rw(ld, _i);
1504 #if __WORDSIZE == 64
1505                 case_rr(ld, _ui);
1506                 case_rw(ld, _ui);
1507                 case_rr(ld, _l);
1508                 case_rw(ld, _l);
1509 #endif
1510                 case_rrr(ldx, _c);
1511                 case_rrw(ldx, _c);
1512                 case_rrr(ldx, _uc);
1513                 case_rrw(ldx, _uc);
1514                 case_rrr(ldx, _s);
1515                 case_rrw(ldx, _s);
1516                 case_rrr(ldx, _us);
1517                 case_rrw(ldx, _us);
1518                 case_rrr(ldx, _i);
1519                 case_rrw(ldx, _i);
1520 #if __WORDSIZE == 64
1521                 case_rrr(ldx, _ui);
1522                 case_rrw(ldx, _ui);
1523                 case_rrr(ldx, _l);
1524                 case_rrw(ldx, _l);
1525 #endif
1526 #define unldr(r0, r1, i0)       fallback_unldr(r0, r1, i0)
1527             case jit_code_unldr:
1528                 unldr(rn(node->u.w), rn(node->v.w), node->w.w);
1529                 break;
1530 #define unldi(r0, i0, i1)       fallback_unldi(r0, i0, i1)
1531             case jit_code_unldi:
1532                 unldi(rn(node->u.w), node->v.w, node->w.w);
1533                 break;
1534 #define unldr_u(r0, r1, i0)     fallback_unldr_u(r0, r1, i0)
1535             case jit_code_unldr_u:
1536                 unldr_u(rn(node->u.w), rn(node->v.w), node->w.w);
1537                 break;
1538 #define unldi_u(r0, i0, i1)     fallback_unldi_u(r0, i0, i1)
1539             case jit_code_unldi_u:
1540                 unldi_u(rn(node->u.w), node->v.w, node->w.w);
1541                 break;
1542                 case_rr(st, _c);
1543                 case_wr(st, _c);
1544                 case_rr(st, _s);
1545                 case_wr(st, _s);
1546                 case_rr(st, _i);
1547                 case_wr(st, _i);
1548 #if __WORDSIZE == 64
1549                 case_rr(st, _l);
1550                 case_wr(st, _l);
1551 #endif
1552                 case_rrr(stx, _c);
1553                 case_wrr(stx, _c);
1554                 case_rrr(stx, _s);
1555                 case_wrr(stx, _s);
1556                 case_rrr(stx, _i);
1557                 case_wrr(stx, _i);
1558 #if __WORDSIZE == 64
1559                 case_rrr(stx, _l);
1560                 case_wrr(stx, _l);
1561 #endif
1562 #define unstr(r0, r1, i0)       fallback_unstr(r0, r1, i0)
1563             case jit_code_unstr:
1564                 unstr(rn(node->u.w), rn(node->v.w), node->w.w);
1565                 break;
1566 #define unsti(i0, r0, i1)       fallback_unsti(i0, r0, i1)
1567             case jit_code_unsti:
1568                 unsti(node->u.w, rn(node->v.w), node->w.w);
1569                 break;
1570                 case_rr(hton, _us);
1571                 case_rr(hton, _ui);
1572 #if __WORDSIZE == 64
1573                 case_rr(hton, _ul);
1574 #endif
1575                 case_rr(bswap, _us);
1576                 case_rr(bswap, _ui);
1577 #if __WORDSIZE == 64
1578                 case_rr(bswap, _ul);
1579 #endif
1580 #define extr(r0, r1, i0, i1)    fallback_ext(r0, r1, i0, i1)
1581 #define extr_u(r0, r1, i0, i1)  fallback_ext_u(r0, r1, i0, i1)
1582 #define depr(r0, r1, i0, i1)    fallback_dep(r0, r1, i0, i1)
1583             case jit_code_extr:
1584                 extr(rn(node->u.w), rn(node->v.w), node->w.q.l, node->w.q.h);
1585                 break;
1586             case jit_code_extr_u:
1587                 extr_u(rn(node->u.w), rn(node->v.w), node->w.q.l, node->w.q.h);
1588                 break;
1589             case jit_code_depr:
1590                 depr(rn(node->u.w), rn(node->v.w), node->w.q.l, node->w.q.h);
1591                 break;
1592             case jit_code_depi:
1593                 depi(rn(node->u.w), node->v.w, node->w.q.l, node->w.q.h);
1594                 break;
1595                 case_rr(ext, _c);
1596                 case_rr(ext, _uc);
1597                 case_rr(ext, _s);
1598                 case_rr(ext, _us);
1599 #if __WORDSIZE == 64
1600                 case_rr(ext, _i);
1601                 case_rr(ext, _ui);
1602 #endif
1603             case jit_code_casr:
1604                 casr(rn(node->u.w), rn(node->v.w),
1605                      rn(node->w.q.l), rn(node->w.q.h));
1606                 break;
1607             case jit_code_casi:
1608                 casi(rn(node->u.w), node->v.w,
1609                      rn(node->w.q.l), rn(node->w.q.h));
1610                 break;
1611                 case_rrr(movn,);
1612                 case_rrr(movz,);
1613                 case_rr(mov,);
1614             case jit_code_movi:
1615                 if (node->flag & jit_flag_node) {
1616                     temp = node->v.n;
1617                     if (temp->code == jit_code_data ||
1618                         (temp->code == jit_code_label &&
1619                          (temp->flag & jit_flag_patch)))
1620                         movi(rn(node->u.w), temp->u.w);
1621                     else {
1622                         assert(temp->code == jit_code_label ||
1623                                temp->code == jit_code_epilog);
1624                         word = movi_p(rn(node->u.w), node->v.w);
1625                         patch(word, node);
1626                     }
1627                 }
1628                 else
1629                     movi(rn(node->u.w), node->v.w);
1630                 break;
1631                 case_rr(neg,);
1632                 case_rr(com,);
1633                 case_rr(clo,);
1634                 case_rr(clz,);
1635                 case_rr(cto,);
1636                 case_rr(ctz,);
1637 #define rbitr(r0, r1)   fallback_rbit(r0, r1)
1638 #define popcntr(r0, r1) fallback_popcnt(r0, r1)
1639                 case_rr(rbit,);
1640                 case_rr(popcnt,);
1641                 case_brr(blt,);
1642                 case_brw(blt,);
1643                 case_brr(blt, _u);
1644                 case_brw(blt, _u);
1645                 case_brr(ble,);
1646                 case_brw(ble,);
1647                 case_brr(ble, _u);
1648                 case_brw(ble, _u);
1649                 case_brr(beq,);
1650                 case_brw(beq,);
1651                 case_brr(bge,);
1652                 case_brw(bge,);
1653                 case_brr(bge, _u);
1654                 case_brw(bge, _u);
1655                 case_brr(bgt,);
1656                 case_brw(bgt,);
1657                 case_brr(bgt, _u);
1658                 case_brw(bgt, _u);
1659                 case_brr(bne,);
1660                 case_brw(bne,);
1661                 case_brr(boadd,);
1662                 case_brw(boadd,);
1663                 case_brr(boadd, _u);
1664                 case_brw(boadd, _u);
1665                 case_brr(bxadd,);
1666                 case_brw(bxadd,);
1667                 case_brr(bxadd, _u);
1668                 case_brw(bxadd, _u);
1669                 case_brr(bosub,);
1670                 case_brw(bosub,);
1671                 case_brr(bosub, _u);
1672                 case_brw(bosub, _u);
1673                 case_brr(bxsub,);
1674                 case_brw(bxsub,);
1675                 case_brr(bxsub, _u);
1676                 case_brw(bxsub, _u);
1677                 case_brr(bms,);
1678                 case_brw(bms,);
1679                 case_brr(bmc,);
1680                 case_brw(bmc,);
1681                 case_rrr(add, _f);
1682                 case_rrf(add, _f, 32);
1683                 case_rrr(sub, _f);
1684                 case_rrf(sub, _f, 32);
1685                 case_rrf(rsb, _f, 32);
1686                 case_rrr(mul, _f);
1687                 case_rrf(mul, _f, 32);
1688                 case_rrr(div, _f);
1689                 case_rrf(div, _f, 32);
1690                 case_rr(abs, _f);
1691                 case_rr(neg, _f);
1692                 case_rr(sqrt, _f);
1693                 case_rqr(fma, _f);
1694                 case_rqr(fms, _f);
1695                 case_rqr(fnma, _f);
1696                 case_rqr(fnms, _f);
1697                 case_rr(ext, _f);
1698                 case_rr(ext, _d_f);
1699                 case_rrr(lt, _f);
1700                 case_rrf(lt, _f, 32);
1701                 case_rrr(le, _f);
1702                 case_rrf(le, _f, 32);
1703                 case_rrr(eq, _f);
1704                 case_rrf(eq, _f, 32);
1705                 case_rrr(ge, _f);
1706                 case_rrf(ge, _f, 32);
1707                 case_rrr(gt, _f);
1708                 case_rrf(gt, _f, 32);
1709                 case_rrr(ne, _f);
1710                 case_rrf(ne, _f, 32);
1711                 case_rrr(unlt, _f);
1712                 case_rrf(unlt, _f, 32);
1713                 case_rrr(unle, _f);
1714                 case_rrf(unle, _f, 32);
1715                 case_rrr(uneq, _f);
1716                 case_rrf(uneq, _f, 32);
1717                 case_rrr(unge, _f);
1718                 case_rrf(unge, _f, 32);
1719                 case_rrr(ungt, _f);
1720                 case_rrf(ungt, _f, 32);
1721                 case_rrr(ltgt, _f);
1722                 case_rrf(ltgt, _f, 32);
1723                 case_rrr(ord, _f);
1724                 case_rrf(ord, _f, 32);
1725                 case_rrr(unord, _f);
1726                 case_rrf(unord, _f, 32);
1727                 case_rr(ld, _f);
1728                 case_rw(ld, _f);
1729                 case_rrr(ldx, _f);
1730                 case_rrw(ldx, _f);
1731 #define unldr_x(r0, r1, i0)     fallback_unldr_x(r0, r1, i0)
1732             case jit_code_unldr_x:
1733                 unldr_x(rn(node->u.w), rn(node->v.w), node->w.w);
1734                 break;
1735 #define unldi_x(r0, i0, i1)     fallback_unldi_x(r0, i0, i1)
1736             case jit_code_unldi_x:
1737                 unldi_x(rn(node->u.w), node->v.w, node->w.w);
1738                 break;
1739                 case_rr(st, _f);
1740                 case_wr(st, _f);
1741                 case_rrr(stx, _f);
1742                 case_wrr(stx, _f);
1743 #define unstr_x(r0, r1, i0)     fallback_unstr_x(r0, r1, i0)
1744             case jit_code_unstr_x:
1745                 unstr_x(rn(node->u.w), rn(node->v.w), node->w.w);
1746                 break;
1747 #define unsti_x(i0, r0, i1)     fallback_unsti_x(i0, r0, i1)
1748             case jit_code_unsti_x:
1749                 unsti_x(node->u.w, rn(node->v.w), node->w.w);
1750                 break;
1751                 case_rr(mov, _f);
1752             case jit_code_movi_f:
1753                 assert(node->flag & jit_flag_data);
1754                 movi_f(rn(node->u.w), (jit_float32_t *)node->v.n->u.w);
1755                 break;
1756                 case_brr(blt, _f);
1757                 case_brf(blt, _f, 32);
1758                 case_brr(ble, _f);
1759                 case_brf(ble, _f, 32);
1760                 case_brr(beq, _f);
1761                 case_brf(beq, _f, 32);
1762                 case_brr(bge, _f);
1763                 case_brf(bge, _f, 32);
1764                 case_brr(bgt, _f);
1765                 case_brf(bgt, _f, 32);
1766                 case_brr(bne, _f);
1767                 case_brf(bne, _f, 32);
1768                 case_brr(bunlt, _f);
1769                 case_brf(bunlt, _f, 32);
1770                 case_brr(bunle, _f);
1771                 case_brf(bunle, _f, 32);
1772                 case_brr(buneq, _f);
1773                 case_brf(buneq, _f, 32);
1774                 case_brr(bunge, _f);
1775                 case_brf(bunge, _f, 32);
1776                 case_brr(bungt, _f);
1777                 case_brf(bungt, _f, 32);
1778                 case_brr(bltgt, _f);
1779                 case_brf(bltgt, _f, 32);
1780                 case_brr(bord, _f);
1781                 case_brf(bord, _f, 32);
1782                 case_brr(bunord, _f);
1783                 case_brf(bunord, _f, 32);
1784                 case_rrr(add, _d);
1785                 case_rrf(add, _d, 64);
1786                 case_rrr(sub, _d);
1787                 case_rrf(sub, _d, 64);
1788                 case_rrf(rsb, _d, 64);
1789                 case_rrr(mul, _d);
1790                 case_rrf(mul, _d, 64);
1791                 case_rrr(div, _d);
1792                 case_rrf(div, _d, 64);
1793                 case_rr(abs, _d);
1794                 case_rr(neg, _d);
1795                 case_rr(sqrt, _d);
1796                 case_rqr(fma, _d);
1797                 case_rqr(fms, _d);
1798                 case_rqr(fnma, _d);
1799                 case_rqr(fnms, _d);
1800                 case_rr(ext, _d);
1801                 case_rr(ext, _f_d);
1802                 case_rrr(lt, _d);
1803                 case_rrf(lt, _d, 64);
1804                 case_rrr(le, _d);
1805                 case_rrf(le, _d, 64);
1806                 case_rrr(eq, _d);
1807                 case_rrf(eq, _d, 64);
1808                 case_rrr(ge, _d);
1809                 case_rrf(ge, _d, 64);
1810                 case_rrr(gt, _d);
1811                 case_rrf(gt, _d, 64);
1812                 case_rrr(ne, _d);
1813                 case_rrf(ne, _d, 64);
1814                 case_rrr(unlt, _d);
1815                 case_rrf(unlt, _d, 64);
1816                 case_rrr(unle, _d);
1817                 case_rrf(unle, _d, 64);
1818                 case_rrr(uneq, _d);
1819                 case_rrf(uneq, _d, 64);
1820                 case_rrr(unge, _d);
1821                 case_rrf(unge, _d, 64);
1822                 case_rrr(ungt, _d);
1823                 case_rrf(ungt, _d, 64);
1824                 case_rrr(ltgt, _d);
1825                 case_rrf(ltgt, _d, 64);
1826                 case_rrr(ord, _d);
1827                 case_rrf(ord, _d, 64);
1828                 case_rrr(unord, _d);
1829                 case_rrf(unord, _d, 64);
1830                 case_rr(ld, _d);
1831                 case_rw(ld, _d);
1832                 case_rrr(ldx, _d);
1833                 case_rrw(ldx, _d);
1834                 case_rr(st, _d);
1835                 case_wr(st, _d);
1836                 case_rrr(stx, _d);
1837                 case_wrr(stx, _d);
1838                 case_rr(mov, _d);
1839             case jit_code_movi_d:
1840                 assert(node->flag & jit_flag_data);
1841                 movi_d(rn(node->u.w), (jit_float64_t *)node->v.n->u.w);
1842                 break;
1843                 case_brr(blt, _d);
1844                 case_brf(blt, _d, 64);
1845                 case_brr(ble, _d);
1846                 case_brf(ble, _d, 64);
1847                 case_brr(beq, _d);
1848                 case_brf(beq, _d, 64);
1849                 case_brr(bge, _d);
1850                 case_brf(bge, _d, 64);
1851                 case_brr(bgt, _d);
1852                 case_brf(bgt, _d, 64);
1853                 case_brr(bne, _d);
1854                 case_brf(bne, _d, 64);
1855                 case_brr(bunlt, _d);
1856                 case_brf(bunlt, _d, 64);
1857                 case_brr(bunle, _d);
1858                 case_brf(bunle, _d, 64);
1859                 case_brr(buneq, _d);
1860                 case_brf(buneq, _d, 64);
1861                 case_brr(bunge, _d);
1862                 case_brf(bunge, _d, 64);
1863                 case_brr(bungt, _d);
1864                 case_brf(bungt, _d, 64);
1865                 case_brr(bltgt, _d);
1866                 case_brf(bltgt, _d, 64);
1867                 case_brr(bord, _d);
1868                 case_brf(bord, _d, 64);
1869                 case_brr(bunord, _d);
1870                 case_brf(bunord, _d, 64);
1871             case jit_code_jmpr:
1872                 jmpr(rn(node->u.w));
1873                 break;
1874             case jit_code_jmpi:
1875                 if (node->flag & jit_flag_node) {
1876                     temp = node->u.n;
1877                     assert(temp->code == jit_code_label ||
1878                            temp->code == jit_code_epilog);
1879                     if (temp->flag & jit_flag_patch)
1880                         jmpi(temp->u.w);
1881                     else {
1882                         word = _jit->code.length -
1883                             (_jit->pc.uc - _jit->code.ptr);
1884                         if (s22_p(word >> 2))
1885                             word = jmpi(_jit->pc.w);
1886                         else
1887                             word = jmpi_p(_jit->pc.w);
1888                         patch(word, node);
1889                     }
1890                 }
1891                 else
1892                     jmpi(node->u.w);
1893                 break;
1894             case jit_code_callr:
1895                 callr(rn(node->u.w));
1896                 break;
1897             case jit_code_calli:
1898                 if (node->flag & jit_flag_node) {
1899                     temp = node->u.n;
1900                     assert(temp->code == jit_code_label ||
1901                            temp->code == jit_code_epilog);
1902                     if (temp->flag & jit_flag_patch)
1903                         calli(temp->u.w);
1904                     else {
1905                         word = _jit->code.length -
1906                             (_jit->pc.uc - _jit->code.ptr);
1907                         if (s30_p(word >> 2))
1908                             word = calli(_jit->pc.w);
1909                         else
1910                             word = calli_p(_jit->pc.w);
1911                         patch(word, node);
1912                     }
1913                 }
1914                 else
1915                     calli(node->u.w);
1916                 break;
1917             case jit_code_prolog:
1918                 _jitc->function = _jitc->functions.ptr + node->w.w;
1919                 undo.node = node;
1920                 undo.word = _jit->pc.w;
1921                 memcpy(&undo.func, _jitc->function, sizeof(undo.func));
1922 #if DEVEL_DISASSEMBLER
1923                 undo.prevw = prevw;
1924 #endif
1925                 undo.patch_offset = _jitc->patches.offset;
1926             restart_function:
1927                 _jitc->again = 0;
1928                 prolog(node);
1929                 break;
1930             case jit_code_epilog:
1931                 assert(_jitc->function == _jitc->functions.ptr + node->w.w);
1932                 if (_jitc->again) {
1933                     for (temp = undo.node->next;
1934                          temp != node; temp = temp->next) {
1935                         if (temp->code == jit_code_label ||
1936                             temp->code == jit_code_epilog)
1937                             temp->flag &= ~jit_flag_patch;
1938                     }
1939                     temp->flag &= ~jit_flag_patch;
1940                     node = undo.node;
1941                     _jit->pc.w = undo.word;
1942                     /* undo.func.self.aoff and undo.func.regset should not
1943                      * be undone, as they will be further updated, and are
1944                      * the reason of the undo. */
1945                     undo.func.self.aoff = _jitc->function->frame +
1946                         _jitc->function->self.aoff;
1947                     jit_regset_set(&undo.func.regset, &_jitc->function->regset);
1948                     /* allocar information also does not need to be undone */
1949                     undo.func.aoffoff = _jitc->function->aoffoff;
1950                     undo.func.allocar = _jitc->function->allocar;
1951                     /* cvt_offset must also not be undone */
1952                     undo.func.cvt_offset = _jitc->function->cvt_offset;
1953                     memcpy(_jitc->function, &undo.func, sizeof(undo.func));
1954 #if DEVEL_DISASSEMBLER
1955                     prevw = undo.prevw;
1956 #endif
1957                     _jitc->patches.offset = undo.patch_offset;
1958                     goto restart_function;
1959                 }
1960                 /* remember label is defined */
1961                 node->flag |= jit_flag_patch;
1962                 node->u.w = _jit->pc.w;
1963                 epilog(node);
1964                 _jitc->function = NULL;
1965                 break;
1966             case jit_code_movr_w_f:
1967                 movr_w_f(rn(node->u.w), rn(node->v.w));
1968                 break;
1969             case jit_code_movr_f_w:
1970                 movr_f_w(rn(node->u.w), rn(node->v.w));
1971                 break;
1972             case jit_code_movi_f_w:
1973                 assert(node->flag & jit_flag_data);
1974                 movi_f_w(rn(node->u.w), *(jit_float32_t *)node->v.n->u.w);
1975                 break;
1976             case jit_code_movi_w_f:
1977                 movi_w_f(rn(node->u.w), node->v.w);
1978                 break;
1979 #if __WORDSIZE == 32
1980             case jit_code_movr_ww_d:
1981                 movr_ww_d(rn(node->u.w), rn(node->v.w), rn(node->w.w));
1982                 break;
1983             case jit_code_movr_d_ww:
1984                 movr_d_ww(rn(node->u.w), rn(node->v.w), rn(node->w.w));
1985                 break;
1986             case jit_code_movi_d_ww:
1987                 assert(node->flag & jit_flag_data);
1988                 movi_d_ww(rn(node->u.w), rn(node->v.w),
1989                           *(jit_float64_t *)node->w.n->u.w);
1990                 break;
1991             case jit_code_movi_ww_d:
1992                 movi_ww_d(rn(node->u.w), node->v.w, node->w.w);
1993                 break;
1994 #else
1995             case jit_code_movr_w_d:
1996                 movr_w_d(rn(node->u.w), rn(node->v.w));
1997                 break;
1998             case jit_code_movr_d_w:
1999                 movr_d_w(rn(node->u.w), rn(node->v.w));
2000                 break;
2001             case jit_code_movi_d_w:
2002                 assert(node->flag & jit_flag_data);
2003                 movi_d_w(rn(node->u.w), *(jit_float64_t *)node->v.n->u.w);
2004                 break;
2005             case jit_code_movi_w_d:
2006                 movi_w_d(rn(node->u.w), node->v.w);
2007                 break;
2008 #endif
2009             case jit_code_va_start:
2010                 vastart(rn(node->u.w));
2011                 break;
2012             case jit_code_va_arg:
2013                 vaarg(rn(node->u.w), rn(node->v.w));
2014                 break;
2015             case jit_code_va_arg_d:
2016                 vaarg_d(rn(node->u.w), rn(node->v.w));
2017                 break;
2018             case jit_code_live:                 case jit_code_ellipsis:
2019             case jit_code_va_push:
2020             case jit_code_allocai:              case jit_code_allocar:
2021             case jit_code_arg_c:                case jit_code_arg_s:
2022             case jit_code_arg_i:
2023 #if __WORDSIZE == 64
2024            case jit_code_arg_l:
2025 #endif
2026             case jit_code_arg_f:                case jit_code_arg_d:
2027             case jit_code_va_end:
2028             case jit_code_ret:
2029             case jit_code_retr_c:               case jit_code_reti_c:
2030             case jit_code_retr_uc:              case jit_code_reti_uc:
2031             case jit_code_retr_s:               case jit_code_reti_s:
2032             case jit_code_retr_us:              case jit_code_reti_us:
2033             case jit_code_retr_i:               case jit_code_reti_i:
2034 #if __WORDSIZE == 64
2035             case jit_code_retr_ui:              case jit_code_reti_ui:
2036             case jit_code_retr_l:               case jit_code_reti_l:
2037 #endif
2038             case jit_code_retr_f:               case jit_code_reti_f:
2039             case jit_code_retr_d:               case jit_code_reti_d:
2040             case jit_code_getarg_c:             case jit_code_getarg_uc:
2041             case jit_code_getarg_s:             case jit_code_getarg_us:
2042             case jit_code_getarg_i:
2043 #if __WORDSIZE == 64
2044             case jit_code_getarg_ui:            case jit_code_getarg_l:
2045 #endif
2046             case jit_code_getarg_f:             case jit_code_getarg_d:
2047             case jit_code_putargr_c:            case jit_code_putargi_c:
2048             case jit_code_putargr_uc:           case jit_code_putargi_uc:
2049             case jit_code_putargr_s:            case jit_code_putargi_s:
2050             case jit_code_putargr_us:           case jit_code_putargi_us:
2051             case jit_code_putargr_i:            case jit_code_putargi_i:
2052 #if __WORDSIZE == 64
2053             case jit_code_putargr_ui:           case jit_code_putargi_ui:
2054             case jit_code_putargr_l:            case jit_code_putargi_l:
2055 #endif
2056             case jit_code_putargr_f:            case jit_code_putargi_f:
2057             case jit_code_putargr_d:            case jit_code_putargi_d:
2058             case jit_code_pushargr_c:           case jit_code_pushargi_c:
2059             case jit_code_pushargr_uc:          case jit_code_pushargi_uc:
2060             case jit_code_pushargr_s:           case jit_code_pushargi_s:
2061             case jit_code_pushargr_us:          case jit_code_pushargi_us:
2062             case jit_code_pushargr_i:           case jit_code_pushargi_i:
2063 #if __WORDSIZE == 64
2064             case jit_code_pushargr_ui:          case jit_code_pushargi_ui:
2065             case jit_code_pushargr_l:           case jit_code_pushargi_l:
2066 #endif
2067             case jit_code_pushargr_f:           case jit_code_pushargi_f:
2068             case jit_code_pushargr_d:           case jit_code_pushargi_d:
2069             case jit_code_retval_c:             case jit_code_retval_uc:
2070             case jit_code_retval_s:             case jit_code_retval_us:
2071             case jit_code_retval_i:
2072 #if __WORDSIZE == 64
2073             case jit_code_retval_ui:            case jit_code_retval_l:
2074 #endif
2075             case jit_code_retval_f:             case jit_code_retval_d:
2076             case jit_code_prepare:
2077             case jit_code_finishr:              case jit_code_finishi:
2078             case jit_code_negi_f:               case jit_code_absi_f:
2079             case jit_code_sqrti_f:              case jit_code_negi_d:
2080             case jit_code_absi_d:               case jit_code_sqrti_d:
2081                 break;
2082             case jit_code_negi:
2083                 negi(rn(node->u.w), node->v.w);
2084                 break;
2085             case jit_code_comi:
2086                 comi(rn(node->u.w), node->v.w);
2087                 break;
2088             case jit_code_exti_c:
2089                 exti_c(rn(node->u.w), node->v.w);
2090                 break;
2091             case jit_code_exti_uc:
2092                 exti_uc(rn(node->u.w), node->v.w);
2093                 break;
2094             case jit_code_exti_s:
2095                 exti_s(rn(node->u.w), node->v.w);
2096                 break;
2097             case jit_code_exti_us:
2098                 exti_us(rn(node->u.w), node->v.w);
2099                 break;
2100             case jit_code_bswapi_us:
2101                 bswapi_us(rn(node->u.w), node->v.w);
2102                 break;
2103             case jit_code_bswapi_ui:
2104                 bswapi_ui(rn(node->u.w), node->v.w);
2105                 break;
2106             case jit_code_htoni_us:
2107                 htoni_us(rn(node->u.w), node->v.w);
2108                 break;
2109             case jit_code_htoni_ui:
2110                 htoni_ui(rn(node->u.w), node->v.w);
2111                 break;
2112 #if __WORDSIZE == 64
2113             case jit_code_exti_i:
2114                 exti_i(rn(node->u.w), node->v.w);
2115                 break;
2116             case jit_code_exti_ui:
2117                 exti_ui(rn(node->u.w), node->v.w);
2118                 break;
2119             case jit_code_bswapi_ul:
2120                 bswapi_ul(rn(node->u.w), node->v.w);
2121                 break;
2122             case jit_code_htoni_ul:
2123                 htoni_ul(rn(node->u.w), node->v.w);
2124                 break;
2125 #endif
2126             case jit_code_cloi:
2127                 cloi(rn(node->u.w), node->v.w);
2128                 break;
2129             case jit_code_clzi:
2130                 clzi(rn(node->u.w), node->v.w);
2131                 break;
2132             case jit_code_ctoi:
2133                 ctoi(rn(node->u.w), node->v.w);
2134                 break;
2135             case jit_code_ctzi:
2136                 ctzi(rn(node->u.w), node->v.w);
2137                 break;
2138             case jit_code_rbiti:
2139                 rbiti(rn(node->u.w), node->v.w);
2140                 break;
2141             case jit_code_popcnti:
2142                 popcnti(rn(node->u.w), node->v.w);
2143                 break;
2144             case jit_code_exti:
2145                 exti(rn(node->u.w), node->v.w, node->w.q.l, node->w.q.h);
2146                 break;
2147             case jit_code_exti_u:
2148                 exti_u(rn(node->u.w), node->v.w, node->w.q.l, node->w.q.h);
2149                 break;
2150             default:
2151                 abort();
2152         }
2153 #  if __WORDSIZE == 64
2154         if (jit_carry != _NOREG) {
2155             switch (node->code) {
2156                 case jit_code_note:
2157                 case jit_code_addcr:            case jit_code_addci:
2158                 case jit_code_addxr:            case jit_code_addxi:
2159                 case jit_code_subcr:            case jit_code_subci:
2160                 case jit_code_subxr:            case jit_code_subxi:
2161                     break;
2162                 default:
2163                     jit_unget_reg(jit_carry);
2164                     jit_carry = _NOREG;
2165                     break;
2166             }
2167         }
2168 #  endif
2169         jit_regarg_clr(node, value);
2170 #  if __WORDSIZE == 64
2171         if (jit_regset_cmp_ui(&_jitc->regarg, 0) != 0) {
2172             assert(jit_regset_scan1(&_jitc->regarg, 0) == jit_carry);
2173             assert(jit_regset_scan1(&_jitc->regarg, jit_carry + 1) == ULONG_MAX);
2174         }
2175         assert(_jitc->synth == 0);
2176 #  else
2177         assert(_jitc->regarg == 0 && _jitc->synth == 0);
2178 #  endif
2179         jit_reglive(node);
2180     }
2181 #undef case_brf
2182 #undef case_brw
2183 #undef case_brr
2184 #undef case_wrr
2185 #undef case_rrf
2186 #undef case_rrrw
2187 #undef case_rrw
2188 #undef case_rrrr
2189 #undef case_rrr
2190 #undef case_rf
2191 #undef case_wr
2192 #undef case_rw
2193 #undef case_rr
2194
2195     for (offset = 0; offset < _jitc->patches.offset; offset++) {
2196         node = _jitc->patches.ptr[offset].node;
2197         word = node->code == jit_code_movi ? node->v.n->u.w : node->u.n->u.w;
2198         patch_at(_jitc->patches.ptr[offset].inst, word);
2199     }
2200
2201     jit_flush(_jit->code.ptr, _jit->pc.uc);
2202
2203     return (_jit->code.ptr);
2204 }
2205
2206 #define CODE                            1
2207 #  include "jit_sparc-cpu.c"
2208 #  include "jit_sparc-fpu.c"
2209 #  include "jit_fallback.c"
2210 #undef CODE
2211
2212 void
2213 jit_flush(void *fptr, void *tptr)
2214 {
2215 }
2216
2217 void
2218 _emit_ldxi(jit_state_t *_jit, jit_gpr_t r0, jit_gpr_t r1, jit_word_t i0)
2219 {
2220     ldxi(rn(r0), rn(r1), i0);
2221 }
2222
2223 void
2224 _emit_stxi(jit_state_t *_jit, jit_word_t i0, jit_gpr_t r0, jit_gpr_t r1)
2225 {
2226     stxi(i0, rn(r0), rn(r1));
2227 }
2228
2229 void
2230 _emit_ldxi_d(jit_state_t *_jit, jit_fpr_t r0, jit_gpr_t r1, jit_word_t i0)
2231 {
2232     ldxi_d(rn(r0), rn(r1), i0);
2233 }
2234
2235 void
2236 _emit_stxi_d(jit_state_t *_jit, jit_word_t i0, jit_gpr_t r0, jit_fpr_t r1)
2237 {
2238     stxi_d(i0, rn(r0), rn(r1));
2239 }
2240
2241 static void
2242 _patch(jit_state_t *_jit, jit_word_t instr, jit_node_t *node)
2243 {
2244     jit_int32_t         flag;
2245
2246     assert(node->flag & jit_flag_node);
2247     if (node->code == jit_code_movi)
2248         flag = node->v.n->flag;
2249     else
2250         flag = node->u.n->flag;
2251     assert(!(flag & jit_flag_patch));
2252     if (_jitc->patches.offset >= _jitc->patches.length) {
2253         jit_realloc((jit_pointer_t *)&_jitc->patches.ptr,
2254                     _jitc->patches.length * sizeof(jit_patch_t),
2255                     (_jitc->patches.length + 1024) * sizeof(jit_patch_t));
2256         memset(_jitc->patches.ptr + _jitc->patches.length, 0,
2257                1024 * sizeof(jit_patch_t));
2258         _jitc->patches.length += 1024;
2259     }
2260     _jitc->patches.ptr[_jitc->patches.offset].inst = instr;
2261     _jitc->patches.ptr[_jitc->patches.offset].node = node;
2262     ++_jitc->patches.offset;
2263 }