add a thp-based huge page alloc fallback
[pcsx_rearmed.git] / deps / lightning / lib / jit_ia64.c
1 /*
2  * Copyright (C) 2013-2023  Free Software Foundation, Inc.
3  *
4  * This file is part of GNU lightning.
5  *
6  * GNU lightning is free software; you can redistribute it and/or modify it
7  * under the terms of the GNU Lesser General Public License as published
8  * by the Free Software Foundation; either version 3, or (at your option)
9  * any later version.
10  *
11  * GNU lightning is distributed in the hope that it will be useful, but
12  * WITHOUT ANY WARRANTY; without even the implied warranty of MERCHANTABILITY
13  * or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU Lesser General Public
14  * License for more details.
15  *
16  * Authors:
17  *      Paulo Cesar Pereira de Andrade
18  */
19
20 #include <lightning.h>
21 #include <lightning/jit_private.h>
22
23 #define jit_arg_reg_p(i)                ((i) >= 0 && (i) < 8)
24 #if __BYTE_ORDER == __LITTLE_ENDIAN
25 #  define C_DISP                        0
26 #  define S_DISP                        0
27 #  define I_DISP                        0
28 #  define F_DISP                        0
29 #else
30 #  define C_DISP                        8 - sizeof(jit_int8_t)
31 #  define S_DISP                        8 - sizeof(jit_int16_t)
32 #  define I_DISP                        8 - sizeof(jit_int32_t)
33 #  define F_DISP                        8 - sizeof(jit_float32_t)
34 #endif
35
36 /*
37  * Types
38  */
39 typedef jit_pointer_t jit_va_list_t;
40
41 /*
42  * Prototypes
43  */
44 #define patch(instr, node)              _patch(_jit, instr, node)
45 static void _patch(jit_state_t*,jit_word_t,jit_node_t*);
46
47 #if defined(__GNUC__)
48 /* libgcc */
49 extern void __clear_cache(void *, void *);
50 #endif
51
52 #define PROTO                           1
53 #  include "jit_ia64-cpu.c"
54 #  include "jit_ia64-fpu.c"
55 #  include "jit_fallback.c"
56 #undef PROTO
57
58 /*
59  * Initialization
60  */
61 jit_cpu_t               jit_cpu;
62 jit_register_t          _rvs[] = {
63     /* Always 0 */
64     { 0,                 "r0"  },
65     /* Global Pointer */
66     { rc(sav)|1,         "r1"  },
67     /* Used when a register cannot be allocated */
68     { 2,                 "r2"  },
69     /* First scratch register */
70     { rc(gpr)|3,         "r3"  },
71     /* Use r4 as lightning fp register */
72     { rc(sav)|4,         "r4"  },
73     /* Do not touch callee save registers not automatically spill/reloaded */
74     { rc(sav)|5,         "r5"  },       { rc(sav)|6,            "r6"  },
75     { rc(sav)|7,         "r7"  },
76     /* Do not touch return register for the sake of simplicity, besides
77      * having JIT_R0 being the same as JIT_RET usually an optimization */
78     { 8,                 "r8"  },
79     /* Return registers, use as temporaries */
80     { rc(gpr)|9,         "r9"  },
81     { rc(gpr)|10,        "r10" },       { rc(gpr)|11,           "r11" },
82     /* Stack pointer */
83     { rc(sav)|12,        "r12" },
84     /* Thread pointer */
85     { rc(sav)|13,        "r13" },
86     /* (Usually) assembly temporaries */
87     { rc(gpr)|31,        "r31" },       { rc(gpr)|30,           "r30" },
88     { rc(gpr)|29,        "r29" },       { rc(gpr)|28,           "r28" },
89     { rc(gpr)|27,        "r27" },       { rc(gpr)|26,           "r26" },
90     { rc(gpr)|25,        "r25" },       { rc(gpr)|24,           "r24" },
91     { rc(gpr)|23,        "r23" },       { rc(gpr)|22,           "r22" },
92     { rc(gpr)|21,        "r21" },       { rc(gpr)|20,           "r20" },
93     { rc(gpr)|19,        "r19" },       { rc(gpr)|18,           "r18" },
94     { rc(gpr)|17,        "r17" },       { rc(gpr)|16,           "r16" },
95     { rc(gpr)|15,        "r15" },       { rc(gpr)|14,           "r14" },
96     /* Do not allow allocating r32-r41 as temoraries for the sake of
97      * avoiding the need of extra complexity  in the non backend code */
98     { rc(arg)|32,        "r32" },       { rc(arg)|33,           "r33" },
99     { rc(arg)|34,        "r34" },       { rc(arg)|35,           "r35" },
100     { rc(arg)|36,        "r36" },       { rc(arg)|37,           "r37" },
101     { rc(arg)|38,        "r38" },       { rc(arg)|39,           "r39" },
102     /* JIT_R0-JIT_V3 */
103     { rc(gpr)|rc(sav)|40, "r40" },      { rc(gpr)|rc(sav)|41,   "r41" },
104     { rc(gpr)|rc(sav)|42, "r42" },      { rc(gpr)|rc(sav)|43,   "r43" },
105     { rc(gpr)|rc(sav)|44, "r44" },      { rc(gpr)|rc(sav)|45,   "r45" },
106     { rc(gpr)|rc(sav)|46, "r46" },      { rc(gpr)|rc(sav)|47,   "r47" },
107     /* Temporaries/locals */
108     { rc(gpr)|rc(sav)|48, "r48" },      { rc(gpr)|rc(sav)|49,   "r49" },
109     { rc(gpr)|rc(sav)|50, "r50" },      { rc(gpr)|rc(sav)|51,   "r51" },
110     { rc(gpr)|rc(sav)|52, "r52" },      { rc(gpr)|rc(sav)|53,   "r53" },
111     { rc(gpr)|rc(sav)|54, "r54" },      { rc(gpr)|rc(sav)|55,   "r55" },
112     { rc(gpr)|rc(sav)|56, "r56" },      { rc(gpr)|rc(sav)|57,   "r57" },
113     { rc(gpr)|rc(sav)|58, "r58" },      { rc(gpr)|rc(sav)|59,   "r59" },
114     { rc(gpr)|rc(sav)|60, "r60" },      { rc(gpr)|rc(sav)|61,   "r61" },
115     { rc(gpr)|rc(sav)|62, "r62" },      { rc(gpr)|rc(sav)|63,   "r63" },
116     { rc(gpr)|rc(sav)|64, "r64" },      { rc(gpr)|rc(sav)|65,   "r65" },
117     { rc(gpr)|rc(sav)|66, "r66" },      { rc(gpr)|rc(sav)|67,   "r67" },
118     { rc(gpr)|rc(sav)|68, "r68" },      { rc(gpr)|rc(sav)|69,   "r69" },
119     { rc(gpr)|rc(sav)|70, "r70" },      { rc(gpr)|rc(sav)|71,   "r71" },
120     { rc(gpr)|rc(sav)|72, "r72" },      { rc(gpr)|rc(sav)|73,   "r73" },
121     { rc(gpr)|rc(sav)|74, "r74" },      { rc(gpr)|rc(sav)|75,   "r75" },
122     { rc(gpr)|rc(sav)|76, "r76" },      { rc(gpr)|rc(sav)|77,   "r77" },
123     { rc(gpr)|rc(sav)|78, "r78" },      { rc(gpr)|rc(sav)|79,   "r79" },
124     { rc(gpr)|rc(sav)|80, "r80" },      { rc(gpr)|rc(sav)|81,   "r81" },
125     { rc(gpr)|rc(sav)|82, "r82" },      { rc(gpr)|rc(sav)|83,   "r83" },
126     { rc(gpr)|rc(sav)|84, "r84" },      { rc(gpr)|rc(sav)|85,   "r85" },
127     { rc(gpr)|rc(sav)|86, "r86" },      { rc(gpr)|rc(sav)|87,   "r87" },
128     { rc(gpr)|rc(sav)|88, "r88" },      { rc(gpr)|rc(sav)|89,   "r89" },
129     { rc(gpr)|rc(sav)|90, "r90" },      { rc(gpr)|rc(sav)|91,   "r91" },
130     { rc(gpr)|rc(sav)|92, "r92" },      { rc(gpr)|rc(sav)|93,   "r93" },
131     { rc(gpr)|rc(sav)|94, "r94" },      { rc(gpr)|rc(sav)|95,   "r95" },
132     { rc(gpr)|rc(sav)|96, "r96" },      { rc(gpr)|rc(sav)|97,   "r97" },
133     { rc(gpr)|rc(sav)|98, "r98" },      { rc(gpr)|rc(sav)|99,   "r99" },
134     { rc(gpr)|rc(sav)|100,"r100"},      { rc(gpr)|rc(sav)|101,  "r101"},
135     { rc(gpr)|rc(sav)|102,"r102"},      { rc(gpr)|rc(sav)|103,  "r103"},
136     { rc(gpr)|rc(sav)|104,"r104"},      { rc(gpr)|rc(sav)|105,  "r105"},
137     { rc(gpr)|rc(sav)|106,"r106"},      { rc(gpr)|rc(sav)|107,  "r107"},
138     { rc(gpr)|rc(sav)|108,"r108"},      { rc(gpr)|rc(sav)|109,  "r109"},
139     { rc(gpr)|rc(sav)|110,"r110"},      { rc(gpr)|rc(sav)|111,  "r111"},
140     { rc(gpr)|rc(sav)|112,"r112"},      { rc(gpr)|rc(sav)|113,  "r113"},
141     { rc(gpr)|rc(sav)|114,"r114"},
142     /* Do not enable these because no matter what, want 13 free registers,
143      * 5 for prolog and epilog and 8 for outgoing arguments */
144     { 115,               "r115"},
145     { 116,               "r116"},       { 117,                  "r117"},
146     { 118,               "r118"},       { 119,                  "r119"},
147     { 120,               "r120"},       { 121,                  "r121"},
148     { 122,               "r122"},       { 123,                  "r123"},
149     { 124,               "r124"},       { 125,                  "r125"},
150     { 126,               "r126"},       { 127,                  "r127"},
151     /* Always 0.0 */
152     { 0,                 "f0"  },
153     /* Always 1.0 */
154     { 1,                 "f1"  },
155     /* Do not touch callee save registers not automatically spill/reloaded */
156     { rc(sav)|2,         "f2"  },       { rc(sav)|3,            "f3"  },
157     { rc(sav)|4,         "f4"  },       { rc(sav)|5,            "f5"  },
158     /* Scratch */
159     { rc(fpr)|6,         "f6"  },       { rc(fpr)|7,            "f7"  },
160     /* Do not allocate for the sake of simplification */
161     { rc(arg)|8,         "f8"  },
162     /* Scratch - Argument/return registers */
163     { rc(arg)|9,         "f9"  },
164     { rc(arg)|10,        "f10" },       { rc(arg)|11,           "f11" },
165     { rc(arg)|12,        "f12" },       { rc(arg)|13,           "f13" },
166     { rc(arg)|14,        "f14" },       { rc(arg)|15,           "f15" },
167     /* Do not touch callee save registers not automatically spill/reloaded */
168     { rc(sav)|16,        "f16" },       { rc(sav)|17,           "f17" },
169     { rc(sav)|18,        "f18" },       { rc(sav)|19,           "f19" },
170     { rc(sav)|20,        "f20" },       { rc(sav)|21,           "f21" },
171     { rc(sav)|22,        "f22" },       { rc(sav)|23,           "f23" },
172     { rc(sav)|24,        "f24" },       { rc(sav)|25,           "f25" },
173     { rc(sav)|26,        "f26" },       { rc(sav)|27,           "f27" },
174     { rc(sav)|28,        "f28" },       { rc(sav)|29,           "f29" },
175     { rc(sav)|30,        "f30" },       { rc(sav)|31,           "f31" },
176     /* Scratch */
177     { rc(fpr)|32,        "f32" },       { rc(fpr)|33,           "f33" },
178     { rc(fpr)|34,        "f34" },       { rc(fpr)|35,           "f35" },
179     { rc(fpr)|36,        "f36" },       { rc(fpr)|37,           "f37" },
180     { rc(fpr)|38,        "f38" },       { rc(fpr)|39,           "f39" },
181     { rc(fpr)|40,        "f40" },       { rc(fpr)|41,           "f41" },
182     { rc(fpr)|42,        "f42" },       { rc(fpr)|43,           "f43" },
183     { rc(fpr)|44,        "f44" },       { rc(fpr)|45,           "f45" },
184     { rc(fpr)|46,        "f46" },       { rc(fpr)|47,           "f47" },
185     { rc(fpr)|48,        "f48" },       { rc(fpr)|49,           "f49" },
186     { rc(fpr)|50,        "f50" },       { rc(fpr)|51,           "f51" },
187     { rc(fpr)|52,        "f52" },       { rc(fpr)|53,           "f53" },
188     { rc(fpr)|54,        "f54" },       { rc(fpr)|55,           "f55" },
189     { rc(fpr)|56,        "f56" },       { rc(fpr)|57,           "f57" },
190     { rc(fpr)|58,        "f58" },       { rc(fpr)|59,           "f59" },
191     { rc(fpr)|60,        "f60" },       { rc(fpr)|61,           "f61" },
192     { rc(fpr)|62,        "f62" },       { rc(fpr)|63,           "f63" },
193     { rc(fpr)|64,        "f64" },       { rc(fpr)|65,           "f65" },
194     { rc(fpr)|66,        "f66" },       { rc(fpr)|67,           "f67" },
195     { rc(fpr)|68,        "f68" },       { rc(fpr)|69,           "f69" },
196     { rc(fpr)|70,        "f70" },       { rc(fpr)|71,           "f71" },
197     { rc(fpr)|72,        "f72" },       { rc(fpr)|73,           "f73" },
198     { rc(fpr)|74,        "f74" },       { rc(fpr)|75,           "f75" },
199     { rc(fpr)|76,        "f76" },       { rc(fpr)|77,           "f77" },
200     { rc(fpr)|78,        "f78" },       { rc(fpr)|79,           "f79" },
201     { rc(fpr)|80,        "f80" },       { rc(fpr)|81,           "f81" },
202     { rc(fpr)|82,        "f82" },       { rc(fpr)|83,           "f83" },
203     { rc(fpr)|84,        "f84" },       { rc(fpr)|85,           "f85" },
204     { rc(fpr)|86,        "f86" },       { rc(fpr)|87,           "f87" },
205     { rc(fpr)|88,        "f88" },       { rc(fpr)|89,           "f89" },
206     { rc(fpr)|90,        "f90" },       { rc(fpr)|91,           "f91" },
207     { rc(fpr)|92,        "f92" },       { rc(fpr)|93,           "f93" },
208     { rc(fpr)|94,        "f94" },       { rc(fpr)|95,           "f95" },
209     { rc(fpr)|96,        "f96" },       { rc(fpr)|97,           "f97" },
210     { rc(fpr)|98,        "f98" },       { rc(fpr)|99,           "f99" },
211     { rc(fpr)|100,       "f100"},       { rc(fpr)|101,          "f101"},
212     { rc(fpr)|102,       "f102"},       { rc(fpr)|103,          "f103"},
213     { rc(fpr)|104,       "f104"},       { rc(fpr)|105,          "f105"},
214     { rc(fpr)|106,       "f106"},       { rc(fpr)|107,          "f107"},
215     { rc(fpr)|108,       "f108"},       { rc(fpr)|109,          "f109"},
216     { rc(fpr)|110,       "f110"},       { rc(fpr)|111,          "f111"},
217     { rc(fpr)|112,       "f112"},       { rc(fpr)|113,          "f113"},
218     { rc(fpr)|114,       "f114"},       { rc(fpr)|115,          "f115"},
219     { rc(fpr)|116,       "f116"},       { rc(fpr)|117,          "f117"},
220     { rc(fpr)|118,       "f118"},       { rc(fpr)|119,          "f119"},
221 #if 0
222     /* commented to fit a jit_regset_t in 256 bits, so that the fake
223      * O0-O7 registers are easily patched when an argument is pushed */
224     { rc(fpr)|120,       "f120"},       { rc(fpr)|121,          "f121"},
225     { rc(fpr)|122,       "f122"},       { rc(fpr)|123,          "f123"},
226     { rc(fpr)|124,       "f124"},       { rc(fpr)|125,          "f125"},
227     { rc(fpr)|126,       "f126"},       { rc(fpr)|127,          "f127"},
228 #endif
229     /* Fake registers to patch in movr and movi arguments */
230     { rc(arg)|120,       "o0"  },       { rc(arg)|121,          "o1"  },
231     { rc(arg)|122,       "o2"  },       { rc(arg)|123,          "o3"  },
232     { rc(arg)|124,       "o4"  },       { rc(arg)|125,          "o5"  },
233     { rc(arg)|126,       "o6"  },       { rc(arg)|127,          "o7"  },
234     { _NOREG,            "<none>" },
235 };
236
237 /*
238  * Implementation
239  */
240 void
241 jit_get_cpu(void)
242 {
243     jit_word_t          clz = -1;
244     __asm__ volatile("tf.nz.unc p6,p7=32;(p6)mov %0=1;(p7)mov %0=0"
245                      : "=r" (clz));
246     assert(clz == 0 || clz == 1);
247     jit_cpu.clz = clz;
248     /* By default assume kernel or cpu will not handle unaligned load/store */
249     jit_cpu.unaligned = 1;
250 }
251
252 void
253 _jit_init(jit_state_t *_jit)
254 {
255     _jitc->reglen = jit_size(_rvs) - 1;
256     jit_regset_new(&_jitc->regs);
257     jit_carry = _NOREG;
258 }
259
260 void
261 _jit_prolog(jit_state_t *_jit)
262 {
263     jit_int32_t         offset;
264
265     if (_jitc->function)
266         jit_epilog();
267     assert(jit_regset_cmp_ui(&_jitc->regarg, 0) == 0);
268     jit_regset_set_ui(&_jitc->regsav, 0);
269     offset = _jitc->functions.offset;
270     if (offset >= _jitc->functions.length) {
271         jit_realloc((jit_pointer_t *)&_jitc->functions.ptr,
272                     _jitc->functions.length * sizeof(jit_function_t),
273                     (_jitc->functions.length + 16) * sizeof(jit_function_t));
274         _jitc->functions.length += 16;
275     }
276     _jitc->function = _jitc->functions.ptr + _jitc->functions.offset++;
277     /* params_offset << 1, once for ours arguments, once for arguments
278      * to any called function; required because offsets are computed
279      * as JIT_FP displacement */
280     _jitc->function->self.size = stack_framesize + (params_offset << 1);
281     _jitc->function->self.argi = _jitc->function->self.argf =
282         _jitc->function->self.aoff = _jitc->function->self.alen = 0;
283     _jitc->function->self.call = jit_call_default;
284     jit_alloc((jit_pointer_t *)&_jitc->function->regoff,
285               _jitc->reglen * sizeof(jit_int32_t));
286
287     /* _no_link here does not mean the jit_link() call can be removed
288      * by rewriting as:
289      * _jitc->function->prolog = jit_new_node(jit_code_prolog);
290      */
291     _jitc->function->prolog = jit_new_node_no_link(jit_code_prolog);
292     jit_link(_jitc->function->prolog);
293     _jitc->function->prolog->w.w = offset;
294     _jitc->function->epilog = jit_new_node_no_link(jit_code_epilog);
295     /*  u:      label value
296      *  v:      offset in blocks vector
297      *  w:      offset in functions vector
298      */
299     _jitc->function->epilog->w.w = offset;
300
301     jit_regset_new(&_jitc->function->regset);
302 }
303
304 jit_int32_t
305 _jit_allocai(jit_state_t *_jit, jit_int32_t length)
306 {
307     assert(_jitc->function);
308     switch (length) {
309         case 0: case 1:                                         break;
310         case 2:         _jitc->function->self.aoff &= -2;       break;
311         case 3: case 4: _jitc->function->self.aoff &= -4;       break;
312         default:        _jitc->function->self.aoff &= -8;       break;
313     }
314     _jitc->function->self.aoff -= length;
315     if (!_jitc->realize) {
316         jit_inc_synth_ww(allocai, _jitc->function->self.aoff, length);
317         jit_dec_synth();
318     }
319     return (_jitc->function->self.aoff);
320 }
321
322 void
323 _jit_allocar(jit_state_t *_jit, jit_int32_t u, jit_int32_t v)
324 {
325     jit_int32_t          reg;
326     assert(_jitc->function);
327     jit_inc_synth_ww(allocar, u, v);
328     if (!_jitc->function->allocar) {
329         _jitc->function->aoffoff = jit_allocai(sizeof(jit_int32_t));
330         _jitc->function->allocar = 1;
331     }
332     reg = jit_get_reg(jit_class_gpr);
333     jit_negr(reg, v);
334     jit_andi(reg, reg, -16);
335     jit_ldxi_i(u, JIT_FP, _jitc->function->aoffoff);
336     jit_addr(u, u, reg);
337     jit_addr(JIT_SP, JIT_SP, reg);
338     jit_stxi_i(_jitc->function->aoffoff, JIT_FP, u);
339     jit_unget_reg(reg);
340     jit_dec_synth();
341 }
342
343 void
344 _jit_ret(jit_state_t *_jit)
345 {
346     jit_node_t          *instr;
347     assert(_jitc->function);
348     jit_inc_synth(ret);
349     /* jump to epilog */
350     instr = jit_jmpi();
351     jit_patch_at(instr, _jitc->function->epilog);
352     jit_dec_synth();
353 }
354
355 void
356 _jit_retr(jit_state_t *_jit, jit_int32_t u, jit_code_t code)
357 {
358     jit_code_inc_synth_w(code, u);
359     jit_movr(JIT_RET, u);
360     jit_ret();
361     jit_dec_synth();
362 }
363
364 void
365 _jit_reti(jit_state_t *_jit, jit_word_t u, jit_code_t code)
366 {
367     jit_code_inc_synth_w(code, u);
368     jit_movi(JIT_RET, u);
369     jit_ret();
370     jit_dec_synth();
371 }
372
373 void
374 _jit_retr_f(jit_state_t *_jit, jit_int32_t u)
375 {
376     jit_inc_synth_w(retr_f, u);
377     jit_movr_f(JIT_FRET, u);
378     jit_ret();
379     jit_dec_synth();
380 }
381
382 void
383 _jit_reti_f(jit_state_t *_jit, jit_float32_t u)
384 {
385     jit_inc_synth_f(reti_f, u);
386     jit_movi_f(JIT_FRET, u);
387     jit_ret();
388     jit_dec_synth();
389 }
390
391 void
392 _jit_retr_d(jit_state_t *_jit, jit_int32_t u)
393 {
394     jit_inc_synth_w(retr_d, u);
395     jit_movr_d(JIT_FRET, u);
396     jit_ret();
397     jit_dec_synth();
398 }
399
400 void
401 _jit_reti_d(jit_state_t *_jit, jit_float64_t u)
402 {
403     jit_inc_synth_d(reti_d, u);
404     jit_movi_d(JIT_FRET, u);
405     jit_ret();
406     jit_dec_synth();
407 }
408
409 void
410 _jit_epilog(jit_state_t *_jit)
411 {
412     assert(_jitc->function);
413     assert(_jitc->function->epilog->next == NULL);
414     jit_link(_jitc->function->epilog);
415     _jitc->function = NULL;
416 }
417
418 jit_bool_t
419 _jit_arg_register_p(jit_state_t *_jit, jit_node_t *u)
420 {
421     if (u->code >= jit_code_arg_c && u->code <= jit_code_arg)
422         return (jit_arg_reg_p(u->u.w));
423     assert(u->code == jit_code_arg_f || u->code == jit_code_arg_d);
424     return (jit_arg_reg_p(u->u.w) || jit_arg_reg_p(u->u.w - 8));
425 }
426
427 void
428 _jit_ellipsis(jit_state_t *_jit)
429 {
430     jit_inc_synth(ellipsis);
431     if (_jitc->prepare) {
432         assert(!(_jitc->function->call.call & jit_call_varargs));
433         _jitc->function->call.call |= jit_call_varargs;
434         jit_link_prepare();
435     }
436     else {
437         assert(!(_jitc->function->self.call & jit_call_varargs));
438         _jitc->function->self.call |= jit_call_varargs;
439         _jitc->function->vagp = _jitc->function->self.argi;
440         jit_link_prolog();
441     }
442     jit_dec_synth();
443 }
444
445 void
446 _jit_va_push(jit_state_t *_jit, jit_int32_t u)
447 {
448     jit_inc_synth_w(va_push, u);
449     jit_pushargr(u);
450     jit_dec_synth();
451 }
452
453 jit_node_t *
454 _jit_arg(jit_state_t *_jit, jit_code_t code)
455 {
456     jit_node_t          *node;
457     jit_int32_t          offset;
458     assert(_jitc->function);
459     assert(!(_jitc->function->self.call & jit_call_varargs));
460 #if STRONG_TYPE_CHECKING
461     assert(code >= jit_code_arg_c && code <= jit_code_arg);
462 #endif
463     if (jit_arg_reg_p(_jitc->function->self.argi))
464         offset = _jitc->function->self.argi++;
465     else {
466         offset = _jitc->function->self.size;
467         _jitc->function->self.size += sizeof(jit_word_t);
468     }
469     node = jit_new_node_ww(code, offset,
470                            ++_jitc->function->self.argn);
471     jit_link_prolog();
472     return (node);
473 }
474
475 jit_node_t *
476 _jit_arg_f(jit_state_t *_jit)
477 {
478     jit_node_t          *node;
479     jit_int32_t          offset;
480     assert(_jitc->function);
481     if (jit_arg_reg_p(_jitc->function->self.argi)) {
482         if (!(_jitc->function->self.call & jit_call_varargs))
483             offset = 8 + _jitc->function->self.argf++;
484         else
485             offset = _jitc->function->self.argi;
486         ++_jitc->function->self.argi;
487     }
488    else {
489         offset = _jitc->function->self.size;
490         _jitc->function->self.size += sizeof(jit_word_t);
491     }
492     node = jit_new_node_ww(jit_code_arg_f, offset,
493                            ++_jitc->function->self.argn);
494     jit_link_prolog();
495     return (node);
496 }
497
498 jit_node_t *
499 _jit_arg_d(jit_state_t *_jit)
500 {
501     jit_node_t          *node;
502     jit_int32_t          offset;
503     assert(_jitc->function);
504     if (jit_arg_reg_p(_jitc->function->self.argi)) {
505         if (!(_jitc->function->self.call & jit_call_varargs))
506             offset = 8 + _jitc->function->self.argf++;
507         else
508             offset = _jitc->function->self.argi;
509         ++_jitc->function->self.argi;
510     }
511    else {
512         offset = _jitc->function->self.size;
513         _jitc->function->self.size += sizeof(jit_float64_t);
514     }
515     node = jit_new_node_ww(jit_code_arg_d, offset,
516                            ++_jitc->function->self.argn);
517     jit_link_prolog();
518     return (node);
519 }
520
521 void
522 _jit_getarg_c(jit_state_t *_jit, jit_int32_t u, jit_node_t *v)
523 {
524     assert_arg_type(v->code, jit_code_arg_c);
525     jit_inc_synth_wp(getarg_c, u, v);
526     if (jit_arg_reg_p(v->u.w))
527         jit_extr_c(u, _R32 + v->u.w);
528     else
529         jit_ldxi_c(u, JIT_FP, v->u.w + C_DISP);
530     jit_dec_synth();
531 }
532
533 void
534 _jit_getarg_uc(jit_state_t *_jit, jit_int32_t u, jit_node_t *v)
535 {
536     assert_arg_type(v->code, jit_code_arg_c);
537     jit_inc_synth_wp(getarg_uc, u, v);
538     if (jit_arg_reg_p(v->u.w))
539         jit_extr_uc(u, _R32 + v->u.w);
540     else
541         jit_ldxi_uc(u, JIT_FP, v->u.w + C_DISP);
542     jit_dec_synth();
543 }
544
545 void
546 _jit_getarg_s(jit_state_t *_jit, jit_int32_t u, jit_node_t *v)
547 {
548     assert_arg_type(v->code, jit_code_arg_s);
549     jit_inc_synth_wp(getarg_s, u, v);
550     if (jit_arg_reg_p(v->u.w))
551         jit_extr_s(u, _R32 + v->u.w);
552     else
553         jit_ldxi_s(u, JIT_FP, v->u.w + S_DISP);
554     jit_dec_synth();
555 }
556
557 void
558 _jit_getarg_us(jit_state_t *_jit, jit_int32_t u, jit_node_t *v)
559 {
560     assert_arg_type(v->code, jit_code_arg_s);
561     jit_inc_synth_wp(getarg_us, u, v);
562     if (jit_arg_reg_p(v->u.w))
563         jit_extr_us(u, _R32 + v->u.w);
564     else
565         jit_ldxi_us(u, JIT_FP, v->u.w + S_DISP);
566     jit_dec_synth();
567 }
568
569 void
570 _jit_getarg_i(jit_state_t *_jit, jit_int32_t u, jit_node_t *v)
571 {
572     assert_arg_type(v->code, jit_code_arg_i);
573     jit_inc_synth_wp(getarg_i, u, v);
574     if (jit_arg_reg_p(v->u.w))
575         jit_extr_i(u, _R32 + v->u.w);
576     else
577         jit_ldxi_i(u, JIT_FP, v->u.w + I_DISP);
578     jit_dec_synth();
579 }
580
581 void
582 _jit_getarg_ui(jit_state_t *_jit, jit_int32_t u, jit_node_t *v)
583 {
584     assert_arg_type(v->code, jit_code_arg_i);
585     jit_inc_synth_wp(getarg_ui, u, v);
586     if (jit_arg_reg_p(v->u.w))
587         jit_extr_ui(u, _R32 + v->u.w);
588     else
589         jit_ldxi_ui(u, JIT_FP, v->u.w + I_DISP);
590     jit_dec_synth();
591 }
592
593 void
594 _jit_getarg_l(jit_state_t *_jit, jit_int32_t u, jit_node_t *v)
595 {
596     assert_arg_type(v->code, jit_code_arg_l);
597     jit_inc_synth_wp(getarg_l, u, v);
598     if (jit_arg_reg_p(v->u.w))
599         jit_movr(u, _R32 + v->u.w);
600     else
601         jit_ldxi(u, JIT_FP, v->u.w);
602     jit_dec_synth();
603 }
604
605 void
606 _jit_putargr(jit_state_t *_jit, jit_int32_t u, jit_node_t *v, jit_code_t code)
607 {
608     assert_putarg_type(code, v->code);
609     jit_code_inc_synth_wp(code, u, v);
610     if (jit_arg_reg_p(v->u.w))
611         jit_movr(_R32 + v->u.w, u);
612     else
613         jit_stxi(v->u.w, JIT_FP, u);
614     jit_dec_synth();
615 }
616
617 void
618 _jit_putargi(jit_state_t *_jit, jit_word_t u, jit_node_t *v, jit_code_t code)
619 {
620     jit_int32_t         regno;
621     assert_putarg_type(code, v->code);
622     jit_code_inc_synth_wp(code, u, v);
623     if (jit_arg_reg_p(v->u.w))
624         jit_movi(_R32 + v->u.w, u);
625     else {
626         regno = jit_get_reg(jit_class_gpr);
627         jit_movi(regno, u);
628         jit_stxi(v->u.w, JIT_FP, regno);
629         jit_unget_reg(regno);
630     }
631     jit_dec_synth();
632 }
633
634 void
635 _jit_getarg_f(jit_state_t *_jit, jit_int32_t u, jit_node_t *v)
636 {
637     assert(v->code == jit_code_arg_f);
638     jit_inc_synth_wp(getarg_f, u, v);
639     if (jit_arg_reg_p(v->u.w))
640         jit_movr_w_f(u, _OUT0 + v->u.w);
641     else if (jit_arg_reg_p(v->u.w - 8))
642         jit_movr_f(u, _F8 + (v->u.w - 8));
643     else
644         jit_ldxi_f(u, JIT_FP, v->u.w + F_DISP);
645     jit_dec_synth();
646 }
647
648 void
649 _jit_putargr_f(jit_state_t *_jit, jit_int32_t u, jit_node_t *v)
650 {
651     assert(v->code == jit_code_arg_f);
652     jit_inc_synth_wp(putargr_f, u, v);
653     if (jit_arg_reg_p(v->u.w))
654         jit_movr_f_w(_OUT0 + v->u.w, u);
655     else if (jit_arg_reg_p(v->u.w - 8))
656         jit_movr_f(_F8 + (v->u.w - 8), u);
657     else
658         jit_stxi_f(v->u.w, JIT_FP, u + F_DISP);
659     jit_dec_synth();
660 }
661
662 void
663 _jit_putargi_f(jit_state_t *_jit, jit_float32_t u, jit_node_t *v)
664 {
665     jit_int32_t         regno;
666     assert(v->code == jit_code_arg_f);
667     jit_inc_synth_fp(putargi_f, u, v);
668     if (jit_arg_reg_p(v->u.w))
669         jit_movi_f_w(_OUT0 + v->u.w, u);
670     else if (jit_arg_reg_p(v->u.w - 8))
671         jit_movi_f(_F8 + (v->u.w - 8), u);
672     else {
673         regno = jit_get_reg(jit_class_fpr);
674         jit_movi_f(regno, u);
675         jit_stxi_f(v->u.w, JIT_FP, regno + F_DISP);
676         jit_unget_reg(regno);
677     }
678     jit_dec_synth();
679 }
680
681 void
682 _jit_getarg_d(jit_state_t *_jit, jit_int32_t u, jit_node_t *v)
683 {
684     assert(v->code == jit_code_arg_d);
685     jit_inc_synth_wp(getarg_d, u, v);
686     if (jit_arg_reg_p(v->u.w))
687         jit_movr_w_d(u, _OUT0 + v->u.w);
688     else if (jit_arg_reg_p(v->u.w - 8))
689         jit_movr_d(u, _F8 + (v->u.w - 8));
690     else
691         jit_ldxi_d(u, JIT_FP, v->u.w);
692     jit_dec_synth();
693 }
694
695 void
696 _jit_putargr_d(jit_state_t *_jit, jit_int32_t u, jit_node_t *v)
697 {
698     assert(v->code == jit_code_arg_d);
699     jit_inc_synth_wp(putargr_d, u, v);
700     if (jit_arg_reg_p(v->u.w))
701         jit_movr_d_w(_OUT0 + v->u.w, u);
702     else if (jit_arg_reg_p(v->u.w - 8))
703         jit_movr_d(_F8 + (v->u.w - 8), u);
704     else
705         jit_stxi_d(v->u.w, JIT_FP, u);
706     jit_dec_synth();
707 }
708
709 void
710 _jit_putargi_d(jit_state_t *_jit, jit_float64_t u, jit_node_t *v)
711 {
712     jit_int32_t         regno;
713     assert(v->code == jit_code_arg_d);
714     jit_inc_synth_dp(putargi_d, u, v);
715     if (jit_arg_reg_p(v->u.w))
716         jit_movi_d_w(_OUT0 + v->u.w, u);
717     else if (jit_arg_reg_p(v->u.w - 8))
718         jit_movi_d(_F8 + (v->u.w - 8), u);
719     else {
720         regno = jit_get_reg(jit_class_fpr);
721         jit_movi_d(regno, u);
722         jit_stxi_d(v->u.w, JIT_FP, regno);
723         jit_unget_reg(regno);
724     }
725     jit_dec_synth();
726 }
727
728 void
729 _jit_pushargr(jit_state_t *_jit, jit_int32_t u, jit_code_t code)
730 {
731     assert(_jitc->function);
732     jit_code_inc_synth_w(code, u);
733     jit_link_prepare();
734     if (jit_arg_reg_p(_jitc->function->call.argi)) {
735         jit_movr(_OUT0 + _jitc->function->call.argi, u);
736         ++_jitc->function->call.argi;
737     }
738     else {
739         jit_stxi(_jitc->function->call.size + params_offset, JIT_SP, u);
740         _jitc->function->call.size += sizeof(jit_word_t);
741     }
742     jit_dec_synth();
743 }
744
745 void
746 _jit_pushargi(jit_state_t *_jit, jit_word_t u, jit_code_t code)
747 {
748     jit_int32_t          regno;
749     assert(_jitc->function);
750     jit_code_inc_synth_w(code, u);
751     jit_link_prepare();
752     if (jit_arg_reg_p(_jitc->function->call.argi)) {
753         jit_movi(_OUT0 + _jitc->function->call.argi, u);
754         ++_jitc->function->call.argi;
755     }
756     else {
757         regno = jit_get_reg(jit_class_gpr);
758         jit_movi(regno, u);
759         jit_stxi(_jitc->function->call.size + params_offset, JIT_SP, regno);
760         _jitc->function->call.size += sizeof(jit_word_t);
761         jit_unget_reg(regno);
762     }
763     jit_dec_synth();
764 }
765
766 void
767 _jit_pushargr_f(jit_state_t *_jit, jit_int32_t u)
768 {
769     assert(_jitc->function);
770     jit_inc_synth_w(pushargr_f, u);
771     jit_link_prepare();
772     if (jit_arg_reg_p(_jitc->function->call.argi)) {
773         if (!(_jitc->function->call.call & jit_call_varargs)) {
774             jit_movr_f(_F8 + _jitc->function->call.argf, u);
775             ++_jitc->function->call.argf;
776         }
777         else
778             jit_movr_f_w(_OUT0 + _jitc->function->call.argi, u);
779         ++_jitc->function->call.argi;
780     }
781     else {
782         jit_stxi_f(_jitc->function->call.size + params_offset + F_DISP,
783                    JIT_SP, u);
784         _jitc->function->call.size += sizeof(jit_word_t);
785     }
786     jit_dec_synth();
787 }
788
789 void
790 _jit_pushargi_f(jit_state_t *_jit, jit_float32_t u)
791 {
792     jit_int32_t          regno;
793     assert(_jitc->function);
794     jit_inc_synth_f(pushargi_f, u);
795     jit_link_prepare();
796     if (jit_arg_reg_p(_jitc->function->call.argi)) {
797         if (!(_jitc->function->call.call & jit_call_varargs)) {
798             jit_movi_f(_F8 + _jitc->function->call.argf, u);
799             ++_jitc->function->call.argf;
800         }
801         else
802             jit_movi_f_w(_OUT0 + _jitc->function->call.argi, u);
803         ++_jitc->function->call.argi;
804     }
805     else {
806         regno = jit_get_reg(jit_class_fpr);
807         jit_movi_f(regno, u);
808         jit_stxi_f(_jitc->function->call.size + params_offset + F_DISP,
809                    JIT_SP, regno);
810         _jitc->function->call.size += sizeof(jit_word_t);
811         jit_unget_reg(regno);
812     }
813     jit_dec_synth();
814 }
815
816 void
817 _jit_pushargr_d(jit_state_t *_jit, jit_int32_t u)
818 {
819     assert(_jitc->function);
820     jit_inc_synth_w(pushargr_d, u);
821     jit_link_prepare();
822     if (jit_arg_reg_p(_jitc->function->call.argi)) {
823         if (!(_jitc->function->call.call & jit_call_varargs)) {
824             jit_movr_d(_F8 + _jitc->function->call.argf, u);
825             ++_jitc->function->call.argf;
826         }
827         else
828             jit_movr_d_w(_OUT0 + _jitc->function->call.argi, u);
829         ++_jitc->function->call.argi;
830     }
831     else {
832         jit_stxi_d(_jitc->function->call.size + params_offset, JIT_SP, u);
833         _jitc->function->call.size += sizeof(jit_float64_t);
834     }
835     jit_dec_synth();
836 }
837
838 void
839 _jit_pushargi_d(jit_state_t *_jit, jit_float64_t u)
840 {
841     jit_int32_t          regno;
842     assert(_jitc->function);
843     jit_inc_synth_d(pushargi_d, u);
844     jit_link_prepare();
845     if (jit_arg_reg_p(_jitc->function->call.argi)) {
846         if (!(_jitc->function->call.call & jit_call_varargs)) {
847             jit_movi_d(_F8 + _jitc->function->call.argf, u);
848             ++_jitc->function->call.argf;
849         }
850         else
851             jit_movi_d_w(_OUT0 + _jitc->function->call.argi, u);
852         ++_jitc->function->call.argi;
853     }
854     else {
855         regno = jit_get_reg(jit_class_fpr);
856         jit_movi_d(regno, u);
857         jit_stxi_d(_jitc->function->call.size + params_offset, JIT_SP, regno);
858         _jitc->function->call.size += sizeof(jit_float64_t);
859         jit_unget_reg(regno);
860     }
861     jit_dec_synth();
862 }
863
864 jit_bool_t
865 _jit_regarg_p(jit_state_t *_jit, jit_node_t *node, jit_int32_t regno)
866 {
867     /* Argument registers are allocated from the pool of unused registers */
868     return (0);
869 }
870
871 void
872 _jit_finishr(jit_state_t *_jit, jit_int32_t r0)
873 {
874     jit_node_t          *call;
875     assert(_jitc->function);
876     jit_inc_synth_w(finishr, r0);
877     if (_jitc->function->self.alen < _jitc->function->call.size)
878         _jitc->function->self.alen = _jitc->function->call.size;
879     call = jit_callr(r0);
880     call->v.w = _jitc->function->call.argi;
881     call->w.w = _jitc->function->call.argf;
882     _jitc->function->call.argi = _jitc->function->call.argf =
883         _jitc->function->call.size = 0;
884     _jitc->prepare = 0;
885     jit_dec_synth();
886 }
887
888 jit_node_t *
889 _jit_finishi(jit_state_t *_jit, jit_pointer_t i0)
890 {
891     jit_node_t          *node;
892     assert(_jitc->function);
893     jit_inc_synth_w(finishi, (jit_word_t)i0);
894     if (_jitc->function->self.alen < _jitc->function->call.size)
895         _jitc->function->self.alen = _jitc->function->call.size;
896     node = jit_calli(i0);
897     node->v.w = _jitc->function->call.argi;
898     node->w.w = _jitc->function->call.argf;
899     _jitc->function->call.argi = _jitc->function->call.argf =
900         _jitc->function->call.size = 0;
901     _jitc->prepare = 0;
902     jit_dec_synth();
903     return (node);
904 }
905
906 void
907 _jit_retval_c(jit_state_t *_jit, jit_int32_t r0)
908 {
909     jit_inc_synth_w(retval_c, r0);
910     jit_extr_c(r0, JIT_RET);
911     jit_dec_synth();
912 }
913
914 void
915 _jit_retval_uc(jit_state_t *_jit, jit_int32_t r0)
916 {
917     jit_inc_synth_w(retval_uc, r0);
918     jit_extr_uc(r0, JIT_RET);
919     jit_dec_synth();
920 }
921
922 void
923 _jit_retval_s(jit_state_t *_jit, jit_int32_t r0)
924 {
925     jit_inc_synth_w(retval_s, r0);
926     jit_extr_s(r0, JIT_RET);
927     jit_dec_synth();
928 }
929
930 void
931 _jit_retval_us(jit_state_t *_jit, jit_int32_t r0)
932 {
933     jit_inc_synth_w(retval_us, r0);
934     jit_extr_us(r0, JIT_RET);
935     jit_dec_synth();
936 }
937
938 void
939 _jit_retval_i(jit_state_t *_jit, jit_int32_t r0)
940 {
941     jit_inc_synth_w(retval_i, r0);
942     jit_extr_i(r0, JIT_RET);
943     jit_dec_synth();
944 }
945
946 void
947 _jit_retval_ui(jit_state_t *_jit, jit_int32_t r0)
948 {
949     jit_inc_synth_w(retval_ui, r0);
950     jit_extr_ui(r0, JIT_RET);
951     jit_dec_synth();
952 }
953
954 void
955 _jit_retval_l(jit_state_t *_jit, jit_int32_t r0)
956 {
957     jit_inc_synth_w(retval_l, r0);
958     jit_movr(r0, JIT_RET);
959     jit_dec_synth();
960 }
961
962 void
963 _jit_retval_f(jit_state_t *_jit, jit_int32_t r0)
964 {
965     jit_inc_synth_w(retval_l, r0);
966     jit_movr_f(r0, JIT_FRET);
967     jit_dec_synth();
968 }
969
970 void
971 _jit_retval_d(jit_state_t *_jit, jit_int32_t r0)
972 {
973     jit_inc_synth_w(retval_d, r0);
974     jit_movr_d(r0, JIT_FRET);
975     jit_dec_synth();
976 }
977
978 jit_pointer_t
979 _emit_code(jit_state_t *_jit)
980 {
981     jit_node_t          *node;
982     jit_node_t          *temp;
983     jit_word_t           word;
984     jit_int32_t          value;
985     jit_int32_t          offset;
986     struct {
987         jit_node_t      *node;
988         jit_word_t       word;
989         jit_function_t   func;
990 #if DEVEL_DISASSEMBLER
991         jit_word_t       prevw;
992 #endif
993         jit_int32_t      patch_offset;
994         jit_word_t       prolog_offset;
995     } undo;
996 #if DEVEL_DISASSEMBLER
997     jit_word_t           prevw;
998 #endif
999
1000     _jitc->function = NULL;
1001
1002     /* If did resize the code buffer, these were not reset */
1003     _jitc->ioff = 0;
1004     jit_regset_set_ui(&_jitc->regs, 0);
1005     _jitc->pred = 0;
1006
1007     jit_reglive_setup();
1008
1009     undo.word = 0;
1010     undo.node = NULL;
1011     undo.patch_offset = 0;
1012
1013 #if DEVEL_DISASSEMBLER
1014     prevw = _jit->pc.w;
1015 #endif
1016     undo.prolog_offset = 0;
1017     for (node = _jitc->head; node; node = node->next)
1018         if (node->code != jit_code_label &&
1019             node->code != jit_code_note &&
1020             node->code != jit_code_name)
1021             break;
1022     if (node && (node->code != jit_code_prolog ||
1023                  !(_jitc->functions.ptr + node->w.w)->assume_frame)) {
1024         /* code may start with a jump so add an initial function descriptor */
1025         word = _jit->pc.w + 16;
1026         il(word);               /* addr */
1027         il(0);                  /* gp */
1028     }
1029 #define case_rr(name, type)                                             \
1030             case jit_code_##name##r##type:                              \
1031                 name##r##type(rn(node->u.w), rn(node->v.w));            \
1032                 break
1033 #define case_rw(name, type)                                             \
1034             case jit_code_##name##i##type:                              \
1035                 name##i##type(rn(node->u.w), node->v.w);                \
1036                 break
1037 #define case_wr(name, type)                                             \
1038             case jit_code_##name##i##type:                              \
1039                 name##i##type(node->u.w, rn(node->v.w));                \
1040                 break
1041 #define case_rrr(name, type)                                            \
1042             case jit_code_##name##r##type:                              \
1043                 name##r##type(rn(node->u.w),                            \
1044                               rn(node->v.w), rn(node->w.w));            \
1045                 break
1046 #define case_rrrr(name, type)                                           \
1047             case jit_code_##name##r##type:                              \
1048                 name##r##type(rn(node->u.q.l), rn(node->u.q.h),         \
1049                               rn(node->v.w), rn(node->w.w));            \
1050                 break
1051 #define case_rqr(name, type)                                            \
1052             case jit_code_##name##r##type:                              \
1053                 name##r##type(rn(node->u.w), rn(node->v.q.l),           \
1054                               rn(node->v.q.h), rn(node->w.w));          \
1055             case jit_code_##name##i##type:                              \
1056                 break;
1057 #define case_rrw(name, type)                                            \
1058             case jit_code_##name##i##type:                              \
1059                 name##i##type(rn(node->u.w), rn(node->v.w), node->w.w); \
1060                 break
1061 #define case_rrrw(name, type)                                           \
1062             case jit_code_##name##i##type:                              \
1063                 name##i##type(rn(node->u.q.l), rn(node->u.q.h),         \
1064                               rn(node->v.w), node->w.w);                \
1065                 break
1066 #define case_rrf(name, type, size)                                      \
1067             case jit_code_##name##i##type:                              \
1068                 assert(node->flag & jit_flag_data);                     \
1069                 name##i##type(rn(node->u.w), rn(node->v.w),             \
1070                               (jit_float##size##_t *)node->w.n->u.w);   \
1071                 break
1072 #define case_wrr(name, type)                                            \
1073             case jit_code_##name##i##type:                              \
1074                 name##i##type(node->u.w, rn(node->v.w), rn(node->w.w)); \
1075                 break
1076 #define case_brr(name, type)                                            \
1077             case jit_code_##name##r##type:                              \
1078                 temp = node->u.n;                                       \
1079                 assert(temp->code == jit_code_label ||                  \
1080                        temp->code == jit_code_epilog);                  \
1081                 if (temp->flag & jit_flag_patch)                        \
1082                     name##r##type(temp->u.w, rn(node->v.w),             \
1083                                   rn(node->w.w));                       \
1084                 else {                                                  \
1085                     word = name##r##type(_jit->pc.w,                    \
1086                                          rn(node->v.w), rn(node->w.w)); \
1087                     patch(word, node);                                  \
1088                 }                                                       \
1089                 break
1090 #define case_brw(name, type)                                            \
1091             case jit_code_##name##i##type:                              \
1092                 temp = node->u.n;                                       \
1093                 assert(temp->code == jit_code_label ||                  \
1094                        temp->code == jit_code_epilog);                  \
1095                 if (temp->flag & jit_flag_patch)                        \
1096                     name##i##type(temp->u.w,                            \
1097                                   rn(node->v.w), node->w.w);            \
1098                 else {                                                  \
1099                     word = name##i##type(_jit->pc.w,                    \
1100                                          rn(node->v.w), node->w.w);     \
1101                     patch(word, node);                                  \
1102                 }                                                       \
1103                 break
1104 #define case_brf(name, type, size)                                      \
1105             case jit_code_##name##i##type:                              \
1106                 temp = node->u.n;                                       \
1107                 assert(temp->code == jit_code_label ||                  \
1108                        temp->code == jit_code_epilog);                  \
1109                 if (temp->flag & jit_flag_patch)                        \
1110                     name##i##type(temp->u.w, rn(node->v.w),             \
1111                                 (jit_float##size##_t *)node->w.n->u.w); \
1112                 else {                                                  \
1113                     word = name##i##type(_jit->pc.w, rn(node->v.w),     \
1114                                 (jit_float##size##_t *)node->w.n->u.w); \
1115                     patch(word, node);                                  \
1116                 }                                                       \
1117                 break
1118     for (node = _jitc->head; node; node = node->next) {
1119         if (_jit->pc.uc >= _jitc->code.end)
1120             return (NULL);
1121
1122         value = jit_classify(node->code);
1123 #if GET_JIT_SIZE
1124         sync();
1125 #endif
1126 #if DEVEL_DISASSEMBLER
1127         /* FIXME DEVEL_DISASSEMBLER should become DISASSEMBLER,
1128          * but a "real" DEVEL_DISASSEMBLER should be required
1129          * to turn the below "#if 0" into "#if 1" */
1130 #  if 0         /* Since disassembly outputs 3 instructions at a time,
1131                  * make it "#if 1" for more clear debug output. */
1132         sync();
1133 #  endif
1134         node->offset = (jit_uword_t)_jit->pc.w - (jit_uword_t)prevw;
1135         prevw = _jit->pc.w;
1136 #endif
1137         jit_regarg_set(node, value);
1138         switch (node->code) {
1139             case jit_code_align:
1140                 assert(!(node->u.w & (node->u.w - 1)));
1141                 sync();
1142                 if (node->u.w > 8)
1143                     nop(node->u.w - 8);
1144                 break;
1145             case jit_code_skip:
1146                 sync();
1147                 nop((node->u.w + 7) & ~7);
1148                 break;
1149             case jit_code_note:         case jit_code_name:
1150                 sync();
1151                 node->u.w = _jit->pc.w;
1152                 break;
1153             case jit_code_label:
1154                 sync();
1155                 /* remember label is defined */
1156                 node->flag |= jit_flag_patch;
1157                 node->u.w = _jit->pc.w;
1158                 break;
1159                 case_rrr(add,);
1160                 case_rrw(add,);
1161                 case_rrr(addc,);
1162                 case_rrw(addc,);
1163                 case_rrr(addx,);
1164                 case_rrw(addx,);
1165                 case_rrr(sub,);
1166                 case_rrw(sub,);
1167                 case_rrr(subx,);
1168                 case_rrw(subx,);
1169                 case_rrw(rsb,);
1170                 case_rrr(subc,);
1171                 case_rrw(subc,);
1172                 case_rrr(mul,);
1173                 case_rrw(mul,);
1174                 case_rrr(hmul,);
1175                 case_rrw(hmul,);
1176                 case_rrr(hmul, _u);
1177                 case_rrw(hmul, _u);
1178                 case_rrrr(qmul,);
1179                 case_rrrw(qmul,);
1180                 case_rrrr(qmul, _u);
1181                 case_rrrw(qmul, _u);
1182                 case_rrr(div,);
1183                 case_rrw(div,);
1184                 case_rrr(div, _u);
1185                 case_rrw(div, _u);
1186                 case_rrr(rem,);
1187                 case_rrw(rem,);
1188                 case_rrr(rem, _u);
1189                 case_rrw(rem, _u);
1190                 case_rrrr(qdiv,);
1191                 case_rrrw(qdiv,);
1192                 case_rrrr(qdiv, _u);
1193                 case_rrrw(qdiv, _u);
1194                 case_rrr(and,);
1195                 case_rrw(and,);
1196                 case_rrr(or,);
1197                 case_rrw(or,);
1198                 case_rrr(xor,);
1199                 case_rrw(xor,);
1200                 case_rrr(lsh,);
1201                 case_rrw(lsh,);
1202                 case_rrrr(qlsh,);
1203                 case_rrrw(qlsh,);
1204                 case_rrrr(qlsh, _u);
1205                 case_rrrw(qlsh, _u);
1206                 case_rrrr(qrsh,);
1207                 case_rrrw(qrsh,);
1208                 case_rrrr(qrsh, _u);
1209                 case_rrrw(qrsh, _u);
1210                 case_rrr(rsh,);
1211                 case_rrw(rsh,);
1212                 case_rrr(rsh, _u);
1213                 case_rrw(rsh, _u);
1214 #define lrotr(r0,r1,r2)         fallback_lrotr(r0,r1,r2)
1215 #define lroti(r0,r1,i0)         fallback_lroti(r0,r1,i0)
1216 #define rrotr(r0,r1,r2)         fallback_rrotr(r0,r1,r2)
1217 #define rroti(r0,r1,i0)         fallback_rroti(r0,r1,i0)
1218                 case_rrr(lrot,);
1219                 case_rrw(lrot,);
1220                 case_rrr(rrot,);
1221                 case_rrw(rrot,);
1222                 case_rr(neg,);
1223                 case_rr(com,);
1224                 case_rr(clo,);
1225                 case_rr(clz,);
1226                 case_rr(cto,);
1227                 case_rr(ctz,);
1228 #define rbitr(r0, r1)   fallback_rbit(r0, r1)
1229                 case_rr(rbit,);
1230                 case_rr(popcnt,);
1231             case jit_code_casr:
1232                 casr(rn(node->u.w), rn(node->v.w),
1233                      rn(node->w.q.l), rn(node->w.q.h));
1234                 break;
1235             case jit_code_casi:
1236                 casi(rn(node->u.w), node->v.w,
1237                      rn(node->w.q.l), rn(node->w.q.h));
1238                 break;
1239                 case_rrr(movn,);
1240                 case_rrr(movz,);
1241                 case_rr(mov,);
1242             case jit_code_movi:
1243                 if (node->flag & jit_flag_node) {
1244                     temp = node->v.n;
1245                     if (temp->code == jit_code_data ||
1246                         (temp->code == jit_code_label &&
1247                          (temp->flag & jit_flag_patch)))
1248                         movi(rn(node->u.w), temp->u.w);
1249                     else {
1250                         assert(temp->code == jit_code_label ||
1251                                temp->code == jit_code_epilog);
1252                         word = movi_p(rn(node->u.w), node->v.w);
1253                         patch(word, node);
1254                     }
1255                 }
1256                 else
1257                     movi(rn(node->u.w), node->v.w);
1258                 break;
1259                 case_rr(hton, _us);
1260                 case_rr(hton, _ui);
1261                 case_rr(hton, _ul);
1262                 case_rr(bswap, _us);
1263                 case_rr(bswap, _ui);
1264                 case_rr(bswap, _ul);
1265             case jit_code_extr:
1266                 extr(rn(node->u.w), rn(node->v.w), node->w.q.l, node->w.q.h);
1267                 break;
1268             case jit_code_extr_u:
1269                 extr_u(rn(node->u.w), rn(node->v.w), node->w.q.l, node->w.q.h);
1270                 break;
1271             case jit_code_depr:
1272                 depr(rn(node->u.w), rn(node->v.w), node->w.q.l, node->w.q.h);
1273                 break;
1274             case jit_code_depi:
1275                 depi(rn(node->u.w), node->v.w, node->w.q.l, node->w.q.h);
1276                 break;
1277                 case_rr(ext, _c);
1278                 case_rr(ext, _uc);
1279                 case_rr(ext, _s);
1280                 case_rr(ext, _us);
1281                 case_rr(ext, _i);
1282                 case_rr(ext, _ui);
1283                 case_rr(trunc, _f_i);
1284                 case_rr(trunc, _d_i);
1285                 case_rr(trunc, _f_l);
1286                 case_rr(trunc, _d_l);
1287                 case_rrr(lt,);
1288                 case_rrw(lt,);
1289                 case_rrr(lt, _u);
1290                 case_rrw(lt, _u);
1291                 case_rrr(le,);
1292                 case_rrw(le,);
1293                 case_rrr(le, _u);
1294                 case_rrw(le, _u);
1295                 case_rrr(eq,);
1296                 case_rrw(eq,);
1297                 case_rrr(ge,);
1298                 case_rrw(ge,);
1299                 case_rrr(ge, _u);
1300                 case_rrw(ge, _u);
1301                 case_rrr(gt,);
1302                 case_rrw(gt,);
1303                 case_rrr(gt, _u);
1304                 case_rrw(gt, _u);
1305                 case_rrr(ne,);
1306                 case_rrw(ne,);
1307                 case_rr(ld, _c);
1308                 case_rw(ld, _c);
1309                 case_rr(ld, _uc);
1310                 case_rw(ld, _uc);
1311                 case_rr(ld, _s);
1312                 case_rw(ld, _s);
1313                 case_rr(ld, _us);
1314                 case_rw(ld, _us);
1315                 case_rr(ld, _i);
1316                 case_rw(ld, _i);
1317                 case_rr(ld, _ui);
1318                 case_rw(ld, _ui);
1319                 case_rr(ld, _l);
1320                 case_rw(ld, _l);
1321                 case_rrr(ldx, _c);
1322                 case_rrw(ldx, _c);
1323                 case_rrr(ldx, _uc);
1324                 case_rrw(ldx, _uc);
1325                 case_rrr(ldx, _s);
1326                 case_rrw(ldx, _s);
1327                 case_rrr(ldx, _us);
1328                 case_rrw(ldx, _us);
1329                 case_rrr(ldx, _i);
1330                 case_rrw(ldx, _i);
1331                 case_rrr(ldx, _ui);
1332                 case_rrw(ldx, _ui);
1333                 case_rrr(ldx, _l);
1334                 case_rrw(ldx, _l);
1335             case jit_code_unldr:
1336                 unldr(rn(node->u.w), rn(node->v.w), node->w.w);
1337                 break;
1338             case jit_code_unldi:
1339                 unldi(rn(node->u.w), node->v.w, node->w.w);
1340                 break;
1341             case jit_code_unldr_u:
1342                 unldr_u(rn(node->u.w), rn(node->v.w), node->w.w);
1343                 break;
1344             case jit_code_unldi_u:
1345                 unldi_u(rn(node->u.w), node->v.w, node->w.w);
1346                 break;
1347                 case_rr(st, _c);
1348                 case_wr(st, _c);
1349                 case_rr(st, _s);
1350                 case_wr(st, _s);
1351                 case_rr(st, _i);
1352                 case_wr(st, _i);
1353                 case_rr(st, _l);
1354                 case_wr(st, _l);
1355                 case_rrr(stx, _c);
1356                 case_wrr(stx, _c);
1357                 case_rrr(stx, _s);
1358                 case_wrr(stx, _s);
1359                 case_rrr(stx, _i);
1360                 case_wrr(stx, _i);
1361                 case_rrr(stx, _l);
1362                 case_wrr(stx, _l);
1363             case jit_code_unstr:
1364                 unstr(rn(node->u.w), rn(node->v.w), node->w.w);
1365                 break;
1366             case jit_code_unsti:
1367                 unsti(node->u.w, rn(node->v.w), node->w.w);
1368                 break;
1369                 case_brr(blt,);
1370                 case_brw(blt,);
1371                 case_brr(blt, _u);
1372                 case_brw(blt, _u);
1373                 case_brr(ble,);
1374                 case_brw(ble,);
1375                 case_brr(ble, _u);
1376                 case_brw(ble, _u);
1377                 case_brr(beq,);
1378                 case_brw(beq,);
1379                 case_brr(bge,);
1380                 case_brw(bge,);
1381                 case_brr(bge, _u);
1382                 case_brw(bge, _u);
1383                 case_brr(bgt,);
1384                 case_brw(bgt,);
1385                 case_brr(bgt, _u);
1386                 case_brw(bgt, _u);
1387                 case_brr(bne,);
1388                 case_brw(bne,);
1389                 case_brr(bms,);
1390                 case_brw(bms,);
1391                 case_brr(bmc,);
1392                 case_brw(bmc,);
1393                 case_brr(boadd,);
1394                 case_brw(boadd,);
1395                 case_brr(boadd, _u);
1396                 case_brw(boadd, _u);
1397                 case_brr(bxadd,);
1398                 case_brw(bxadd,);
1399                 case_brr(bxadd, _u);
1400                 case_brw(bxadd, _u);
1401                 case_brr(bosub,);
1402                 case_brw(bosub,);
1403                 case_brr(bosub, _u);
1404                 case_brw(bosub, _u);
1405                 case_brr(bxsub,);
1406                 case_brw(bxsub,);
1407                 case_brr(bxsub, _u);
1408                 case_brw(bxsub, _u);
1409                 case_rrr(add, _f);
1410                 case_rrf(add, _f, 32);
1411                 case_rrr(sub, _f);
1412                 case_rrf(sub, _f, 32);
1413                 case_rrf(rsb, _f, 32);
1414                 case_rrr(mul, _f);
1415                 case_rrf(mul, _f, 32);
1416                 case_rrr(div, _f);
1417                 case_rrf(div, _f, 32);
1418                 case_rr(ext, _f);
1419                 case_rr(abs, _f);
1420                 case_rr(neg, _f);
1421                 case_rr(sqrt, _f);
1422                 case_rqr(fma, _f);
1423                 case_rqr(fms, _f);
1424                 case_rqr(fnma, _f);
1425                 case_rqr(fnms, _f);
1426                 case_rr(ld, _f);
1427                 case_rw(ld, _f);
1428                 case_rrr(ldx, _f);
1429                 case_rrw(ldx, _f);
1430             case jit_code_unldr_x:
1431                 unldr_x(rn(node->u.w), rn(node->v.w), node->w.w);
1432                 break;
1433             case jit_code_unldi_x:
1434                 unldi_x(rn(node->u.w), node->v.w, node->w.w);
1435                 break;
1436             case jit_code_unstr_x:
1437                 unstr_x(rn(node->u.w), rn(node->v.w), node->w.w);
1438                 break;
1439             case jit_code_unsti_x:
1440                 unsti_x(node->u.w, rn(node->v.w), node->w.w);
1441                 break;
1442                 case_rr(st, _f);
1443                 case_wr(st, _f);
1444                 case_rrr(stx, _f);
1445                 case_wrr(stx, _f);
1446                 case_rr(mov, _f);
1447             case jit_code_movi_f:
1448                 assert(node->flag & jit_flag_data);
1449                 movi_f(rn(node->u.w), (jit_float32_t *)node->v.n->u.w);
1450                 break;
1451                 case_rr(ext, _d_f);
1452                 case_rrr(lt, _f);
1453                 case_rrf(lt, _f, 32);
1454                 case_rrr(le, _f);
1455                 case_rrf(le, _f, 32);
1456                 case_rrr(eq, _f);
1457                 case_rrf(eq, _f, 32);
1458                 case_rrr(ge, _f);
1459                 case_rrf(ge, _f, 32);
1460                 case_rrr(gt, _f);
1461                 case_rrf(gt, _f, 32);
1462                 case_rrr(ne, _f);
1463                 case_rrf(ne, _f, 32);
1464                 case_rrr(unlt, _f);
1465                 case_rrf(unlt, _f, 32);
1466                 case_rrr(unle, _f);
1467                 case_rrf(unle, _f, 32);
1468                 case_rrr(uneq, _f);
1469                 case_rrf(uneq, _f, 32);
1470                 case_rrr(unge, _f);
1471                 case_rrf(unge, _f, 32);
1472                 case_rrr(ungt, _f);
1473                 case_rrf(ungt, _f, 32);
1474                 case_rrr(ltgt, _f);
1475                 case_rrf(ltgt, _f, 32);
1476                 case_rrr(ord, _f);
1477                 case_rrf(ord, _f, 32);
1478                 case_rrr(unord, _f);
1479                 case_rrf(unord, _f, 32);
1480                 case_brr(blt, _f);
1481                 case_brf(blt, _f, 32);
1482                 case_brr(ble, _f);
1483                 case_brf(ble, _f, 32);
1484                 case_brr(beq, _f);
1485                 case_brf(beq, _f, 32);
1486                 case_brr(bge, _f);
1487                 case_brf(bge, _f, 32);
1488                 case_brr(bgt, _f);
1489                 case_brf(bgt, _f, 32);
1490                 case_brr(bne, _f);
1491                 case_brf(bne, _f, 32);
1492                 case_brr(bunlt, _f);
1493                 case_brf(bunlt, _f, 32);
1494                 case_brr(bunle, _f);
1495                 case_brf(bunle, _f, 32);
1496                 case_brr(buneq, _f);
1497                 case_brf(buneq, _f, 32);
1498                 case_brr(bunge, _f);
1499                 case_brf(bunge, _f, 32);
1500                 case_brr(bungt, _f);
1501                 case_brf(bungt, _f, 32);
1502                 case_brr(bltgt, _f);
1503                 case_brf(bltgt, _f, 32);
1504                 case_brr(bord, _f);
1505                 case_brf(bord, _f, 32);
1506                 case_brr(bunord, _f);
1507                 case_brf(bunord, _f, 32);
1508                 case_rrr(add, _d);
1509                 case_rrf(add, _d, 64);
1510                 case_rrr(sub, _d);
1511                 case_rrf(sub, _d, 64);
1512                 case_rrf(rsb, _d, 64);
1513                 case_rrr(mul, _d);
1514                 case_rrf(mul, _d, 64);
1515                 case_rrr(div, _d);
1516                 case_rrf(div, _d, 64);
1517                 case_rr(ext, _d);
1518                 case_rr(abs, _d);
1519                 case_rr(neg, _d);
1520                 case_rr(sqrt, _d);
1521                 case_rqr(fma, _d);
1522                 case_rqr(fms, _d);
1523                 case_rqr(fnma, _d);
1524                 case_rqr(fnms, _d);
1525                 case_rr(ld, _d);
1526                 case_rw(ld, _d);
1527                 case_rrr(ldx, _d);
1528                 case_rrw(ldx, _d);
1529                 case_rr(st, _d);
1530                 case_wr(st, _d);
1531                 case_rrr(stx, _d);
1532                 case_wrr(stx, _d);
1533                 case_rr(mov, _d);
1534             case jit_code_movi_d:
1535                 assert(node->flag & jit_flag_data);
1536                 movi_d(rn(node->u.w), (jit_float64_t *)node->v.n->u.w);
1537                 break;
1538                 case_rr(ext, _f_d);
1539                 case_rrr(lt, _d);
1540                 case_rrf(lt, _d, 64);
1541                 case_rrr(le, _d);
1542                 case_rrf(le, _d, 64);
1543                 case_rrr(eq, _d);
1544                 case_rrf(eq, _d, 64);
1545                 case_rrr(ge, _d);
1546                 case_rrf(ge, _d, 64);
1547                 case_rrr(gt, _d);
1548                 case_rrf(gt, _d, 64);
1549                 case_rrr(ne, _d);
1550                 case_rrf(ne, _d, 64);
1551                 case_rrr(unlt, _d);
1552                 case_rrf(unlt, _d, 64);
1553                 case_rrr(unle, _d);
1554                 case_rrf(unle, _d, 64);
1555                 case_rrr(uneq, _d);
1556                 case_rrf(uneq, _d, 64);
1557                 case_rrr(unge, _d);
1558                 case_rrf(unge, _d, 64);
1559                 case_rrr(ungt, _d);
1560                 case_rrf(ungt, _d, 64);
1561                 case_rrr(ltgt, _d);
1562                 case_rrf(ltgt, _d, 64);
1563                 case_rrr(ord, _d);
1564                 case_rrf(ord, _d, 64);
1565                 case_rrr(unord, _d);
1566                 case_rrf(unord, _d, 64);
1567                 case_brr(blt, _d);
1568                 case_brf(blt, _d, 64);
1569                 case_brr(ble, _d);
1570                 case_brf(ble, _d, 64);
1571                 case_brr(beq, _d);
1572                 case_brf(beq, _d, 64);
1573                 case_brr(bge, _d);
1574                 case_brf(bge, _d, 64);
1575                 case_brr(bgt, _d);
1576                 case_brf(bgt, _d, 64);
1577                 case_brr(bne, _d);
1578                 case_brf(bne, _d, 64);
1579                 case_brr(bunlt, _d);
1580                 case_brf(bunlt, _d, 64);
1581                 case_brr(bunle, _d);
1582                 case_brf(bunle, _d, 64);
1583                 case_brr(buneq, _d);
1584                 case_brf(buneq, _d, 64);
1585                 case_brr(bunge, _d);
1586                 case_brf(bunge, _d, 64);
1587                 case_brr(bungt, _d);
1588                 case_brf(bungt, _d, 64);
1589                 case_brr(bltgt, _d);
1590                 case_brf(bltgt, _d, 64);
1591                 case_brr(bord, _d);
1592                 case_brf(bord, _d, 64);
1593                 case_brr(bunord, _d);
1594                 case_brf(bunord, _d, 64);
1595             case jit_code_jmpr:
1596                 jmpr(rn(node->u.w));
1597                 break;
1598             case jit_code_jmpi:
1599                 if (node->flag & jit_flag_node) {
1600                     if (_jit->pc.uc == _jit->code.ptr + 16)
1601                         _jitc->jump = 1;
1602                     temp = node->u.n;
1603                     assert(temp->code == jit_code_label ||
1604                            temp->code == jit_code_epilog);
1605                     if (temp->flag & jit_flag_patch)
1606                         jmpi(temp->u.w);
1607                     else {
1608                         word = _jit->code.length -
1609                             (_jit->pc.uc - _jit->code.ptr);
1610                         if (word  >= -16777216 && word <= 16777215)
1611                             word = jmpi(_jit->pc.w);
1612                         else
1613                             word = jmpi_p(_jit->pc.w);
1614                         patch(word, node);
1615                     }
1616                 }
1617                 else
1618                     jmpi(node->u.w);
1619                 break;
1620             case jit_code_callr:
1621                 callr(rn(node->u.w));
1622                 break;
1623             case jit_code_calli:
1624                 if (node->flag & jit_flag_node) {
1625                     temp = node->u.n;
1626                     assert(temp->code == jit_code_label ||
1627                            temp->code == jit_code_epilog);
1628                     word = calli_p(temp->u.w);
1629                     if (!(temp->flag & jit_flag_patch))
1630                         patch(word, node);
1631                 }
1632                 else
1633                     calli(node->u.w);
1634                 break;
1635             case jit_code_prolog:
1636                 jit_regset_set_ui(&_jitc->regs, 0);
1637                 _jitc->pred = 0;
1638                 sync();
1639                 _jitc->function = _jitc->functions.ptr + node->w.w;
1640                 undo.node = node;
1641                 undo.word = _jit->pc.w;
1642                 memcpy(&undo.func, _jitc->function, sizeof(undo.func));
1643 #if DEVEL_DISASSEMBLER
1644                 undo.prevw = prevw;
1645 #endif
1646                 undo.patch_offset = _jitc->patches.offset;
1647                 undo.prolog_offset = _jitc->prolog.offset;
1648             restart_function:
1649                 _jitc->again = 0;
1650                 if (_jitc->jump && !_jitc->function->assume_frame) {
1651                     /* remember prolog to hide offset adjustment for a jump
1652                      * to the start of a function, what is expected to be
1653                      * a common practice as first jit instruction */
1654                     if (_jitc->prolog.offset >= _jitc->prolog.length) {
1655                         _jitc->prolog.length += 16;
1656                         jit_realloc((jit_pointer_t *)&_jitc->prolog.ptr,
1657                                     (_jitc->prolog.length - 16) *
1658                                     sizeof(jit_word_t),
1659                                     _jitc->prolog.length * sizeof(jit_word_t));
1660                     }
1661                     _jitc->prolog.ptr[_jitc->prolog.offset++] = _jit->pc.w;
1662                     /* function descriptor */
1663                     word = _jit->pc.w + 16;
1664                     il(word);                   /* addr */
1665                     il(0);                      /* gp */
1666                 }
1667                 prolog(node);
1668                 break;
1669             case jit_code_epilog:
1670                 assert(_jitc->function == _jitc->functions.ptr + node->w.w);
1671                 if (_jitc->again) {
1672                     for (temp = undo.node->next;
1673                          temp != node; temp = temp->next) {
1674                         if (temp->code == jit_code_label ||
1675                             temp->code == jit_code_epilog)
1676                             temp->flag &= ~jit_flag_patch;
1677                     }
1678                     temp->flag &= ~jit_flag_patch;
1679                     node = undo.node;
1680                     _jit->pc.w = undo.word;
1681                     /* undo.func.self.aoff and undo.func.regset should not
1682                      * be undone, as they will be further updated, and are
1683                      * the reason of the undo. */
1684                     undo.func.self.aoff = _jitc->function->frame +
1685                         _jitc->function->self.aoff;
1686                     jit_regset_set(&undo.func.regset, &_jitc->function->regset);
1687                     /* allocar information also does not need to be undone */
1688                     undo.func.aoffoff = _jitc->function->aoffoff;
1689                     undo.func.allocar = _jitc->function->allocar;
1690                     memcpy(_jitc->function, &undo.func, sizeof(undo.func));
1691 #if DEVEL_DISASSEMBLER
1692                     prevw = undo.prevw;
1693 #endif
1694                     _jitc->patches.offset = undo.patch_offset;
1695                     _jitc->prolog.offset = undo.prolog_offset;
1696                     _jitc->ioff = 0;
1697                     jit_regset_set_ui(&_jitc->regs, 0);
1698                     _jitc->pred = 0;
1699                     goto restart_function;
1700                 }
1701                 /* remember label is defined */
1702                 sync();
1703                 node->flag |= jit_flag_patch;
1704                 node->u.w = _jit->pc.w;
1705                 epilog(node);
1706                 jit_regset_set_ui(&_jitc->regs, 0);
1707                 _jitc->pred = 0;
1708                 _jitc->function = NULL;
1709                 break;
1710             case jit_code_va_start:
1711                 vastart(rn(node->u.w));
1712                 break;
1713             case jit_code_va_arg:
1714                 vaarg(rn(node->u.w), rn(node->v.w));
1715                 break;
1716             case jit_code_va_arg_d:
1717                 vaarg_d(rn(node->u.w), rn(node->v.w));
1718                 break;
1719             case jit_code_live:                 case jit_code_ellipsis:
1720             case jit_code_va_push:
1721             case jit_code_allocai:              case jit_code_allocar:
1722             case jit_code_arg_c:                case jit_code_arg_s:
1723             case jit_code_arg_i:                case jit_code_arg_l:
1724             case jit_code_arg_f:                case jit_code_arg_d:
1725             case jit_code_va_end:
1726             case jit_code_ret:
1727             case jit_code_retr_c:               case jit_code_reti_c:
1728             case jit_code_retr_uc:              case jit_code_reti_uc:
1729             case jit_code_retr_s:               case jit_code_reti_s:
1730             case jit_code_retr_us:              case jit_code_reti_us:
1731             case jit_code_retr_i:               case jit_code_reti_i:
1732             case jit_code_retr_ui:              case jit_code_reti_ui:
1733             case jit_code_retr_l:               case jit_code_reti_l:
1734             case jit_code_retr_f:               case jit_code_reti_f:
1735             case jit_code_retr_d:               case jit_code_reti_d:
1736             case jit_code_getarg_c:             case jit_code_getarg_uc:
1737             case jit_code_getarg_s:             case jit_code_getarg_us:
1738             case jit_code_getarg_i:             case jit_code_getarg_ui:
1739             case jit_code_getarg_l:
1740             case jit_code_getarg_f:             case jit_code_getarg_d:
1741             case jit_code_putargr_c:            case jit_code_putargi_c:
1742             case jit_code_putargr_uc:           case jit_code_putargi_uc:
1743             case jit_code_putargr_s:            case jit_code_putargi_s:
1744             case jit_code_putargr_us:           case jit_code_putargi_us:
1745             case jit_code_putargr_i:            case jit_code_putargi_i:
1746             case jit_code_putargr_ui:           case jit_code_putargi_ui:
1747             case jit_code_putargr_l:            case jit_code_putargi_l:
1748             case jit_code_putargr_f:            case jit_code_putargi_f:
1749             case jit_code_putargr_d:            case jit_code_putargi_d:
1750             case jit_code_pushargr_c:           case jit_code_pushargi_c:
1751             case jit_code_pushargr_uc:          case jit_code_pushargi_uc:
1752             case jit_code_pushargr_s:           case jit_code_pushargi_s:
1753             case jit_code_pushargr_us:          case jit_code_pushargi_us:
1754             case jit_code_pushargr_i:           case jit_code_pushargi_i:
1755             case jit_code_pushargr_ui:          case jit_code_pushargi_ui:
1756             case jit_code_pushargr_l:           case jit_code_pushargi_l:
1757             case jit_code_pushargr_f:           case jit_code_pushargi_f:
1758             case jit_code_pushargr_d:           case jit_code_pushargi_d:
1759             case jit_code_retval_c:             case jit_code_retval_uc:
1760             case jit_code_retval_s:             case jit_code_retval_us:
1761             case jit_code_retval_i:
1762             case jit_code_retval_ui:            case jit_code_retval_l:
1763             case jit_code_retval_f:             case jit_code_retval_d:
1764             case jit_code_prepare:
1765             case jit_code_finishr:              case jit_code_finishi:
1766             case jit_code_negi_f:               case jit_code_absi_f:
1767             case jit_code_sqrti_f:              case jit_code_negi_d:
1768             case jit_code_absi_d:               case jit_code_sqrti_d:
1769                 break;
1770             case jit_code_movr_w_f:
1771                 movr_w_f(rn(node->u.w), rn(node->v.w));
1772                 break;
1773             case jit_code_movr_f_w:
1774                 movr_f_w(rn(node->u.w), rn(node->v.w));
1775                 break;
1776             case jit_code_movr_w_d:
1777                 movr_w_d(rn(node->u.w), rn(node->v.w));
1778                 break;
1779             case jit_code_movr_d_w:
1780                 movr_d_w(rn(node->u.w), rn(node->v.w));
1781                 break;
1782             case jit_code_movi_f_w:
1783                 assert(node->flag & jit_flag_data);
1784                 movi_f_w(rn(node->u.w), *(jit_float32_t *)node->v.n->u.w);
1785                 break;
1786             case jit_code_movi_w_f:
1787                 movi_w_f(rn(node->u.w), node->v.w);
1788                 break;
1789             case jit_code_movi_d_w:
1790                 assert(node->flag & jit_flag_data);
1791                 movi_d_w(rn(node->u.w), *(jit_float64_t *)node->v.n->u.w);
1792                 break;
1793             case jit_code_movi_w_d:
1794                 movi_w_d(rn(node->u.w), node->v.w);
1795                 break;
1796             case jit_code_negi:
1797                 negi(rn(node->u.w), node->v.w);
1798                 break;
1799             case jit_code_comi:
1800                 comi(rn(node->u.w), node->v.w);
1801                 break;
1802             case jit_code_exti_c:
1803                 exti_c(rn(node->u.w), node->v.w);
1804                 break;
1805             case jit_code_exti_uc:
1806                 exti_uc(rn(node->u.w), node->v.w);
1807                 break;
1808             case jit_code_exti_s:
1809                 exti_s(rn(node->u.w), node->v.w);
1810                 break;
1811             case jit_code_exti_us:
1812                 exti_us(rn(node->u.w), node->v.w);
1813                 break;
1814             case jit_code_bswapi_us:
1815                 bswapi_us(rn(node->u.w), node->v.w);
1816                 break;
1817             case jit_code_bswapi_ui:
1818                 bswapi_ui(rn(node->u.w), node->v.w);
1819                 break;
1820             case jit_code_htoni_us:
1821                 htoni_us(rn(node->u.w), node->v.w);
1822                 break;
1823             case jit_code_htoni_ui:
1824                 htoni_ui(rn(node->u.w), node->v.w);
1825                 break;
1826             case jit_code_exti_i:
1827                 exti_i(rn(node->u.w), node->v.w);
1828                 break;
1829             case jit_code_exti_ui:
1830                 exti_ui(rn(node->u.w), node->v.w);
1831                 break;
1832             case jit_code_bswapi_ul:
1833                 bswapi_ul(rn(node->u.w), node->v.w);
1834                 break;
1835             case jit_code_htoni_ul:
1836                 htoni_ul(rn(node->u.w), node->v.w);
1837                 break;
1838             case jit_code_cloi:
1839                 cloi(rn(node->u.w), node->v.w);
1840                 break;
1841             case jit_code_clzi:
1842                 clzi(rn(node->u.w), node->v.w);
1843                 break;
1844             case jit_code_ctoi:
1845                 ctoi(rn(node->u.w), node->v.w);
1846                 break;
1847             case jit_code_ctzi:
1848                 ctzi(rn(node->u.w), node->v.w);
1849                 break;
1850             case jit_code_rbiti:
1851                 rbiti(rn(node->u.w), node->v.w);
1852                 break;
1853             case jit_code_popcnti:
1854                 popcnti(rn(node->u.w), node->v.w);
1855                 break;
1856             case jit_code_exti:
1857                 exti(rn(node->u.w), node->v.w, node->w.q.l, node->w.q.h);
1858                 break;
1859             case jit_code_exti_u:
1860                 exti_u(rn(node->u.w), node->v.w, node->w.q.l, node->w.q.h);
1861                 break;
1862             default:
1863                 abort();
1864         }
1865         if (jit_carry != _NOREG) {
1866             switch (node->code) {
1867                 case jit_code_note:
1868                 case jit_code_addcr:            case jit_code_addci:
1869                 case jit_code_addxr:            case jit_code_addxi:
1870                 case jit_code_subcr:            case jit_code_subci:
1871                 case jit_code_subxr:            case jit_code_subxi:
1872                     break;
1873                 default:
1874                     jit_unget_reg(jit_carry);
1875                     jit_carry = _NOREG;
1876                     break;
1877             }
1878         }
1879 #if GET_JIT_SIZE
1880         sync();
1881 #endif
1882         jit_regarg_clr(node, value);
1883         if (jit_regset_cmp_ui(&_jitc->regarg, 0) != 0) {
1884             assert(jit_regset_scan1(&_jitc->regarg, 0) == jit_carry);
1885             assert(jit_regset_scan1(&_jitc->regarg, jit_carry + 1) == ULONG_MAX);
1886         }
1887         assert(_jitc->synth == 0);
1888         /* update register live state */
1889         jit_reglive(node);
1890     }
1891     sync();
1892 #undef case_brf
1893 #undef case_brw
1894 #undef case_brr
1895 #undef case_wrr
1896 #undef case_rrrw
1897 #undef case_rrf
1898 #undef case_rrw
1899 #undef case_rrrr
1900 #undef case_rrr
1901 #undef case_wr
1902 #undef case_rw
1903 #undef case_rr
1904
1905     for (offset = 0; offset < _jitc->patches.offset; offset++) {
1906         node = _jitc->patches.ptr[offset].node;
1907         word = node->code == jit_code_movi ? node->v.n->u.w : node->u.n->u.w;
1908         patch_at(node->code, _jitc->patches.ptr[offset].inst, word);
1909     }
1910
1911     jit_flush(_jit->code.ptr, _jit->pc.uc);
1912
1913     return (_jit->code.ptr);
1914 }
1915
1916 #define CODE                            1
1917 #  include "jit_ia64-cpu.c"
1918 #  include "jit_ia64-fpu.c"
1919 #  include "jit_fallback.c"
1920 #undef CODE
1921
1922 void
1923 jit_flush(void *fptr, void *tptr)
1924 {
1925 #if defined(__GNUC__)
1926     jit_word_t          f, t, s;
1927
1928     s = sysconf(_SC_PAGE_SIZE);
1929     f = (jit_word_t)fptr & -s;
1930     t = (((jit_word_t)tptr) + s - 1) & -s;
1931 #  if 0
1932     __clear_cache((void *)f, (void *)t);
1933 #  else
1934     /* __clear_cache is a noop in (old?) gcc, but cache flush is
1935      * required on a multi processor Linux system. */
1936     for (s = f; s < t; s += 32)
1937         asm volatile("fc %0" :: "r"(s) : "memory");
1938 #  endif
1939 #endif
1940 }
1941
1942 /* Use r2 that is reserved to not require a jit_get_reg call, also note
1943  * that addil needs a register that first in 2 bits, so, if using a
1944  * register other than r2 must be less than r8 (or check for a smaller
1945  * immediate, i.e. i0 >= -8192 && i0 <= 8191) */
1946 void
1947 _emit_ldxi(jit_state_t *_jit, jit_gpr_t r0, jit_gpr_t r1, jit_word_t i0)
1948 {
1949     assert(i0 >= -2097152 && i0 < 2097151);
1950     addi(GR_2, rn(r1), i0);
1951     ldr(rn(r0), GR_2);
1952 }
1953
1954 void
1955 _emit_stxi(jit_state_t *_jit, jit_word_t i0, jit_gpr_t r0, jit_gpr_t r1)
1956 {
1957     assert(i0 >= -2097152 && i0 < 2097151);
1958     addi(GR_2, rn(r0), i0);
1959     str(GR_2, rn(r1));
1960 }
1961
1962 void
1963 _emit_ldxi_d(jit_state_t *_jit, jit_fpr_t r0, jit_gpr_t r1, jit_word_t i0)
1964 {
1965     assert(i0 >= -2097152 && i0 < 2097151);
1966     addi(GR_2, rn(r1), i0);
1967     ldr_d(rn(r0), GR_2);
1968 }
1969
1970 void
1971 _emit_stxi_d(jit_state_t *_jit, jit_word_t i0, jit_gpr_t r0, jit_fpr_t r1)
1972 {
1973     assert(i0 >= -2097152 && i0 < 2097151);
1974     addi(GR_2, rn(r0), i0);
1975     str_d(GR_2, rn(r1));
1976 }
1977
1978 static void
1979 _patch(jit_state_t *_jit, jit_word_t instr, jit_node_t *node)
1980 {
1981     jit_int32_t         flag;
1982
1983     assert(node->flag & jit_flag_node);
1984     if (node->code == jit_code_movi)
1985         flag = node->v.n->flag;
1986     else
1987         flag = node->u.n->flag;
1988     assert(!(flag & jit_flag_patch));
1989     if (_jitc->patches.offset >= _jitc->patches.length) {
1990         jit_realloc((jit_pointer_t *)&_jitc->patches.ptr,
1991                     _jitc->patches.length * sizeof(jit_patch_t),
1992                     (_jitc->patches.length + 1024) * sizeof(jit_patch_t));
1993         _jitc->patches.length += 1024;
1994     }
1995     _jitc->patches.ptr[_jitc->patches.offset].inst = instr;
1996     _jitc->patches.ptr[_jitc->patches.offset].node = node;
1997     ++_jitc->patches.offset;
1998 }