deps/lightning/lib/jit_ia64.c

   1 /*
   2  * Copyright (C) 2013-2019  Free Software Foundation, Inc.
   3  *
   4  * This file is part of GNU lightning.
   5  *
   6  * GNU lightning is free software; you can redistribute it and/or modify it
   7  * under the terms of the GNU Lesser General Public License as published
   8  * by the Free Software Foundation; either version 3, or (at your option)
   9  * any later version.
  10  *
  11  * GNU lightning is distributed in the hope that it will be useful, but
  12  * WITHOUT ANY WARRANTY; without even the implied warranty of MERCHANTABILITY
  13  * or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU Lesser General Public
  14  * License for more details.
  15  *
  16  * Authors:
  17  *      Paulo Cesar Pereira de Andrade
  18  */
  19
  20 #include <lightning.h>
  21 #include <lightning/jit_private.h>
  22
  23 #define jit_arg_reg_p(i)                ((i) >= 0 && (i) < 8)
  24 #if __BYTE_ORDER == __LITTLE_ENDIAN
  25 #  define C_DISP                        0
  26 #  define S_DISP                        0
  27 #  define I_DISP                        0
  28 #  define F_DISP                        0
  29 #else
  30 #  define C_DISP                        8 - sizeof(jit_int8_t)
  31 #  define S_DISP                        8 - sizeof(jit_int16_t)
  32 #  define I_DISP                        8 - sizeof(jit_int32_t)
  33 #  define F_DISP                        8 - sizeof(jit_float32_t)
  34 #endif
  35
  36 /*
  37  * Types
  38  */
  39 typedef jit_pointer_t jit_va_list_t;
  40
  41 /*
  42  * Prototypes
  43  */
  44 #define patch(instr, node)              _patch(_jit, instr, node)
  45 static void _patch(jit_state_t*,jit_word_t,jit_node_t*);
  46
  47 #if defined(__GNUC__)
  48 /* libgcc */
  49 extern void __clear_cache(void *, void *);
  50 #endif
  51
  52 #define PROTO                           1
  53 #  include "jit_ia64-cpu.c"
  54 #  include "jit_ia64-fpu.c"
  55 #  include "jit_fallback.c"
  56 #undef PROTO
  57
  58 /*
  59  * Initialization
  60  */
  61 jit_register_t          _rvs[] = {
  62     /* Always 0 */
  63     { 0,                 "r0"  },
  64     /* Global Pointer */
  65     { rc(sav)|1,         "r1"  },
  66     /* Used when a register cannot be allocated */
  67     { 2,                 "r2"  },
  68     /* First scratch register */
  69     { rc(gpr)|3,         "r3"  },
  70     /* Use r4 as lightning fp register */
  71     { rc(sav)|4,         "r4"  },
  72     /* Do not touch callee save registers not automatically spill/reloaded */
  73     { rc(sav)|5,         "r5"  },       { rc(sav)|6,            "r6"  },
  74     { rc(sav)|7,         "r7"  },
  75     /* Do not touch return register for the sake of simplicity, besides
  76      * having JIT_R0 being the same as JIT_RET usually an optimization */
  77     { 8,                 "r8"  },
  78     /* Return registers, use as temporaries */
  79     { rc(gpr)|9,         "r9"  },
  80     { rc(gpr)|10,        "r10" },       { rc(gpr)|11,           "r11" },
  81     /* Stack pointer */
  82     { rc(sav)|12,        "r12" },
  83     /* Thread pointer */
  84     { rc(sav)|13,        "r13" },
  85     /* (Usually) assembly temporaries */
  86     { rc(gpr)|31,        "r31" },       { rc(gpr)|30,           "r30" },
  87     { rc(gpr)|29,        "r29" },       { rc(gpr)|28,           "r28" },
  88     { rc(gpr)|27,        "r27" },       { rc(gpr)|26,           "r26" },
  89     { rc(gpr)|25,        "r25" },       { rc(gpr)|24,           "r24" },
  90     { rc(gpr)|23,        "r23" },       { rc(gpr)|22,           "r22" },
  91     { rc(gpr)|21,        "r21" },       { rc(gpr)|20,           "r20" },
  92     { rc(gpr)|19,        "r19" },       { rc(gpr)|18,           "r18" },
  93     { rc(gpr)|17,        "r17" },       { rc(gpr)|16,           "r16" },
  94     { rc(gpr)|15,        "r15" },       { rc(gpr)|14,           "r14" },
  95     /* Do not allow allocating r32-r41 as temoraries for the sake of
  96      * avoiding the need of extra complexity  in the non backend code */
  97     { rc(arg)|32,        "r32" },       { rc(arg)|33,           "r33" },
  98     { rc(arg)|34,        "r34" },       { rc(arg)|35,           "r35" },
  99     { rc(arg)|36,        "r36" },       { rc(arg)|37,           "r37" },
 100     { rc(arg)|38,        "r38" },       { rc(arg)|39,           "r39" },
 101     /* JIT_R0-JIT_V3 */
 102     { rc(gpr)|rc(sav)|40, "r40" },      { rc(gpr)|rc(sav)|41,   "r41" },
 103     { rc(gpr)|rc(sav)|42, "r42" },      { rc(gpr)|rc(sav)|43,   "r43" },
 104     { rc(gpr)|rc(sav)|44, "r44" },      { rc(gpr)|rc(sav)|45,   "r45" },
 105     { rc(gpr)|rc(sav)|46, "r46" },      { rc(gpr)|rc(sav)|47,   "r47" },
 106     /* Temporaries/locals */
 107     { rc(gpr)|rc(sav)|48, "r48" },      { rc(gpr)|rc(sav)|49,   "r49" },
 108     { rc(gpr)|rc(sav)|50, "r50" },      { rc(gpr)|rc(sav)|51,   "r51" },
 109     { rc(gpr)|rc(sav)|52, "r52" },      { rc(gpr)|rc(sav)|53,   "r53" },
 110     { rc(gpr)|rc(sav)|54, "r54" },      { rc(gpr)|rc(sav)|55,   "r55" },
 111     { rc(gpr)|rc(sav)|56, "r56" },      { rc(gpr)|rc(sav)|57,   "r57" },
 112     { rc(gpr)|rc(sav)|58, "r58" },      { rc(gpr)|rc(sav)|59,   "r59" },
 113     { rc(gpr)|rc(sav)|60, "r60" },      { rc(gpr)|rc(sav)|61,   "r61" },
 114     { rc(gpr)|rc(sav)|62, "r62" },      { rc(gpr)|rc(sav)|63,   "r63" },
 115     { rc(gpr)|rc(sav)|64, "r64" },      { rc(gpr)|rc(sav)|65,   "r65" },
 116     { rc(gpr)|rc(sav)|66, "r66" },      { rc(gpr)|rc(sav)|67,   "r67" },
 117     { rc(gpr)|rc(sav)|68, "r68" },      { rc(gpr)|rc(sav)|69,   "r69" },
 118     { rc(gpr)|rc(sav)|70, "r70" },      { rc(gpr)|rc(sav)|71,   "r71" },
 119     { rc(gpr)|rc(sav)|72, "r72" },      { rc(gpr)|rc(sav)|73,   "r73" },
 120     { rc(gpr)|rc(sav)|74, "r74" },      { rc(gpr)|rc(sav)|75,   "r75" },
 121     { rc(gpr)|rc(sav)|76, "r76" },      { rc(gpr)|rc(sav)|77,   "r77" },
 122     { rc(gpr)|rc(sav)|78, "r78" },      { rc(gpr)|rc(sav)|79,   "r79" },
 123     { rc(gpr)|rc(sav)|80, "r80" },      { rc(gpr)|rc(sav)|81,   "r81" },
 124     { rc(gpr)|rc(sav)|82, "r82" },      { rc(gpr)|rc(sav)|83,   "r83" },
 125     { rc(gpr)|rc(sav)|84, "r84" },      { rc(gpr)|rc(sav)|85,   "r85" },
 126     { rc(gpr)|rc(sav)|86, "r86" },      { rc(gpr)|rc(sav)|87,   "r87" },
 127     { rc(gpr)|rc(sav)|88, "r88" },      { rc(gpr)|rc(sav)|89,   "r89" },
 128     { rc(gpr)|rc(sav)|90, "r90" },      { rc(gpr)|rc(sav)|91,   "r91" },
 129     { rc(gpr)|rc(sav)|92, "r92" },      { rc(gpr)|rc(sav)|93,   "r93" },
 130     { rc(gpr)|rc(sav)|94, "r94" },      { rc(gpr)|rc(sav)|95,   "r95" },
 131     { rc(gpr)|rc(sav)|96, "r96" },      { rc(gpr)|rc(sav)|97,   "r97" },
 132     { rc(gpr)|rc(sav)|98, "r98" },      { rc(gpr)|rc(sav)|99,   "r99" },
 133     { rc(gpr)|rc(sav)|100,"r100"},      { rc(gpr)|rc(sav)|101,  "r101"},
 134     { rc(gpr)|rc(sav)|102,"r102"},      { rc(gpr)|rc(sav)|103,  "r103"},
 135     { rc(gpr)|rc(sav)|104,"r104"},      { rc(gpr)|rc(sav)|105,  "r105"},
 136     { rc(gpr)|rc(sav)|106,"r106"},      { rc(gpr)|rc(sav)|107,  "r107"},
 137     { rc(gpr)|rc(sav)|108,"r108"},      { rc(gpr)|rc(sav)|109,  "r109"},
 138     { rc(gpr)|rc(sav)|110,"r110"},      { rc(gpr)|rc(sav)|111,  "r111"},
 139     { rc(gpr)|rc(sav)|112,"r112"},      { rc(gpr)|rc(sav)|113,  "r113"},
 140     { rc(gpr)|rc(sav)|114,"r114"},
 141     /* Do not enable these because no matter what, want 13 free registers,
 142      * 5 for prolog and epilog and 8 for outgoing arguments */
 143     { 115,               "r115"},
 144     { 116,               "r116"},       { 117,                  "r117"},
 145     { 118,               "r118"},       { 119,                  "r119"},
 146     { 120,               "r120"},       { 121,                  "r121"},
 147     { 122,               "r122"},       { 123,                  "r123"},
 148     { 124,               "r124"},       { 125,                  "r125"},
 149     { 126,               "r126"},       { 127,                  "r127"},
 150     /* Always 0.0 */
 151     { 0,                 "f0"  },
 152     /* Always 1.0 */
 153     { 1,                 "f1"  },
 154     /* Do not touch callee save registers not automatically spill/reloaded */
 155     { rc(sav)|2,         "f2"  },       { rc(sav)|3,            "f3"  },
 156     { rc(sav)|4,         "f4"  },       { rc(sav)|5,            "f5"  },
 157     /* Scratch */
 158     { rc(fpr)|6,         "f6"  },       { rc(fpr)|7,            "f7"  },
 159     /* Do not allocate for the sake of simplification */
 160     { rc(arg)|8,         "f8"  },
 161     /* Scratch - Argument/return registers */
 162     { rc(arg)|9,         "f9"  },
 163     { rc(arg)|10,        "f10" },       { rc(arg)|11,           "f11" },
 164     { rc(arg)|12,        "f12" },       { rc(arg)|13,           "f13" },
 165     { rc(arg)|14,        "f14" },       { rc(arg)|15,           "f15" },
 166     /* Do not touch callee save registers not automatically spill/reloaded */
 167     { rc(sav)|16,        "f16" },       { rc(sav)|17,           "f17" },
 168     { rc(sav)|18,        "f18" },       { rc(sav)|19,           "f19" },
 169     { rc(sav)|20,        "f20" },       { rc(sav)|21,           "f21" },
 170     { rc(sav)|22,        "f22" },       { rc(sav)|23,           "f23" },
 171     { rc(sav)|24,        "f24" },       { rc(sav)|25,           "f25" },
 172     { rc(sav)|26,        "f26" },       { rc(sav)|27,           "f27" },
 173     { rc(sav)|28,        "f28" },       { rc(sav)|29,           "f29" },
 174     { rc(sav)|30,        "f30" },       { rc(sav)|31,           "f31" },
 175     /* Scratch */
 176     { rc(fpr)|32,        "f32" },       { rc(fpr)|33,           "f33" },
 177     { rc(fpr)|34,        "f34" },       { rc(fpr)|35,           "f35" },
 178     { rc(fpr)|36,        "f36" },       { rc(fpr)|37,           "f37" },
 179     { rc(fpr)|38,        "f38" },       { rc(fpr)|39,           "f39" },
 180     { rc(fpr)|40,        "f40" },       { rc(fpr)|41,           "f41" },
 181     { rc(fpr)|42,        "f42" },       { rc(fpr)|43,           "f43" },
 182     { rc(fpr)|44,        "f44" },       { rc(fpr)|45,           "f45" },
 183     { rc(fpr)|46,        "f46" },       { rc(fpr)|47,           "f47" },
 184     { rc(fpr)|48,        "f48" },       { rc(fpr)|49,           "f49" },
 185     { rc(fpr)|50,        "f50" },       { rc(fpr)|51,           "f51" },
 186     { rc(fpr)|52,        "f52" },       { rc(fpr)|53,           "f53" },
 187     { rc(fpr)|54,        "f54" },       { rc(fpr)|55,           "f55" },
 188     { rc(fpr)|56,        "f56" },       { rc(fpr)|57,           "f57" },
 189     { rc(fpr)|58,        "f58" },       { rc(fpr)|59,           "f59" },
 190     { rc(fpr)|60,        "f60" },       { rc(fpr)|61,           "f61" },
 191     { rc(fpr)|62,        "f62" },       { rc(fpr)|63,           "f63" },
 192     { rc(fpr)|64,        "f64" },       { rc(fpr)|65,           "f65" },
 193     { rc(fpr)|66,        "f66" },       { rc(fpr)|67,           "f67" },
 194     { rc(fpr)|68,        "f68" },       { rc(fpr)|69,           "f69" },
 195     { rc(fpr)|70,        "f70" },       { rc(fpr)|71,           "f71" },
 196     { rc(fpr)|72,        "f72" },       { rc(fpr)|73,           "f73" },
 197     { rc(fpr)|74,        "f74" },       { rc(fpr)|75,           "f75" },
 198     { rc(fpr)|76,        "f76" },       { rc(fpr)|77,           "f77" },
 199     { rc(fpr)|78,        "f78" },       { rc(fpr)|79,           "f79" },
 200     { rc(fpr)|80,        "f80" },       { rc(fpr)|81,           "f81" },
 201     { rc(fpr)|82,        "f82" },       { rc(fpr)|83,           "f83" },
 202     { rc(fpr)|84,        "f84" },       { rc(fpr)|85,           "f85" },
 203     { rc(fpr)|86,        "f86" },       { rc(fpr)|87,           "f87" },
 204     { rc(fpr)|88,        "f88" },       { rc(fpr)|89,           "f89" },
 205     { rc(fpr)|90,        "f90" },       { rc(fpr)|91,           "f91" },
 206     { rc(fpr)|92,        "f92" },       { rc(fpr)|93,           "f93" },
 207     { rc(fpr)|94,        "f94" },       { rc(fpr)|95,           "f95" },
 208     { rc(fpr)|96,        "f96" },       { rc(fpr)|97,           "f97" },
 209     { rc(fpr)|98,        "f98" },       { rc(fpr)|99,           "f99" },
 210     { rc(fpr)|100,       "f100"},       { rc(fpr)|101,          "f101"},
 211     { rc(fpr)|102,       "f102"},       { rc(fpr)|103,          "f103"},
 212     { rc(fpr)|104,       "f104"},       { rc(fpr)|105,          "f105"},
 213     { rc(fpr)|106,       "f106"},       { rc(fpr)|107,          "f107"},
 214     { rc(fpr)|108,       "f108"},       { rc(fpr)|109,          "f109"},
 215     { rc(fpr)|110,       "f110"},       { rc(fpr)|111,          "f111"},
 216     { rc(fpr)|112,       "f112"},       { rc(fpr)|113,          "f113"},
 217     { rc(fpr)|114,       "f114"},       { rc(fpr)|115,          "f115"},
 218     { rc(fpr)|116,       "f116"},       { rc(fpr)|117,          "f117"},
 219     { rc(fpr)|118,       "f118"},       { rc(fpr)|119,          "f119"},
 220 #if 0
 221     /* commented to fit a jit_regset_t in 256 bits, so that the fake
 222      * O0-O7 registers are easily patched when an argument is pushed */
 223     { rc(fpr)|120,       "f120"},       { rc(fpr)|121,          "f121"},
 224     { rc(fpr)|122,       "f122"},       { rc(fpr)|123,          "f123"},
 225     { rc(fpr)|124,       "f124"},       { rc(fpr)|125,          "f125"},
 226     { rc(fpr)|126,       "f126"},       { rc(fpr)|127,          "f127"},
 227 #endif
 228     /* Fake registers to patch in movr and movi arguments */
 229     { rc(arg)|120,       "o0"  },       { rc(arg)|121,          "o1"  },
 230     { rc(arg)|122,       "o2"  },       { rc(arg)|123,          "o3"  },
 231     { rc(arg)|124,       "o4"  },       { rc(arg)|125,          "o5"  },
 232     { rc(arg)|126,       "o6"  },       { rc(arg)|127,          "o7"  },
 233     { _NOREG,            "<none>" },
 234 };
 235
 236 /*
 237  * Implementation
 238  */
 239 void
 240 jit_get_cpu(void)
 241 {
 242 }
 243
 244 void
 245 _jit_init(jit_state_t *_jit)
 246 {
 247     _jitc->reglen = jit_size(_rvs) - 1;
 248     jit_regset_new(&_jitc->regs);
 249     jit_carry = _NOREG;
 250 }
 251
 252 void
 253 _jit_prolog(jit_state_t *_jit)
 254 {
 255     jit_int32_t         offset;
 256
 257     if (_jitc->function)
 258         jit_epilog();
 259     assert(jit_regset_cmp_ui(&_jitc->regarg, 0) == 0);
 260     jit_regset_set_ui(&_jitc->regsav, 0);
 261     offset = _jitc->functions.offset;
 262     if (offset >= _jitc->functions.length) {
 263         jit_realloc((jit_pointer_t *)&_jitc->functions.ptr,
 264                     _jitc->functions.length * sizeof(jit_function_t),
 265                     (_jitc->functions.length + 16) * sizeof(jit_function_t));
 266         _jitc->functions.length += 16;
 267     }
 268     _jitc->function = _jitc->functions.ptr + _jitc->functions.offset++;
 269     /* params_offset << 1, once for ours arguments, once for arguments
 270      * to any called function; required because offsets are computed
 271      * as JIT_FP displacement */
 272     _jitc->function->self.size = stack_framesize + (params_offset << 1);
 273     _jitc->function->self.argi = _jitc->function->self.argf =
 274         _jitc->function->self.aoff = _jitc->function->self.alen = 0;
 275     _jitc->function->self.call = jit_call_default;
 276     jit_alloc((jit_pointer_t *)&_jitc->function->regoff,
 277               _jitc->reglen * sizeof(jit_int32_t));
 278
 279     /* _no_link here does not mean the jit_link() call can be removed
 280      * by rewriting as:
 281      * _jitc->function->prolog = jit_new_node(jit_code_prolog);
 282      */
 283     _jitc->function->prolog = jit_new_node_no_link(jit_code_prolog);
 284     jit_link(_jitc->function->prolog);
 285     _jitc->function->prolog->w.w = offset;
 286     _jitc->function->epilog = jit_new_node_no_link(jit_code_epilog);
 287     /*  u:      label value
 288      *  v:      offset in blocks vector
 289      *  w:      offset in functions vector
 290      */
 291     _jitc->function->epilog->w.w = offset;
 292
 293     jit_regset_new(&_jitc->function->regset);
 294 }
 295
 296 jit_int32_t
 297 _jit_allocai(jit_state_t *_jit, jit_int32_t length)
 298 {
 299     assert(_jitc->function);
 300     switch (length) {
 301         case 0: case 1:                                         break;
 302         case 2:         _jitc->function->self.aoff &= -2;       break;
 303         case 3: case 4: _jitc->function->self.aoff &= -4;       break;
 304         default:        _jitc->function->self.aoff &= -8;       break;
 305     }
 306     _jitc->function->self.aoff -= length;
 307     if (!_jitc->realize) {
 308         jit_inc_synth_ww(allocai, _jitc->function->self.aoff, length);
 309         jit_dec_synth();
 310     }
 311     return (_jitc->function->self.aoff);
 312 }
 313
 314 void
 315 _jit_allocar(jit_state_t *_jit, jit_int32_t u, jit_int32_t v)
 316 {
 317     jit_int32_t          reg;
 318     assert(_jitc->function);
 319     jit_inc_synth_ww(allocar, u, v);
 320     if (!_jitc->function->allocar) {
 321         _jitc->function->aoffoff = jit_allocai(sizeof(jit_int32_t));
 322         _jitc->function->allocar = 1;
 323     }
 324     reg = jit_get_reg(jit_class_gpr);
 325     jit_negr(reg, v);
 326     jit_andi(reg, reg, -16);
 327     jit_ldxi_i(u, JIT_FP, _jitc->function->aoffoff);
 328     jit_addr(u, u, reg);
 329     jit_addr(JIT_SP, JIT_SP, reg);
 330     jit_stxi_i(_jitc->function->aoffoff, JIT_FP, u);
 331     jit_unget_reg(reg);
 332     jit_dec_synth();
 333 }
 334
 335 void
 336 _jit_ret(jit_state_t *_jit)
 337 {
 338     jit_node_t          *instr;
 339     assert(_jitc->function);
 340     jit_inc_synth(ret);
 341     /* jump to epilog */
 342     instr = jit_jmpi();
 343     jit_patch_at(instr, _jitc->function->epilog);
 344     jit_dec_synth();
 345 }
 346
 347 void
 348 _jit_retr(jit_state_t *_jit, jit_int32_t u)
 349 {
 350     jit_inc_synth_w(retr, u);
 351     jit_movr(JIT_RET, u);
 352     jit_ret();
 353     jit_dec_synth();
 354 }
 355
 356 void
 357 _jit_reti(jit_state_t *_jit, jit_word_t u)
 358 {
 359     jit_inc_synth_w(reti, u);
 360     jit_movi(JIT_RET, u);
 361     jit_ret();
 362     jit_dec_synth();
 363 }
 364
 365 void
 366 _jit_retr_f(jit_state_t *_jit, jit_int32_t u)
 367 {
 368     jit_inc_synth_w(retr_f, u);
 369     jit_movr_f(JIT_FRET, u);
 370     jit_ret();
 371     jit_dec_synth();
 372 }
 373
 374 void
 375 _jit_reti_f(jit_state_t *_jit, jit_float32_t u)
 376 {
 377     jit_inc_synth_f(reti_f, u);
 378     jit_movi_f(JIT_FRET, u);
 379     jit_ret();
 380     jit_dec_synth();
 381 }
 382
 383 void
 384 _jit_retr_d(jit_state_t *_jit, jit_int32_t u)
 385 {
 386     jit_inc_synth_w(retr_d, u);
 387     jit_movr_d(JIT_FRET, u);
 388     jit_ret();
 389     jit_dec_synth();
 390 }
 391
 392 void
 393 _jit_reti_d(jit_state_t *_jit, jit_float64_t u)
 394 {
 395     jit_inc_synth_d(reti_d, u);
 396     jit_movi_d(JIT_FRET, u);
 397     jit_ret();
 398     jit_dec_synth();
 399 }
 400
 401 void
 402 _jit_epilog(jit_state_t *_jit)
 403 {
 404     assert(_jitc->function);
 405     assert(_jitc->function->epilog->next == NULL);
 406     jit_link(_jitc->function->epilog);
 407     _jitc->function = NULL;
 408 }
 409
 410 jit_bool_t
 411 _jit_arg_register_p(jit_state_t *_jit, jit_node_t *u)
 412 {
 413     assert(u->code == jit_code_arg ||
 414            u->code == jit_code_arg_f || u->code == jit_code_arg_d);
 415     return (jit_arg_reg_p(u->u.w));
 416 }
 417
 418 void
 419 _jit_ellipsis(jit_state_t *_jit)
 420 {
 421     jit_inc_synth(ellipsis);
 422     if (_jitc->prepare) {
 423         assert(!(_jitc->function->call.call & jit_call_varargs));
 424         _jitc->function->call.call |= jit_call_varargs;
 425         jit_link_prepare();
 426     }
 427     else {
 428         assert(!(_jitc->function->self.call & jit_call_varargs));
 429         _jitc->function->self.call |= jit_call_varargs;
 430         _jitc->function->vagp = _jitc->function->self.argi;
 431         jit_link_prolog();
 432     }
 433     jit_dec_synth();
 434 }
 435
 436 void
 437 _jit_va_push(jit_state_t *_jit, jit_int32_t u)
 438 {
 439     jit_inc_synth_w(va_push, u);
 440     jit_pushargr(u);
 441     jit_dec_synth();
 442 }
 443
 444 jit_node_t *
 445 _jit_arg(jit_state_t *_jit)
 446 {
 447     jit_node_t          *node;
 448     jit_int32_t          offset;
 449     assert(_jitc->function);
 450     if (jit_arg_reg_p(_jitc->function->self.argi))
 451         offset = _jitc->function->self.argi++;
 452     else {
 453         offset = _jitc->function->self.size;
 454         _jitc->function->self.size += sizeof(jit_word_t);
 455     }
 456     node = jit_new_node_ww(jit_code_arg, offset,
 457                            ++_jitc->function->self.argn);
 458     jit_link_prolog();
 459     return (node);
 460 }
 461
 462 jit_node_t *
 463 _jit_arg_f(jit_state_t *_jit)
 464 {
 465     jit_node_t          *node;
 466     jit_int32_t          offset;
 467     assert(_jitc->function);
 468     if (jit_arg_reg_p(_jitc->function->self.argi)) {
 469         if (!(_jitc->function->self.call & jit_call_varargs))
 470             offset = 8 + _jitc->function->self.argf++;
 471         else
 472             offset = _jitc->function->self.argi;
 473         ++_jitc->function->self.argi;
 474     }
 475    else {
 476         offset = _jitc->function->self.size;
 477         _jitc->function->self.size += sizeof(jit_word_t);
 478     }
 479     node = jit_new_node_ww(jit_code_arg_f, offset,
 480                            ++_jitc->function->self.argn);
 481     jit_link_prolog();
 482     return (node);
 483 }
 484
 485 jit_node_t *
 486 _jit_arg_d(jit_state_t *_jit)
 487 {
 488     jit_node_t          *node;
 489     jit_int32_t          offset;
 490     assert(_jitc->function);
 491     if (jit_arg_reg_p(_jitc->function->self.argi)) {
 492         if (!(_jitc->function->self.call & jit_call_varargs))
 493             offset = 8 + _jitc->function->self.argf++;
 494         else
 495             offset = _jitc->function->self.argi;
 496         ++_jitc->function->self.argi;
 497     }
 498    else {
 499         offset = _jitc->function->self.size;
 500         _jitc->function->self.size += sizeof(jit_float64_t);
 501     }
 502     node = jit_new_node_ww(jit_code_arg_d, offset,
 503                            ++_jitc->function->self.argn);
 504     jit_link_prolog();
 505     return (node);
 506 }
 507
 508 void
 509 _jit_getarg_c(jit_state_t *_jit, jit_int32_t u, jit_node_t *v)
 510 {
 511     assert(v->code == jit_code_arg);
 512     jit_inc_synth_wp(getarg_c, u, v);
 513     if (jit_arg_reg_p(v->u.w))
 514         jit_extr_c(u, _R32 + v->u.w);
 515     else
 516         jit_ldxi_c(u, JIT_FP, v->u.w + C_DISP);
 517     jit_dec_synth();
 518 }
 519
 520 void
 521 _jit_getarg_uc(jit_state_t *_jit, jit_int32_t u, jit_node_t *v)
 522 {
 523     assert(v->code == jit_code_arg);
 524     jit_inc_synth_wp(getarg_uc, u, v);
 525     if (jit_arg_reg_p(v->u.w))
 526         jit_extr_uc(u, _R32 + v->u.w);
 527     else
 528         jit_ldxi_uc(u, JIT_FP, v->u.w + C_DISP);
 529     jit_dec_synth();
 530 }
 531
 532 void
 533 _jit_getarg_s(jit_state_t *_jit, jit_int32_t u, jit_node_t *v)
 534 {
 535     assert(v->code == jit_code_arg);
 536     jit_inc_synth_wp(getarg_s, u, v);
 537     if (jit_arg_reg_p(v->u.w))
 538         jit_extr_s(u, _R32 + v->u.w);
 539     else
 540         jit_ldxi_s(u, JIT_FP, v->u.w + S_DISP);
 541     jit_dec_synth();
 542 }
 543
 544 void
 545 _jit_getarg_us(jit_state_t *_jit, jit_int32_t u, jit_node_t *v)
 546 {
 547     assert(v->code == jit_code_arg);
 548     jit_inc_synth_wp(getarg_us, u, v);
 549     if (jit_arg_reg_p(v->u.w))
 550         jit_extr_us(u, _R32 + v->u.w);
 551     else
 552         jit_ldxi_us(u, JIT_FP, v->u.w + S_DISP);
 553     jit_dec_synth();
 554 }
 555
 556 void
 557 _jit_getarg_i(jit_state_t *_jit, jit_int32_t u, jit_node_t *v)
 558 {
 559     assert(v->code == jit_code_arg);
 560     jit_inc_synth_wp(getarg_i, u, v);
 561     if (jit_arg_reg_p(v->u.w))
 562         jit_extr_i(u, _R32 + v->u.w);
 563     else
 564         jit_ldxi_i(u, JIT_FP, v->u.w + I_DISP);
 565     jit_dec_synth();
 566 }
 567
 568 void
 569 _jit_getarg_ui(jit_state_t *_jit, jit_int32_t u, jit_node_t *v)
 570 {
 571     assert(v->code == jit_code_arg);
 572     jit_inc_synth_wp(getarg_ui, u, v);
 573     if (jit_arg_reg_p(v->u.w))
 574         jit_extr_ui(u, _R32 + v->u.w);
 575     else
 576         jit_ldxi_ui(u, JIT_FP, v->u.w + I_DISP);
 577     jit_dec_synth();
 578 }
 579
 580 void
 581 _jit_getarg_l(jit_state_t *_jit, jit_int32_t u, jit_node_t *v)
 582 {
 583     assert(v->code == jit_code_arg);
 584     jit_inc_synth_wp(getarg_l, u, v);
 585     if (jit_arg_reg_p(v->u.w))
 586         jit_movr(u, _R32 + v->u.w);
 587     else
 588         jit_ldxi(u, JIT_FP, v->u.w);
 589     jit_dec_synth();
 590 }
 591
 592 void
 593 _jit_putargr(jit_state_t *_jit, jit_int32_t u, jit_node_t *v)
 594 {
 595     assert(v->code == jit_code_arg);
 596     jit_inc_synth_wp(putargr, u, v);
 597     if (jit_arg_reg_p(v->u.w))
 598         jit_movr(_R32 + v->u.w, u);
 599     else
 600         jit_stxi(v->u.w, JIT_FP, u);
 601     jit_dec_synth();
 602 }
 603
 604 void
 605 _jit_putargi(jit_state_t *_jit, jit_word_t u, jit_node_t *v)
 606 {
 607     jit_int32_t         regno;
 608     assert(v->code == jit_code_arg);
 609     jit_inc_synth_wp(putargi, u, v);
 610     if (jit_arg_reg_p(v->u.w))
 611         jit_movi(_R32 + v->u.w, u);
 612     else {
 613         regno = jit_get_reg(jit_class_gpr);
 614         jit_movi(regno, u);
 615         jit_stxi(v->u.w, JIT_FP, regno);
 616         jit_unget_reg(regno);
 617     }
 618     jit_dec_synth();
 619 }
 620
 621 void
 622 _jit_getarg_f(jit_state_t *_jit, jit_int32_t u, jit_node_t *v)
 623 {
 624     assert(v->code == jit_code_arg_f);
 625     jit_inc_synth_wp(getarg_f, u, v);
 626     if (jit_arg_reg_p(v->u.w))
 627         jit_movr_w_f(u, _OUT0 + v->u.w);
 628     else if (jit_arg_reg_p(v->u.w - 8))
 629         jit_movr_f(u, _F8 + (v->u.w - 8));
 630     else
 631         jit_ldxi_f(u, JIT_FP, v->u.w + F_DISP);
 632     jit_dec_synth();
 633 }
 634
 635 void
 636 _jit_putargr_f(jit_state_t *_jit, jit_int32_t u, jit_node_t *v)
 637 {
 638     assert(v->code == jit_code_arg_f);
 639     jit_inc_synth_wp(putargr_f, u, v);
 640     if (jit_arg_reg_p(v->u.w))
 641         jit_movr_f_w(_OUT0 + v->u.w, u);
 642     else if (jit_arg_reg_p(v->u.w - 8))
 643         jit_movr_f(_F8 + (v->u.w - 8), u);
 644     else
 645         jit_stxi_f(v->u.w, JIT_FP, u + F_DISP);
 646     jit_dec_synth();
 647 }
 648
 649 void
 650 _jit_putargi_f(jit_state_t *_jit, jit_float32_t u, jit_node_t *v)
 651 {
 652     jit_int32_t         regno;
 653     assert(v->code == jit_code_arg_f);
 654     jit_inc_synth_fp(putargi_f, u, v);
 655     if (jit_arg_reg_p(v->u.w))
 656         jit_movi_f_w(_OUT0 + v->u.w, u);
 657     else if (jit_arg_reg_p(v->u.w - 8))
 658         jit_movi_f(_F8 + (v->u.w - 8), u);
 659     else {
 660         regno = jit_get_reg(jit_class_fpr);
 661         jit_movi_f(regno, u);
 662         jit_stxi_f(v->u.w, JIT_FP, regno + F_DISP);
 663         jit_unget_reg(regno);
 664     }
 665     jit_dec_synth();
 666 }
 667
 668 void
 669 _jit_getarg_d(jit_state_t *_jit, jit_int32_t u, jit_node_t *v)
 670 {
 671     assert(v->code == jit_code_arg_d);
 672     jit_inc_synth_wp(getarg_d, u, v);
 673     if (jit_arg_reg_p(v->u.w))
 674         jit_movr_w_d(u, _OUT0 + v->u.w);
 675     else if (jit_arg_reg_p(v->u.w - 8))
 676         jit_movr_d(u, _F8 + (v->u.w - 8));
 677     else
 678         jit_ldxi_d(u, JIT_FP, v->u.w);
 679     jit_dec_synth();
 680 }
 681
 682 void
 683 _jit_putargr_d(jit_state_t *_jit, jit_int32_t u, jit_node_t *v)
 684 {
 685     assert(v->code == jit_code_arg_d);
 686     jit_inc_synth_wp(putargr_d, u, v);
 687     if (jit_arg_reg_p(v->u.w))
 688         jit_movr_d_w(_OUT0 + v->u.w, u);
 689     else if (jit_arg_reg_p(v->u.w - 8))
 690         jit_movr_d(_F8 + (v->u.w - 8), u);
 691     else
 692         jit_stxi_d(v->u.w, JIT_FP, u);
 693     jit_dec_synth();
 694 }
 695
 696 void
 697 _jit_putargi_d(jit_state_t *_jit, jit_float64_t u, jit_node_t *v)
 698 {
 699     jit_int32_t         regno;
 700     assert(v->code == jit_code_arg_d);
 701     jit_inc_synth_dp(putargi_d, u, v);
 702     if (jit_arg_reg_p(v->u.w))
 703         jit_movi_d_w(_OUT0 + v->u.w, u);
 704     else if (jit_arg_reg_p(v->u.w - 8))
 705         jit_movi_d(_F8 + (v->u.w - 8), u);
 706     else {
 707         regno = jit_get_reg(jit_class_fpr);
 708         jit_movi_d(regno, u);
 709         jit_stxi_d(v->u.w, JIT_FP, regno);
 710         jit_unget_reg(regno);
 711     }
 712     jit_dec_synth();
 713 }
 714
 715 void
 716 _jit_pushargr(jit_state_t *_jit, jit_int32_t u)
 717 {
 718     assert(_jitc->function);
 719     jit_inc_synth_w(pushargr, u);
 720     jit_link_prepare();
 721     if (jit_arg_reg_p(_jitc->function->call.argi)) {
 722         jit_movr(_OUT0 + _jitc->function->call.argi, u);
 723         ++_jitc->function->call.argi;
 724     }
 725     else {
 726         jit_stxi(_jitc->function->call.size + params_offset, JIT_SP, u);
 727         _jitc->function->call.size += sizeof(jit_word_t);
 728     }
 729     jit_dec_synth();
 730 }
 731
 732 void
 733 _jit_pushargi(jit_state_t *_jit, jit_word_t u)
 734 {
 735     jit_int32_t          regno;
 736     assert(_jitc->function);
 737     jit_inc_synth_w(pushargi, u);
 738     jit_link_prepare();
 739     if (jit_arg_reg_p(_jitc->function->call.argi)) {
 740         jit_movi(_OUT0 + _jitc->function->call.argi, u);
 741         ++_jitc->function->call.argi;
 742     }
 743     else {
 744         regno = jit_get_reg(jit_class_gpr);
 745         jit_movi(regno, u);
 746         jit_stxi(_jitc->function->call.size + params_offset, JIT_SP, regno);
 747         _jitc->function->call.size += sizeof(jit_word_t);
 748         jit_unget_reg(regno);
 749     }
 750     jit_dec_synth();
 751 }
 752
 753 void
 754 _jit_pushargr_f(jit_state_t *_jit, jit_int32_t u)
 755 {
 756     assert(_jitc->function);
 757     jit_inc_synth_w(pushargr_f, u);
 758     jit_link_prepare();
 759     if (jit_arg_reg_p(_jitc->function->call.argi)) {
 760         if (!(_jitc->function->call.call & jit_call_varargs)) {
 761             jit_movr_f(_F8 + _jitc->function->call.argf, u);
 762             ++_jitc->function->call.argf;
 763         }
 764         else
 765             jit_movr_f_w(_OUT0 + _jitc->function->call.argi, u);
 766         ++_jitc->function->call.argi;
 767     }
 768     else {
 769         jit_stxi_f(_jitc->function->call.size + params_offset + F_DISP,
 770                    JIT_SP, u);
 771         _jitc->function->call.size += sizeof(jit_word_t);
 772     }
 773     jit_dec_synth();
 774 }
 775
 776 void
 777 _jit_pushargi_f(jit_state_t *_jit, jit_float32_t u)
 778 {
 779     jit_int32_t          regno;
 780     assert(_jitc->function);
 781     jit_inc_synth_f(pushargi_f, u);
 782     jit_link_prepare();
 783     if (jit_arg_reg_p(_jitc->function->call.argi)) {
 784         if (!(_jitc->function->call.call & jit_call_varargs)) {
 785             jit_movi_f(_F8 + _jitc->function->call.argf, u);
 786             ++_jitc->function->call.argf;
 787         }
 788         else
 789             jit_movi_f_w(_OUT0 + _jitc->function->call.argi, u);
 790         ++_jitc->function->call.argi;
 791     }
 792     else {
 793         regno = jit_get_reg(jit_class_fpr);
 794         jit_movi_f(regno, u);
 795         jit_stxi_f(_jitc->function->call.size + params_offset + F_DISP,
 796                    JIT_SP, regno);
 797         _jitc->function->call.size += sizeof(jit_word_t);
 798         jit_unget_reg(regno);
 799     }
 800     jit_dec_synth();
 801 }
 802
 803 void
 804 _jit_pushargr_d(jit_state_t *_jit, jit_int32_t u)
 805 {
 806     assert(_jitc->function);
 807     jit_inc_synth_w(pushargr_d, u);
 808     jit_link_prepare();
 809     if (jit_arg_reg_p(_jitc->function->call.argi)) {
 810         if (!(_jitc->function->call.call & jit_call_varargs)) {
 811             jit_movr_d(_F8 + _jitc->function->call.argf, u);
 812             ++_jitc->function->call.argf;
 813         }
 814         else
 815             jit_movr_d_w(_OUT0 + _jitc->function->call.argi, u);
 816         ++_jitc->function->call.argi;
 817     }
 818     else {
 819         jit_stxi_d(_jitc->function->call.size + params_offset, JIT_SP, u);
 820         _jitc->function->call.size += sizeof(jit_float64_t);
 821     }
 822     jit_dec_synth();
 823 }
 824
 825 void
 826 _jit_pushargi_d(jit_state_t *_jit, jit_float64_t u)
 827 {
 828     jit_int32_t          regno;
 829     assert(_jitc->function);
 830     jit_inc_synth_d(pushargi_d, u);
 831     jit_link_prepare();
 832     if (jit_arg_reg_p(_jitc->function->call.argi)) {
 833         if (!(_jitc->function->call.call & jit_call_varargs)) {
 834             jit_movi_d(_F8 + _jitc->function->call.argf, u);
 835             ++_jitc->function->call.argf;
 836         }
 837         else
 838             jit_movi_d_w(_OUT0 + _jitc->function->call.argi, u);
 839         ++_jitc->function->call.argi;
 840     }
 841     else {
 842         regno = jit_get_reg(jit_class_fpr);
 843         jit_movi_d(regno, u);
 844         jit_stxi_d(_jitc->function->call.size + params_offset, JIT_SP, regno);
 845         _jitc->function->call.size += sizeof(jit_float64_t);
 846         jit_unget_reg(regno);
 847     }
 848     jit_dec_synth();
 849 }
 850
 851 jit_bool_t
 852 _jit_regarg_p(jit_state_t *_jit, jit_node_t *node, jit_int32_t regno)
 853 {
 854     /* Argument registers are allocated from the pool of unused registers */
 855     return (0);
 856 }
 857
 858 void
 859 _jit_finishr(jit_state_t *_jit, jit_int32_t r0)
 860 {
 861     jit_node_t          *call;
 862     assert(_jitc->function);
 863     jit_inc_synth_w(finishr, r0);
 864     if (_jitc->function->self.alen < _jitc->function->call.size)
 865         _jitc->function->self.alen = _jitc->function->call.size;
 866     call = jit_callr(r0);
 867     call->v.w = _jitc->function->call.argi;
 868     call->w.w = _jitc->function->call.argf;
 869     _jitc->function->call.argi = _jitc->function->call.argf =
 870         _jitc->function->call.size = 0;
 871     _jitc->prepare = 0;
 872     jit_dec_synth();
 873 }
 874
 875 jit_node_t *
 876 _jit_finishi(jit_state_t *_jit, jit_pointer_t i0)
 877 {
 878     jit_node_t          *node;
 879     assert(_jitc->function);
 880     jit_inc_synth_w(finishi, (jit_word_t)i0);
 881     if (_jitc->function->self.alen < _jitc->function->call.size)
 882         _jitc->function->self.alen = _jitc->function->call.size;
 883     node = jit_calli(i0);
 884     node->v.w = _jitc->function->call.argi;
 885     node->w.w = _jitc->function->call.argf;
 886     _jitc->function->call.argi = _jitc->function->call.argf =
 887         _jitc->function->call.size = 0;
 888     _jitc->prepare = 0;
 889     jit_dec_synth();
 890     return (node);
 891 }
 892
 893 void
 894 _jit_retval_c(jit_state_t *_jit, jit_int32_t r0)
 895 {
 896     jit_inc_synth_w(retval_c, r0);
 897     jit_extr_c(r0, JIT_RET);
 898     jit_dec_synth();
 899 }
 900
 901 void
 902 _jit_retval_uc(jit_state_t *_jit, jit_int32_t r0)
 903 {
 904     jit_inc_synth_w(retval_uc, r0);
 905     jit_extr_uc(r0, JIT_RET);
 906     jit_dec_synth();
 907 }
 908
 909 void
 910 _jit_retval_s(jit_state_t *_jit, jit_int32_t r0)
 911 {
 912     jit_inc_synth_w(retval_s, r0);
 913     jit_extr_s(r0, JIT_RET);
 914     jit_dec_synth();
 915 }
 916
 917 void
 918 _jit_retval_us(jit_state_t *_jit, jit_int32_t r0)
 919 {
 920     jit_inc_synth_w(retval_us, r0);
 921     jit_extr_us(r0, JIT_RET);
 922     jit_dec_synth();
 923 }
 924
 925 void
 926 _jit_retval_i(jit_state_t *_jit, jit_int32_t r0)
 927 {
 928     jit_inc_synth_w(retval_i, r0);
 929     jit_extr_i(r0, JIT_RET);
 930     jit_dec_synth();
 931 }
 932
 933 void
 934 _jit_retval_ui(jit_state_t *_jit, jit_int32_t r0)
 935 {
 936     jit_inc_synth_w(retval_ui, r0);
 937     jit_extr_ui(r0, JIT_RET);
 938     jit_dec_synth();
 939 }
 940
 941 void
 942 _jit_retval_l(jit_state_t *_jit, jit_int32_t r0)
 943 {
 944     jit_inc_synth_w(retval_l, r0);
 945     jit_movr(r0, JIT_RET);
 946     jit_dec_synth();
 947 }
 948
 949 void
 950 _jit_retval_f(jit_state_t *_jit, jit_int32_t r0)
 951 {
 952     jit_inc_synth_w(retval_l, r0);
 953     jit_movr_f(r0, JIT_FRET);
 954     jit_dec_synth();
 955 }
 956
 957 void
 958 _jit_retval_d(jit_state_t *_jit, jit_int32_t r0)
 959 {
 960     jit_inc_synth_w(retval_d, r0);
 961     jit_movr_d(r0, JIT_FRET);
 962     jit_dec_synth();
 963 }
 964
 965 jit_pointer_t
 966 _emit_code(jit_state_t *_jit)
 967 {
 968     jit_node_t          *node;
 969     jit_node_t          *temp;
 970     jit_word_t           word;
 971     jit_int32_t          value;
 972     jit_int32_t          offset;
 973     struct {
 974         jit_node_t      *node;
 975         jit_word_t       word;
 976 #if DEVEL_DISASSEMBLER
 977         jit_word_t       prevw;
 978 #endif
 979         jit_int32_t      patch_offset;
 980         jit_word_t       prolog_offset;
 981     } undo;
 982 #if DEVEL_DISASSEMBLER
 983     jit_word_t           prevw;
 984 #endif
 985
 986     _jitc->function = NULL;
 987
 988     /* If did resize the code buffer, these were not reset */
 989     _jitc->ioff = 0;
 990     jit_regset_set_ui(&_jitc->regs, 0);
 991     _jitc->pred = 0;
 992
 993     jit_reglive_setup();
 994
 995     undo.word = 0;
 996     undo.node = NULL;
 997     undo.patch_offset = 0;
 998
 999 #if DEVEL_DISASSEMBLER
1000     prevw = _jit->pc.w;
1001 #endif
1002     undo.prolog_offset = 0;
1003     for (node = _jitc->head; node; node = node->next)
1004         if (node->code != jit_code_label &&
1005             node->code != jit_code_note &&
1006             node->code != jit_code_name)
1007             break;
1008     if (node && (node->code != jit_code_prolog ||
1009                  !(_jitc->functions.ptr + node->w.w)->assume_frame)) {
1010         /* code may start with a jump so add an initial function descriptor */
1011         word = _jit->pc.w + 16;
1012         il(word);               /* addr */
1013         il(0);                  /* gp */
1014     }
1015 #define case_rr(name, type)                                             \
1016             case jit_code_##name##r##type:                              \
1017                 name##r##type(rn(node->u.w), rn(node->v.w));            \
1018                 break
1019 #define case_rw(name, type)                                             \
1020             case jit_code_##name##i##type:                              \
1021                 name##i##type(rn(node->u.w), node->v.w);                \
1022                 break
1023 #define case_wr(name, type)                                             \
1024             case jit_code_##name##i##type:                              \
1025                 name##i##type(node->u.w, rn(node->v.w));                \
1026                 break
1027 #define case_rrr(name, type)                                            \
1028             case jit_code_##name##r##type:                              \
1029                 name##r##type(rn(node->u.w),                            \
1030                               rn(node->v.w), rn(node->w.w));            \
1031                 break
1032 #define case_rrrr(name, type)                                           \
1033             case jit_code_##name##r##type:                              \
1034                 name##r##type(rn(node->u.q.l), rn(node->u.q.h),         \
1035                               rn(node->v.w), rn(node->w.w));            \
1036                 break
1037 #define case_rrw(name, type)                                            \
1038             case jit_code_##name##i##type:                              \
1039                 name##i##type(rn(node->u.w), rn(node->v.w), node->w.w); \
1040                 break
1041 #define case_rrrw(name, type)                                           \
1042             case jit_code_##name##i##type:                              \
1043                 name##i##type(rn(node->u.q.l), rn(node->u.q.h),         \
1044                               rn(node->v.w), node->w.w);                \
1045                 break
1046 #define case_rrf(name, type, size)                                      \
1047             case jit_code_##name##i##type:                              \
1048                 assert(node->flag & jit_flag_data);                     \
1049                 name##i##type(rn(node->u.w), rn(node->v.w),             \
1050                               (jit_float##size##_t *)node->w.n->u.w);   \
1051                 break
1052 #define case_wrr(name, type)                                            \
1053             case jit_code_##name##i##type:                              \
1054                 name##i##type(node->u.w, rn(node->v.w), rn(node->w.w)); \
1055                 break
1056 #define case_brr(name, type)                                            \
1057             case jit_code_##name##r##type:                              \
1058                 temp = node->u.n;                                       \
1059                 assert(temp->code == jit_code_label ||                  \
1060                        temp->code == jit_code_epilog);                  \
1061                 if (temp->flag & jit_flag_patch)                        \
1062                     name##r##type(temp->u.w, rn(node->v.w),             \
1063                                   rn(node->w.w));                       \
1064                 else {                                                  \
1065                     word = name##r##type(_jit->pc.w,                    \
1066                                          rn(node->v.w), rn(node->w.w)); \
1067                     patch(word, node);                                  \
1068                 }                                                       \
1069                 break
1070 #define case_brw(name, type)                                            \
1071             case jit_code_##name##i##type:                              \
1072                 temp = node->u.n;                                       \
1073                 assert(temp->code == jit_code_label ||                  \
1074                        temp->code == jit_code_epilog);                  \
1075                 if (temp->flag & jit_flag_patch)                        \
1076                     name##i##type(temp->u.w,                            \
1077                                   rn(node->v.w), node->w.w);            \
1078                 else {                                                  \
1079                     word = name##i##type(_jit->pc.w,                    \
1080                                          rn(node->v.w), node->w.w);     \
1081                     patch(word, node);                                  \
1082                 }                                                       \
1083                 break
1084 #define case_brf(name, type, size)                                      \
1085             case jit_code_##name##i##type:                              \
1086                 temp = node->u.n;                                       \
1087                 assert(temp->code == jit_code_label ||                  \
1088                        temp->code == jit_code_epilog);                  \
1089                 if (temp->flag & jit_flag_patch)                        \
1090                     name##i##type(temp->u.w, rn(node->v.w),             \
1091                                 (jit_float##size##_t *)node->w.n->u.w); \
1092                 else {                                                  \
1093                     word = name##i##type(_jit->pc.w, rn(node->v.w),     \
1094                                 (jit_float##size##_t *)node->w.n->u.w); \
1095                     patch(word, node);                                  \
1096                 }                                                       \
1097                 break
1098     for (node = _jitc->head; node; node = node->next) {
1099         if (_jit->pc.uc >= _jitc->code.end)
1100             return (NULL);
1101
1102         value = jit_classify(node->code);
1103 #if GET_JIT_SIZE
1104         sync();
1105 #endif
1106 #if DEVEL_DISASSEMBLER
1107         /* FIXME DEVEL_DISASSEMBLER should become DISASSEMBLER,
1108          * but a "real" DEVEL_DISASSEMBLER should be required
1109          * to turn the below "#if 0" into "#if 1" */
1110 #  if 0         /* Since disassembly outputs 3 instructions at a time,
1111                  * make it "#if 1" for more clear debug output. */
1112         sync();
1113 #  endif
1114         node->offset = (jit_uword_t)_jit->pc.w - (jit_uword_t)prevw;
1115         prevw = _jit->pc.w;
1116 #endif
1117         jit_regarg_set(node, value);
1118         switch (node->code) {
1119             case jit_code_align:
1120                 assert(!(node->u.w & (node->u.w - 1)) &&
1121                        node->u.w <= sizeof(jit_word_t));
1122                 /* nothing done */
1123                 break;
1124             case jit_code_note:         case jit_code_name:
1125                 sync();
1126                 node->u.w = _jit->pc.w;
1127                 break;
1128             case jit_code_label:
1129                 sync();
1130                 /* remember label is defined */
1131                 node->flag |= jit_flag_patch;
1132                 node->u.w = _jit->pc.w;
1133                 break;
1134                 case_rrr(add,);
1135                 case_rrw(add,);
1136                 case_rrr(addc,);
1137                 case_rrw(addc,);
1138                 case_rrr(addx,);
1139                 case_rrw(addx,);
1140                 case_rrr(sub,);
1141                 case_rrw(sub,);
1142                 case_rrr(subx,);
1143                 case_rrw(subx,);
1144                 case_rrw(rsb,);
1145                 case_rrr(subc,);
1146                 case_rrw(subc,);
1147                 case_rrr(mul,);
1148                 case_rrw(mul,);
1149                 case_rrrr(qmul,);
1150                 case_rrrw(qmul,);
1151                 case_rrrr(qmul, _u);
1152                 case_rrrw(qmul, _u);
1153                 case_rrr(div,);
1154                 case_rrw(div,);
1155                 case_rrr(div, _u);
1156                 case_rrw(div, _u);
1157                 case_rrr(rem,);
1158                 case_rrw(rem,);
1159                 case_rrr(rem, _u);
1160                 case_rrw(rem, _u);
1161                 case_rrrr(qdiv,);
1162                 case_rrrw(qdiv,);
1163                 case_rrrr(qdiv, _u);
1164                 case_rrrw(qdiv, _u);
1165                 case_rrr(and,);
1166                 case_rrw(and,);
1167                 case_rrr(or,);
1168                 case_rrw(or,);
1169                 case_rrr(xor,);
1170                 case_rrw(xor,);
1171                 case_rrr(lsh,);
1172                 case_rrw(lsh,);
1173                 case_rrr(rsh,);
1174                 case_rrw(rsh,);
1175                 case_rrr(rsh, _u);
1176                 case_rrw(rsh, _u);
1177                 case_rr(neg,);
1178                 case_rr(com,);
1179             case jit_code_casr:
1180                 casr(rn(node->u.w), rn(node->v.w),
1181                      rn(node->w.q.l), rn(node->w.q.h));
1182                 break;
1183             case jit_code_casi:
1184                 casi(rn(node->u.w), node->v.w,
1185                      rn(node->w.q.l), rn(node->w.q.h));
1186                 break;
1187                 case_rrr(movn,);
1188                 case_rrr(movz,);
1189                 case_rr(mov,);
1190             case jit_code_movi:
1191                 if (node->flag & jit_flag_node) {
1192                     temp = node->v.n;
1193                     if (temp->code == jit_code_data ||
1194                         (temp->code == jit_code_label &&
1195                          (temp->flag & jit_flag_patch)))
1196                         movi(rn(node->u.w), temp->u.w);
1197                     else {
1198                         assert(temp->code == jit_code_label ||
1199                                temp->code == jit_code_epilog);
1200                         word = movi_p(rn(node->u.w), node->v.w);
1201                         patch(word, node);
1202                     }
1203                 }
1204                 else
1205                     movi(rn(node->u.w), node->v.w);
1206                 break;
1207                 case_rr(hton, _us);
1208                 case_rr(hton, _ui);
1209                 case_rr(hton, _ul);
1210                 case_rr(bswap, _us);
1211                 case_rr(bswap, _ui);
1212                 case_rr(bswap, _ul);
1213                 case_rr(ext, _c);
1214                 case_rr(ext, _uc);
1215                 case_rr(ext, _s);
1216                 case_rr(ext, _us);
1217                 case_rr(ext, _i);
1218                 case_rr(ext, _ui);
1219                 case_rr(trunc, _f_i);
1220                 case_rr(trunc, _d_i);
1221                 case_rr(trunc, _f_l);
1222                 case_rr(trunc, _d_l);
1223                 case_rrr(lt,);
1224                 case_rrw(lt,);
1225                 case_rrr(lt, _u);
1226                 case_rrw(lt, _u);
1227                 case_rrr(le,);
1228                 case_rrw(le,);
1229                 case_rrr(le, _u);
1230                 case_rrw(le, _u);
1231                 case_rrr(eq,);
1232                 case_rrw(eq,);
1233                 case_rrr(ge,);
1234                 case_rrw(ge,);
1235                 case_rrr(ge, _u);
1236                 case_rrw(ge, _u);
1237                 case_rrr(gt,);
1238                 case_rrw(gt,);
1239                 case_rrr(gt, _u);
1240                 case_rrw(gt, _u);
1241                 case_rrr(ne,);
1242                 case_rrw(ne,);
1243                 case_rr(ld, _c);
1244                 case_rw(ld, _c);
1245                 case_rr(ld, _uc);
1246                 case_rw(ld, _uc);
1247                 case_rr(ld, _s);
1248                 case_rw(ld, _s);
1249                 case_rr(ld, _us);
1250                 case_rw(ld, _us);
1251                 case_rr(ld, _i);
1252                 case_rw(ld, _i);
1253                 case_rr(ld, _ui);
1254                 case_rw(ld, _ui);
1255                 case_rr(ld, _l);
1256                 case_rw(ld, _l);
1257                 case_rrr(ldx, _c);
1258                 case_rrw(ldx, _c);
1259                 case_rrr(ldx, _uc);
1260                 case_rrw(ldx, _uc);
1261                 case_rrr(ldx, _s);
1262                 case_rrw(ldx, _s);
1263                 case_rrr(ldx, _us);
1264                 case_rrw(ldx, _us);
1265                 case_rrr(ldx, _i);
1266                 case_rrw(ldx, _i);
1267                 case_rrr(ldx, _ui);
1268                 case_rrw(ldx, _ui);
1269                 case_rrr(ldx, _l);
1270                 case_rrw(ldx, _l);
1271                 case_rr(st, _c);
1272                 case_wr(st, _c);
1273                 case_rr(st, _s);
1274                 case_wr(st, _s);
1275                 case_rr(st, _i);
1276                 case_wr(st, _i);
1277                 case_rr(st, _l);
1278                 case_wr(st, _l);
1279                 case_rrr(stx, _c);
1280                 case_wrr(stx, _c);
1281                 case_rrr(stx, _s);
1282                 case_wrr(stx, _s);
1283                 case_rrr(stx, _i);
1284                 case_wrr(stx, _i);
1285                 case_rrr(stx, _l);
1286                 case_wrr(stx, _l);
1287                 case_brr(blt,);
1288                 case_brw(blt,);
1289                 case_brr(blt, _u);
1290                 case_brw(blt, _u);
1291                 case_brr(ble,);
1292                 case_brw(ble,);
1293                 case_brr(ble, _u);
1294                 case_brw(ble, _u);
1295                 case_brr(beq,);
1296                 case_brw(beq,);
1297                 case_brr(bge,);
1298                 case_brw(bge,);
1299                 case_brr(bge, _u);
1300                 case_brw(bge, _u);
1301                 case_brr(bgt,);
1302                 case_brw(bgt,);
1303                 case_brr(bgt, _u);
1304                 case_brw(bgt, _u);
1305                 case_brr(bne,);
1306                 case_brw(bne,);
1307                 case_brr(bms,);
1308                 case_brw(bms,);
1309                 case_brr(bmc,);
1310                 case_brw(bmc,);
1311                 case_brr(boadd,);
1312                 case_brw(boadd,);
1313                 case_brr(boadd, _u);
1314                 case_brw(boadd, _u);
1315                 case_brr(bxadd,);
1316                 case_brw(bxadd,);
1317                 case_brr(bxadd, _u);
1318                 case_brw(bxadd, _u);
1319                 case_brr(bosub,);
1320                 case_brw(bosub,);
1321                 case_brr(bosub, _u);
1322                 case_brw(bosub, _u);
1323                 case_brr(bxsub,);
1324                 case_brw(bxsub,);
1325                 case_brr(bxsub, _u);
1326                 case_brw(bxsub, _u);
1327                 case_rrr(add, _f);
1328                 case_rrf(add, _f, 32);
1329                 case_rrr(sub, _f);
1330                 case_rrf(sub, _f, 32);
1331                 case_rrf(rsb, _f, 32);
1332                 case_rrr(mul, _f);
1333                 case_rrf(mul, _f, 32);
1334                 case_rrr(div, _f);
1335                 case_rrf(div, _f, 32);
1336                 case_rr(ext, _f);
1337                 case_rr(abs, _f);
1338                 case_rr(neg, _f);
1339                 case_rr(sqrt, _f);
1340                 case_rr(ld, _f);
1341                 case_rw(ld, _f);
1342                 case_rrr(ldx, _f);
1343                 case_rrw(ldx, _f);
1344                 case_rr(st, _f);
1345                 case_wr(st, _f);
1346                 case_rrr(stx, _f);
1347                 case_wrr(stx, _f);
1348                 case_rr(mov, _f);
1349             case jit_code_movi_f:
1350                 assert(node->flag & jit_flag_data);
1351                 movi_f(rn(node->u.w), (jit_float32_t *)node->v.n->u.w);
1352                 break;
1353                 case_rr(ext, _d_f);
1354                 case_rrr(lt, _f);
1355                 case_rrf(lt, _f, 32);
1356                 case_rrr(le, _f);
1357                 case_rrf(le, _f, 32);
1358                 case_rrr(eq, _f);
1359                 case_rrf(eq, _f, 32);
1360                 case_rrr(ge, _f);
1361                 case_rrf(ge, _f, 32);
1362                 case_rrr(gt, _f);
1363                 case_rrf(gt, _f, 32);
1364                 case_rrr(ne, _f);
1365                 case_rrf(ne, _f, 32);
1366                 case_rrr(unlt, _f);
1367                 case_rrf(unlt, _f, 32);
1368                 case_rrr(unle, _f);
1369                 case_rrf(unle, _f, 32);
1370                 case_rrr(uneq, _f);
1371                 case_rrf(uneq, _f, 32);
1372                 case_rrr(unge, _f);
1373                 case_rrf(unge, _f, 32);
1374                 case_rrr(ungt, _f);
1375                 case_rrf(ungt, _f, 32);
1376                 case_rrr(ltgt, _f);
1377                 case_rrf(ltgt, _f, 32);
1378                 case_rrr(ord, _f);
1379                 case_rrf(ord, _f, 32);
1380                 case_rrr(unord, _f);
1381                 case_rrf(unord, _f, 32);
1382                 case_brr(blt, _f);
1383                 case_brf(blt, _f, 32);
1384                 case_brr(ble, _f);
1385                 case_brf(ble, _f, 32);
1386                 case_brr(beq, _f);
1387                 case_brf(beq, _f, 32);
1388                 case_brr(bge, _f);
1389                 case_brf(bge, _f, 32);
1390                 case_brr(bgt, _f);
1391                 case_brf(bgt, _f, 32);
1392                 case_brr(bne, _f);
1393                 case_brf(bne, _f, 32);
1394                 case_brr(bunlt, _f);
1395                 case_brf(bunlt, _f, 32);
1396                 case_brr(bunle, _f);
1397                 case_brf(bunle, _f, 32);
1398                 case_brr(buneq, _f);
1399                 case_brf(buneq, _f, 32);
1400                 case_brr(bunge, _f);
1401                 case_brf(bunge, _f, 32);
1402                 case_brr(bungt, _f);
1403                 case_brf(bungt, _f, 32);
1404                 case_brr(bltgt, _f);
1405                 case_brf(bltgt, _f, 32);
1406                 case_brr(bord, _f);
1407                 case_brf(bord, _f, 32);
1408                 case_brr(bunord, _f);
1409                 case_brf(bunord, _f, 32);
1410                 case_rrr(add, _d);
1411                 case_rrf(add, _d, 64);
1412                 case_rrr(sub, _d);
1413                 case_rrf(sub, _d, 64);
1414                 case_rrf(rsb, _d, 64);
1415                 case_rrr(mul, _d);
1416                 case_rrf(mul, _d, 64);
1417                 case_rrr(div, _d);
1418                 case_rrf(div, _d, 64);
1419                 case_rr(ext, _d);
1420                 case_rr(abs, _d);
1421                 case_rr(neg, _d);
1422                 case_rr(sqrt, _d);
1423                 case_rr(ld, _d);
1424                 case_rw(ld, _d);
1425                 case_rrr(ldx, _d);
1426                 case_rrw(ldx, _d);
1427                 case_rr(st, _d);
1428                 case_wr(st, _d);
1429                 case_rrr(stx, _d);
1430                 case_wrr(stx, _d);
1431                 case_rr(mov, _d);
1432             case jit_code_movi_d:
1433                 assert(node->flag & jit_flag_data);
1434                 movi_d(rn(node->u.w), (jit_float64_t *)node->v.n->u.w);
1435                 break;
1436                 case_rr(ext, _f_d);
1437                 case_rrr(lt, _d);
1438                 case_rrf(lt, _d, 64);
1439                 case_rrr(le, _d);
1440                 case_rrf(le, _d, 64);
1441                 case_rrr(eq, _d);
1442                 case_rrf(eq, _d, 64);
1443                 case_rrr(ge, _d);
1444                 case_rrf(ge, _d, 64);
1445                 case_rrr(gt, _d);
1446                 case_rrf(gt, _d, 64);
1447                 case_rrr(ne, _d);
1448                 case_rrf(ne, _d, 64);
1449                 case_rrr(unlt, _d);
1450                 case_rrf(unlt, _d, 64);
1451                 case_rrr(unle, _d);
1452                 case_rrf(unle, _d, 64);
1453                 case_rrr(uneq, _d);
1454                 case_rrf(uneq, _d, 64);
1455                 case_rrr(unge, _d);
1456                 case_rrf(unge, _d, 64);
1457                 case_rrr(ungt, _d);
1458                 case_rrf(ungt, _d, 64);
1459                 case_rrr(ltgt, _d);
1460                 case_rrf(ltgt, _d, 64);
1461                 case_rrr(ord, _d);
1462                 case_rrf(ord, _d, 64);
1463                 case_rrr(unord, _d);
1464                 case_rrf(unord, _d, 64);
1465                 case_brr(blt, _d);
1466                 case_brf(blt, _d, 64);
1467                 case_brr(ble, _d);
1468                 case_brf(ble, _d, 64);
1469                 case_brr(beq, _d);
1470                 case_brf(beq, _d, 64);
1471                 case_brr(bge, _d);
1472                 case_brf(bge, _d, 64);
1473                 case_brr(bgt, _d);
1474                 case_brf(bgt, _d, 64);
1475                 case_brr(bne, _d);
1476                 case_brf(bne, _d, 64);
1477                 case_brr(bunlt, _d);
1478                 case_brf(bunlt, _d, 64);
1479                 case_brr(bunle, _d);
1480                 case_brf(bunle, _d, 64);
1481                 case_brr(buneq, _d);
1482                 case_brf(buneq, _d, 64);
1483                 case_brr(bunge, _d);
1484                 case_brf(bunge, _d, 64);
1485                 case_brr(bungt, _d);
1486                 case_brf(bungt, _d, 64);
1487                 case_brr(bltgt, _d);
1488                 case_brf(bltgt, _d, 64);
1489                 case_brr(bord, _d);
1490                 case_brf(bord, _d, 64);
1491                 case_brr(bunord, _d);
1492                 case_brf(bunord, _d, 64);
1493             case jit_code_jmpr:
1494                 jmpr(rn(node->u.w));
1495                 break;
1496             case jit_code_jmpi:
1497                 if (node->flag & jit_flag_node) {
1498                     if (_jit->pc.uc == _jit->code.ptr + 16)
1499                         _jitc->jump = 1;
1500                     temp = node->u.n;
1501                     assert(temp->code == jit_code_label ||
1502                            temp->code == jit_code_epilog);
1503                     if (temp->flag & jit_flag_patch)
1504                         jmpi(temp->u.w);
1505                     else {
1506                         word = jmpi_p(_jit->pc.w);
1507                         patch(word, node);
1508                     }
1509                 }
1510                 else
1511                     jmpi(node->u.w);
1512                 break;
1513             case jit_code_callr:
1514                 callr(rn(node->u.w));
1515                 break;
1516             case jit_code_calli:
1517                 if (node->flag & jit_flag_node) {
1518                     temp = node->u.n;
1519                     assert(temp->code == jit_code_label ||
1520                            temp->code == jit_code_epilog);
1521                     word = calli_p(temp->u.w);
1522                     if (!(temp->flag & jit_flag_patch))
1523                         patch(word, node);
1524                 }
1525                 else
1526                     calli(node->u.w);
1527                 break;
1528             case jit_code_prolog:
1529                 jit_regset_set_ui(&_jitc->regs, 0);
1530                 _jitc->pred = 0;
1531                 sync();
1532                 _jitc->function = _jitc->functions.ptr + node->w.w;
1533                 undo.node = node;
1534                 undo.word = _jit->pc.w;
1535 #if DEVEL_DISASSEMBLER
1536                 undo.prevw = prevw;
1537 #endif
1538                 undo.patch_offset = _jitc->patches.offset;
1539                 undo.prolog_offset = _jitc->prolog.offset;
1540             restart_function:
1541                 _jitc->again = 0;
1542                 if (_jitc->jump && !_jitc->function->assume_frame) {
1543                     /* remember prolog to hide offset adjustment for a jump
1544                      * to the start of a function, what is expected to be
1545                      * a common practice as first jit instruction */
1546                     if (_jitc->prolog.offset >= _jitc->prolog.length) {
1547                         _jitc->prolog.length += 16;
1548                         jit_realloc((jit_pointer_t *)&_jitc->prolog.ptr,
1549                                     (_jitc->prolog.length - 16) *
1550                                     sizeof(jit_word_t),
1551                                     _jitc->prolog.length * sizeof(jit_word_t));
1552                     }
1553                     _jitc->prolog.ptr[_jitc->prolog.offset++] = _jit->pc.w;
1554                     /* function descriptor */
1555                     word = _jit->pc.w + 16;
1556                     il(word);                   /* addr */
1557                     il(0);                      /* gp */
1558                 }
1559                 prolog(node);
1560                 break;
1561             case jit_code_epilog:
1562                 assert(_jitc->function == _jitc->functions.ptr + node->w.w);
1563                 if (_jitc->again) {
1564                     for (temp = undo.node->next;
1565                          temp != node; temp = temp->next) {
1566                         if (temp->code == jit_code_label ||
1567                             temp->code == jit_code_epilog)
1568                             temp->flag &= ~jit_flag_patch;
1569                     }
1570                     temp->flag &= ~jit_flag_patch;
1571                     node = undo.node;
1572                     _jit->pc.w = undo.word;
1573 #if DEVEL_DISASSEMBLER
1574                     prevw = undo.prevw;
1575 #endif
1576                     _jitc->patches.offset = undo.patch_offset;
1577                     _jitc->prolog.offset = undo.prolog_offset;
1578                     _jitc->ioff = 0;
1579                     jit_regset_set_ui(&_jitc->regs, 0);
1580                     _jitc->pred = 0;
1581                     goto restart_function;
1582                 }
1583                 /* remember label is defined */
1584                 sync();
1585                 node->flag |= jit_flag_patch;
1586                 node->u.w = _jit->pc.w;
1587                 epilog(node);
1588                 jit_regset_set_ui(&_jitc->regs, 0);
1589                 _jitc->pred = 0;
1590                 _jitc->function = NULL;
1591                 break;
1592             case jit_code_va_start:
1593                 vastart(rn(node->u.w));
1594                 break;
1595             case jit_code_va_arg:
1596                 vaarg(rn(node->u.w), rn(node->v.w));
1597                 break;
1598             case jit_code_va_arg_d:
1599                 vaarg_d(rn(node->u.w), rn(node->v.w));
1600                 break;
1601             case jit_code_live:
1602             case jit_code_arg:                  case jit_code_ellipsis:
1603             case jit_code_va_push:
1604             case jit_code_allocai:              case jit_code_allocar:
1605             case jit_code_arg_f:                case jit_code_arg_d:
1606             case jit_code_va_end:
1607             case jit_code_ret:
1608             case jit_code_retr:                 case jit_code_reti:
1609             case jit_code_retr_f:               case jit_code_reti_f:
1610             case jit_code_retr_d:               case jit_code_reti_d:
1611             case jit_code_getarg_c:             case jit_code_getarg_uc:
1612             case jit_code_getarg_s:             case jit_code_getarg_us:
1613             case jit_code_getarg_i:             case jit_code_getarg_ui:
1614             case jit_code_getarg_l:
1615             case jit_code_getarg_f:             case jit_code_getarg_d:
1616             case jit_code_putargr:              case jit_code_putargi:
1617             case jit_code_putargr_f:            case jit_code_putargi_f:
1618             case jit_code_putargr_d:            case jit_code_putargi_d:
1619             case jit_code_pushargr:             case jit_code_pushargi:
1620             case jit_code_pushargr_f:           case jit_code_pushargi_f:
1621             case jit_code_pushargr_d:           case jit_code_pushargi_d:
1622             case jit_code_retval_c:             case jit_code_retval_uc:
1623             case jit_code_retval_s:             case jit_code_retval_us:
1624             case jit_code_retval_i:
1625             case jit_code_retval_ui:            case jit_code_retval_l:
1626             case jit_code_retval_f:             case jit_code_retval_d:
1627             case jit_code_prepare:
1628             case jit_code_finishr:              case jit_code_finishi:
1629                 break;
1630             case jit_code_movr_w_f:
1631                 movr_w_f(rn(node->u.w), rn(node->v.w));
1632                 break;
1633             case jit_code_movr_f_w:
1634                 movr_f_w(rn(node->u.w), rn(node->v.w));
1635                 break;
1636             case jit_code_movr_w_d:
1637                 movr_w_d(rn(node->u.w), rn(node->v.w));
1638                 break;
1639             case jit_code_movr_d_w:
1640                 movr_d_w(rn(node->u.w), rn(node->v.w));
1641                 break;
1642             case jit_code_movi_f_w:
1643                 movi_f_w(rn(node->u.w), node->v.n->u.p);
1644                 break;
1645             case jit_code_movi_d_w:
1646                 movi_d_w(rn(node->u.w), node->v.n->u.p);
1647                 break;
1648             default:
1649                 abort();
1650         }
1651         if (jit_carry != _NOREG) {
1652             switch (node->code) {
1653                 case jit_code_note:
1654                 case jit_code_addcr:            case jit_code_addci:
1655                 case jit_code_addxr:            case jit_code_addxi:
1656                 case jit_code_subcr:            case jit_code_subci:
1657                 case jit_code_subxr:            case jit_code_subxi:
1658                     break;
1659                 default:
1660                     jit_unget_reg(jit_carry);
1661                     jit_carry = _NOREG;
1662                     break;
1663             }
1664         }
1665 #if GET_JIT_SIZE
1666         sync();
1667 #endif
1668         jit_regarg_clr(node, value);
1669         if (jit_regset_cmp_ui(&_jitc->regarg, 0) != 0) {
1670             assert(jit_regset_scan1(&_jitc->regarg, 0) == jit_carry);
1671             assert(jit_regset_scan1(&_jitc->regarg, jit_carry + 1) == ULONG_MAX);
1672         }
1673         assert(_jitc->synth == 0);
1674         /* update register live state */
1675         jit_reglive(node);
1676     }
1677     sync();
1678 #undef case_brf
1679 #undef case_brw
1680 #undef case_brr
1681 #undef case_wrr
1682 #undef case_rrrw
1683 #undef case_rrf
1684 #undef case_rrw
1685 #undef case_rrrr
1686 #undef case_rrr
1687 #undef case_wr
1688 #undef case_rw
1689 #undef case_rr
1690
1691     for (offset = 0; offset < _jitc->patches.offset; offset++) {
1692         node = _jitc->patches.ptr[offset].node;
1693         word = node->code == jit_code_movi ? node->v.n->u.w : node->u.n->u.w;
1694         patch_at(node->code, _jitc->patches.ptr[offset].inst, word);
1695     }
1696
1697     jit_flush(_jit->code.ptr, _jit->pc.uc);
1698
1699     return (_jit->code.ptr);
1700 }
1701
1702 #define CODE                            1
1703 #  include "jit_ia64-cpu.c"
1704 #  include "jit_ia64-fpu.c"
1705 #  include "jit_fallback.c"
1706 #undef CODE
1707
1708 void
1709 jit_flush(void *fptr, void *tptr)
1710 {
1711 #if defined(__GNUC__)
1712     jit_word_t          f, t, s;
1713
1714     s = sysconf(_SC_PAGE_SIZE);
1715     f = (jit_word_t)fptr & -s;
1716     t = (((jit_word_t)tptr) + s - 1) & -s;
1717 #  if 0
1718     __clear_cache((void *)f, (void *)t);
1719 #  else
1720     /* __clear_cache is a noop in (old?) gcc, but cache flush is
1721      * required on a multi processor Linux system. */
1722     for (s = f; s < t; s += 32)
1723         asm volatile("fc %0" :: "r"(s) : "memory");
1724 #  endif
1725 #endif
1726 }
1727
1728 /* Use r2 that is reserved to not require a jit_get_reg call, also note
1729  * that addil needs a register that first in 2 bits, so, if using a
1730  * register other than r2 must be less than r8 (or check for a smaller
1731  * immediate, i.e. i0 >= -8192 && i0 <= 8191) */
1732 void
1733 _emit_ldxi(jit_state_t *_jit, jit_gpr_t r0, jit_gpr_t r1, jit_word_t i0)
1734 {
1735     assert(i0 >= -2097152 && i0 < 2097151);
1736     addi(GR_2, rn(r1), i0);
1737     ldr(rn(r0), GR_2);
1738 }
1739
1740 void
1741 _emit_stxi(jit_state_t *_jit, jit_word_t i0, jit_gpr_t r0, jit_gpr_t r1)
1742 {
1743     assert(i0 >= -2097152 && i0 < 2097151);
1744     addi(GR_2, rn(r0), i0);
1745     str(GR_2, rn(r1));
1746 }
1747
1748 void
1749 _emit_ldxi_d(jit_state_t *_jit, jit_fpr_t r0, jit_gpr_t r1, jit_word_t i0)
1750 {
1751     assert(i0 >= -2097152 && i0 < 2097151);
1752     addi(GR_2, rn(r1), i0);
1753     ldr_d(rn(r0), GR_2);
1754 }
1755
1756 void
1757 _emit_stxi_d(jit_state_t *_jit, jit_word_t i0, jit_gpr_t r0, jit_fpr_t r1)
1758 {
1759     assert(i0 >= -2097152 && i0 < 2097151);
1760     addi(GR_2, rn(r0), i0);
1761     str_d(GR_2, rn(r1));
1762 }
1763
1764 static void
1765 _patch(jit_state_t *_jit, jit_word_t instr, jit_node_t *node)
1766 {
1767     jit_int32_t         flag;
1768
1769     assert(node->flag & jit_flag_node);
1770     if (node->code == jit_code_movi)
1771         flag = node->v.n->flag;
1772     else
1773         flag = node->u.n->flag;
1774     assert(!(flag & jit_flag_patch));
1775     if (_jitc->patches.offset >= _jitc->patches.length) {
1776         jit_realloc((jit_pointer_t *)&_jitc->patches.ptr,
1777                     _jitc->patches.length * sizeof(jit_patch_t),
1778                     (_jitc->patches.length + 1024) * sizeof(jit_patch_t));
1779         _jitc->patches.length += 1024;
1780     }
1781     _jitc->patches.ptr[_jitc->patches.offset].inst = instr;
1782     _jitc->patches.ptr[_jitc->patches.offset].node = node;
1783     ++_jitc->patches.offset;
1784 }