deps/lightning/lib/jit_sparc.c

   1 /*
   2  * Copyright (C) 2013-2019  Free Software Foundation, Inc.
   3  *
   4  * This file is part of GNU lightning.
   5  *
   6  * GNU lightning is free software; you can redistribute it and/or modify it
   7  * under the terms of the GNU Lesser General Public License as published
   8  * by the Free Software Foundation; either version 3, or (at your option)
   9  * any later version.
  10  *
  11  * GNU lightning is distributed in the hope that it will be useful, but
  12  * WITHOUT ANY WARRANTY; without even the implied warranty of MERCHANTABILITY
  13  * or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU Lesser General Public
  14  * License for more details.
  15  *
  16  * Authors:
  17  *      Paulo Cesar Pereira de Andrade
  18  */
  19
  20 #define jit_arg_reg_p(i)                ((i) >= 0 && (i) < 6)
  21 #if __WORDSIZE == 32
  22 #  define jit_arg_d_reg_p(i)            ((i) >= 0 && (i) < 5)
  23 #  define BIAS(n)                       (n)
  24 #else
  25 #  define jit_arg_d_reg_p(i)            ((i) >= 0 && (i) < 16)
  26 #  define BIAS(n)                       ((n) + 2047)
  27 #endif
  28
  29 /*
  30  * Types
  31  */
  32 typedef jit_pointer_t   jit_va_list_t;
  33
  34 /*
  35  * Prototypes
  36  */
  37 #define patch(instr, node)              _patch(_jit, instr, node)
  38 static void _patch(jit_state_t*,jit_word_t,jit_node_t*);
  39
  40 #define PROTO                           1
  41 #  include "jit_sparc-cpu.c"
  42 #  include "jit_sparc-fpu.c"
  43 #  include "jit_fallback.c"
  44 #undef PROTO
  45
  46 /*
  47  * Initialization
  48  */
  49 jit_register_t          _rvs[] = {
  50     { 0x00,                             "%g0" },
  51     { 0x01,                             "%g1" },
  52     { rc(gpr) | 0x02,                   "%g2" },
  53     { rc(gpr) | 0x03,                   "%g3" },
  54     { rc(gpr) | 0x04,                   "%g4" },
  55     { 0x05,                             "%g5" },
  56     { 0x06,                             "%g6" },
  57     { 0x07,                             "%g7" },
  58     { rc(arg) | rc(gpr) | 0x08,         "%o0" },
  59     { rc(arg) | rc(gpr) | 0x09,         "%o1" },
  60     { rc(arg) | rc(gpr) | 0x0a,         "%o2" },
  61     { rc(arg) | rc(gpr) | 0x0b,         "%o3" },
  62     { rc(arg) | rc(gpr) | 0x0c,         "%o4" },
  63     { rc(arg) | rc(gpr) | 0x0d,         "%o5" },
  64     { rc(sav) | 0x0e,                   "%sp" },
  65     { 0x0f,                             "%o7" },
  66     { rc(sav) | rc(gpr) | 0x10,         "%l0" },
  67     { rc(sav) | rc(gpr) | 0x11,         "%l1" },
  68     { rc(sav) | rc(gpr) | 0x12,         "%l2" },
  69     { rc(sav) | rc(gpr) | 0x13,         "%l3" },
  70     { rc(sav) | rc(gpr) | 0x14,         "%l4" },
  71     { rc(sav) | rc(gpr) | 0x15,         "%l5" },
  72     { rc(sav) | rc(gpr) | 0x16,         "%l6" },
  73     { rc(sav) | rc(gpr) | 0x17,         "%l7" },
  74     { 0x18,                             "%i0" },
  75     { 0x19,                             "%i1" },
  76     { 0x1a,                             "%i2" },
  77     { 0x1b,                             "%i3" },
  78     { 0x1c,                             "%i4" },
  79     { 0x1d,                             "%i5" },
  80     { rc(sav) | 0x1e,                   "%fp" },
  81     { 0x1f,                             "%i7" },
  82 #  if __WORDSIZE == 32
  83     { rc(fpr) | 0x00,                   "%f0" },
  84     { 0x01,                             "%f1" },
  85     { rc(fpr) | 0x02,                   "%f2" },
  86     { 0x03,                             "%f3" },
  87     { rc(fpr) | 0x04,                   "%f4" },
  88     { 0x05,                             "%f5" },
  89     { rc(fpr) | 0x06,                   "%f6" },
  90     { 0x07,                             "%f7" },
  91     { rc(fpr) | 0x08,                   "%f8" },
  92     { 0x09,                             "%f9" },
  93     { rc(fpr) | 0x0a,                   "%f10" },
  94     { 0x0b,                             "%f11" },
  95     { rc(fpr) | 0x0c,                   "%f12" },
  96     { 0x0d,                             "%f13" },
  97     { rc(fpr) | 0x0e,                   "%f14" },
  98     { 0x0f,                             "%f15" },
  99 #  else
 100     { rc(fpr) | rc(dbl) | 0x3e,         "%f62" },
 101     { rc(fpr) | rc(dbl) | 0x3c,         "%f60" },
 102     { rc(fpr) | rc(dbl) | 0x3a,         "%f58" },
 103     { rc(fpr) | rc(dbl) | 0x38,         "%f56" },
 104     { rc(fpr) | rc(dbl) | 0x36,         "%f54" },
 105     { rc(fpr) | rc(dbl) | 0x34,         "%f52" },
 106     { rc(fpr) | rc(dbl) | 0x32,         "%f50" },
 107     { rc(fpr) | rc(dbl) | 0x30,         "%f48" },
 108     { rc(fpr) | rc(dbl) | 0x2e,         "%f46" },
 109     { rc(fpr) | rc(dbl) | 0x2c,         "%f44" },
 110     { rc(fpr) | rc(dbl) | 0x2a,         "%f42" },
 111     { rc(fpr) | rc(dbl) | 0x28,         "%f40" },
 112     { rc(fpr) | rc(dbl) | 0x26,         "%f38" },
 113     { rc(fpr) | rc(dbl) | 0x24,         "%f36" },
 114     { rc(fpr) | rc(dbl) | 0x22,         "%f34" },
 115     { rc(fpr) | rc(dbl) | 0x20,         "%f32" },
 116     { 0x1f,                             "%f31" },
 117     { rc(arg)|rc(fpr)|rc(sng)|0x1e,     "%f30" },
 118     { 0x1d,                             "%f29" },
 119     { rc(arg)|rc(fpr)|rc(sng)|0x1c,     "%f28" },
 120     { 0x1b,                             "%f27" },
 121     { rc(arg)|rc(fpr)|rc(sng)|0x1a,     "%f26" },
 122     { 0x19,                             "%f25" },
 123     { rc(arg)|rc(fpr)|rc(sng)|0x18,     "%f24" },
 124     { 0x17,                             "%f23" },
 125     { rc(arg)|rc(fpr)|rc(sng)|0x16,     "%f22" },
 126     { 0x15,                             "%f21" },
 127     { rc(arg)|rc(fpr)|rc(sng)|0x14,     "%f20" },
 128     { 0x13,                             "%f19" },
 129     { rc(arg)|rc(fpr)|rc(sng)|0x12,     "%f18" },
 130     { 0x11,                             "%f17" },
 131     { rc(arg)|rc(fpr)|rc(sng)|0x10,     "%f16" },
 132     { 0x0f,                             "%f15" },
 133     { rc(arg)|rc(fpr)|rc(sng)|0x0e,     "%f14" },
 134     { 0x0d,                             "%f13" },
 135     { rc(arg)|rc(fpr)|rc(sng)|0x0c,     "%f12" },
 136     { 0x0b,                             "%f11" },
 137     { rc(arg)|rc(fpr)|rc(sng)|0x0a,     "%f10" },
 138     { 0x09,                             "%f9" },
 139     { rc(arg)|rc(fpr)|rc(sng)|0x08,     "%f8" },
 140     { 0x07,                             "%f7" },
 141     { rc(arg)|rc(fpr)|rc(sng)|0x06,     "%f6" },
 142     { 0x05,                             "%f5" },
 143     { rc(arg)|rc(fpr)|rc(sng)|0x04,     "%f4" },
 144     { 0x03,                             "%f3" },
 145     { rc(arg)|rc(fpr)|rc(sng)|0x02,     "%f2" },
 146     { 0x01,                             "%f1" },
 147     { rc(arg)|rc(fpr)|rc(sng)|0x00,     "%f0" },
 148 #  endif
 149     { _NOREG,                           "<none>" },
 150 };
 151
 152 /*
 153  * Implementation
 154  */
 155 void
 156 jit_get_cpu(void)
 157 {
 158 }
 159
 160 void
 161 _jit_init(jit_state_t *_jit)
 162 {
 163     _jitc->reglen = jit_size(_rvs) - 1;
 164 #  if __WORDSIZE == 64
 165     jit_carry = _NOREG;
 166 #  endif
 167 }
 168
 169 void
 170 _jit_prolog(jit_state_t *_jit)
 171 {
 172     jit_int32_t          offset;
 173
 174     if (_jitc->function)
 175         jit_epilog();
 176     assert(jit_regset_cmp_ui(&_jitc->regarg, 0) == 0);
 177     jit_regset_set_ui(&_jitc->regsav, 0);
 178     offset = _jitc->functions.offset;
 179     if (offset >= _jitc->functions.length) {
 180         jit_realloc((jit_pointer_t *)&_jitc->functions.ptr,
 181                     _jitc->functions.length * sizeof(jit_function_t),
 182                     (_jitc->functions.length + 16) * sizeof(jit_function_t));
 183         _jitc->functions.length += 16;
 184     }
 185     _jitc->function = _jitc->functions.ptr + _jitc->functions.offset++;
 186     _jitc->function->self.size = stack_framesize;
 187     _jitc->function->self.argi = _jitc->function->self.argf =
 188         _jitc->function->self.aoff = _jitc->function->self.alen = 0;
 189     /* float conversion */
 190 #  if __WORDSIZE == 32
 191     _jitc->function->self.aoff = -8;
 192 #  else
 193     /* extra slots in case qmul is called */
 194     _jitc->function->self.aoff = -24;
 195 #  endif
 196      _jitc->function->self.call = jit_call_default;
 197     jit_alloc((jit_pointer_t *)&_jitc->function->regoff,
 198               _jitc->reglen * sizeof(jit_int32_t));
 199
 200     /* _no_link here does not mean the jit_link() call can be removed
 201      * by rewriting as:
 202      * _jitc->function->prolog = jit_new_node(jit_code_prolog);
 203      */
 204     _jitc->function->prolog = jit_new_node_no_link(jit_code_prolog);
 205     jit_link(_jitc->function->prolog);
 206     _jitc->function->prolog->w.w = offset;
 207     _jitc->function->epilog = jit_new_node_no_link(jit_code_epilog);
 208     /*  u:      label value
 209      *  v:      offset in blocks vector
 210      *  w:      offset in functions vector
 211      */
 212     _jitc->function->epilog->w.w = offset;
 213
 214     jit_regset_new(&_jitc->function->regset);
 215 }
 216
 217 jit_int32_t
 218 _jit_allocai(jit_state_t *_jit, jit_int32_t length)
 219 {
 220     assert(_jitc->function);
 221     switch (length) {
 222         case 0: case 1:                                         break;
 223         case 2:         _jitc->function->self.aoff &= -2;       break;
 224         case 3: case 4: _jitc->function->self.aoff &= -4;       break;
 225         default:        _jitc->function->self.aoff &= -8;       break;
 226     }
 227     _jitc->function->self.aoff -= length;
 228     if (!_jitc->realize) {
 229         jit_inc_synth_ww(allocai, _jitc->function->self.aoff, length);
 230         jit_dec_synth();
 231     }
 232     return (BIAS(_jitc->function->self.aoff));
 233 }
 234
 235 void
 236 _jit_allocar(jit_state_t *_jit, jit_int32_t u, jit_int32_t v)
 237 {
 238     jit_int32_t          reg;
 239     assert(_jitc->function);
 240     jit_inc_synth_ww(allocar, u, v);
 241     if (!_jitc->function->allocar) {
 242         _jitc->function->aoffoff = jit_allocai(sizeof(jit_int32_t));
 243         _jitc->function->allocar = 1;
 244     }
 245     reg = jit_get_reg(jit_class_gpr);
 246     jit_negr(reg, v);
 247     jit_andi(reg, reg, -16);
 248     jit_ldxi_i(u, JIT_FP, _jitc->function->aoffoff);
 249     jit_addr(u, u, reg);
 250     jit_addr(_SP, _SP, reg);
 251     jit_stxi_i(_jitc->function->aoffoff, JIT_FP, u);
 252     jit_unget_reg(reg);
 253     jit_dec_synth();
 254 }
 255
 256 void
 257 _jit_ret(jit_state_t *_jit)
 258 {
 259     jit_node_t          *instr;
 260     assert(_jitc->function);
 261     jit_inc_synth(ret);
 262     /* jump to epilog */
 263     instr = jit_jmpi();
 264     jit_patch_at(instr, _jitc->function->epilog);
 265     jit_dec_synth();
 266 }
 267
 268 void
 269 _jit_retr(jit_state_t *_jit, jit_int32_t u)
 270 {
 271     jit_inc_synth_w(retr, u);
 272     if (JIT_RET != u)
 273         jit_movr(JIT_RET, u);
 274     jit_live(JIT_RET);
 275     jit_ret();
 276     jit_dec_synth();
 277 }
 278
 279 void
 280 _jit_reti(jit_state_t *_jit, jit_word_t u)
 281 {
 282     jit_inc_synth_w(reti, u);
 283     jit_movi(JIT_RET, u);
 284     jit_ret();
 285     jit_dec_synth();
 286 }
 287
 288 void
 289 _jit_retr_f(jit_state_t *_jit, jit_int32_t u)
 290 {
 291     jit_inc_synth_w(retr_f, u);
 292     if (JIT_FRET != u)
 293         jit_movr_f(JIT_FRET, u);
 294     else
 295         jit_live(JIT_FRET);
 296     jit_ret();
 297     jit_dec_synth();
 298 }
 299
 300 void
 301 _jit_reti_f(jit_state_t *_jit, jit_float32_t u)
 302 {
 303     jit_inc_synth_f(reti_f, u);
 304     jit_movi_f(JIT_FRET, u);
 305     jit_ret();
 306     jit_dec_synth();
 307 }
 308
 309 void
 310 _jit_retr_d(jit_state_t *_jit, jit_int32_t u)
 311 {
 312     jit_inc_synth_w(retr_d, u);
 313     if (JIT_FRET != u)
 314         jit_movr_d(JIT_FRET, u);
 315     else
 316         jit_live(JIT_FRET);
 317     jit_ret();
 318     jit_dec_synth();
 319 }
 320
 321 void
 322 _jit_reti_d(jit_state_t *_jit, jit_float64_t u)
 323 {
 324     jit_inc_synth_d(reti_d, u);
 325     jit_movi_d(JIT_FRET, u);
 326     jit_ret();
 327     jit_dec_synth();
 328 }
 329
 330 void
 331 _jit_epilog(jit_state_t *_jit)
 332 {
 333     assert(_jitc->function);
 334     assert(_jitc->function->epilog->next == NULL);
 335     jit_link(_jitc->function->epilog);
 336     _jitc->function = NULL;
 337 }
 338
 339 jit_bool_t
 340 _jit_arg_register_p(jit_state_t *_jit, jit_node_t *u)
 341 {
 342 #  if __WORDSIZE == 32
 343     if (u->code == jit_code_arg || u->code == jit_code_arg_f)
 344         return (jit_arg_reg_p(u->u.w));
 345     assert(u->code == jit_code_arg_d);
 346     return (jit_arg_d_reg_p(u->u.w));
 347 #  else
 348     if (u->code == jit_code_arg)
 349         return (jit_arg_reg_p(u->u.w));
 350     assert(u->code == jit_code_arg_d || u->code == jit_code_arg_f);
 351     return (jit_arg_d_reg_p(u->u.w));
 352 #  endif
 353 }
 354
 355 void
 356 _jit_ellipsis(jit_state_t *_jit)
 357 {
 358     jit_inc_synth(ellipsis);
 359     if (_jitc->prepare) {
 360         jit_link_prepare();
 361         assert(!(_jitc->function->call.call & jit_call_varargs));
 362         _jitc->function->call.call |= jit_call_varargs;
 363     }
 364     else {
 365         jit_link_prolog();
 366         assert(!(_jitc->function->self.call & jit_call_varargs));
 367         _jitc->function->self.call |= jit_call_varargs;
 368
 369         _jitc->function->vagp = _jitc->function->self.argi;
 370     }
 371     jit_dec_synth();
 372 }
 373
 374 void
 375 _jit_va_push(jit_state_t *_jit, jit_int32_t u)
 376 {
 377     jit_inc_synth_w(va_push, u);
 378     jit_pushargr(u);
 379     jit_dec_synth();
 380 }
 381
 382 jit_node_t *
 383 _jit_arg(jit_state_t *_jit)
 384 {
 385     jit_node_t          *node;
 386     jit_int32_t          offset;
 387     assert(_jitc->function);
 388     if (jit_arg_reg_p(_jitc->function->self.argi))
 389         offset = _jitc->function->self.argi++;
 390     else {
 391 #  if __WORDSIZE == 64
 392         if (jit_arg_d_reg_p(_jitc->function->self.argi))
 393             ++_jitc->function->self.argi;
 394 #   endif
 395         offset = BIAS(_jitc->function->self.size);
 396         _jitc->function->self.size += sizeof(jit_word_t);
 397     }
 398     node = jit_new_node_ww(jit_code_arg, offset,
 399                            ++_jitc->function->self.argn);
 400     jit_link_prolog();
 401     return (node);
 402 }
 403
 404 jit_node_t *
 405 _jit_arg_f(jit_state_t *_jit)
 406 {
 407     jit_node_t          *node;
 408     jit_int32_t          offset;
 409 #  if __WORDSIZE == 64
 410     jit_bool_t           inc;
 411 #  endif
 412     assert(_jitc->function);
 413 #  if __WORDSIZE == 32
 414     if (jit_arg_reg_p(_jitc->function->self.argi))
 415         offset = _jitc->function->self.argi++;
 416     else {
 417         offset = _jitc->function->self.size;
 418         _jitc->function->self.size += sizeof(jit_word_t);
 419     }
 420 #  else
 421     inc = !jit_arg_reg_p(_jitc->function->self.argi);
 422     if (jit_arg_d_reg_p(_jitc->function->self.argi))
 423         offset = _jitc->function->self.argi++;
 424     else
 425         offset = BIAS(_jitc->function->self.size);
 426     if (inc)
 427         _jitc->function->self.size += sizeof(jit_word_t);
 428 #  endif
 429     node = jit_new_node_ww(jit_code_arg_f, offset,
 430                            ++_jitc->function->self.argn);
 431     jit_link_prolog();
 432     return (node);
 433 }
 434
 435 jit_node_t *
 436 _jit_arg_d(jit_state_t *_jit)
 437 {
 438     jit_node_t          *node;
 439     jit_int32_t          offset;
 440 #  if __WORDSIZE == 64
 441     jit_bool_t           inc;
 442 #  endif
 443     assert(_jitc->function);
 444 #  if __WORDSIZE == 32
 445     if (jit_arg_d_reg_p(_jitc->function->self.argi)) {
 446         offset = _jitc->function->self.argi;
 447         _jitc->function->self.argi += 2;
 448     }
 449     else if (jit_arg_reg_p(_jitc->function->self.argi)) {
 450         offset = _jitc->function->self.argi++;
 451         _jitc->function->self.size += sizeof(jit_float32_t);
 452     }
 453     else {
 454         offset = _jitc->function->self.size;
 455         _jitc->function->self.size += sizeof(jit_float64_t);
 456     }
 457 #  else
 458     inc = !jit_arg_reg_p(_jitc->function->self.argi);
 459     if (jit_arg_d_reg_p(_jitc->function->self.argi))
 460         offset = _jitc->function->self.argi++;
 461     else
 462         offset = BIAS(_jitc->function->self.size);
 463     if (inc)
 464         _jitc->function->self.size += sizeof(jit_word_t);
 465 #  endif
 466     node = jit_new_node_ww(jit_code_arg_d, offset,
 467                            ++_jitc->function->self.argn);
 468     jit_link_prolog();
 469     return (node);
 470 }
 471
 472 void
 473 _jit_getarg_c(jit_state_t *_jit, jit_int32_t u, jit_node_t *v)
 474 {
 475     assert(v->code == jit_code_arg);
 476     jit_inc_synth_wp(getarg_c, u, v);
 477     if (jit_arg_reg_p(v->u.w))
 478         jit_extr_c(u, _I0 + v->u.w);
 479     else
 480         jit_ldxi_c(u, JIT_FP,
 481                    v->u.w + (__WORDSIZE >> 3) - sizeof(jit_int8_t));
 482     jit_dec_synth();
 483 }
 484
 485 void
 486 _jit_getarg_uc(jit_state_t *_jit, jit_int32_t u, jit_node_t *v)
 487 {
 488     assert(v->code == jit_code_arg);
 489     jit_inc_synth_wp(getarg_uc, u, v);
 490     if (jit_arg_reg_p(v->u.w))
 491         jit_extr_uc(u, _I0 + v->u.w);
 492     else
 493         jit_ldxi_uc(u, JIT_FP,
 494                     v->u.w + (__WORDSIZE >> 3) - sizeof(jit_uint8_t));
 495     jit_dec_synth();
 496 }
 497
 498 void
 499 _jit_getarg_s(jit_state_t *_jit, jit_int32_t u, jit_node_t *v)
 500 {
 501     assert(v->code == jit_code_arg);
 502     jit_inc_synth_wp(getarg_s, u, v);
 503     if (jit_arg_reg_p(v->u.w))
 504         jit_extr_s(u, _I0 + v->u.w);
 505     else
 506         jit_ldxi_s(u, JIT_FP,
 507                    v->u.w + (__WORDSIZE >> 3) - sizeof(jit_int16_t));
 508     jit_dec_synth();
 509 }
 510
 511 void
 512 _jit_getarg_us(jit_state_t *_jit, jit_int32_t u, jit_node_t *v)
 513 {
 514     assert(v->code == jit_code_arg);
 515     jit_inc_synth_wp(getarg_us, u, v);
 516     if (jit_arg_reg_p(v->u.w))
 517         jit_extr_us(u, _I0 + v->u.w);
 518     else
 519         jit_ldxi_us(u, JIT_FP,
 520                     v->u.w + (__WORDSIZE >> 3) - sizeof(jit_uint16_t));
 521     jit_dec_synth();
 522 }
 523
 524 void
 525 _jit_getarg_i(jit_state_t *_jit, jit_int32_t u, jit_node_t *v)
 526 {
 527     assert(v->code == jit_code_arg);
 528     jit_inc_synth_wp(getarg_i, u, v);
 529     if (jit_arg_reg_p(v->u.w)) {
 530 #  if __WORDSIZE == 64
 531         jit_extr_i(u, _I0 + v->u.w);
 532 #  else
 533         jit_movr(u, _I0 + v->u.w);
 534 #  endif
 535     }
 536     else
 537         jit_ldxi_i(u, JIT_FP,
 538                    v->u.w + (__WORDSIZE >> 3) - sizeof(jit_int32_t));
 539     jit_dec_synth();
 540 }
 541
 542 #  if __WORDSIZE == 64
 543 void
 544 _jit_getarg_ui(jit_state_t *_jit, jit_int32_t u, jit_node_t *v)
 545 {
 546     assert(v->code == jit_code_arg);
 547     jit_inc_synth_wp(getarg_i, u, v);
 548     if (jit_arg_reg_p(v->u.w))
 549         jit_extr_ui(u, _I0 + v->u.w);
 550     else
 551         jit_ldxi_ui(u, JIT_FP,
 552                     v->u.w + (__WORDSIZE >> 3) - sizeof(jit_int32_t));
 553     jit_dec_synth();
 554 }
 555
 556 void
 557 _jit_getarg_l(jit_state_t *_jit, jit_int32_t u, jit_node_t *v)
 558 {
 559     assert(v->code == jit_code_arg);
 560     jit_inc_synth_wp(getarg_i, u, v);
 561     if (jit_arg_reg_p(v->u.w))
 562         jit_movr(u, _I0 + v->u.w);
 563     else
 564         jit_ldxi_l(u, JIT_FP, v->u.w);
 565     jit_dec_synth();
 566 }
 567 #  endif
 568
 569 void
 570 _jit_putargr(jit_state_t *_jit, jit_int32_t u, jit_node_t *v)
 571 {
 572     assert(v->code == jit_code_arg);
 573     jit_inc_synth_wp(putargr, u, v);
 574     if (jit_arg_reg_p(v->u.w))
 575         jit_movr(_I0 + v->u.w, u);
 576     else
 577         jit_stxi(v->u.w, JIT_FP, u);
 578     jit_dec_synth();
 579 }
 580
 581 void
 582 _jit_putargi(jit_state_t *_jit, jit_word_t u, jit_node_t *v)
 583 {
 584     jit_int32_t         regno;
 585     assert(v->code == jit_code_arg);
 586     jit_inc_synth_wp(putargi, u, v);
 587     if (jit_arg_reg_p(v->u.w))
 588         jit_movi(_I0 + v->u.w, u);
 589     else {
 590         regno = jit_get_reg(jit_class_gpr);
 591         jit_movi(regno, u);
 592         jit_stxi(v->u.w, JIT_FP, regno);
 593         jit_unget_reg(regno);
 594     }
 595     jit_dec_synth();
 596 }
 597
 598 void
 599 _jit_getarg_f(jit_state_t *_jit, jit_int32_t u, jit_node_t *v)
 600 {
 601     assert(v->code == jit_code_arg_f);
 602     assert(_jitc->function);
 603     jit_inc_synth_wp(getarg_f, u, v);
 604 #  if __WORDSIZE == 32
 605     if (jit_arg_reg_p(v->u.w)) {
 606         jit_stxi_i(-4, JIT_FP, _I0 + v->u.w);
 607         jit_ldxi_f(u, JIT_FP, -4);
 608     }
 609 #  else
 610     if (jit_arg_d_reg_p(v->u.w)) {
 611         jit_live(_F0 - (v->u.w << 1));  /* pair of registers is live */
 612         jit_movr_f(u, (_F0 - (v->u.w << 1)) - 1);
 613     }
 614 #  endif
 615     else
 616         jit_ldxi_f(u, JIT_FP, v->u.w + (__WORDSIZE >> 3) -
 617                    sizeof(jit_float32_t));
 618     jit_dec_synth();
 619 }
 620
 621 void
 622 _jit_putargr_f(jit_state_t *_jit, jit_int32_t u, jit_node_t *v)
 623 {
 624     assert(v->code == jit_code_arg_f);
 625     jit_inc_synth_wp(putargr_f, u, v);
 626 #  if __WORDSIZE == 32
 627     if (jit_arg_reg_p(v->u.w)) {
 628         jit_stxi_f(-4, JIT_FP, u);
 629         jit_ldxi_i(_I0 + v->u.w, JIT_FP, -4);
 630     }
 631 #  else
 632     if (jit_arg_d_reg_p(v->u.w)) {
 633         jit_live(_F0 - (v->u.w << 1));  /* pair of registers is live */
 634         jit_movr_f((_F0 - (v->u.w << 1)) - 1, u);
 635     }
 636 #  endif
 637     else
 638         jit_stxi_f(v->u.w + (__WORDSIZE >> 3) -
 639                    sizeof(jit_float32_t), JIT_FP, u);
 640     jit_dec_synth();
 641 }
 642
 643 void
 644 _jit_putargi_f(jit_state_t *_jit, jit_float32_t u, jit_node_t *v)
 645 {
 646     jit_int32_t         regno;
 647     assert(v->code == jit_code_arg_f);
 648     jit_inc_synth_fp(putargi_f, u, v);
 649 #  if __WORDSIZE == 32
 650     regno = jit_get_reg(jit_class_fpr);
 651     jit_movi_f(regno, u);
 652     if (jit_arg_reg_p(v->u.w)) {
 653         jit_stxi_f(-4, JIT_FP, regno);
 654         jit_ldxi_i(_I0 + v->u.w, JIT_FP, -4);
 655     }
 656     else
 657         jit_stxi_f(v->u.w, JIT_FP, regno);
 658     jit_unget_reg(regno);
 659 #  else
 660     if (jit_arg_d_reg_p(v->u.w)) {
 661         jit_live(_F0 - (v->u.w << 1));  /* pair of registers is live */
 662         jit_movi_f((_F0 - (v->u.w << 1)) - 1, u);
 663     }
 664     else {
 665         regno = jit_get_reg(jit_class_fpr | jit_class_sng);
 666         jit_movi_f(regno, u);
 667         jit_stxi_f(v->u.w + (__WORDSIZE >> 3) -
 668                    sizeof(jit_float32_t), JIT_FP, regno);
 669         jit_unget_reg(regno);
 670     }
 671 #  endif
 672     jit_dec_synth();
 673 }
 674
 675 void
 676 _jit_getarg_d(jit_state_t *_jit, jit_int32_t u, jit_node_t *v)
 677 {
 678     assert(v->code == jit_code_arg_d);
 679     assert(_jitc->function);
 680     jit_inc_synth_wp(getarg_d, u, v);
 681     if (jit_arg_d_reg_p(v->u.w)) {
 682 #  if __WORDSIZE == 32
 683         jit_stxi(-8, JIT_FP, _I0 + v->u.w);
 684         jit_stxi(-4, JIT_FP, _I0 + v->u.w + 1);
 685         jit_ldxi_d(u, JIT_FP, -8);
 686 #  else
 687         jit_movr_d(u, _F0 - (v->u.w << 1));
 688 #  endif
 689     }
 690 #  if __WORDSIZE == 32
 691     else if (jit_arg_reg_p(v->u.w)) {
 692         jit_stxi(-8, JIT_FP, _I0 + v->u.w);
 693         jit_ldxi_f(u, JIT_FP, -8);
 694         jit_ldxi_f(u + 1, JIT_FP, stack_framesize);
 695     }
 696 #  endif
 697     else {
 698 #  if __WORDSIZE == 32
 699         jit_ldxi_f(u, JIT_FP, v->u.w);
 700         jit_ldxi_f(u + 1, JIT_FP, v->u.w + 4);
 701 #  else
 702         jit_ldxi_d(u, JIT_FP, v->u.w);
 703 #  endif
 704     }
 705     jit_dec_synth();
 706 }
 707
 708 void
 709 _jit_putargr_d(jit_state_t *_jit, jit_int32_t u, jit_node_t *v)
 710 {
 711    jit_int32_t          regno;
 712     assert(v->code == jit_code_arg_d);
 713     jit_inc_synth_wp(putargr_d, u, v);
 714 #  if __WORDSIZE == 32
 715     if (jit_arg_d_reg_p(v->u.w)) {
 716         jit_stxi_d(-8, JIT_FP, u);
 717         jit_ldxi(_I0 + v->u.w, JIT_FP, -8);
 718         jit_ldxi(_I0 + v->u.w + 1, JIT_FP, -4);
 719     }
 720     else if (jit_arg_reg_p(v->u.w)) {
 721         regno = jit_get_reg(jit_class_gpr);
 722         jit_stxi_d(-8, JIT_FP, u);
 723         jit_ldxi(_I0 + v->u.w, JIT_FP, -8);
 724         jit_ldxi(regno, JIT_FP, -4);
 725         jit_stxi(stack_framesize, JIT_FP, regno);
 726         jit_unget_reg(regno);
 727     }
 728     else if ((v->u.w & 7) == 0)
 729         jit_stxi_d(v->u.w, JIT_FP, u);
 730     else {
 731         jit_stxi_d(-8, JIT_FP, u);
 732         regno = jit_get_reg(jit_class_gpr);
 733         jit_ldxi(regno, JIT_FP, -8);
 734         jit_stxi(v->u.w, JIT_FP, regno);
 735         jit_ldxi(regno, JIT_FP, -4);
 736         jit_stxi(v->u.w + 4, JIT_FP, regno);
 737         jit_unget_reg(regno);
 738     }
 739 #  else
 740     if (jit_arg_d_reg_p(v->u.w))
 741         jit_movr_d(_F0 - (v->u.w << 1), u);
 742     else
 743         jit_stxi_d(v->u.w, JIT_FP, u);
 744 #  endif
 745     jit_dec_synth();
 746 }
 747
 748 void
 749 _jit_putargi_d(jit_state_t *_jit, jit_float64_t u, jit_node_t *v)
 750 {
 751 #  if __WORDSIZE == 32
 752     jit_int32_t         gpr;
 753 #  endif
 754    jit_int32_t          regno;
 755     assert(v->code == jit_code_arg_d);
 756     jit_inc_synth_dp(putargi_d, u, v);
 757 #  if __WORDSIZE == 32
 758     regno = jit_get_reg(jit_class_fpr);
 759     jit_movi_d(regno, u);
 760     if (jit_arg_d_reg_p(v->u.w)) {
 761         jit_stxi_d(-8, JIT_FP, regno);
 762         jit_ldxi(_I0 + v->u.w, JIT_FP, -8);
 763         jit_ldxi(_I0 + v->u.w + 1, JIT_FP, -4);
 764     }
 765     else if (jit_arg_reg_p(v->u.w)) {
 766         gpr = jit_get_reg(jit_class_gpr);
 767         jit_stxi_d(-8, JIT_FP, regno);
 768         jit_ldxi(_I0 + v->u.w, JIT_FP, -8);
 769         jit_ldxi(gpr, JIT_FP, -4);
 770         jit_stxi(stack_framesize, JIT_FP, gpr);
 771         jit_unget_reg(gpr);
 772     }
 773     else if ((v->u.w & 7) == 0)
 774         jit_stxi_d(v->u.w, JIT_FP, regno);
 775     else {
 776         jit_stxi_d(-8, JIT_FP, regno);
 777         gpr = jit_get_reg(jit_class_gpr);
 778         jit_ldxi(gpr, JIT_FP, -8);
 779         jit_stxi(v->u.w, JIT_FP, gpr);
 780         jit_ldxi(gpr, JIT_FP, -4);
 781         jit_stxi(v->u.w + 4, JIT_FP, gpr);
 782         jit_unget_reg(gpr);
 783     }
 784     jit_unget_reg(regno);
 785 #  else
 786     if (jit_arg_d_reg_p(v->u.w))
 787         jit_movi_d(_F0 - (v->u.w << 1), u);
 788     else {
 789         regno = jit_get_reg(jit_class_fpr | jit_class_dbl);
 790         jit_movi_d(regno, u);
 791         jit_stxi_d(v->u.w, JIT_FP, regno);
 792         jit_unget_reg(regno);
 793     }
 794 #  endif
 795     jit_dec_synth();
 796 }
 797
 798 void
 799 _jit_pushargr(jit_state_t *_jit, jit_int32_t u)
 800 {
 801     jit_inc_synth_w(pushargr, u);
 802     jit_link_prepare();
 803     if (jit_arg_reg_p(_jitc->function->call.argi)) {
 804         jit_movr(_O0 + _jitc->function->call.argi, u);
 805         ++_jitc->function->call.argi;
 806     }
 807     else {
 808 #  if __WORDSIZE == 64
 809         if (jit_arg_d_reg_p(_jitc->function->call.argi))
 810             ++_jitc->function->call.argi;
 811 #  endif
 812         jit_stxi(BIAS(_jitc->function->call.size + stack_framesize),
 813                  JIT_SP, u);
 814         _jitc->function->call.size += sizeof(jit_word_t);
 815     }
 816     jit_dec_synth();
 817 }
 818
 819 void
 820 _jit_pushargi(jit_state_t *_jit, jit_word_t u)
 821 {
 822     jit_int32_t         regno;
 823     jit_inc_synth_w(pushargi, u);
 824     jit_link_prepare();
 825     if (jit_arg_reg_p(_jitc->function->call.argi)) {
 826         jit_movi(_O0 + _jitc->function->call.argi, u);
 827         ++_jitc->function->call.argi;
 828     }
 829     else {
 830 #  if __WORDSIZE == 64
 831         if (jit_arg_d_reg_p(_jitc->function->call.argi))
 832             ++_jitc->function->call.argi;
 833 #  endif
 834         regno = jit_get_reg(jit_class_gpr);
 835         jit_movi(regno, u);
 836         jit_stxi(BIAS(_jitc->function->call.size + stack_framesize),
 837                  JIT_SP, regno);
 838         jit_unget_reg(regno);
 839         _jitc->function->call.size += sizeof(jit_word_t);
 840     }
 841     jit_dec_synth();
 842 }
 843
 844 void
 845 _jit_pushargr_f(jit_state_t *_jit, jit_int32_t u)
 846 {
 847     jit_inc_synth_w(pushargr_f, u);
 848     jit_link_prepare();
 849 #  if __WORDSIZE == 32
 850     if (jit_arg_reg_p(_jitc->function->call.argi)) {
 851         jit_stxi_f(-8, JIT_FP, u);
 852         jit_ldxi(_O0 + _jitc->function->call.argi, JIT_FP, -8);
 853         ++_jitc->function->call.argi;
 854     }
 855     else {
 856         jit_stxi_f(_jitc->function->call.size + stack_framesize,
 857                    JIT_SP, u);
 858         _jitc->function->call.size += sizeof(jit_float32_t);
 859     }
 860 #  else
 861     if ((_jitc->function->call.call & jit_call_varargs) &&
 862         jit_arg_reg_p(_jitc->function->call.argi)) {
 863         jit_stxi_f(BIAS(-8), JIT_FP, u);
 864         jit_ldxi_i(_O0 + _jitc->function->call.argi, JIT_FP, BIAS(-8));
 865         ++_jitc->function->call.argi;
 866     }
 867     else if (!(_jitc->function->call.call & jit_call_varargs) &&
 868              jit_arg_d_reg_p(_jitc->function->call.argi)) {
 869         /* pair of registers is live */
 870         jit_live(_F0 - (_jitc->function->call.argi << 1));
 871         jit_movr_f((_F0 - (_jitc->function->call.argi << 1)) - 1, u);
 872         if (!jit_arg_reg_p(_jitc->function->call.argi))
 873             _jitc->function->call.size += sizeof(jit_float64_t);
 874         ++_jitc->function->call.argi;
 875     }
 876     else {
 877         jit_stxi_f(BIAS(_jitc->function->call.size + stack_framesize + 4),
 878                    JIT_SP, u);
 879         _jitc->function->call.size += sizeof(jit_float64_t);
 880     }
 881 #  endif
 882     jit_dec_synth();
 883 }
 884
 885 void
 886 _jit_pushargi_f(jit_state_t *_jit, jit_float32_t u)
 887 {
 888     jit_int32_t         regno;
 889     jit_inc_synth_f(pushargi_f, u);
 890     jit_link_prepare();
 891 #  if __WORDSIZE == 32
 892     regno = jit_get_reg(jit_class_fpr);
 893     jit_movi_f(regno, u);
 894     if (jit_arg_reg_p(_jitc->function->call.argi)) {
 895         jit_stxi_f(-8, JIT_FP, regno);
 896         jit_ldxi(_O0 + _jitc->function->call.argi, JIT_FP, -8);
 897         _jitc->function->call.argi++;
 898     }
 899     else {
 900         jit_stxi_f(_jitc->function->call.size + stack_framesize,
 901                    JIT_SP, regno);
 902         _jitc->function->call.size += sizeof(jit_float32_t);
 903     }
 904     jit_unget_reg(regno);
 905 #  else
 906     if ((_jitc->function->call.call & jit_call_varargs) &&
 907         jit_arg_reg_p(_jitc->function->call.argi)) {
 908         regno = jit_get_reg(jit_class_fpr | jit_class_sng);
 909         jit_movi_f(regno, u);
 910         jit_stxi_f(BIAS(-8), JIT_FP, regno);
 911         jit_ldxi_i(_O0 + _jitc->function->call.argi, JIT_FP, BIAS(-8));
 912         ++_jitc->function->call.argi;
 913         jit_unget_reg(regno);
 914     }
 915     else if (!(_jitc->function->call.call & jit_call_varargs) &&
 916              jit_arg_d_reg_p(_jitc->function->call.argi)) {
 917         /* pair of registers is live */
 918         jit_live(_F0 - (_jitc->function->call.argi << 1));
 919         jit_movi_f((_F0 - (_jitc->function->call.argi << 1)) - 1, u);
 920         if (!jit_arg_reg_p(_jitc->function->call.argi))
 921             _jitc->function->call.size += sizeof(jit_float64_t);
 922         ++_jitc->function->call.argi;
 923     }
 924     else {
 925         regno = jit_get_reg(jit_class_fpr | jit_class_sng);
 926         jit_movi_f(regno, u);
 927         jit_stxi_f(BIAS(_jitc->function->call.size + stack_framesize + 4),
 928                    JIT_SP, regno);
 929         jit_unget_reg(regno);
 930         _jitc->function->call.size += sizeof(jit_float64_t);
 931     }
 932 #  endif
 933     jit_dec_synth();
 934 }
 935
 936 void
 937 _jit_pushargr_d(jit_state_t *_jit, jit_int32_t u)
 938 {
 939     jit_inc_synth_w(pushargr_d, u);
 940     jit_link_prepare();
 941 #  if __WORDSIZE == 32
 942     if (jit_arg_d_reg_p(_jitc->function->call.argi)) {
 943         jit_stxi_d(BIAS(-8), JIT_FP, u);
 944         jit_ldxi(_O0 + _jitc->function->call.argi, JIT_FP, BIAS(-8));
 945         jit_ldxi(_O0 + _jitc->function->call.argi + 1, JIT_FP, -4);
 946         _jitc->function->call.argi += 2;
 947     }
 948     else if (jit_arg_reg_p(_jitc->function->call.argi)) {
 949         jit_stxi_f(-8, JIT_FP, u);
 950         jit_ldxi(_O0 + _jitc->function->call.argi, JIT_FP, -8);
 951         ++_jitc->function->call.argi;
 952         jit_stxi_f(stack_framesize, JIT_SP, u + 1);
 953         _jitc->function->call.size += sizeof(jit_float32_t);
 954     }
 955     else {
 956         jit_stxi_f(_jitc->function->call.size + stack_framesize,
 957                    JIT_SP, u);
 958         jit_stxi_f(_jitc->function->call.size + stack_framesize + 4,
 959                    JIT_SP, u + 1);
 960         _jitc->function->call.size += sizeof(jit_float64_t);
 961     }
 962 #  else
 963     if ((_jitc->function->call.call & jit_call_varargs) &&
 964         jit_arg_reg_p(_jitc->function->call.argi)) {
 965         jit_stxi_d(BIAS(-8), JIT_FP, u);
 966         jit_ldxi(_O0 + _jitc->function->call.argi, JIT_FP, BIAS(-8));
 967         ++_jitc->function->call.argi;
 968     }
 969     else if (!(_jitc->function->call.call & jit_call_varargs) &&
 970              jit_arg_d_reg_p(_jitc->function->call.argi)) {
 971         jit_movr_d(_F0 - (_jitc->function->call.argi << 1), u);
 972         if (!jit_arg_reg_p(_jitc->function->call.argi))
 973             _jitc->function->call.size += sizeof(jit_float64_t);
 974         ++_jitc->function->call.argi;
 975     }
 976     else {
 977         jit_stxi_d(BIAS(_jitc->function->call.size + stack_framesize),
 978                    JIT_SP, u);
 979         _jitc->function->call.size += sizeof(jit_float64_t);
 980     }
 981 #  endif
 982     jit_dec_synth();
 983 }
 984
 985 void
 986 _jit_pushargi_d(jit_state_t *_jit, jit_float64_t u)
 987 {
 988     jit_int32_t         regno;
 989     jit_inc_synth_d(pushargi_d, u);
 990     jit_link_prepare();
 991 #  if __WORDSIZE == 32
 992     regno = jit_get_reg(jit_class_fpr);
 993     jit_movi_d(regno, u);
 994     if (jit_arg_d_reg_p(_jitc->function->call.argi)) {
 995         jit_stxi_d(BIAS(-8), JIT_FP, regno);
 996         jit_ldxi(_O0 + _jitc->function->call.argi, JIT_FP, BIAS(-8));
 997         jit_ldxi(_O0 + _jitc->function->call.argi + 1, JIT_FP, -4);
 998         _jitc->function->call.argi += 2;
 999     }
1000     else if (jit_arg_reg_p(_jitc->function->call.argi)) {
1001         jit_stxi_f(-8, JIT_FP, regno);
1002         jit_ldxi(_O0 + _jitc->function->call.argi, JIT_FP, -8);
1003         ++_jitc->function->call.argi;
1004         jit_stxi_f(stack_framesize, JIT_SP, u + 1);
1005         _jitc->function->call.size += sizeof(jit_float32_t);
1006     }
1007     else {
1008         jit_stxi_f(_jitc->function->call.size + stack_framesize,
1009                    JIT_SP, regno);
1010         jit_stxi_f(_jitc->function->call.size + stack_framesize + 4,
1011                    JIT_SP, regno + 1);
1012         _jitc->function->call.size += sizeof(jit_float64_t);
1013     }
1014     jit_unget_reg(regno);
1015 #  else
1016     if ((_jitc->function->call.call & jit_call_varargs) &&
1017         jit_arg_reg_p(_jitc->function->call.argi)) {
1018         regno = jit_get_reg(jit_class_fpr | jit_class_dbl);
1019         jit_movi_d(regno, u);
1020         jit_stxi_d(BIAS(-8), JIT_FP, regno);
1021         jit_ldxi(_O0 + _jitc->function->call.argi, JIT_FP, BIAS(-8));
1022         ++_jitc->function->call.argi;
1023         jit_unget_reg(regno);
1024     }
1025     else if (!(_jitc->function->call.call & jit_call_varargs) &&
1026              jit_arg_d_reg_p(_jitc->function->call.argi)) {
1027         jit_movi_d(_F0 - (_jitc->function->call.argi << 1), u);
1028         if (!jit_arg_reg_p(_jitc->function->call.argi))
1029             _jitc->function->call.size += sizeof(jit_float64_t);
1030         ++_jitc->function->call.argi;
1031     }
1032     else {
1033         regno = jit_get_reg(jit_class_fpr | jit_class_dbl);
1034         jit_movi_d(regno, u);
1035         jit_stxi_d(BIAS(_jitc->function->call.size + stack_framesize),
1036                    JIT_SP, regno);
1037         jit_unget_reg(regno);
1038         _jitc->function->call.size += sizeof(jit_float64_t);
1039     }
1040 #  endif
1041     jit_dec_synth();
1042 }
1043
1044 jit_bool_t
1045 _jit_regarg_p(jit_state_t *_jit, jit_node_t *node, jit_int32_t regno)
1046 {
1047     jit_int32_t         spec;
1048
1049     spec = jit_class(_rvs[regno].spec);
1050     if ((spec & (jit_class_arg|jit_class_gpr)) ==
1051         (jit_class_arg|jit_class_gpr)) {
1052         regno -= _O0;
1053         if (regno >= 0 && regno < node->v.w)
1054             return (1);
1055     }
1056 #  if __WORDSIZE == 64
1057     if ((spec & (jit_class_arg|jit_class_fpr)) ==
1058         (jit_class_arg|jit_class_fpr)) {
1059         regno = _F0 - (regno >> 1);
1060         if (regno >= 0 && regno < node->v.w)
1061             return (1);
1062     }
1063 #  endif
1064
1065     return (0);
1066 }
1067
1068 void
1069 _jit_finishr(jit_state_t *_jit, jit_int32_t r0)
1070 {
1071     jit_node_t          *call;
1072     assert(_jitc->function);
1073     jit_inc_synth_w(finishr, r0);
1074     if (_jitc->function->self.alen < _jitc->function->call.size)
1075         _jitc->function->self.alen = _jitc->function->call.size;
1076     call = jit_callr(r0);
1077     call->v.w = _jitc->function->self.argi;
1078     call->w.w = _jitc->function->self.argf;
1079     _jitc->function->call.argi = _jitc->function->call.argf =
1080         _jitc->function->call.size = 0;
1081     _jitc->prepare = 0;
1082     jit_dec_synth();
1083 }
1084
1085 jit_node_t *
1086 _jit_finishi(jit_state_t *_jit, jit_pointer_t i0)
1087 {
1088     jit_node_t          *node;
1089     assert(_jitc->function);
1090     jit_inc_synth_w(finishi, (jit_word_t)i0);
1091     if (_jitc->function->self.alen < _jitc->function->call.size)
1092         _jitc->function->self.alen = _jitc->function->call.size;
1093     node = jit_calli(i0);
1094     node->v.w = _jitc->function->call.argi;
1095     node->w.w = _jitc->function->call.argf;
1096     _jitc->function->call.argi = _jitc->function->call.argf =
1097         _jitc->function->call.size = 0;
1098     _jitc->prepare = 0;
1099     jit_dec_synth();
1100     return (node);
1101 }
1102
1103 void
1104 _jit_retval_c(jit_state_t *_jit, jit_int32_t r0)
1105 {
1106     jit_inc_synth_w(retval_c, r0);
1107     jit_extr_c(r0, _O0);
1108     jit_dec_synth();
1109 }
1110
1111 void
1112 _jit_retval_uc(jit_state_t *_jit, jit_int32_t r0)
1113 {
1114     jit_inc_synth_w(retval_uc, r0);
1115     jit_extr_uc(r0, _O0);
1116     jit_dec_synth();
1117 }
1118
1119 void
1120 _jit_retval_s(jit_state_t *_jit, jit_int32_t r0)
1121 {
1122     jit_inc_synth_w(retval_s, r0);
1123     jit_extr_s(r0, _O0);
1124     jit_dec_synth();
1125 }
1126
1127 void
1128 _jit_retval_us(jit_state_t *_jit, jit_int32_t r0)
1129 {
1130     jit_inc_synth_w(retval_us, r0);
1131     jit_extr_us(r0, _O0);
1132     jit_dec_synth();
1133 }
1134
1135 void
1136 _jit_retval_i(jit_state_t *_jit, jit_int32_t r0)
1137 {
1138     jit_inc_synth_w(retval_i, r0);
1139 #  if __WORDSIZE == 32
1140     if (r0 != _O0)
1141         jit_movr(r0, _O0);
1142 #  else
1143     jit_extr_i(r0, _O0);
1144 #  endif
1145     jit_dec_synth();
1146 }
1147
1148 #  if __WORDSIZE == 64
1149 void
1150 _jit_retval_ui(jit_state_t *_jit, jit_int32_t r0)
1151 {
1152     jit_inc_synth_w(retval_i, r0);
1153     if (r0 != _O0)
1154         jit_extr_ui(r0, _O0);
1155     jit_dec_synth();
1156 }
1157
1158 void
1159 _jit_retval_l(jit_state_t *_jit, jit_int32_t r0)
1160 {
1161     jit_inc_synth_w(retval_i, r0);
1162     if (r0 != _O0)
1163         jit_movr(r0, _O0);
1164     jit_dec_synth();
1165 }
1166 #  endif
1167
1168 void
1169 _jit_retval_f(jit_state_t *_jit, jit_int32_t r0)
1170 {
1171     jit_inc_synth_w(retval_f, r0);
1172     if (r0 != JIT_FRET)
1173         jit_movr_f(r0, JIT_FRET);
1174     jit_dec_synth();
1175 }
1176
1177 void
1178 _jit_retval_d(jit_state_t *_jit, jit_int32_t r0)
1179 {
1180     jit_inc_synth_w(retval_d, r0);
1181     if (r0 != JIT_FRET)
1182         jit_movr_d(r0, JIT_FRET);
1183     jit_dec_synth();
1184 }
1185
1186 jit_pointer_t
1187 _emit_code(jit_state_t *_jit)
1188 {
1189     jit_node_t          *node;
1190     jit_node_t          *temp;
1191     jit_word_t           word;
1192     jit_int32_t          value;
1193     jit_int32_t          offset;
1194     struct {
1195         jit_node_t      *node;
1196         jit_word_t       word;
1197 #if DEVEL_DISASSEMBLER
1198         jit_word_t       prevw;
1199 #endif
1200         jit_int32_t      patch_offset;
1201     } undo;
1202 #if DEVEL_DISASSEMBLER
1203     jit_word_t           prevw;
1204 #endif
1205
1206     _jitc->function = NULL;
1207
1208     jit_reglive_setup();
1209
1210     undo.word = 0;
1211     undo.node = NULL;
1212     undo.patch_offset = 0;
1213
1214 #define case_rr(name, type)                                             \
1215             case jit_code_##name##r##type:                              \
1216                 name##r##type(rn(node->u.w), rn(node->v.w));            \
1217                 break
1218 #define case_rw(name, type)                                             \
1219             case jit_code_##name##i##type:                              \
1220                 name##i##type(rn(node->u.w), node->v.w);                \
1221                 break
1222 #define case_wr(name, type)                                             \
1223             case jit_code_##name##i##type:                              \
1224                 name##i##type(node->u.w, rn(node->v.w));                \
1225                 break
1226 #define case_rf(name)                                                   \
1227             case jit_code_##name##i##type:                              \
1228                 assert(node->flag & jit_flag_data);                     \
1229                 name##_f(rn(node->u.w),                                 \
1230                 (jit_float32_t *)node->v.n->u.w);                       \
1231                 break
1232 #define case_rd(name)                                                   \
1233             case jit_code_##name##i_d:                                  \
1234                 assert(node->flag & jit_flag_data);                     \
1235                 name##_d(rn(node->u.w),                                 \
1236                          (jit_float64_t *)node->v.n->u.w);              \
1237                 break
1238 #define case_rrr(name, type)                                            \
1239             case jit_code_##name##r##type:                              \
1240                 name##r##type(rn(node->u.w),                            \
1241                               rn(node->v.w), rn(node->w.w));            \
1242                 break
1243 #define case_rrrr(name, type)                                           \
1244             case jit_code_##name##r##type:                              \
1245                 name##r##type(rn(node->u.q.l), rn(node->u.q.h),         \
1246                               rn(node->v.w), rn(node->w.w));            \
1247                 break
1248 #define case_rrw(name, type)                                            \
1249             case jit_code_##name##i##type:                              \
1250                 name##i##type(rn(node->u.w),                            \
1251                               rn(node->v.w), node->w.w);                \
1252                 break
1253 #define case_rrrw(name, type)                                           \
1254             case jit_code_##name##i##type:                              \
1255                 name##i##type(rn(node->u.q.l), rn(node->u.q.h),         \
1256                               rn(node->v.w), node->w.w);                \
1257                 break
1258 #define case_rrf(name, type, size)                                      \
1259             case jit_code_##name##i##type:                              \
1260                 assert(node->flag & jit_flag_data);                     \
1261                 name##i##type(rn(node->u.w), rn(node->v.w),             \
1262                               (jit_float##size##_t *)node->w.n->u.w);   \
1263                 break
1264 #define case_wrr(name, type)                                            \
1265             case jit_code_##name##i##type:                              \
1266                 name##i##type(node->u.w, rn(node->v.w), rn(node->w.w)); \
1267                 break
1268 #define case_brr(name, type)                                            \
1269             case jit_code_##name##r##type:                              \
1270                 temp = node->u.n;                                       \
1271                 assert(temp->code == jit_code_label ||                  \
1272                        temp->code == jit_code_epilog);                  \
1273                 if (temp->flag & jit_flag_patch)                        \
1274                     name##r##type(temp->u.w, rn(node->v.w),             \
1275                                   rn(node->w.w));                       \
1276                 else {                                                  \
1277                     word = name##r##type(_jit->pc.w,                    \
1278                                          rn(node->v.w), rn(node->w.w)); \
1279                     patch(word, node);                                  \
1280                 }                                                       \
1281                 break
1282 #define case_brw(name, type)                                            \
1283             case jit_code_##name##i##type:                              \
1284                 temp = node->u.n;                                       \
1285                 assert(temp->code == jit_code_label ||                  \
1286                        temp->code == jit_code_epilog);                  \
1287                 if (temp->flag & jit_flag_patch)                        \
1288                     name##i##type(temp->u.w,                            \
1289                                   rn(node->v.w), node->w.w);            \
1290                 else {                                                  \
1291                     word = name##i##type(_jit->pc.w,                    \
1292                                          rn(node->v.w), node->w.w);     \
1293                     patch(word, node);                                  \
1294                 }                                                       \
1295                 break
1296 #define case_brf(name, type, size)                                      \
1297             case jit_code_##name##i##type:                              \
1298                 temp = node->u.n;                                       \
1299                 assert(temp->code == jit_code_label ||                  \
1300                        temp->code == jit_code_epilog);                  \
1301                 if (temp->flag & jit_flag_patch)                        \
1302                     name##i##type(temp->u.w, rn(node->v.w),             \
1303                                 (jit_float##size##_t *)node->w.n->u.w); \
1304                 else {                                                  \
1305                     word = name##i##type(_jit->pc.w, rn(node->v.w),     \
1306                                 (jit_float##size##_t *)node->w.n->u.w); \
1307                     patch(word, node);                                  \
1308                 }                                                       \
1309                 break
1310 #if DEVEL_DISASSEMBLER
1311     prevw = _jit->pc.w;
1312 #endif
1313     for (node = _jitc->head; node; node = node->next) {
1314         if (_jit->pc.uc >= _jitc->code.end)
1315             return (NULL);
1316
1317 #if DEVEL_DISASSEMBLER
1318         node->offset = (jit_uword_t)_jit->pc.w - (jit_uword_t)prevw;
1319         prevw = _jit->pc.w;
1320 #endif
1321         value = jit_classify(node->code);
1322         jit_regarg_set(node, value);
1323         switch (node->code) {
1324             case jit_code_align:
1325                 assert(!(node->u.w & (node->u.w - 1)) &&
1326                        node->u.w <= sizeof(jit_word_t));
1327                 if (node->u.w == sizeof(jit_word_t) &&
1328                     (word = _jit->pc.w & (sizeof(jit_word_t) - 1)))
1329                     nop(sizeof(jit_word_t) - word);
1330                 break;
1331             case jit_code_note:         case jit_code_name:
1332                 node->u.w = _jit->pc.w;
1333                 break;
1334             case jit_code_label:
1335                 if ((node->link || (node->flag & jit_flag_use)) &&
1336                     (word = _jit->pc.w & (sizeof(jit_word_t) - 1)))
1337                     nop(sizeof(jit_word_t) - word);
1338                 /* remember label is defined */
1339                 node->flag |= jit_flag_patch;
1340                 node->u.w = _jit->pc.w;
1341                 break;
1342                 case_rrr(add,);
1343                 case_rrw(add,);
1344                 case_rrr(addc,);
1345                 case_rrw(addc,);
1346                 case_rrr(addx,);
1347                 case_rrw(addx,);
1348                 case_rrr(sub,);
1349                 case_rrw(sub,);
1350                 case_rrr(subc,);
1351                 case_rrw(subc,);
1352                 case_rrr(subx,);
1353                 case_rrw(subx,);
1354                 case_rrw(rsb,);
1355                 case_rrr(mul,);
1356                 case_rrw(mul,);
1357                 case_rrrr(qmul,);
1358                 case_rrrw(qmul,);
1359                 case_rrrr(qmul, _u);
1360                 case_rrrw(qmul, _u);
1361                 case_rrr(div,);
1362                 case_rrw(div,);
1363                 case_rrr(div, _u);
1364                 case_rrw(div, _u);
1365                 case_rrrr(qdiv,);
1366                 case_rrrw(qdiv,);
1367                 case_rrrr(qdiv, _u);
1368                 case_rrrw(qdiv, _u);
1369                 case_rrr(rem,);
1370                 case_rrw(rem,);
1371                 case_rrr(rem, _u);
1372                 case_rrw(rem, _u);
1373                 case_rrr(and,);
1374                 case_rrw(and,);
1375                 case_rrr(or,);
1376                 case_rrw(or,);
1377                 case_rrr(xor,);
1378                 case_rrw(xor,);
1379                 case_rrr(lsh,);
1380                 case_rrw(lsh,);
1381                 case_rrr(rsh,);
1382                 case_rrw(rsh,);
1383                 case_rrr(rsh, _u);
1384                 case_rrw(rsh, _u);
1385                 case_rr(trunc, _f_i);
1386                 case_rr(trunc, _d_i);
1387 #if __WORDSIZE == 64
1388                 case_rr(trunc, _f_l);
1389                 case_rr(trunc, _d_l);
1390 #endif
1391                 case_rrr(lt,);
1392                 case_rrw(lt,);
1393                 case_rrr(lt, _u);
1394                 case_rrw(lt, _u);
1395                 case_rrr(le,);
1396                 case_rrw(le,);
1397                 case_rrr(le, _u);
1398                 case_rrw(le, _u);
1399                 case_rrr(eq,);
1400                 case_rrw(eq,);
1401                 case_rrr(ge,);
1402                 case_rrw(ge,);
1403                 case_rrr(ge, _u);
1404                 case_rrw(ge, _u);
1405                 case_rrr(gt,);
1406                 case_rrw(gt,);
1407                 case_rrr(gt, _u);
1408                 case_rrw(gt, _u);
1409                 case_rrr(ne,);
1410                 case_rrw(ne,);
1411                 case_rr(ld, _c);
1412                 case_rw(ld, _c);
1413                 case_rr(ld, _uc);
1414                 case_rw(ld, _uc);
1415                 case_rr(ld, _s);
1416                 case_rw(ld, _s);
1417                 case_rr(ld, _us);
1418                 case_rw(ld, _us);
1419                 case_rr(ld, _i);
1420                 case_rw(ld, _i);
1421 #if __WORDSIZE == 64
1422                 case_rr(ld, _ui);
1423                 case_rw(ld, _ui);
1424                 case_rr(ld, _l);
1425                 case_rw(ld, _l);
1426 #endif
1427                 case_rrr(ldx, _c);
1428                 case_rrw(ldx, _c);
1429                 case_rrr(ldx, _uc);
1430                 case_rrw(ldx, _uc);
1431                 case_rrr(ldx, _s);
1432                 case_rrw(ldx, _s);
1433                 case_rrr(ldx, _us);
1434                 case_rrw(ldx, _us);
1435                 case_rrr(ldx, _i);
1436                 case_rrw(ldx, _i);
1437 #if __WORDSIZE == 64
1438                 case_rrr(ldx, _ui);
1439                 case_rrw(ldx, _ui);
1440                 case_rrr(ldx, _l);
1441                 case_rrw(ldx, _l);
1442 #endif
1443                 case_rr(st, _c);
1444                 case_wr(st, _c);
1445                 case_rr(st, _s);
1446                 case_wr(st, _s);
1447                 case_rr(st, _i);
1448                 case_wr(st, _i);
1449 #if __WORDSIZE == 64
1450                 case_rr(st, _l);
1451                 case_wr(st, _l);
1452 #endif
1453                 case_rrr(stx, _c);
1454                 case_wrr(stx, _c);
1455                 case_rrr(stx, _s);
1456                 case_wrr(stx, _s);
1457                 case_rrr(stx, _i);
1458                 case_wrr(stx, _i);
1459 #if __WORDSIZE == 64
1460                 case_rrr(stx, _l);
1461                 case_wrr(stx, _l);
1462 #endif
1463                 case_rr(hton, _us);
1464                 case_rr(hton, _ui);
1465 #if __WORDSIZE == 64
1466                 case_rr(hton, _ul);
1467 #endif
1468                 case_rr(bswap, _us);
1469                 case_rr(bswap, _ui);
1470 #if __WORDSIZE == 64
1471                 case_rr(bswap, _ul);
1472 #endif
1473                 case_rr(ext, _c);
1474                 case_rr(ext, _uc);
1475                 case_rr(ext, _s);
1476                 case_rr(ext, _us);
1477 #if __WORDSIZE == 64
1478                 case_rr(ext, _i);
1479                 case_rr(ext, _ui);
1480 #endif
1481             case jit_code_casr:
1482                 casr(rn(node->u.w), rn(node->v.w),
1483                      rn(node->w.q.l), rn(node->w.q.h));
1484                 break;
1485             case jit_code_casi:
1486                 casi(rn(node->u.w), node->v.w,
1487                      rn(node->w.q.l), rn(node->w.q.h));
1488                 break;
1489                 case_rrr(movn,);
1490                 case_rrr(movz,);
1491                 case_rr(mov,);
1492             case jit_code_movi:
1493                 if (node->flag & jit_flag_node) {
1494                     temp = node->v.n;
1495                     if (temp->code == jit_code_data ||
1496                         (temp->code == jit_code_label &&
1497                          (temp->flag & jit_flag_patch)))
1498                         movi(rn(node->u.w), temp->u.w);
1499                     else {
1500                         assert(temp->code == jit_code_label ||
1501                                temp->code == jit_code_epilog);
1502                         word = movi_p(rn(node->u.w), node->v.w);
1503                         patch(word, node);
1504                     }
1505                 }
1506                 else
1507                     movi(rn(node->u.w), node->v.w);
1508                 break;
1509                 case_rr(neg,);
1510                 case_rr(com,);
1511                 case_brr(blt,);
1512                 case_brw(blt,);
1513                 case_brr(blt, _u);
1514                 case_brw(blt, _u);
1515                 case_brr(ble,);
1516                 case_brw(ble,);
1517                 case_brr(ble, _u);
1518                 case_brw(ble, _u);
1519                 case_brr(beq,);
1520                 case_brw(beq,);
1521                 case_brr(bge,);
1522                 case_brw(bge,);
1523                 case_brr(bge, _u);
1524                 case_brw(bge, _u);
1525                 case_brr(bgt,);
1526                 case_brw(bgt,);
1527                 case_brr(bgt, _u);
1528                 case_brw(bgt, _u);
1529                 case_brr(bne,);
1530                 case_brw(bne,);
1531                 case_brr(boadd,);
1532                 case_brw(boadd,);
1533                 case_brr(boadd, _u);
1534                 case_brw(boadd, _u);
1535                 case_brr(bxadd,);
1536                 case_brw(bxadd,);
1537                 case_brr(bxadd, _u);
1538                 case_brw(bxadd, _u);
1539                 case_brr(bosub,);
1540                 case_brw(bosub,);
1541                 case_brr(bosub, _u);
1542                 case_brw(bosub, _u);
1543                 case_brr(bxsub,);
1544                 case_brw(bxsub,);
1545                 case_brr(bxsub, _u);
1546                 case_brw(bxsub, _u);
1547                 case_brr(bms,);
1548                 case_brw(bms,);
1549                 case_brr(bmc,);
1550                 case_brw(bmc,);
1551                 case_rrr(add, _f);
1552                 case_rrf(add, _f, 32);
1553                 case_rrr(sub, _f);
1554                 case_rrf(sub, _f, 32);
1555                 case_rrf(rsb, _f, 32);
1556                 case_rrr(mul, _f);
1557                 case_rrf(mul, _f, 32);
1558                 case_rrr(div, _f);
1559                 case_rrf(div, _f, 32);
1560                 case_rr(abs, _f);
1561                 case_rr(neg, _f);
1562                 case_rr(sqrt, _f);
1563                 case_rr(ext, _f);
1564                 case_rr(ext, _d_f);
1565                 case_rrr(lt, _f);
1566                 case_rrf(lt, _f, 32);
1567                 case_rrr(le, _f);
1568                 case_rrf(le, _f, 32);
1569                 case_rrr(eq, _f);
1570                 case_rrf(eq, _f, 32);
1571                 case_rrr(ge, _f);
1572                 case_rrf(ge, _f, 32);
1573                 case_rrr(gt, _f);
1574                 case_rrf(gt, _f, 32);
1575                 case_rrr(ne, _f);
1576                 case_rrf(ne, _f, 32);
1577                 case_rrr(unlt, _f);
1578                 case_rrf(unlt, _f, 32);
1579                 case_rrr(unle, _f);
1580                 case_rrf(unle, _f, 32);
1581                 case_rrr(uneq, _f);
1582                 case_rrf(uneq, _f, 32);
1583                 case_rrr(unge, _f);
1584                 case_rrf(unge, _f, 32);
1585                 case_rrr(ungt, _f);
1586                 case_rrf(ungt, _f, 32);
1587                 case_rrr(ltgt, _f);
1588                 case_rrf(ltgt, _f, 32);
1589                 case_rrr(ord, _f);
1590                 case_rrf(ord, _f, 32);
1591                 case_rrr(unord, _f);
1592                 case_rrf(unord, _f, 32);
1593                 case_rr(ld, _f);
1594                 case_rw(ld, _f);
1595                 case_rrr(ldx, _f);
1596                 case_rrw(ldx, _f);
1597                 case_rr(st, _f);
1598                 case_wr(st, _f);
1599                 case_rrr(stx, _f);
1600                 case_wrr(stx, _f);
1601                 case_rr(mov, _f);
1602             case jit_code_movi_f:
1603                 assert(node->flag & jit_flag_data);
1604                 movi_f(rn(node->u.w), (jit_float32_t *)node->v.n->u.w);
1605                 break;
1606                 case_brr(blt, _f);
1607                 case_brf(blt, _f, 32);
1608                 case_brr(ble, _f);
1609                 case_brf(ble, _f, 32);
1610                 case_brr(beq, _f);
1611                 case_brf(beq, _f, 32);
1612                 case_brr(bge, _f);
1613                 case_brf(bge, _f, 32);
1614                 case_brr(bgt, _f);
1615                 case_brf(bgt, _f, 32);
1616                 case_brr(bne, _f);
1617                 case_brf(bne, _f, 32);
1618                 case_brr(bunlt, _f);
1619                 case_brf(bunlt, _f, 32);
1620                 case_brr(bunle, _f);
1621                 case_brf(bunle, _f, 32);
1622                 case_brr(buneq, _f);
1623                 case_brf(buneq, _f, 32);
1624                 case_brr(bunge, _f);
1625                 case_brf(bunge, _f, 32);
1626                 case_brr(bungt, _f);
1627                 case_brf(bungt, _f, 32);
1628                 case_brr(bltgt, _f);
1629                 case_brf(bltgt, _f, 32);
1630                 case_brr(bord, _f);
1631                 case_brf(bord, _f, 32);
1632                 case_brr(bunord, _f);
1633                 case_brf(bunord, _f, 32);
1634                 case_rrr(add, _d);
1635                 case_rrf(add, _d, 64);
1636                 case_rrr(sub, _d);
1637                 case_rrf(sub, _d, 64);
1638                 case_rrf(rsb, _d, 64);
1639                 case_rrr(mul, _d);
1640                 case_rrf(mul, _d, 64);
1641                 case_rrr(div, _d);
1642                 case_rrf(div, _d, 64);
1643                 case_rr(abs, _d);
1644                 case_rr(neg, _d);
1645                 case_rr(sqrt, _d);
1646                 case_rr(ext, _d);
1647                 case_rr(ext, _f_d);
1648                 case_rrr(lt, _d);
1649                 case_rrf(lt, _d, 64);
1650                 case_rrr(le, _d);
1651                 case_rrf(le, _d, 64);
1652                 case_rrr(eq, _d);
1653                 case_rrf(eq, _d, 64);
1654                 case_rrr(ge, _d);
1655                 case_rrf(ge, _d, 64);
1656                 case_rrr(gt, _d);
1657                 case_rrf(gt, _d, 64);
1658                 case_rrr(ne, _d);
1659                 case_rrf(ne, _d, 64);
1660                 case_rrr(unlt, _d);
1661                 case_rrf(unlt, _d, 64);
1662                 case_rrr(unle, _d);
1663                 case_rrf(unle, _d, 64);
1664                 case_rrr(uneq, _d);
1665                 case_rrf(uneq, _d, 64);
1666                 case_rrr(unge, _d);
1667                 case_rrf(unge, _d, 64);
1668                 case_rrr(ungt, _d);
1669                 case_rrf(ungt, _d, 64);
1670                 case_rrr(ltgt, _d);
1671                 case_rrf(ltgt, _d, 64);
1672                 case_rrr(ord, _d);
1673                 case_rrf(ord, _d, 64);
1674                 case_rrr(unord, _d);
1675                 case_rrf(unord, _d, 64);
1676                 case_rr(ld, _d);
1677                 case_rw(ld, _d);
1678                 case_rrr(ldx, _d);
1679                 case_rrw(ldx, _d);
1680                 case_rr(st, _d);
1681                 case_wr(st, _d);
1682                 case_rrr(stx, _d);
1683                 case_wrr(stx, _d);
1684                 case_rr(mov, _d);
1685             case jit_code_movi_d:
1686                 assert(node->flag & jit_flag_data);
1687                 movi_d(rn(node->u.w), (jit_float64_t *)node->v.n->u.w);
1688                 break;
1689                 case_brr(blt, _d);
1690                 case_brf(blt, _d, 64);
1691                 case_brr(ble, _d);
1692                 case_brf(ble, _d, 64);
1693                 case_brr(beq, _d);
1694                 case_brf(beq, _d, 64);
1695                 case_brr(bge, _d);
1696                 case_brf(bge, _d, 64);
1697                 case_brr(bgt, _d);
1698                 case_brf(bgt, _d, 64);
1699                 case_brr(bne, _d);
1700                 case_brf(bne, _d, 64);
1701                 case_brr(bunlt, _d);
1702                 case_brf(bunlt, _d, 64);
1703                 case_brr(bunle, _d);
1704                 case_brf(bunle, _d, 64);
1705                 case_brr(buneq, _d);
1706                 case_brf(buneq, _d, 64);
1707                 case_brr(bunge, _d);
1708                 case_brf(bunge, _d, 64);
1709                 case_brr(bungt, _d);
1710                 case_brf(bungt, _d, 64);
1711                 case_brr(bltgt, _d);
1712                 case_brf(bltgt, _d, 64);
1713                 case_brr(bord, _d);
1714                 case_brf(bord, _d, 64);
1715                 case_brr(bunord, _d);
1716                 case_brf(bunord, _d, 64);
1717             case jit_code_jmpr:
1718                 jmpr(rn(node->u.w));
1719                 break;
1720             case jit_code_jmpi:
1721                 if (node->flag & jit_flag_node) {
1722                     temp = node->u.n;
1723                     assert(temp->code == jit_code_label ||
1724                            temp->code == jit_code_epilog);
1725                     if (temp->flag & jit_flag_patch)
1726                         jmpi(temp->u.w);
1727                     else {
1728                         word = jmpi_p(_jit->pc.w);
1729                         patch(word, node);
1730                     }
1731                 }
1732                 else
1733                     jmpi(node->u.w);
1734                 break;
1735             case jit_code_callr:
1736                 callr(rn(node->u.w));
1737                 break;
1738             case jit_code_calli:
1739                 if (node->flag & jit_flag_node) {
1740                     temp = node->u.n;
1741                     assert(temp->code == jit_code_label ||
1742                            temp->code == jit_code_epilog);
1743                     word = calli_p(temp->u.w);
1744                     if (!(temp->flag & jit_flag_patch))
1745                         patch(word, node);
1746                 }
1747                 else
1748                     calli(node->u.w);
1749                 break;
1750             case jit_code_prolog:
1751                 _jitc->function = _jitc->functions.ptr + node->w.w;
1752                 undo.node = node;
1753                 undo.word = _jit->pc.w;
1754 #if DEVEL_DISASSEMBLER
1755                 undo.prevw = prevw;
1756 #endif
1757                 undo.patch_offset = _jitc->patches.offset;
1758             restart_function:
1759                 _jitc->again = 0;
1760                 prolog(node);
1761                 break;
1762             case jit_code_epilog:
1763                 assert(_jitc->function == _jitc->functions.ptr + node->w.w);
1764                 if (_jitc->again) {
1765                     for (temp = undo.node->next;
1766                          temp != node; temp = temp->next) {
1767                         if (temp->code == jit_code_label ||
1768                             temp->code == jit_code_epilog)
1769                             temp->flag &= ~jit_flag_patch;
1770                     }
1771                     temp->flag &= ~jit_flag_patch;
1772                     node = undo.node;
1773                     _jit->pc.w = undo.word;
1774 #if DEVEL_DISASSEMBLER
1775                     prevw = undo.prevw;
1776 #endif
1777                     _jitc->patches.offset = undo.patch_offset;
1778                     goto restart_function;
1779                 }
1780                 /* remember label is defined */
1781                 node->flag |= jit_flag_patch;
1782                 node->u.w = _jit->pc.w;
1783                 epilog(node);
1784                 _jitc->function = NULL;
1785                 break;
1786             case jit_code_va_start:
1787                 vastart(rn(node->u.w));
1788                 break;
1789             case jit_code_va_arg:
1790                 vaarg(rn(node->u.w), rn(node->v.w));
1791                 break;
1792             case jit_code_va_arg_d:
1793                 vaarg_d(rn(node->u.w), rn(node->v.w));
1794                 break;
1795             case jit_code_live:                 case jit_code_ellipsis:
1796             case jit_code_va_push:
1797             case jit_code_allocai:              case jit_code_allocar:
1798             case jit_code_arg:
1799             case jit_code_arg_f:                case jit_code_arg_d:
1800             case jit_code_va_end:
1801             case jit_code_ret:
1802             case jit_code_retr:                 case jit_code_reti:
1803             case jit_code_retr_f:               case jit_code_reti_f:
1804             case jit_code_retr_d:               case jit_code_reti_d:
1805             case jit_code_getarg_c:             case jit_code_getarg_uc:
1806             case jit_code_getarg_s:             case jit_code_getarg_us:
1807             case jit_code_getarg_i:
1808 #if __WORDSIZE == 64
1809             case jit_code_getarg_ui:            case jit_code_getarg_l:
1810 #endif
1811             case jit_code_getarg_f:             case jit_code_getarg_d:
1812             case jit_code_putargr:              case jit_code_putargi:
1813             case jit_code_putargr_f:            case jit_code_putargi_f:
1814             case jit_code_putargr_d:            case jit_code_putargi_d:
1815             case jit_code_pushargr:             case jit_code_pushargi:
1816             case jit_code_pushargr_f:           case jit_code_pushargi_f:
1817             case jit_code_pushargr_d:           case jit_code_pushargi_d:
1818             case jit_code_retval_c:             case jit_code_retval_uc:
1819             case jit_code_retval_s:             case jit_code_retval_us:
1820             case jit_code_retval_i:
1821 #if __WORDSIZE == 64
1822             case jit_code_retval_ui:            case jit_code_retval_l:
1823 #endif
1824             case jit_code_retval_f:             case jit_code_retval_d:
1825             case jit_code_prepare:
1826             case jit_code_finishr:              case jit_code_finishi:
1827                 break;
1828             default:
1829                 abort();
1830         }
1831 #  if __WORDSIZE == 64
1832         if (jit_carry != _NOREG) {
1833             switch (node->code) {
1834                 case jit_code_note:
1835                 case jit_code_addcr:            case jit_code_addci:
1836                 case jit_code_addxr:            case jit_code_addxi:
1837                 case jit_code_subcr:            case jit_code_subci:
1838                 case jit_code_subxr:            case jit_code_subxi:
1839                     break;
1840                 default:
1841                     jit_unget_reg(jit_carry);
1842                     jit_carry = _NOREG;
1843                     break;
1844             }
1845         }
1846 #  endif
1847         jit_regarg_clr(node, value);
1848 #  if __WORDSIZE == 64
1849         if (jit_regset_cmp_ui(&_jitc->regarg, 0) != 0) {
1850             assert(jit_regset_scan1(&_jitc->regarg, 0) == jit_carry);
1851             assert(jit_regset_scan1(&_jitc->regarg, jit_carry + 1) == ULONG_MAX);
1852         }
1853         assert(_jitc->synth == 0);
1854 #  else
1855         assert(_jitc->regarg == 0 && _jitc->synth == 0);
1856 #  endif
1857         jit_reglive(node);
1858     }
1859 #undef case_brf
1860 #undef case_brw
1861 #undef case_brr
1862 #undef case_wrr
1863 #undef case_rrf
1864 #undef case_rrrw
1865 #undef case_rrw
1866 #undef case_rrrr
1867 #undef case_rrr
1868 #undef case_rf
1869 #undef case_wr
1870 #undef case_rw
1871 #undef case_rr
1872
1873     for (offset = 0; offset < _jitc->patches.offset; offset++) {
1874         node = _jitc->patches.ptr[offset].node;
1875         word = node->code == jit_code_movi ? node->v.n->u.w : node->u.n->u.w;
1876         patch_at(_jitc->patches.ptr[offset].inst, word);
1877     }
1878
1879     jit_flush(_jit->code.ptr, _jit->pc.uc);
1880
1881     return (_jit->code.ptr);
1882 }
1883
1884 #define CODE                            1
1885 #  include "jit_sparc-cpu.c"
1886 #  include "jit_sparc-fpu.c"
1887 #  include "jit_fallback.c"
1888 #undef CODE
1889
1890 void
1891 jit_flush(void *fptr, void *tptr)
1892 {
1893 }
1894
1895 void
1896 _emit_ldxi(jit_state_t *_jit, jit_gpr_t r0, jit_gpr_t r1, jit_word_t i0)
1897 {
1898     ldxi(rn(r0), rn(r1), i0);
1899 }
1900
1901 void
1902 _emit_stxi(jit_state_t *_jit, jit_word_t i0, jit_gpr_t r0, jit_gpr_t r1)
1903 {
1904     stxi(i0, rn(r0), rn(r1));
1905 }
1906
1907 void
1908 _emit_ldxi_d(jit_state_t *_jit, jit_fpr_t r0, jit_gpr_t r1, jit_word_t i0)
1909 {
1910     ldxi_d(rn(r0), rn(r1), i0);
1911 }
1912
1913 void
1914 _emit_stxi_d(jit_state_t *_jit, jit_word_t i0, jit_gpr_t r0, jit_fpr_t r1)
1915 {
1916     stxi_d(i0, rn(r0), rn(r1));
1917 }
1918
1919 static void
1920 _patch(jit_state_t *_jit, jit_word_t instr, jit_node_t *node)
1921 {
1922     jit_int32_t         flag;
1923
1924     assert(node->flag & jit_flag_node);
1925     if (node->code == jit_code_movi)
1926         flag = node->v.n->flag;
1927     else
1928         flag = node->u.n->flag;
1929     assert(!(flag & jit_flag_patch));
1930     if (_jitc->patches.offset >= _jitc->patches.length) {
1931         jit_realloc((jit_pointer_t *)&_jitc->patches.ptr,
1932                     _jitc->patches.length * sizeof(jit_patch_t),
1933                     (_jitc->patches.length + 1024) * sizeof(jit_patch_t));
1934         memset(_jitc->patches.ptr + _jitc->patches.length, 0,
1935                1024 * sizeof(jit_patch_t));
1936         _jitc->patches.length += 1024;
1937     }
1938     _jitc->patches.ptr[_jitc->patches.offset].inst = instr;
1939     _jitc->patches.ptr[_jitc->patches.offset].node = node;
1940     ++_jitc->patches.offset;
1941 }