deps/lightning/lib/jit_sparc.c

   1 /*
   2  * Copyright (C) 2013-2023  Free Software Foundation, Inc.
   3  *
   4  * This file is part of GNU lightning.
   5  *
   6  * GNU lightning is free software; you can redistribute it and/or modify it
   7  * under the terms of the GNU Lesser General Public License as published
   8  * by the Free Software Foundation; either version 3, or (at your option)
   9  * any later version.
  10  *
  11  * GNU lightning is distributed in the hope that it will be useful, but
  12  * WITHOUT ANY WARRANTY; without even the implied warranty of MERCHANTABILITY
  13  * or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU Lesser General Public
  14  * License for more details.
  15  *
  16  * Authors:
  17  *      Paulo Cesar Pereira de Andrade
  18  */
  19
  20 /* Handling SIGILL should not be done by Lightning, but can either use
  21  * sample, or use another approach to set jit_cpu.lzcnt
  22  */
  23 #define CHECK_LZCNT     0
  24
  25 #if CHECK_LZCNT
  26 #include <signal.h>
  27 #include <setjmp.h>
  28 #endif
  29
  30 #define jit_arg_reg_p(i)                ((i) >= 0 && (i) < 6)
  31 #if __WORDSIZE == 32
  32 #  define jit_arg_d_reg_p(i)            ((i) >= 0 && (i) < 5)
  33 #  define BIAS(n)                       (n)
  34 #else
  35 #  define jit_arg_d_reg_p(i)            ((i) >= 0 && (i) < 16)
  36 #  define BIAS(n)                       ((n) + 2047)
  37 #endif
  38
  39 /*
  40  * Types
  41  */
  42 typedef jit_pointer_t   jit_va_list_t;
  43
  44 /*
  45  * Prototypes
  46  */
  47 #define patch(instr, node)              _patch(_jit, instr, node)
  48 static void _patch(jit_state_t*,jit_word_t,jit_node_t*);
  49
  50 #define PROTO                           1
  51 #  include "jit_sparc-cpu.c"
  52 #  include "jit_sparc-fpu.c"
  53 #  include "jit_fallback.c"
  54 #undef PROTO
  55
  56 /*
  57  * Initialization
  58  */
  59 jit_cpu_t               jit_cpu;
  60 jit_register_t          _rvs[] = {
  61     { 0x00,                             "%g0" },
  62     { 0x01,                             "%g1" },
  63     { rc(gpr) | 0x02,                   "%g2" },
  64     { rc(gpr) | 0x03,                   "%g3" },
  65     { rc(gpr) | 0x04,                   "%g4" },
  66     { 0x05,                             "%g5" },
  67     { 0x06,                             "%g6" },
  68     { 0x07,                             "%g7" },
  69     { rc(arg) | rc(gpr) | 0x08,         "%o0" },
  70     { rc(arg) | rc(gpr) | 0x09,         "%o1" },
  71     { rc(arg) | rc(gpr) | 0x0a,         "%o2" },
  72     { rc(arg) | rc(gpr) | 0x0b,         "%o3" },
  73     { rc(arg) | rc(gpr) | 0x0c,         "%o4" },
  74     { rc(arg) | rc(gpr) | 0x0d,         "%o5" },
  75     { rc(sav) | 0x0e,                   "%sp" },
  76     { 0x0f,                             "%o7" },
  77     { rc(sav) | rc(gpr) | 0x10,         "%l0" },
  78     { rc(sav) | rc(gpr) | 0x11,         "%l1" },
  79     { rc(sav) | rc(gpr) | 0x12,         "%l2" },
  80     { rc(sav) | rc(gpr) | 0x13,         "%l3" },
  81     { rc(sav) | rc(gpr) | 0x14,         "%l4" },
  82     { rc(sav) | rc(gpr) | 0x15,         "%l5" },
  83     { rc(sav) | rc(gpr) | 0x16,         "%l6" },
  84     { rc(sav) | rc(gpr) | 0x17,         "%l7" },
  85     { 0x18,                             "%i0" },
  86     { 0x19,                             "%i1" },
  87     { 0x1a,                             "%i2" },
  88     { 0x1b,                             "%i3" },
  89     { 0x1c,                             "%i4" },
  90     { 0x1d,                             "%i5" },
  91     { rc(sav) | 0x1e,                   "%fp" },
  92     { 0x1f,                             "%i7" },
  93 #  if __WORDSIZE == 32
  94     { rc(fpr) | 0x00,                   "%f0" },
  95     { 0x01,                             "%f1" },
  96     { rc(fpr) | 0x02,                   "%f2" },
  97     { 0x03,                             "%f3" },
  98     { rc(fpr) | 0x04,                   "%f4" },
  99     { 0x05,                             "%f5" },
 100     { rc(fpr) | 0x06,                   "%f6" },
 101     { 0x07,                             "%f7" },
 102     { rc(fpr) | 0x08,                   "%f8" },
 103     { 0x09,                             "%f9" },
 104     { rc(fpr) | 0x0a,                   "%f10" },
 105     { 0x0b,                             "%f11" },
 106     { rc(fpr) | 0x0c,                   "%f12" },
 107     { 0x0d,                             "%f13" },
 108     { rc(fpr) | 0x0e,                   "%f14" },
 109     { 0x0f,                             "%f15" },
 110 #  else
 111     { rc(fpr) | rc(dbl) | 0x3e,         "%f62" },
 112     { rc(fpr) | rc(dbl) | 0x3c,         "%f60" },
 113     { rc(fpr) | rc(dbl) | 0x3a,         "%f58" },
 114     { rc(fpr) | rc(dbl) | 0x38,         "%f56" },
 115     { rc(fpr) | rc(dbl) | 0x36,         "%f54" },
 116     { rc(fpr) | rc(dbl) | 0x34,         "%f52" },
 117     { rc(fpr) | rc(dbl) | 0x32,         "%f50" },
 118     { rc(fpr) | rc(dbl) | 0x30,         "%f48" },
 119     { rc(fpr) | rc(dbl) | 0x2e,         "%f46" },
 120     { rc(fpr) | rc(dbl) | 0x2c,         "%f44" },
 121     { rc(fpr) | rc(dbl) | 0x2a,         "%f42" },
 122     { rc(fpr) | rc(dbl) | 0x28,         "%f40" },
 123     { rc(fpr) | rc(dbl) | 0x26,         "%f38" },
 124     { rc(fpr) | rc(dbl) | 0x24,         "%f36" },
 125     { rc(fpr) | rc(dbl) | 0x22,         "%f34" },
 126     { rc(fpr) | rc(dbl) | 0x20,         "%f32" },
 127     { 0x1f,                             "%f31" },
 128     { rc(arg)|rc(fpr)|rc(sng)|0x1e,     "%f30" },
 129     { 0x1d,                             "%f29" },
 130     { rc(arg)|rc(fpr)|rc(sng)|0x1c,     "%f28" },
 131     { 0x1b,                             "%f27" },
 132     { rc(arg)|rc(fpr)|rc(sng)|0x1a,     "%f26" },
 133     { 0x19,                             "%f25" },
 134     { rc(arg)|rc(fpr)|rc(sng)|0x18,     "%f24" },
 135     { 0x17,                             "%f23" },
 136     { rc(arg)|rc(fpr)|rc(sng)|0x16,     "%f22" },
 137     { 0x15,                             "%f21" },
 138     { rc(arg)|rc(fpr)|rc(sng)|0x14,     "%f20" },
 139     { 0x13,                             "%f19" },
 140     { rc(arg)|rc(fpr)|rc(sng)|0x12,     "%f18" },
 141     { 0x11,                             "%f17" },
 142     { rc(arg)|rc(fpr)|rc(sng)|0x10,     "%f16" },
 143     { 0x0f,                             "%f15" },
 144     { rc(arg)|rc(fpr)|rc(sng)|0x0e,     "%f14" },
 145     { 0x0d,                             "%f13" },
 146     { rc(arg)|rc(fpr)|rc(sng)|0x0c,     "%f12" },
 147     { 0x0b,                             "%f11" },
 148     { rc(arg)|rc(fpr)|rc(sng)|0x0a,     "%f10" },
 149     { 0x09,                             "%f9" },
 150     { rc(arg)|rc(fpr)|rc(sng)|0x08,     "%f8" },
 151     { 0x07,                             "%f7" },
 152     { rc(arg)|rc(fpr)|rc(sng)|0x06,     "%f6" },
 153     { 0x05,                             "%f5" },
 154     { rc(arg)|rc(fpr)|rc(sng)|0x04,     "%f4" },
 155     { 0x03,                             "%f3" },
 156     { rc(arg)|rc(fpr)|rc(sng)|0x02,     "%f2" },
 157     { 0x01,                             "%f1" },
 158     { rc(arg)|rc(fpr)|rc(sng)|0x00,     "%f0" },
 159 #  endif
 160     { _NOREG,                           "<none>" },
 161 };
 162 #if CHECK_LZCNT
 163 sigjmp_buf              jit_env;
 164 #endif
 165
 166 /*
 167  * Implementation
 168  */
 169 #if CHECK_LZCNT
 170 static void
 171 sigill_handler(int signum)
 172 {
 173     jit_cpu.lzcnt = 0;
 174     siglongjmp(jit_env, 1);
 175 }
 176 #endif
 177
 178 void
 179 jit_get_cpu(void)
 180 {
 181 #if CHECK_LZCNT
 182     int                 g2;
 183     struct              sigaction new_action, old_action;
 184     new_action.sa_handler = sigill_handler;
 185     sigemptyset(&new_action.sa_mask);
 186     new_action.sa_flags = 0;
 187     sigaction(SIGILL, NULL, &old_action);
 188     if (old_action.sa_handler != SIG_IGN) {
 189         sigaction(SIGILL, &new_action, NULL);
 190         if (!sigsetjmp(jit_env, 1)) {
 191             jit_cpu.lzcnt = 1;
 192             /* lzcnt %g2, %g2 */
 193             __asm__ volatile("mov %%g2, %0; .long 0xa3b0021; mov %0, %%g2"
 194                              : "=r" (g2));
 195             sigaction(SIGILL, &old_action, NULL);
 196         }
 197     }
 198 #else
 199     jit_cpu.lzcnt = 0;
 200 #endif
 201 }
 202
 203 void
 204 _jit_init(jit_state_t *_jit)
 205 {
 206     _jitc->reglen = jit_size(_rvs) - 1;
 207 #  if __WORDSIZE == 64
 208     jit_carry = _NOREG;
 209 #  endif
 210 }
 211
 212 void
 213 _jit_prolog(jit_state_t *_jit)
 214 {
 215     jit_int32_t          offset;
 216
 217     if (_jitc->function)
 218         jit_epilog();
 219     assert(jit_regset_cmp_ui(&_jitc->regarg, 0) == 0);
 220     jit_regset_set_ui(&_jitc->regsav, 0);
 221     offset = _jitc->functions.offset;
 222     if (offset >= _jitc->functions.length) {
 223         jit_realloc((jit_pointer_t *)&_jitc->functions.ptr,
 224                     _jitc->functions.length * sizeof(jit_function_t),
 225                     (_jitc->functions.length + 16) * sizeof(jit_function_t));
 226         _jitc->functions.length += 16;
 227     }
 228     _jitc->function = _jitc->functions.ptr + _jitc->functions.offset++;
 229     _jitc->function->self.size = stack_framesize;
 230     _jitc->function->self.argi = _jitc->function->self.argf =
 231         _jitc->function->self.alen = 0;
 232     /* float conversion */
 233 #  if __WORDSIZE == 32
 234     _jitc->function->self.aoff = -8;
 235 #  else
 236     /* extra slots in case qmul is called */
 237     _jitc->function->self.aoff = -24;
 238 #  endif
 239      _jitc->function->self.call = jit_call_default;
 240     jit_alloc((jit_pointer_t *)&_jitc->function->regoff,
 241               _jitc->reglen * sizeof(jit_int32_t));
 242
 243     /* _no_link here does not mean the jit_link() call can be removed
 244      * by rewriting as:
 245      * _jitc->function->prolog = jit_new_node(jit_code_prolog);
 246      */
 247     _jitc->function->prolog = jit_new_node_no_link(jit_code_prolog);
 248     jit_link(_jitc->function->prolog);
 249     _jitc->function->prolog->w.w = offset;
 250     _jitc->function->epilog = jit_new_node_no_link(jit_code_epilog);
 251     /*  u:      label value
 252      *  v:      offset in blocks vector
 253      *  w:      offset in functions vector
 254      */
 255     _jitc->function->epilog->w.w = offset;
 256
 257     jit_regset_new(&_jitc->function->regset);
 258 }
 259
 260 jit_int32_t
 261 _jit_allocai(jit_state_t *_jit, jit_int32_t length)
 262 {
 263     assert(_jitc->function);
 264     switch (length) {
 265         case 0: case 1:                                         break;
 266         case 2:         _jitc->function->self.aoff &= -2;       break;
 267         case 3: case 4: _jitc->function->self.aoff &= -4;       break;
 268         default:        _jitc->function->self.aoff &= -8;       break;
 269     }
 270     _jitc->function->self.aoff -= length;
 271     if (!_jitc->realize) {
 272         jit_inc_synth_ww(allocai, _jitc->function->self.aoff, length);
 273         jit_dec_synth();
 274     }
 275     return (BIAS(_jitc->function->self.aoff));
 276 }
 277
 278 void
 279 _jit_allocar(jit_state_t *_jit, jit_int32_t u, jit_int32_t v)
 280 {
 281     jit_int32_t          reg;
 282     assert(_jitc->function);
 283     jit_inc_synth_ww(allocar, u, v);
 284     if (!_jitc->function->allocar) {
 285         _jitc->function->aoffoff = jit_allocai(sizeof(jit_int32_t));
 286         _jitc->function->allocar = 1;
 287     }
 288     reg = jit_get_reg(jit_class_gpr);
 289     jit_negr(reg, v);
 290     jit_andi(reg, reg, -16);
 291     jit_ldxi_i(u, JIT_FP, _jitc->function->aoffoff);
 292     jit_addr(u, u, reg);
 293     jit_addr(_SP, _SP, reg);
 294     jit_stxi_i(_jitc->function->aoffoff, JIT_FP, u);
 295     jit_unget_reg(reg);
 296     jit_dec_synth();
 297 }
 298
 299 void
 300 _jit_ret(jit_state_t *_jit)
 301 {
 302     jit_node_t          *instr;
 303     assert(_jitc->function);
 304     jit_inc_synth(ret);
 305     /* jump to epilog */
 306     instr = jit_jmpi();
 307     jit_patch_at(instr, _jitc->function->epilog);
 308     jit_dec_synth();
 309 }
 310
 311 void
 312 _jit_retr(jit_state_t *_jit, jit_int32_t u, jit_code_t code)
 313 {
 314     jit_code_inc_synth_w(code, u);
 315     jit_movr(JIT_RET, u);
 316     jit_ret();
 317     jit_dec_synth();
 318 }
 319
 320 void
 321 _jit_reti(jit_state_t *_jit, jit_word_t u, jit_code_t code)
 322 {
 323     jit_code_inc_synth_w(code, u);
 324     jit_movi(JIT_RET, u);
 325     jit_ret();
 326     jit_dec_synth();
 327 }
 328
 329 void
 330 _jit_retr_f(jit_state_t *_jit, jit_int32_t u)
 331 {
 332     jit_inc_synth_w(retr_f, u);
 333     if (JIT_FRET != u)
 334         jit_movr_f(JIT_FRET, u);
 335     else
 336         jit_live(JIT_FRET);
 337     jit_ret();
 338     jit_dec_synth();
 339 }
 340
 341 void
 342 _jit_reti_f(jit_state_t *_jit, jit_float32_t u)
 343 {
 344     jit_inc_synth_f(reti_f, u);
 345     jit_movi_f(JIT_FRET, u);
 346     jit_ret();
 347     jit_dec_synth();
 348 }
 349
 350 void
 351 _jit_retr_d(jit_state_t *_jit, jit_int32_t u)
 352 {
 353     jit_inc_synth_w(retr_d, u);
 354     if (JIT_FRET != u)
 355         jit_movr_d(JIT_FRET, u);
 356     else
 357         jit_live(JIT_FRET);
 358     jit_ret();
 359     jit_dec_synth();
 360 }
 361
 362 void
 363 _jit_reti_d(jit_state_t *_jit, jit_float64_t u)
 364 {
 365     jit_inc_synth_d(reti_d, u);
 366     jit_movi_d(JIT_FRET, u);
 367     jit_ret();
 368     jit_dec_synth();
 369 }
 370
 371 void
 372 _jit_epilog(jit_state_t *_jit)
 373 {
 374     assert(_jitc->function);
 375     assert(_jitc->function->epilog->next == NULL);
 376     jit_link(_jitc->function->epilog);
 377     _jitc->function = NULL;
 378 }
 379
 380 jit_bool_t
 381 _jit_arg_register_p(jit_state_t *_jit, jit_node_t *u)
 382 {
 383 #  if __WORDSIZE == 32
 384     if ((u->code >= jit_code_arg_c && u->code <= jit_code_arg) ||
 385         u->code == jit_code_arg_f)
 386         return (jit_arg_reg_p(u->u.w));
 387     assert(u->code == jit_code_arg_d);
 388     return (jit_arg_d_reg_p(u->u.w));
 389 #  else
 390     if (u->code >= jit_code_arg_c && u->code <= jit_code_arg)
 391         return (jit_arg_reg_p(u->u.w));
 392     assert(u->code == jit_code_arg_d || u->code == jit_code_arg_f);
 393     return (jit_arg_d_reg_p(u->u.w));
 394 #  endif
 395 }
 396
 397 void
 398 _jit_ellipsis(jit_state_t *_jit)
 399 {
 400     jit_inc_synth(ellipsis);
 401     if (_jitc->prepare) {
 402         jit_link_prepare();
 403         assert(!(_jitc->function->call.call & jit_call_varargs));
 404         _jitc->function->call.call |= jit_call_varargs;
 405     }
 406     else {
 407         jit_link_prolog();
 408         assert(!(_jitc->function->self.call & jit_call_varargs));
 409         _jitc->function->self.call |= jit_call_varargs;
 410
 411         _jitc->function->vagp = _jitc->function->self.argi;
 412     }
 413     jit_dec_synth();
 414 }
 415
 416 void
 417 _jit_va_push(jit_state_t *_jit, jit_int32_t u)
 418 {
 419     jit_inc_synth_w(va_push, u);
 420     jit_pushargr(u);
 421     jit_dec_synth();
 422 }
 423
 424 jit_node_t *
 425 _jit_arg(jit_state_t *_jit, jit_code_t code)
 426 {
 427     jit_node_t          *node;
 428     jit_int32_t          offset;
 429     assert(_jitc->function);
 430     assert(!(_jitc->function->self.call & jit_call_varargs));
 431 #if STRONG_TYPE_CHECKING
 432     assert(code >= jit_code_arg_c && code <= jit_code_arg);
 433 #endif
 434     if (jit_arg_reg_p(_jitc->function->self.argi))
 435         offset = _jitc->function->self.argi++;
 436     else {
 437 #  if __WORDSIZE == 64
 438         if (jit_arg_d_reg_p(_jitc->function->self.argi))
 439             ++_jitc->function->self.argi;
 440 #   endif
 441         offset = BIAS(_jitc->function->self.size);
 442         _jitc->function->self.size += sizeof(jit_word_t);
 443     }
 444     node = jit_new_node_ww(code, offset,
 445                            ++_jitc->function->self.argn);
 446     jit_link_prolog();
 447     return (node);
 448 }
 449
 450 jit_node_t *
 451 _jit_arg_f(jit_state_t *_jit)
 452 {
 453     jit_node_t          *node;
 454     jit_int32_t          offset;
 455 #  if __WORDSIZE == 64
 456     jit_bool_t           inc;
 457 #  endif
 458     assert(_jitc->function);
 459 #  if __WORDSIZE == 32
 460     if (jit_arg_reg_p(_jitc->function->self.argi))
 461         offset = _jitc->function->self.argi++;
 462     else {
 463         offset = _jitc->function->self.size;
 464         _jitc->function->self.size += sizeof(jit_word_t);
 465     }
 466 #  else
 467     inc = !jit_arg_reg_p(_jitc->function->self.argi);
 468     if (jit_arg_d_reg_p(_jitc->function->self.argi))
 469         offset = _jitc->function->self.argi++;
 470     else
 471         offset = BIAS(_jitc->function->self.size);
 472     if (inc)
 473         _jitc->function->self.size += sizeof(jit_word_t);
 474 #  endif
 475     node = jit_new_node_ww(jit_code_arg_f, offset,
 476                            ++_jitc->function->self.argn);
 477     jit_link_prolog();
 478     return (node);
 479 }
 480
 481 jit_node_t *
 482 _jit_arg_d(jit_state_t *_jit)
 483 {
 484     jit_node_t          *node;
 485     jit_int32_t          offset;
 486 #  if __WORDSIZE == 64
 487     jit_bool_t           inc;
 488 #  endif
 489     assert(_jitc->function);
 490 #  if __WORDSIZE == 32
 491     if (jit_arg_d_reg_p(_jitc->function->self.argi)) {
 492         offset = _jitc->function->self.argi;
 493         _jitc->function->self.argi += 2;
 494     }
 495     else if (jit_arg_reg_p(_jitc->function->self.argi)) {
 496         offset = _jitc->function->self.argi++;
 497         _jitc->function->self.size += sizeof(jit_float32_t);
 498     }
 499     else {
 500         offset = _jitc->function->self.size;
 501         _jitc->function->self.size += sizeof(jit_float64_t);
 502     }
 503 #  else
 504     inc = !jit_arg_reg_p(_jitc->function->self.argi);
 505     if (jit_arg_d_reg_p(_jitc->function->self.argi))
 506         offset = _jitc->function->self.argi++;
 507     else
 508         offset = BIAS(_jitc->function->self.size);
 509     if (inc)
 510         _jitc->function->self.size += sizeof(jit_word_t);
 511 #  endif
 512     node = jit_new_node_ww(jit_code_arg_d, offset,
 513                            ++_jitc->function->self.argn);
 514     jit_link_prolog();
 515     return (node);
 516 }
 517
 518 void
 519 _jit_getarg_c(jit_state_t *_jit, jit_int32_t u, jit_node_t *v)
 520 {
 521     assert_arg_type(v->code, jit_code_arg_c);
 522     jit_inc_synth_wp(getarg_c, u, v);
 523     if (jit_arg_reg_p(v->u.w))
 524         jit_extr_c(u, _I0 + v->u.w);
 525     else
 526         jit_ldxi_c(u, JIT_FP,
 527                    v->u.w + (__WORDSIZE >> 3) - sizeof(jit_int8_t));
 528     jit_dec_synth();
 529 }
 530
 531 void
 532 _jit_getarg_uc(jit_state_t *_jit, jit_int32_t u, jit_node_t *v)
 533 {
 534     assert_arg_type(v->code, jit_code_arg_c);
 535     jit_inc_synth_wp(getarg_uc, u, v);
 536     if (jit_arg_reg_p(v->u.w))
 537         jit_extr_uc(u, _I0 + v->u.w);
 538     else
 539         jit_ldxi_uc(u, JIT_FP,
 540                     v->u.w + (__WORDSIZE >> 3) - sizeof(jit_uint8_t));
 541     jit_dec_synth();
 542 }
 543
 544 void
 545 _jit_getarg_s(jit_state_t *_jit, jit_int32_t u, jit_node_t *v)
 546 {
 547     assert_arg_type(v->code, jit_code_arg_s);
 548     jit_inc_synth_wp(getarg_s, u, v);
 549     if (jit_arg_reg_p(v->u.w))
 550         jit_extr_s(u, _I0 + v->u.w);
 551     else
 552         jit_ldxi_s(u, JIT_FP,
 553                    v->u.w + (__WORDSIZE >> 3) - sizeof(jit_int16_t));
 554     jit_dec_synth();
 555 }
 556
 557 void
 558 _jit_getarg_us(jit_state_t *_jit, jit_int32_t u, jit_node_t *v)
 559 {
 560     assert_arg_type(v->code, jit_code_arg_s);
 561     jit_inc_synth_wp(getarg_us, u, v);
 562     if (jit_arg_reg_p(v->u.w))
 563         jit_extr_us(u, _I0 + v->u.w);
 564     else
 565         jit_ldxi_us(u, JIT_FP,
 566                     v->u.w + (__WORDSIZE >> 3) - sizeof(jit_uint16_t));
 567     jit_dec_synth();
 568 }
 569
 570 void
 571 _jit_getarg_i(jit_state_t *_jit, jit_int32_t u, jit_node_t *v)
 572 {
 573     assert_arg_type(v->code, jit_code_arg_i);
 574     jit_inc_synth_wp(getarg_i, u, v);
 575     if (jit_arg_reg_p(v->u.w)) {
 576 #  if __WORDSIZE == 64
 577         jit_extr_i(u, _I0 + v->u.w);
 578 #  else
 579         jit_movr(u, _I0 + v->u.w);
 580 #  endif
 581     }
 582     else
 583         jit_ldxi_i(u, JIT_FP,
 584                    v->u.w + (__WORDSIZE >> 3) - sizeof(jit_int32_t));
 585     jit_dec_synth();
 586 }
 587
 588 #  if __WORDSIZE == 64
 589 void
 590 _jit_getarg_ui(jit_state_t *_jit, jit_int32_t u, jit_node_t *v)
 591 {
 592     assert_arg_type(v->code, jit_code_arg_i);
 593     jit_inc_synth_wp(getarg_i, u, v);
 594     if (jit_arg_reg_p(v->u.w))
 595         jit_extr_ui(u, _I0 + v->u.w);
 596     else
 597         jit_ldxi_ui(u, JIT_FP,
 598                     v->u.w + (__WORDSIZE >> 3) - sizeof(jit_int32_t));
 599     jit_dec_synth();
 600 }
 601
 602 void
 603 _jit_getarg_l(jit_state_t *_jit, jit_int32_t u, jit_node_t *v)
 604 {
 605     assert_arg_type(v->code, jit_code_arg_l);
 606     jit_inc_synth_wp(getarg_i, u, v);
 607     if (jit_arg_reg_p(v->u.w))
 608         jit_movr(u, _I0 + v->u.w);
 609     else
 610         jit_ldxi_l(u, JIT_FP, v->u.w);
 611     jit_dec_synth();
 612 }
 613 #  endif
 614
 615 void
 616 _jit_putargr(jit_state_t *_jit, jit_int32_t u, jit_node_t *v, jit_code_t code)
 617 {
 618     assert_putarg_type(code, v->code);
 619     jit_code_inc_synth_wp(code, u, v);
 620     if (jit_arg_reg_p(v->u.w))
 621         jit_movr(_I0 + v->u.w, u);
 622     else
 623         jit_stxi(v->u.w, JIT_FP, u);
 624     jit_dec_synth();
 625 }
 626
 627 void
 628 _jit_putargi(jit_state_t *_jit, jit_word_t u, jit_node_t *v, jit_code_t code)
 629 {
 630     jit_int32_t         regno;
 631     assert_putarg_type(code, v->code);
 632     jit_code_inc_synth_wp(code, u, v);
 633     if (jit_arg_reg_p(v->u.w))
 634         jit_movi(_I0 + v->u.w, u);
 635     else {
 636         regno = jit_get_reg(jit_class_gpr);
 637         jit_movi(regno, u);
 638         jit_stxi(v->u.w, JIT_FP, regno);
 639         jit_unget_reg(regno);
 640     }
 641     jit_dec_synth();
 642 }
 643
 644 void
 645 _jit_getarg_f(jit_state_t *_jit, jit_int32_t u, jit_node_t *v)
 646 {
 647     assert(v->code == jit_code_arg_f);
 648     assert(_jitc->function);
 649     jit_inc_synth_wp(getarg_f, u, v);
 650 #  if __WORDSIZE == 32
 651     if (jit_arg_reg_p(v->u.w)) {
 652         jit_stxi_i(-4, JIT_FP, _I0 + v->u.w);
 653         jit_ldxi_f(u, JIT_FP, -4);
 654     }
 655 #  else
 656     if (jit_arg_d_reg_p(v->u.w)) {
 657         jit_live(_F0 - (v->u.w << 1));  /* pair of registers is live */
 658         jit_movr_f(u, (_F0 - (v->u.w << 1)) - 1);
 659     }
 660 #  endif
 661     else
 662         jit_ldxi_f(u, JIT_FP, v->u.w + (__WORDSIZE >> 3) -
 663                    sizeof(jit_float32_t));
 664     jit_dec_synth();
 665 }
 666
 667 void
 668 _jit_putargr_f(jit_state_t *_jit, jit_int32_t u, jit_node_t *v)
 669 {
 670     assert(v->code == jit_code_arg_f);
 671     jit_inc_synth_wp(putargr_f, u, v);
 672 #  if __WORDSIZE == 32
 673     if (jit_arg_reg_p(v->u.w)) {
 674         jit_stxi_f(-4, JIT_FP, u);
 675         jit_ldxi_i(_I0 + v->u.w, JIT_FP, -4);
 676     }
 677 #  else
 678     if (jit_arg_d_reg_p(v->u.w)) {
 679         jit_live(_F0 - (v->u.w << 1));  /* pair of registers is live */
 680         jit_movr_f((_F0 - (v->u.w << 1)) - 1, u);
 681     }
 682 #  endif
 683     else
 684         jit_stxi_f(v->u.w + (__WORDSIZE >> 3) -
 685                    sizeof(jit_float32_t), JIT_FP, u);
 686     jit_dec_synth();
 687 }
 688
 689 void
 690 _jit_putargi_f(jit_state_t *_jit, jit_float32_t u, jit_node_t *v)
 691 {
 692     jit_int32_t         regno;
 693     assert(v->code == jit_code_arg_f);
 694     jit_inc_synth_fp(putargi_f, u, v);
 695 #  if __WORDSIZE == 32
 696     regno = jit_get_reg(jit_class_fpr);
 697     jit_movi_f(regno, u);
 698     if (jit_arg_reg_p(v->u.w)) {
 699         jit_stxi_f(-4, JIT_FP, regno);
 700         jit_ldxi_i(_I0 + v->u.w, JIT_FP, -4);
 701     }
 702     else
 703         jit_stxi_f(v->u.w, JIT_FP, regno);
 704     jit_unget_reg(regno);
 705 #  else
 706     if (jit_arg_d_reg_p(v->u.w)) {
 707         jit_live(_F0 - (v->u.w << 1));  /* pair of registers is live */
 708         jit_movi_f((_F0 - (v->u.w << 1)) - 1, u);
 709     }
 710     else {
 711         regno = jit_get_reg(jit_class_fpr | jit_class_sng);
 712         jit_movi_f(regno, u);
 713         jit_stxi_f(v->u.w + (__WORDSIZE >> 3) -
 714                    sizeof(jit_float32_t), JIT_FP, regno);
 715         jit_unget_reg(regno);
 716     }
 717 #  endif
 718     jit_dec_synth();
 719 }
 720
 721 void
 722 _jit_getarg_d(jit_state_t *_jit, jit_int32_t u, jit_node_t *v)
 723 {
 724     assert(v->code == jit_code_arg_d);
 725     assert(_jitc->function);
 726     jit_inc_synth_wp(getarg_d, u, v);
 727     if (jit_arg_d_reg_p(v->u.w)) {
 728 #  if __WORDSIZE == 32
 729         jit_stxi(-8, JIT_FP, _I0 + v->u.w);
 730         jit_stxi(-4, JIT_FP, _I0 + v->u.w + 1);
 731         jit_ldxi_d(u, JIT_FP, -8);
 732 #  else
 733         jit_movr_d(u, _F0 - (v->u.w << 1));
 734 #  endif
 735     }
 736 #  if __WORDSIZE == 32
 737     else if (jit_arg_reg_p(v->u.w)) {
 738         jit_stxi(-8, JIT_FP, _I0 + v->u.w);
 739         jit_ldxi_f(u, JIT_FP, -8);
 740         jit_ldxi_f(u + 1, JIT_FP, stack_framesize);
 741     }
 742 #  endif
 743     else {
 744 #  if __WORDSIZE == 32
 745         jit_ldxi_f(u, JIT_FP, v->u.w);
 746         jit_ldxi_f(u + 1, JIT_FP, v->u.w + 4);
 747 #  else
 748         jit_ldxi_d(u, JIT_FP, v->u.w);
 749 #  endif
 750     }
 751     jit_dec_synth();
 752 }
 753
 754 void
 755 _jit_putargr_d(jit_state_t *_jit, jit_int32_t u, jit_node_t *v)
 756 {
 757    jit_int32_t          regno;
 758     assert(v->code == jit_code_arg_d);
 759     jit_inc_synth_wp(putargr_d, u, v);
 760 #  if __WORDSIZE == 32
 761     if (jit_arg_d_reg_p(v->u.w)) {
 762         jit_stxi_d(-8, JIT_FP, u);
 763         jit_ldxi(_I0 + v->u.w, JIT_FP, -8);
 764         jit_ldxi(_I0 + v->u.w + 1, JIT_FP, -4);
 765     }
 766     else if (jit_arg_reg_p(v->u.w)) {
 767         regno = jit_get_reg(jit_class_gpr);
 768         jit_stxi_d(-8, JIT_FP, u);
 769         jit_ldxi(_I0 + v->u.w, JIT_FP, -8);
 770         jit_ldxi(regno, JIT_FP, -4);
 771         jit_stxi(stack_framesize, JIT_FP, regno);
 772         jit_unget_reg(regno);
 773     }
 774     else if ((v->u.w & 7) == 0)
 775         jit_stxi_d(v->u.w, JIT_FP, u);
 776     else {
 777         jit_stxi_d(-8, JIT_FP, u);
 778         regno = jit_get_reg(jit_class_gpr);
 779         jit_ldxi(regno, JIT_FP, -8);
 780         jit_stxi(v->u.w, JIT_FP, regno);
 781         jit_ldxi(regno, JIT_FP, -4);
 782         jit_stxi(v->u.w + 4, JIT_FP, regno);
 783         jit_unget_reg(regno);
 784     }
 785 #  else
 786     if (jit_arg_d_reg_p(v->u.w))
 787         jit_movr_d(_F0 - (v->u.w << 1), u);
 788     else
 789         jit_stxi_d(v->u.w, JIT_FP, u);
 790 #  endif
 791     jit_dec_synth();
 792 }
 793
 794 void
 795 _jit_putargi_d(jit_state_t *_jit, jit_float64_t u, jit_node_t *v)
 796 {
 797 #  if __WORDSIZE == 32
 798     jit_int32_t         gpr;
 799 #  endif
 800    jit_int32_t          regno;
 801     assert(v->code == jit_code_arg_d);
 802     jit_inc_synth_dp(putargi_d, u, v);
 803 #  if __WORDSIZE == 32
 804     regno = jit_get_reg(jit_class_fpr);
 805     jit_movi_d(regno, u);
 806     if (jit_arg_d_reg_p(v->u.w)) {
 807         jit_stxi_d(-8, JIT_FP, regno);
 808         jit_ldxi(_I0 + v->u.w, JIT_FP, -8);
 809         jit_ldxi(_I0 + v->u.w + 1, JIT_FP, -4);
 810     }
 811     else if (jit_arg_reg_p(v->u.w)) {
 812         gpr = jit_get_reg(jit_class_gpr);
 813         jit_stxi_d(-8, JIT_FP, regno);
 814         jit_ldxi(_I0 + v->u.w, JIT_FP, -8);
 815         jit_ldxi(gpr, JIT_FP, -4);
 816         jit_stxi(stack_framesize, JIT_FP, gpr);
 817         jit_unget_reg(gpr);
 818     }
 819     else if ((v->u.w & 7) == 0)
 820         jit_stxi_d(v->u.w, JIT_FP, regno);
 821     else {
 822         jit_stxi_d(-8, JIT_FP, regno);
 823         gpr = jit_get_reg(jit_class_gpr);
 824         jit_ldxi(gpr, JIT_FP, -8);
 825         jit_stxi(v->u.w, JIT_FP, gpr);
 826         jit_ldxi(gpr, JIT_FP, -4);
 827         jit_stxi(v->u.w + 4, JIT_FP, gpr);
 828         jit_unget_reg(gpr);
 829     }
 830     jit_unget_reg(regno);
 831 #  else
 832     if (jit_arg_d_reg_p(v->u.w))
 833         jit_movi_d(_F0 - (v->u.w << 1), u);
 834     else {
 835         regno = jit_get_reg(jit_class_fpr | jit_class_dbl);
 836         jit_movi_d(regno, u);
 837         jit_stxi_d(v->u.w, JIT_FP, regno);
 838         jit_unget_reg(regno);
 839     }
 840 #  endif
 841     jit_dec_synth();
 842 }
 843
 844 void
 845 _jit_pushargr(jit_state_t *_jit, jit_int32_t u, jit_code_t code)
 846 {
 847     jit_code_inc_synth_w(code, u);
 848     jit_link_prepare();
 849     if (jit_arg_reg_p(_jitc->function->call.argi)) {
 850         jit_movr(_O0 + _jitc->function->call.argi, u);
 851         ++_jitc->function->call.argi;
 852     }
 853     else {
 854 #  if __WORDSIZE == 64
 855         if (jit_arg_d_reg_p(_jitc->function->call.argi))
 856             ++_jitc->function->call.argi;
 857 #  endif
 858         jit_stxi(BIAS(_jitc->function->call.size + stack_framesize),
 859                  JIT_SP, u);
 860         _jitc->function->call.size += sizeof(jit_word_t);
 861     }
 862     jit_dec_synth();
 863 }
 864
 865 void
 866 _jit_pushargi(jit_state_t *_jit, jit_word_t u, jit_code_t code)
 867 {
 868     jit_int32_t         regno;
 869     jit_code_inc_synth_w(code, u);
 870     jit_link_prepare();
 871     if (jit_arg_reg_p(_jitc->function->call.argi)) {
 872         jit_movi(_O0 + _jitc->function->call.argi, u);
 873         ++_jitc->function->call.argi;
 874     }
 875     else {
 876 #  if __WORDSIZE == 64
 877         if (jit_arg_d_reg_p(_jitc->function->call.argi))
 878             ++_jitc->function->call.argi;
 879 #  endif
 880         regno = jit_get_reg(jit_class_gpr);
 881         jit_movi(regno, u);
 882         jit_stxi(BIAS(_jitc->function->call.size + stack_framesize),
 883                  JIT_SP, regno);
 884         jit_unget_reg(regno);
 885         _jitc->function->call.size += sizeof(jit_word_t);
 886     }
 887     jit_dec_synth();
 888 }
 889
 890 void
 891 _jit_pushargr_f(jit_state_t *_jit, jit_int32_t u)
 892 {
 893     jit_inc_synth_w(pushargr_f, u);
 894     jit_link_prepare();
 895 #  if __WORDSIZE == 32
 896     if (jit_arg_reg_p(_jitc->function->call.argi)) {
 897         jit_stxi_f(-8, JIT_FP, u);
 898         jit_ldxi(_O0 + _jitc->function->call.argi, JIT_FP, -8);
 899         ++_jitc->function->call.argi;
 900     }
 901     else {
 902         jit_stxi_f(_jitc->function->call.size + stack_framesize,
 903                    JIT_SP, u);
 904         _jitc->function->call.size += sizeof(jit_float32_t);
 905     }
 906 #  else
 907     if ((_jitc->function->call.call & jit_call_varargs) &&
 908         jit_arg_reg_p(_jitc->function->call.argi)) {
 909         jit_stxi_f(BIAS(-8), JIT_FP, u);
 910         jit_ldxi_i(_O0 + _jitc->function->call.argi, JIT_FP, BIAS(-8));
 911         ++_jitc->function->call.argi;
 912     }
 913     else if (!(_jitc->function->call.call & jit_call_varargs) &&
 914              jit_arg_d_reg_p(_jitc->function->call.argi)) {
 915         /* pair of registers is live */
 916         jit_live(_F0 - (_jitc->function->call.argi << 1));
 917         jit_movr_f((_F0 - (_jitc->function->call.argi << 1)) - 1, u);
 918         if (!jit_arg_reg_p(_jitc->function->call.argi))
 919             _jitc->function->call.size += sizeof(jit_float64_t);
 920         ++_jitc->function->call.argi;
 921     }
 922     else {
 923         jit_stxi_f(BIAS(_jitc->function->call.size + stack_framesize + 4),
 924                    JIT_SP, u);
 925         _jitc->function->call.size += sizeof(jit_float64_t);
 926     }
 927 #  endif
 928     jit_dec_synth();
 929 }
 930
 931 void
 932 _jit_pushargi_f(jit_state_t *_jit, jit_float32_t u)
 933 {
 934     jit_int32_t         regno;
 935     jit_inc_synth_f(pushargi_f, u);
 936     jit_link_prepare();
 937 #  if __WORDSIZE == 32
 938     regno = jit_get_reg(jit_class_fpr);
 939     jit_movi_f(regno, u);
 940     if (jit_arg_reg_p(_jitc->function->call.argi)) {
 941         jit_stxi_f(-8, JIT_FP, regno);
 942         jit_ldxi(_O0 + _jitc->function->call.argi, JIT_FP, -8);
 943         _jitc->function->call.argi++;
 944     }
 945     else {
 946         jit_stxi_f(_jitc->function->call.size + stack_framesize,
 947                    JIT_SP, regno);
 948         _jitc->function->call.size += sizeof(jit_float32_t);
 949     }
 950     jit_unget_reg(regno);
 951 #  else
 952     if ((_jitc->function->call.call & jit_call_varargs) &&
 953         jit_arg_reg_p(_jitc->function->call.argi)) {
 954         regno = jit_get_reg(jit_class_fpr | jit_class_sng);
 955         jit_movi_f(regno, u);
 956         jit_stxi_f(BIAS(-8), JIT_FP, regno);
 957         jit_ldxi_i(_O0 + _jitc->function->call.argi, JIT_FP, BIAS(-8));
 958         ++_jitc->function->call.argi;
 959         jit_unget_reg(regno);
 960     }
 961     else if (!(_jitc->function->call.call & jit_call_varargs) &&
 962              jit_arg_d_reg_p(_jitc->function->call.argi)) {
 963         /* pair of registers is live */
 964         jit_live(_F0 - (_jitc->function->call.argi << 1));
 965         jit_movi_f((_F0 - (_jitc->function->call.argi << 1)) - 1, u);
 966         if (!jit_arg_reg_p(_jitc->function->call.argi))
 967             _jitc->function->call.size += sizeof(jit_float64_t);
 968         ++_jitc->function->call.argi;
 969     }
 970     else {
 971         regno = jit_get_reg(jit_class_fpr | jit_class_sng);
 972         jit_movi_f(regno, u);
 973         jit_stxi_f(BIAS(_jitc->function->call.size + stack_framesize + 4),
 974                    JIT_SP, regno);
 975         jit_unget_reg(regno);
 976         _jitc->function->call.size += sizeof(jit_float64_t);
 977     }
 978 #  endif
 979     jit_dec_synth();
 980 }
 981
 982 void
 983 _jit_pushargr_d(jit_state_t *_jit, jit_int32_t u)
 984 {
 985     jit_inc_synth_w(pushargr_d, u);
 986     jit_link_prepare();
 987 #  if __WORDSIZE == 32
 988     if (jit_arg_d_reg_p(_jitc->function->call.argi)) {
 989         jit_stxi_d(BIAS(-8), JIT_FP, u);
 990         jit_ldxi(_O0 + _jitc->function->call.argi, JIT_FP, BIAS(-8));
 991         jit_ldxi(_O0 + _jitc->function->call.argi + 1, JIT_FP, -4);
 992         _jitc->function->call.argi += 2;
 993     }
 994     else if (jit_arg_reg_p(_jitc->function->call.argi)) {
 995         jit_stxi_f(-8, JIT_FP, u);
 996         jit_ldxi(_O0 + _jitc->function->call.argi, JIT_FP, -8);
 997         ++_jitc->function->call.argi;
 998         jit_stxi_f(stack_framesize, JIT_SP, u + 1);
 999         _jitc->function->call.size += sizeof(jit_float32_t);
1000     }
1001     else {
1002         jit_stxi_f(_jitc->function->call.size + stack_framesize,
1003                    JIT_SP, u);
1004         jit_stxi_f(_jitc->function->call.size + stack_framesize + 4,
1005                    JIT_SP, u + 1);
1006         _jitc->function->call.size += sizeof(jit_float64_t);
1007     }
1008 #  else
1009     if ((_jitc->function->call.call & jit_call_varargs) &&
1010         jit_arg_reg_p(_jitc->function->call.argi)) {
1011         jit_stxi_d(BIAS(-8), JIT_FP, u);
1012         jit_ldxi(_O0 + _jitc->function->call.argi, JIT_FP, BIAS(-8));
1013         ++_jitc->function->call.argi;
1014     }
1015     else if (!(_jitc->function->call.call & jit_call_varargs) &&
1016              jit_arg_d_reg_p(_jitc->function->call.argi)) {
1017         jit_movr_d(_F0 - (_jitc->function->call.argi << 1), u);
1018         if (!jit_arg_reg_p(_jitc->function->call.argi))
1019             _jitc->function->call.size += sizeof(jit_float64_t);
1020         ++_jitc->function->call.argi;
1021     }
1022     else {
1023         jit_stxi_d(BIAS(_jitc->function->call.size + stack_framesize),
1024                    JIT_SP, u);
1025         _jitc->function->call.size += sizeof(jit_float64_t);
1026     }
1027 #  endif
1028     jit_dec_synth();
1029 }
1030
1031 void
1032 _jit_pushargi_d(jit_state_t *_jit, jit_float64_t u)
1033 {
1034     jit_int32_t         regno;
1035     jit_inc_synth_d(pushargi_d, u);
1036     jit_link_prepare();
1037 #  if __WORDSIZE == 32
1038     regno = jit_get_reg(jit_class_fpr);
1039     jit_movi_d(regno, u);
1040     if (jit_arg_d_reg_p(_jitc->function->call.argi)) {
1041         jit_stxi_d(BIAS(-8), JIT_FP, regno);
1042         jit_ldxi(_O0 + _jitc->function->call.argi, JIT_FP, BIAS(-8));
1043         jit_ldxi(_O0 + _jitc->function->call.argi + 1, JIT_FP, -4);
1044         _jitc->function->call.argi += 2;
1045     }
1046     else if (jit_arg_reg_p(_jitc->function->call.argi)) {
1047         jit_stxi_f(-8, JIT_FP, regno);
1048         jit_ldxi(_O0 + _jitc->function->call.argi, JIT_FP, -8);
1049         ++_jitc->function->call.argi;
1050         jit_stxi_f(stack_framesize, JIT_SP, u + 1);
1051         _jitc->function->call.size += sizeof(jit_float32_t);
1052     }
1053     else {
1054         jit_stxi_f(_jitc->function->call.size + stack_framesize,
1055                    JIT_SP, regno);
1056         jit_stxi_f(_jitc->function->call.size + stack_framesize + 4,
1057                    JIT_SP, regno + 1);
1058         _jitc->function->call.size += sizeof(jit_float64_t);
1059     }
1060     jit_unget_reg(regno);
1061 #  else
1062     if ((_jitc->function->call.call & jit_call_varargs) &&
1063         jit_arg_reg_p(_jitc->function->call.argi)) {
1064         regno = jit_get_reg(jit_class_fpr | jit_class_dbl);
1065         jit_movi_d(regno, u);
1066         jit_stxi_d(BIAS(-8), JIT_FP, regno);
1067         jit_ldxi(_O0 + _jitc->function->call.argi, JIT_FP, BIAS(-8));
1068         ++_jitc->function->call.argi;
1069         jit_unget_reg(regno);
1070     }
1071     else if (!(_jitc->function->call.call & jit_call_varargs) &&
1072              jit_arg_d_reg_p(_jitc->function->call.argi)) {
1073         jit_movi_d(_F0 - (_jitc->function->call.argi << 1), u);
1074         if (!jit_arg_reg_p(_jitc->function->call.argi))
1075             _jitc->function->call.size += sizeof(jit_float64_t);
1076         ++_jitc->function->call.argi;
1077     }
1078     else {
1079         regno = jit_get_reg(jit_class_fpr | jit_class_dbl);
1080         jit_movi_d(regno, u);
1081         jit_stxi_d(BIAS(_jitc->function->call.size + stack_framesize),
1082                    JIT_SP, regno);
1083         jit_unget_reg(regno);
1084         _jitc->function->call.size += sizeof(jit_float64_t);
1085     }
1086 #  endif
1087     jit_dec_synth();
1088 }
1089
1090 jit_bool_t
1091 _jit_regarg_p(jit_state_t *_jit, jit_node_t *node, jit_int32_t regno)
1092 {
1093     jit_int32_t         spec;
1094
1095     spec = jit_class(_rvs[regno].spec);
1096     if ((spec & (jit_class_arg|jit_class_gpr)) ==
1097         (jit_class_arg|jit_class_gpr)) {
1098         regno -= _O0;
1099         if (regno >= 0 && regno < node->v.w)
1100             return (1);
1101     }
1102 #  if __WORDSIZE == 64
1103     if ((spec & (jit_class_arg|jit_class_fpr)) ==
1104         (jit_class_arg|jit_class_fpr)) {
1105         regno = _F0 - (regno >> 1);
1106         if (regno >= 0 && regno < node->v.w)
1107             return (1);
1108     }
1109 #  endif
1110
1111     return (0);
1112 }
1113
1114 void
1115 _jit_finishr(jit_state_t *_jit, jit_int32_t r0)
1116 {
1117     jit_node_t          *call;
1118     assert(_jitc->function);
1119     jit_inc_synth_w(finishr, r0);
1120     if (_jitc->function->self.alen < _jitc->function->call.size)
1121         _jitc->function->self.alen = _jitc->function->call.size;
1122     call = jit_callr(r0);
1123     call->v.w = _jitc->function->self.argi;
1124     call->w.w = _jitc->function->self.argf;
1125     _jitc->function->call.argi = _jitc->function->call.argf =
1126         _jitc->function->call.size = 0;
1127     _jitc->prepare = 0;
1128     jit_dec_synth();
1129 }
1130
1131 jit_node_t *
1132 _jit_finishi(jit_state_t *_jit, jit_pointer_t i0)
1133 {
1134     jit_node_t          *node;
1135     assert(_jitc->function);
1136     jit_inc_synth_w(finishi, (jit_word_t)i0);
1137     if (_jitc->function->self.alen < _jitc->function->call.size)
1138         _jitc->function->self.alen = _jitc->function->call.size;
1139     node = jit_calli(i0);
1140     node->v.w = _jitc->function->call.argi;
1141     node->w.w = _jitc->function->call.argf;
1142     _jitc->function->call.argi = _jitc->function->call.argf =
1143         _jitc->function->call.size = 0;
1144     _jitc->prepare = 0;
1145     jit_dec_synth();
1146     return (node);
1147 }
1148
1149 void
1150 _jit_retval_c(jit_state_t *_jit, jit_int32_t r0)
1151 {
1152     jit_inc_synth_w(retval_c, r0);
1153     jit_extr_c(r0, _O0);
1154     jit_dec_synth();
1155 }
1156
1157 void
1158 _jit_retval_uc(jit_state_t *_jit, jit_int32_t r0)
1159 {
1160     jit_inc_synth_w(retval_uc, r0);
1161     jit_extr_uc(r0, _O0);
1162     jit_dec_synth();
1163 }
1164
1165 void
1166 _jit_retval_s(jit_state_t *_jit, jit_int32_t r0)
1167 {
1168     jit_inc_synth_w(retval_s, r0);
1169     jit_extr_s(r0, _O0);
1170     jit_dec_synth();
1171 }
1172
1173 void
1174 _jit_retval_us(jit_state_t *_jit, jit_int32_t r0)
1175 {
1176     jit_inc_synth_w(retval_us, r0);
1177     jit_extr_us(r0, _O0);
1178     jit_dec_synth();
1179 }
1180
1181 void
1182 _jit_retval_i(jit_state_t *_jit, jit_int32_t r0)
1183 {
1184     jit_inc_synth_w(retval_i, r0);
1185 #  if __WORDSIZE == 32
1186     if (r0 != _O0)
1187         jit_movr(r0, _O0);
1188 #  else
1189     jit_extr_i(r0, _O0);
1190 #  endif
1191     jit_dec_synth();
1192 }
1193
1194 #  if __WORDSIZE == 64
1195 void
1196 _jit_retval_ui(jit_state_t *_jit, jit_int32_t r0)
1197 {
1198     jit_inc_synth_w(retval_i, r0);
1199     if (r0 != _O0)
1200         jit_extr_ui(r0, _O0);
1201     jit_dec_synth();
1202 }
1203
1204 void
1205 _jit_retval_l(jit_state_t *_jit, jit_int32_t r0)
1206 {
1207     jit_inc_synth_w(retval_i, r0);
1208     if (r0 != _O0)
1209         jit_movr(r0, _O0);
1210     jit_dec_synth();
1211 }
1212 #  endif
1213
1214 void
1215 _jit_retval_f(jit_state_t *_jit, jit_int32_t r0)
1216 {
1217     jit_inc_synth_w(retval_f, r0);
1218     if (r0 != JIT_FRET)
1219         jit_movr_f(r0, JIT_FRET);
1220     jit_dec_synth();
1221 }
1222
1223 void
1224 _jit_retval_d(jit_state_t *_jit, jit_int32_t r0)
1225 {
1226     jit_inc_synth_w(retval_d, r0);
1227     if (r0 != JIT_FRET)
1228         jit_movr_d(r0, JIT_FRET);
1229     jit_dec_synth();
1230 }
1231
1232 jit_pointer_t
1233 _emit_code(jit_state_t *_jit)
1234 {
1235     jit_node_t          *node;
1236     jit_node_t          *temp;
1237     jit_word_t           word;
1238     jit_int32_t          value;
1239     jit_int32_t          offset;
1240     struct {
1241         jit_node_t      *node;
1242         jit_word_t       word;
1243         jit_function_t   func;
1244 #if DEVEL_DISASSEMBLER
1245         jit_word_t       prevw;
1246 #endif
1247         jit_int32_t      patch_offset;
1248     } undo;
1249 #if DEVEL_DISASSEMBLER
1250     jit_word_t           prevw;
1251 #endif
1252
1253     _jitc->function = NULL;
1254
1255     jit_reglive_setup();
1256
1257     undo.word = 0;
1258     undo.node = NULL;
1259     undo.patch_offset = 0;
1260
1261 #define case_rr(name, type)                                             \
1262             case jit_code_##name##r##type:                              \
1263                 name##r##type(rn(node->u.w), rn(node->v.w));            \
1264                 break
1265 #define case_rw(name, type)                                             \
1266             case jit_code_##name##i##type:                              \
1267                 name##i##type(rn(node->u.w), node->v.w);                \
1268                 break
1269 #define case_wr(name, type)                                             \
1270             case jit_code_##name##i##type:                              \
1271                 name##i##type(node->u.w, rn(node->v.w));                \
1272                 break
1273 #define case_rf(name)                                                   \
1274             case jit_code_##name##i##type:                              \
1275                 assert(node->flag & jit_flag_data);                     \
1276                 name##_f(rn(node->u.w),                                 \
1277                 (jit_float32_t *)node->v.n->u.w);                       \
1278                 break
1279 #define case_rd(name)                                                   \
1280             case jit_code_##name##i_d:                                  \
1281                 assert(node->flag & jit_flag_data);                     \
1282                 name##_d(rn(node->u.w),                                 \
1283                          (jit_float64_t *)node->v.n->u.w);              \
1284                 break
1285 #define case_rrr(name, type)                                            \
1286             case jit_code_##name##r##type:                              \
1287                 name##r##type(rn(node->u.w),                            \
1288                               rn(node->v.w), rn(node->w.w));            \
1289                 break
1290 #define case_rrrr(name, type)                                           \
1291             case jit_code_##name##r##type:                              \
1292                 name##r##type(rn(node->u.q.l), rn(node->u.q.h),         \
1293                               rn(node->v.w), rn(node->w.w));            \
1294                 break
1295 #define case_rqr(name, type)                                            \
1296             case jit_code_##name##r##type:                              \
1297                 name##r##type(rn(node->u.w), rn(node->v.q.l),           \
1298                               rn(node->v.q.h), rn(node->w.w));          \
1299             case jit_code_##name##i##type:                              \
1300                 break;
1301 #define case_rrw(name, type)                                            \
1302             case jit_code_##name##i##type:                              \
1303                 name##i##type(rn(node->u.w),                            \
1304                               rn(node->v.w), node->w.w);                \
1305                 break
1306 #define case_rrrw(name, type)                                           \
1307             case jit_code_##name##i##type:                              \
1308                 name##i##type(rn(node->u.q.l), rn(node->u.q.h),         \
1309                               rn(node->v.w), node->w.w);                \
1310                 break
1311 #define case_rrf(name, type, size)                                      \
1312             case jit_code_##name##i##type:                              \
1313                 assert(node->flag & jit_flag_data);                     \
1314                 name##i##type(rn(node->u.w), rn(node->v.w),             \
1315                               (jit_float##size##_t *)node->w.n->u.w);   \
1316                 break
1317 #define case_wrr(name, type)                                            \
1318             case jit_code_##name##i##type:                              \
1319                 name##i##type(node->u.w, rn(node->v.w), rn(node->w.w)); \
1320                 break
1321 #define case_brr(name, type)                                            \
1322             case jit_code_##name##r##type:                              \
1323                 temp = node->u.n;                                       \
1324                 assert(temp->code == jit_code_label ||                  \
1325                        temp->code == jit_code_epilog);                  \
1326                 if (temp->flag & jit_flag_patch)                        \
1327                     name##r##type(temp->u.w, rn(node->v.w),             \
1328                                   rn(node->w.w));                       \
1329                 else {                                                  \
1330                     word = name##r##type(_jit->pc.w,                    \
1331                                          rn(node->v.w), rn(node->w.w)); \
1332                     patch(word, node);                                  \
1333                 }                                                       \
1334                 break
1335 #define case_brw(name, type)                                            \
1336             case jit_code_##name##i##type:                              \
1337                 temp = node->u.n;                                       \
1338                 assert(temp->code == jit_code_label ||                  \
1339                        temp->code == jit_code_epilog);                  \
1340                 if (temp->flag & jit_flag_patch)                        \
1341                     name##i##type(temp->u.w,                            \
1342                                   rn(node->v.w), node->w.w);            \
1343                 else {                                                  \
1344                     word = name##i##type(_jit->pc.w,                    \
1345                                          rn(node->v.w), node->w.w);     \
1346                     patch(word, node);                                  \
1347                 }                                                       \
1348                 break
1349 #define case_brf(name, type, size)                                      \
1350             case jit_code_##name##i##type:                              \
1351                 temp = node->u.n;                                       \
1352                 assert(temp->code == jit_code_label ||                  \
1353                        temp->code == jit_code_epilog);                  \
1354                 if (temp->flag & jit_flag_patch)                        \
1355                     name##i##type(temp->u.w, rn(node->v.w),             \
1356                                 (jit_float##size##_t *)node->w.n->u.w); \
1357                 else {                                                  \
1358                     word = name##i##type(_jit->pc.w, rn(node->v.w),     \
1359                                 (jit_float##size##_t *)node->w.n->u.w); \
1360                     patch(word, node);                                  \
1361                 }                                                       \
1362                 break
1363 #if DEVEL_DISASSEMBLER
1364     prevw = _jit->pc.w;
1365 #endif
1366     for (node = _jitc->head; node; node = node->next) {
1367         if (_jit->pc.uc >= _jitc->code.end)
1368             return (NULL);
1369
1370 #if DEVEL_DISASSEMBLER
1371         node->offset = (jit_uword_t)_jit->pc.w - (jit_uword_t)prevw;
1372         prevw = _jit->pc.w;
1373 #endif
1374         value = jit_classify(node->code);
1375         jit_regarg_set(node, value);
1376         switch (node->code) {
1377             case jit_code_align:
1378                 /* Must align to a power of two */
1379                 assert(!(node->u.w & (node->u.w - 1)));
1380                 if ((word = _jit->pc.w & (node->u.w - 1)))
1381                     nop(node->u.w - word);
1382                 break;
1383             case jit_code_skip:
1384                 nop((node->u.w + 3) & ~3);
1385                 break;
1386             case jit_code_note:         case jit_code_name:
1387                 node->u.w = _jit->pc.w;
1388                 break;
1389             case jit_code_label:
1390                 if ((node->link || (node->flag & jit_flag_use)) &&
1391                     (word = _jit->pc.w & (sizeof(jit_word_t) - 1)))
1392                     nop(sizeof(jit_word_t) - word);
1393                 /* remember label is defined */
1394                 node->flag |= jit_flag_patch;
1395                 node->u.w = _jit->pc.w;
1396                 break;
1397                 case_rrr(add,);
1398                 case_rrw(add,);
1399                 case_rrr(addc,);
1400                 case_rrw(addc,);
1401                 case_rrr(addx,);
1402                 case_rrw(addx,);
1403                 case_rrr(sub,);
1404                 case_rrw(sub,);
1405                 case_rrr(subc,);
1406                 case_rrw(subc,);
1407                 case_rrr(subx,);
1408                 case_rrw(subx,);
1409                 case_rrw(rsb,);
1410                 case_rrr(mul,);
1411                 case_rrw(mul,);
1412                 case_rrr(hmul,);
1413                 case_rrw(hmul,);
1414                 case_rrr(hmul, _u);
1415                 case_rrw(hmul, _u);
1416                 case_rrrr(qmul,);
1417                 case_rrrw(qmul,);
1418                 case_rrrr(qmul, _u);
1419                 case_rrrw(qmul, _u);
1420                 case_rrr(div,);
1421                 case_rrw(div,);
1422                 case_rrr(div, _u);
1423                 case_rrw(div, _u);
1424                 case_rrrr(qdiv,);
1425                 case_rrrw(qdiv,);
1426                 case_rrrr(qdiv, _u);
1427                 case_rrrw(qdiv, _u);
1428                 case_rrr(rem,);
1429                 case_rrw(rem,);
1430                 case_rrr(rem, _u);
1431                 case_rrw(rem, _u);
1432                 case_rrr(and,);
1433                 case_rrw(and,);
1434                 case_rrr(or,);
1435                 case_rrw(or,);
1436                 case_rrr(xor,);
1437                 case_rrw(xor,);
1438                 case_rrr(lsh,);
1439                 case_rrw(lsh,);
1440 #define qlshr(r0, r1, r2, r3)   fallback_qlshr(r0, r1, r2, r3)
1441 #define qlshi(r0, r1, r2, i0)   fallback_qlshi(r0, r1, r2, i0)
1442 #define qlshr_u(r0, r1, r2, r3) fallback_qlshr_u(r0, r1, r2, r3)
1443 #define qlshi_u(r0, r1, r2, i0) fallback_qlshi_u(r0, r1, r2, i0)
1444                 case_rrrr(qlsh,);
1445                 case_rrrw(qlsh,);
1446                 case_rrrr(qlsh, _u);
1447                 case_rrrw(qlsh, _u);
1448                 case_rrr(rsh,);
1449                 case_rrw(rsh,);
1450                 case_rrr(rsh, _u);
1451                 case_rrw(rsh, _u);
1452 #define qrshr(r0, r1, r2, r3)   fallback_qrshr(r0, r1, r2, r3)
1453 #define qrshi(r0, r1, r2, i0)   fallback_qrshi(r0, r1, r2, i0)
1454 #define qrshr_u(r0, r1, r2, r3) fallback_qrshr_u(r0, r1, r2, r3)
1455 #define qrshi_u(r0, r1, r2, i0) fallback_qrshi_u(r0, r1, r2, i0)
1456                 case_rrrr(qrsh,);
1457                 case_rrrw(qrsh,);
1458                 case_rrrr(qrsh, _u);
1459                 case_rrrw(qrsh, _u);
1460 #define lrotr(r0,r1,r2) fallback_lrotr(r0,r1,r2)
1461 #define lroti(r0,r1,i0) fallback_lroti(r0,r1,i0)
1462 #define rrotr(r0,r1,r2) fallback_rrotr(r0,r1,r2)
1463 #define rroti(r0,r1,i0) fallback_rroti(r0,r1,i0)
1464                 case_rrr(lrot,);
1465                 case_rrw(lrot,);
1466                 case_rrr(rrot,);
1467                 case_rrw(rrot,);
1468                 case_rr(trunc, _f_i);
1469                 case_rr(trunc, _d_i);
1470 #if __WORDSIZE == 64
1471                 case_rr(trunc, _f_l);
1472                 case_rr(trunc, _d_l);
1473 #endif
1474                 case_rrr(lt,);
1475                 case_rrw(lt,);
1476                 case_rrr(lt, _u);
1477                 case_rrw(lt, _u);
1478                 case_rrr(le,);
1479                 case_rrw(le,);
1480                 case_rrr(le, _u);
1481                 case_rrw(le, _u);
1482                 case_rrr(eq,);
1483                 case_rrw(eq,);
1484                 case_rrr(ge,);
1485                 case_rrw(ge,);
1486                 case_rrr(ge, _u);
1487                 case_rrw(ge, _u);
1488                 case_rrr(gt,);
1489                 case_rrw(gt,);
1490                 case_rrr(gt, _u);
1491                 case_rrw(gt, _u);
1492                 case_rrr(ne,);
1493                 case_rrw(ne,);
1494                 case_rr(ld, _c);
1495                 case_rw(ld, _c);
1496                 case_rr(ld, _uc);
1497                 case_rw(ld, _uc);
1498                 case_rr(ld, _s);
1499                 case_rw(ld, _s);
1500                 case_rr(ld, _us);
1501                 case_rw(ld, _us);
1502                 case_rr(ld, _i);
1503                 case_rw(ld, _i);
1504 #if __WORDSIZE == 64
1505                 case_rr(ld, _ui);
1506                 case_rw(ld, _ui);
1507                 case_rr(ld, _l);
1508                 case_rw(ld, _l);
1509 #endif
1510                 case_rrr(ldx, _c);
1511                 case_rrw(ldx, _c);
1512                 case_rrr(ldx, _uc);
1513                 case_rrw(ldx, _uc);
1514                 case_rrr(ldx, _s);
1515                 case_rrw(ldx, _s);
1516                 case_rrr(ldx, _us);
1517                 case_rrw(ldx, _us);
1518                 case_rrr(ldx, _i);
1519                 case_rrw(ldx, _i);
1520 #if __WORDSIZE == 64
1521                 case_rrr(ldx, _ui);
1522                 case_rrw(ldx, _ui);
1523                 case_rrr(ldx, _l);
1524                 case_rrw(ldx, _l);
1525 #endif
1526 #define unldr(r0, r1, i0)       fallback_unldr(r0, r1, i0)
1527             case jit_code_unldr:
1528                 unldr(rn(node->u.w), rn(node->v.w), node->w.w);
1529                 break;
1530 #define unldi(r0, i0, i1)       fallback_unldi(r0, i0, i1)
1531             case jit_code_unldi:
1532                 unldi(rn(node->u.w), node->v.w, node->w.w);
1533                 break;
1534 #define unldr_u(r0, r1, i0)     fallback_unldr_u(r0, r1, i0)
1535             case jit_code_unldr_u:
1536                 unldr_u(rn(node->u.w), rn(node->v.w), node->w.w);
1537                 break;
1538 #define unldi_u(r0, i0, i1)     fallback_unldi_u(r0, i0, i1)
1539             case jit_code_unldi_u:
1540                 unldi_u(rn(node->u.w), node->v.w, node->w.w);
1541                 break;
1542                 case_rr(st, _c);
1543                 case_wr(st, _c);
1544                 case_rr(st, _s);
1545                 case_wr(st, _s);
1546                 case_rr(st, _i);
1547                 case_wr(st, _i);
1548 #if __WORDSIZE == 64
1549                 case_rr(st, _l);
1550                 case_wr(st, _l);
1551 #endif
1552                 case_rrr(stx, _c);
1553                 case_wrr(stx, _c);
1554                 case_rrr(stx, _s);
1555                 case_wrr(stx, _s);
1556                 case_rrr(stx, _i);
1557                 case_wrr(stx, _i);
1558 #if __WORDSIZE == 64
1559                 case_rrr(stx, _l);
1560                 case_wrr(stx, _l);
1561 #endif
1562 #define unstr(r0, r1, i0)       fallback_unstr(r0, r1, i0)
1563             case jit_code_unstr:
1564                 unstr(rn(node->u.w), rn(node->v.w), node->w.w);
1565                 break;
1566 #define unsti(i0, r0, i1)       fallback_unsti(i0, r0, i1)
1567             case jit_code_unsti:
1568                 unsti(node->u.w, rn(node->v.w), node->w.w);
1569                 break;
1570                 case_rr(hton, _us);
1571                 case_rr(hton, _ui);
1572 #if __WORDSIZE == 64
1573                 case_rr(hton, _ul);
1574 #endif
1575                 case_rr(bswap, _us);
1576                 case_rr(bswap, _ui);
1577 #if __WORDSIZE == 64
1578                 case_rr(bswap, _ul);
1579 #endif
1580 #define extr(r0, r1, i0, i1)    fallback_ext(r0, r1, i0, i1)
1581 #define extr_u(r0, r1, i0, i1)  fallback_ext_u(r0, r1, i0, i1)
1582 #define depr(r0, r1, i0, i1)    fallback_dep(r0, r1, i0, i1)
1583             case jit_code_extr:
1584                 extr(rn(node->u.w), rn(node->v.w), node->w.q.l, node->w.q.h);
1585                 break;
1586             case jit_code_extr_u:
1587                 extr_u(rn(node->u.w), rn(node->v.w), node->w.q.l, node->w.q.h);
1588                 break;
1589             case jit_code_depr:
1590                 depr(rn(node->u.w), rn(node->v.w), node->w.q.l, node->w.q.h);
1591                 break;
1592             case jit_code_depi:
1593                 depi(rn(node->u.w), node->v.w, node->w.q.l, node->w.q.h);
1594                 break;
1595                 case_rr(ext, _c);
1596                 case_rr(ext, _uc);
1597                 case_rr(ext, _s);
1598                 case_rr(ext, _us);
1599 #if __WORDSIZE == 64
1600                 case_rr(ext, _i);
1601                 case_rr(ext, _ui);
1602 #endif
1603             case jit_code_casr:
1604                 casr(rn(node->u.w), rn(node->v.w),
1605                      rn(node->w.q.l), rn(node->w.q.h));
1606                 break;
1607             case jit_code_casi:
1608                 casi(rn(node->u.w), node->v.w,
1609                      rn(node->w.q.l), rn(node->w.q.h));
1610                 break;
1611                 case_rrr(movn,);
1612                 case_rrr(movz,);
1613                 case_rr(mov,);
1614             case jit_code_movi:
1615                 if (node->flag & jit_flag_node) {
1616                     temp = node->v.n;
1617                     if (temp->code == jit_code_data ||
1618                         (temp->code == jit_code_label &&
1619                          (temp->flag & jit_flag_patch)))
1620                         movi(rn(node->u.w), temp->u.w);
1621                     else {
1622                         assert(temp->code == jit_code_label ||
1623                                temp->code == jit_code_epilog);
1624                         word = movi_p(rn(node->u.w), node->v.w);
1625                         patch(word, node);
1626                     }
1627                 }
1628                 else
1629                     movi(rn(node->u.w), node->v.w);
1630                 break;
1631                 case_rr(neg,);
1632                 case_rr(com,);
1633                 case_rr(clo,);
1634                 case_rr(clz,);
1635                 case_rr(cto,);
1636                 case_rr(ctz,);
1637 #define rbitr(r0, r1)   fallback_rbit(r0, r1)
1638 #define popcntr(r0, r1) fallback_popcnt(r0, r1)
1639                 case_rr(rbit,);
1640                 case_rr(popcnt,);
1641                 case_brr(blt,);
1642                 case_brw(blt,);
1643                 case_brr(blt, _u);
1644                 case_brw(blt, _u);
1645                 case_brr(ble,);
1646                 case_brw(ble,);
1647                 case_brr(ble, _u);
1648                 case_brw(ble, _u);
1649                 case_brr(beq,);
1650                 case_brw(beq,);
1651                 case_brr(bge,);
1652                 case_brw(bge,);
1653                 case_brr(bge, _u);
1654                 case_brw(bge, _u);
1655                 case_brr(bgt,);
1656                 case_brw(bgt,);
1657                 case_brr(bgt, _u);
1658                 case_brw(bgt, _u);
1659                 case_brr(bne,);
1660                 case_brw(bne,);
1661                 case_brr(boadd,);
1662                 case_brw(boadd,);
1663                 case_brr(boadd, _u);
1664                 case_brw(boadd, _u);
1665                 case_brr(bxadd,);
1666                 case_brw(bxadd,);
1667                 case_brr(bxadd, _u);
1668                 case_brw(bxadd, _u);
1669                 case_brr(bosub,);
1670                 case_brw(bosub,);
1671                 case_brr(bosub, _u);
1672                 case_brw(bosub, _u);
1673                 case_brr(bxsub,);
1674                 case_brw(bxsub,);
1675                 case_brr(bxsub, _u);
1676                 case_brw(bxsub, _u);
1677                 case_brr(bms,);
1678                 case_brw(bms,);
1679                 case_brr(bmc,);
1680                 case_brw(bmc,);
1681                 case_rrr(add, _f);
1682                 case_rrf(add, _f, 32);
1683                 case_rrr(sub, _f);
1684                 case_rrf(sub, _f, 32);
1685                 case_rrf(rsb, _f, 32);
1686                 case_rrr(mul, _f);
1687                 case_rrf(mul, _f, 32);
1688                 case_rrr(div, _f);
1689                 case_rrf(div, _f, 32);
1690                 case_rr(abs, _f);
1691                 case_rr(neg, _f);
1692                 case_rr(sqrt, _f);
1693                 case_rqr(fma, _f);
1694                 case_rqr(fms, _f);
1695                 case_rqr(fnma, _f);
1696                 case_rqr(fnms, _f);
1697                 case_rr(ext, _f);
1698                 case_rr(ext, _d_f);
1699                 case_rrr(lt, _f);
1700                 case_rrf(lt, _f, 32);
1701                 case_rrr(le, _f);
1702                 case_rrf(le, _f, 32);
1703                 case_rrr(eq, _f);
1704                 case_rrf(eq, _f, 32);
1705                 case_rrr(ge, _f);
1706                 case_rrf(ge, _f, 32);
1707                 case_rrr(gt, _f);
1708                 case_rrf(gt, _f, 32);
1709                 case_rrr(ne, _f);
1710                 case_rrf(ne, _f, 32);
1711                 case_rrr(unlt, _f);
1712                 case_rrf(unlt, _f, 32);
1713                 case_rrr(unle, _f);
1714                 case_rrf(unle, _f, 32);
1715                 case_rrr(uneq, _f);
1716                 case_rrf(uneq, _f, 32);
1717                 case_rrr(unge, _f);
1718                 case_rrf(unge, _f, 32);
1719                 case_rrr(ungt, _f);
1720                 case_rrf(ungt, _f, 32);
1721                 case_rrr(ltgt, _f);
1722                 case_rrf(ltgt, _f, 32);
1723                 case_rrr(ord, _f);
1724                 case_rrf(ord, _f, 32);
1725                 case_rrr(unord, _f);
1726                 case_rrf(unord, _f, 32);
1727                 case_rr(ld, _f);
1728                 case_rw(ld, _f);
1729                 case_rrr(ldx, _f);
1730                 case_rrw(ldx, _f);
1731 #define unldr_x(r0, r1, i0)     fallback_unldr_x(r0, r1, i0)
1732             case jit_code_unldr_x:
1733                 unldr_x(rn(node->u.w), rn(node->v.w), node->w.w);
1734                 break;
1735 #define unldi_x(r0, i0, i1)     fallback_unldi_x(r0, i0, i1)
1736             case jit_code_unldi_x:
1737                 unldi_x(rn(node->u.w), node->v.w, node->w.w);
1738                 break;
1739                 case_rr(st, _f);
1740                 case_wr(st, _f);
1741                 case_rrr(stx, _f);
1742                 case_wrr(stx, _f);
1743 #define unstr_x(r0, r1, i0)     fallback_unstr_x(r0, r1, i0)
1744             case jit_code_unstr_x:
1745                 unstr_x(rn(node->u.w), rn(node->v.w), node->w.w);
1746                 break;
1747 #define unsti_x(i0, r0, i1)     fallback_unsti_x(i0, r0, i1)
1748             case jit_code_unsti_x:
1749                 unsti_x(node->u.w, rn(node->v.w), node->w.w);
1750                 break;
1751                 case_rr(mov, _f);
1752             case jit_code_movi_f:
1753                 assert(node->flag & jit_flag_data);
1754                 movi_f(rn(node->u.w), (jit_float32_t *)node->v.n->u.w);
1755                 break;
1756                 case_brr(blt, _f);
1757                 case_brf(blt, _f, 32);
1758                 case_brr(ble, _f);
1759                 case_brf(ble, _f, 32);
1760                 case_brr(beq, _f);
1761                 case_brf(beq, _f, 32);
1762                 case_brr(bge, _f);
1763                 case_brf(bge, _f, 32);
1764                 case_brr(bgt, _f);
1765                 case_brf(bgt, _f, 32);
1766                 case_brr(bne, _f);
1767                 case_brf(bne, _f, 32);
1768                 case_brr(bunlt, _f);
1769                 case_brf(bunlt, _f, 32);
1770                 case_brr(bunle, _f);
1771                 case_brf(bunle, _f, 32);
1772                 case_brr(buneq, _f);
1773                 case_brf(buneq, _f, 32);
1774                 case_brr(bunge, _f);
1775                 case_brf(bunge, _f, 32);
1776                 case_brr(bungt, _f);
1777                 case_brf(bungt, _f, 32);
1778                 case_brr(bltgt, _f);
1779                 case_brf(bltgt, _f, 32);
1780                 case_brr(bord, _f);
1781                 case_brf(bord, _f, 32);
1782                 case_brr(bunord, _f);
1783                 case_brf(bunord, _f, 32);
1784                 case_rrr(add, _d);
1785                 case_rrf(add, _d, 64);
1786                 case_rrr(sub, _d);
1787                 case_rrf(sub, _d, 64);
1788                 case_rrf(rsb, _d, 64);
1789                 case_rrr(mul, _d);
1790                 case_rrf(mul, _d, 64);
1791                 case_rrr(div, _d);
1792                 case_rrf(div, _d, 64);
1793                 case_rr(abs, _d);
1794                 case_rr(neg, _d);
1795                 case_rr(sqrt, _d);
1796                 case_rqr(fma, _d);
1797                 case_rqr(fms, _d);
1798                 case_rqr(fnma, _d);
1799                 case_rqr(fnms, _d);
1800                 case_rr(ext, _d);
1801                 case_rr(ext, _f_d);
1802                 case_rrr(lt, _d);
1803                 case_rrf(lt, _d, 64);
1804                 case_rrr(le, _d);
1805                 case_rrf(le, _d, 64);
1806                 case_rrr(eq, _d);
1807                 case_rrf(eq, _d, 64);
1808                 case_rrr(ge, _d);
1809                 case_rrf(ge, _d, 64);
1810                 case_rrr(gt, _d);
1811                 case_rrf(gt, _d, 64);
1812                 case_rrr(ne, _d);
1813                 case_rrf(ne, _d, 64);
1814                 case_rrr(unlt, _d);
1815                 case_rrf(unlt, _d, 64);
1816                 case_rrr(unle, _d);
1817                 case_rrf(unle, _d, 64);
1818                 case_rrr(uneq, _d);
1819                 case_rrf(uneq, _d, 64);
1820                 case_rrr(unge, _d);
1821                 case_rrf(unge, _d, 64);
1822                 case_rrr(ungt, _d);
1823                 case_rrf(ungt, _d, 64);
1824                 case_rrr(ltgt, _d);
1825                 case_rrf(ltgt, _d, 64);
1826                 case_rrr(ord, _d);
1827                 case_rrf(ord, _d, 64);
1828                 case_rrr(unord, _d);
1829                 case_rrf(unord, _d, 64);
1830                 case_rr(ld, _d);
1831                 case_rw(ld, _d);
1832                 case_rrr(ldx, _d);
1833                 case_rrw(ldx, _d);
1834                 case_rr(st, _d);
1835                 case_wr(st, _d);
1836                 case_rrr(stx, _d);
1837                 case_wrr(stx, _d);
1838                 case_rr(mov, _d);
1839             case jit_code_movi_d:
1840                 assert(node->flag & jit_flag_data);
1841                 movi_d(rn(node->u.w), (jit_float64_t *)node->v.n->u.w);
1842                 break;
1843                 case_brr(blt, _d);
1844                 case_brf(blt, _d, 64);
1845                 case_brr(ble, _d);
1846                 case_brf(ble, _d, 64);
1847                 case_brr(beq, _d);
1848                 case_brf(beq, _d, 64);
1849                 case_brr(bge, _d);
1850                 case_brf(bge, _d, 64);
1851                 case_brr(bgt, _d);
1852                 case_brf(bgt, _d, 64);
1853                 case_brr(bne, _d);
1854                 case_brf(bne, _d, 64);
1855                 case_brr(bunlt, _d);
1856                 case_brf(bunlt, _d, 64);
1857                 case_brr(bunle, _d);
1858                 case_brf(bunle, _d, 64);
1859                 case_brr(buneq, _d);
1860                 case_brf(buneq, _d, 64);
1861                 case_brr(bunge, _d);
1862                 case_brf(bunge, _d, 64);
1863                 case_brr(bungt, _d);
1864                 case_brf(bungt, _d, 64);
1865                 case_brr(bltgt, _d);
1866                 case_brf(bltgt, _d, 64);
1867                 case_brr(bord, _d);
1868                 case_brf(bord, _d, 64);
1869                 case_brr(bunord, _d);
1870                 case_brf(bunord, _d, 64);
1871             case jit_code_jmpr:
1872                 jmpr(rn(node->u.w));
1873                 break;
1874             case jit_code_jmpi:
1875                 if (node->flag & jit_flag_node) {
1876                     temp = node->u.n;
1877                     assert(temp->code == jit_code_label ||
1878                            temp->code == jit_code_epilog);
1879                     if (temp->flag & jit_flag_patch)
1880                         jmpi(temp->u.w);
1881                     else {
1882                         word = _jit->code.length -
1883                             (_jit->pc.uc - _jit->code.ptr);
1884                         if (s22_p(word >> 2))
1885                             word = jmpi(_jit->pc.w);
1886                         else
1887                             word = jmpi_p(_jit->pc.w);
1888                         patch(word, node);
1889                     }
1890                 }
1891                 else
1892                     jmpi(node->u.w);
1893                 break;
1894             case jit_code_callr:
1895                 callr(rn(node->u.w));
1896                 break;
1897             case jit_code_calli:
1898                 if (node->flag & jit_flag_node) {
1899                     temp = node->u.n;
1900                     assert(temp->code == jit_code_label ||
1901                            temp->code == jit_code_epilog);
1902                     if (temp->flag & jit_flag_patch)
1903                         calli(temp->u.w);
1904                     else {
1905                         word = _jit->code.length -
1906                             (_jit->pc.uc - _jit->code.ptr);
1907                         if (s30_p(word >> 2))
1908                             word = calli(_jit->pc.w);
1909                         else
1910                             word = calli_p(_jit->pc.w);
1911                         patch(word, node);
1912                     }
1913                 }
1914                 else
1915                     calli(node->u.w);
1916                 break;
1917             case jit_code_prolog:
1918                 _jitc->function = _jitc->functions.ptr + node->w.w;
1919                 undo.node = node;
1920                 undo.word = _jit->pc.w;
1921                 memcpy(&undo.func, _jitc->function, sizeof(undo.func));
1922 #if DEVEL_DISASSEMBLER
1923                 undo.prevw = prevw;
1924 #endif
1925                 undo.patch_offset = _jitc->patches.offset;
1926             restart_function:
1927                 _jitc->again = 0;
1928                 prolog(node);
1929                 break;
1930             case jit_code_epilog:
1931                 assert(_jitc->function == _jitc->functions.ptr + node->w.w);
1932                 if (_jitc->again) {
1933                     for (temp = undo.node->next;
1934                          temp != node; temp = temp->next) {
1935                         if (temp->code == jit_code_label ||
1936                             temp->code == jit_code_epilog)
1937                             temp->flag &= ~jit_flag_patch;
1938                     }
1939                     temp->flag &= ~jit_flag_patch;
1940                     node = undo.node;
1941                     _jit->pc.w = undo.word;
1942                     /* undo.func.self.aoff and undo.func.regset should not
1943                      * be undone, as they will be further updated, and are
1944                      * the reason of the undo. */
1945                     undo.func.self.aoff = _jitc->function->frame +
1946                         _jitc->function->self.aoff;
1947                     jit_regset_set(&undo.func.regset, &_jitc->function->regset);
1948                     /* allocar information also does not need to be undone */
1949                     undo.func.aoffoff = _jitc->function->aoffoff;
1950                     undo.func.allocar = _jitc->function->allocar;
1951                     /* cvt_offset must also not be undone */
1952                     undo.func.cvt_offset = _jitc->function->cvt_offset;
1953                     memcpy(_jitc->function, &undo.func, sizeof(undo.func));
1954 #if DEVEL_DISASSEMBLER
1955                     prevw = undo.prevw;
1956 #endif
1957                     _jitc->patches.offset = undo.patch_offset;
1958                     goto restart_function;
1959                 }
1960                 /* remember label is defined */
1961                 node->flag |= jit_flag_patch;
1962                 node->u.w = _jit->pc.w;
1963                 epilog(node);
1964                 _jitc->function = NULL;
1965                 break;
1966             case jit_code_movr_w_f:
1967                 movr_w_f(rn(node->u.w), rn(node->v.w));
1968                 break;
1969             case jit_code_movr_f_w:
1970                 movr_f_w(rn(node->u.w), rn(node->v.w));
1971                 break;
1972             case jit_code_movi_f_w:
1973                 assert(node->flag & jit_flag_data);
1974                 movi_f_w(rn(node->u.w), *(jit_float32_t *)node->v.n->u.w);
1975                 break;
1976             case jit_code_movi_w_f:
1977                 movi_w_f(rn(node->u.w), node->v.w);
1978                 break;
1979 #if __WORDSIZE == 32
1980             case jit_code_movr_ww_d:
1981                 movr_ww_d(rn(node->u.w), rn(node->v.w), rn(node->w.w));
1982                 break;
1983             case jit_code_movr_d_ww:
1984                 movr_d_ww(rn(node->u.w), rn(node->v.w), rn(node->w.w));
1985                 break;
1986             case jit_code_movi_d_ww:
1987                 assert(node->flag & jit_flag_data);
1988                 movi_d_ww(rn(node->u.w), rn(node->v.w),
1989                           *(jit_float64_t *)node->w.n->u.w);
1990                 break;
1991             case jit_code_movi_ww_d:
1992                 movi_ww_d(rn(node->u.w), node->v.w, node->w.w);
1993                 break;
1994 #else
1995             case jit_code_movr_w_d:
1996                 movr_w_d(rn(node->u.w), rn(node->v.w));
1997                 break;
1998             case jit_code_movr_d_w:
1999                 movr_d_w(rn(node->u.w), rn(node->v.w));
2000                 break;
2001             case jit_code_movi_d_w:
2002                 assert(node->flag & jit_flag_data);
2003                 movi_d_w(rn(node->u.w), *(jit_float64_t *)node->v.n->u.w);
2004                 break;
2005             case jit_code_movi_w_d:
2006                 movi_w_d(rn(node->u.w), node->v.w);
2007                 break;
2008 #endif
2009             case jit_code_va_start:
2010                 vastart(rn(node->u.w));
2011                 break;
2012             case jit_code_va_arg:
2013                 vaarg(rn(node->u.w), rn(node->v.w));
2014                 break;
2015             case jit_code_va_arg_d:
2016                 vaarg_d(rn(node->u.w), rn(node->v.w));
2017                 break;
2018             case jit_code_live:                 case jit_code_ellipsis:
2019             case jit_code_va_push:
2020             case jit_code_allocai:              case jit_code_allocar:
2021             case jit_code_arg_c:                case jit_code_arg_s:
2022             case jit_code_arg_i:
2023 #if __WORDSIZE == 64
2024            case jit_code_arg_l:
2025 #endif
2026             case jit_code_arg_f:                case jit_code_arg_d:
2027             case jit_code_va_end:
2028             case jit_code_ret:
2029             case jit_code_retr_c:               case jit_code_reti_c:
2030             case jit_code_retr_uc:              case jit_code_reti_uc:
2031             case jit_code_retr_s:               case jit_code_reti_s:
2032             case jit_code_retr_us:              case jit_code_reti_us:
2033             case jit_code_retr_i:               case jit_code_reti_i:
2034 #if __WORDSIZE == 64
2035             case jit_code_retr_ui:              case jit_code_reti_ui:
2036             case jit_code_retr_l:               case jit_code_reti_l:
2037 #endif
2038             case jit_code_retr_f:               case jit_code_reti_f:
2039             case jit_code_retr_d:               case jit_code_reti_d:
2040             case jit_code_getarg_c:             case jit_code_getarg_uc:
2041             case jit_code_getarg_s:             case jit_code_getarg_us:
2042             case jit_code_getarg_i:
2043 #if __WORDSIZE == 64
2044             case jit_code_getarg_ui:            case jit_code_getarg_l:
2045 #endif
2046             case jit_code_getarg_f:             case jit_code_getarg_d:
2047             case jit_code_putargr_c:            case jit_code_putargi_c:
2048             case jit_code_putargr_uc:           case jit_code_putargi_uc:
2049             case jit_code_putargr_s:            case jit_code_putargi_s:
2050             case jit_code_putargr_us:           case jit_code_putargi_us:
2051             case jit_code_putargr_i:            case jit_code_putargi_i:
2052 #if __WORDSIZE == 64
2053             case jit_code_putargr_ui:           case jit_code_putargi_ui:
2054             case jit_code_putargr_l:            case jit_code_putargi_l:
2055 #endif
2056             case jit_code_putargr_f:            case jit_code_putargi_f:
2057             case jit_code_putargr_d:            case jit_code_putargi_d:
2058             case jit_code_pushargr_c:           case jit_code_pushargi_c:
2059             case jit_code_pushargr_uc:          case jit_code_pushargi_uc:
2060             case jit_code_pushargr_s:           case jit_code_pushargi_s:
2061             case jit_code_pushargr_us:          case jit_code_pushargi_us:
2062             case jit_code_pushargr_i:           case jit_code_pushargi_i:
2063 #if __WORDSIZE == 64
2064             case jit_code_pushargr_ui:          case jit_code_pushargi_ui:
2065             case jit_code_pushargr_l:           case jit_code_pushargi_l:
2066 #endif
2067             case jit_code_pushargr_f:           case jit_code_pushargi_f:
2068             case jit_code_pushargr_d:           case jit_code_pushargi_d:
2069             case jit_code_retval_c:             case jit_code_retval_uc:
2070             case jit_code_retval_s:             case jit_code_retval_us:
2071             case jit_code_retval_i:
2072 #if __WORDSIZE == 64
2073             case jit_code_retval_ui:            case jit_code_retval_l:
2074 #endif
2075             case jit_code_retval_f:             case jit_code_retval_d:
2076             case jit_code_prepare:
2077             case jit_code_finishr:              case jit_code_finishi:
2078             case jit_code_negi_f:               case jit_code_absi_f:
2079             case jit_code_sqrti_f:              case jit_code_negi_d:
2080             case jit_code_absi_d:               case jit_code_sqrti_d:
2081                 break;
2082             case jit_code_negi:
2083                 negi(rn(node->u.w), node->v.w);
2084                 break;
2085             case jit_code_comi:
2086                 comi(rn(node->u.w), node->v.w);
2087                 break;
2088             case jit_code_exti_c:
2089                 exti_c(rn(node->u.w), node->v.w);
2090                 break;
2091             case jit_code_exti_uc:
2092                 exti_uc(rn(node->u.w), node->v.w);
2093                 break;
2094             case jit_code_exti_s:
2095                 exti_s(rn(node->u.w), node->v.w);
2096                 break;
2097             case jit_code_exti_us:
2098                 exti_us(rn(node->u.w), node->v.w);
2099                 break;
2100             case jit_code_bswapi_us:
2101                 bswapi_us(rn(node->u.w), node->v.w);
2102                 break;
2103             case jit_code_bswapi_ui:
2104                 bswapi_ui(rn(node->u.w), node->v.w);
2105                 break;
2106             case jit_code_htoni_us:
2107                 htoni_us(rn(node->u.w), node->v.w);
2108                 break;
2109             case jit_code_htoni_ui:
2110                 htoni_ui(rn(node->u.w), node->v.w);
2111                 break;
2112 #if __WORDSIZE == 64
2113             case jit_code_exti_i:
2114                 exti_i(rn(node->u.w), node->v.w);
2115                 break;
2116             case jit_code_exti_ui:
2117                 exti_ui(rn(node->u.w), node->v.w);
2118                 break;
2119             case jit_code_bswapi_ul:
2120                 bswapi_ul(rn(node->u.w), node->v.w);
2121                 break;
2122             case jit_code_htoni_ul:
2123                 htoni_ul(rn(node->u.w), node->v.w);
2124                 break;
2125 #endif
2126             case jit_code_cloi:
2127                 cloi(rn(node->u.w), node->v.w);
2128                 break;
2129             case jit_code_clzi:
2130                 clzi(rn(node->u.w), node->v.w);
2131                 break;
2132             case jit_code_ctoi:
2133                 ctoi(rn(node->u.w), node->v.w);
2134                 break;
2135             case jit_code_ctzi:
2136                 ctzi(rn(node->u.w), node->v.w);
2137                 break;
2138             case jit_code_rbiti:
2139                 rbiti(rn(node->u.w), node->v.w);
2140                 break;
2141             case jit_code_popcnti:
2142                 popcnti(rn(node->u.w), node->v.w);
2143                 break;
2144             case jit_code_exti:
2145                 exti(rn(node->u.w), node->v.w, node->w.q.l, node->w.q.h);
2146                 break;
2147             case jit_code_exti_u:
2148                 exti_u(rn(node->u.w), node->v.w, node->w.q.l, node->w.q.h);
2149                 break;
2150             default:
2151                 abort();
2152         }
2153 #  if __WORDSIZE == 64
2154         if (jit_carry != _NOREG) {
2155             switch (node->code) {
2156                 case jit_code_note:
2157                 case jit_code_addcr:            case jit_code_addci:
2158                 case jit_code_addxr:            case jit_code_addxi:
2159                 case jit_code_subcr:            case jit_code_subci:
2160                 case jit_code_subxr:            case jit_code_subxi:
2161                     break;
2162                 default:
2163                     jit_unget_reg(jit_carry);
2164                     jit_carry = _NOREG;
2165                     break;
2166             }
2167         }
2168 #  endif
2169         jit_regarg_clr(node, value);
2170 #  if __WORDSIZE == 64
2171         if (jit_regset_cmp_ui(&_jitc->regarg, 0) != 0) {
2172             assert(jit_regset_scan1(&_jitc->regarg, 0) == jit_carry);
2173             assert(jit_regset_scan1(&_jitc->regarg, jit_carry + 1) == ULONG_MAX);
2174         }
2175         assert(_jitc->synth == 0);
2176 #  else
2177         assert(_jitc->regarg == 0 && _jitc->synth == 0);
2178 #  endif
2179         jit_reglive(node);
2180     }
2181 #undef case_brf
2182 #undef case_brw
2183 #undef case_brr
2184 #undef case_wrr
2185 #undef case_rrf
2186 #undef case_rrrw
2187 #undef case_rrw
2188 #undef case_rrrr
2189 #undef case_rrr
2190 #undef case_rf
2191 #undef case_wr
2192 #undef case_rw
2193 #undef case_rr
2194
2195     for (offset = 0; offset < _jitc->patches.offset; offset++) {
2196         node = _jitc->patches.ptr[offset].node;
2197         word = node->code == jit_code_movi ? node->v.n->u.w : node->u.n->u.w;
2198         patch_at(_jitc->patches.ptr[offset].inst, word);
2199     }
2200
2201     jit_flush(_jit->code.ptr, _jit->pc.uc);
2202
2203     return (_jit->code.ptr);
2204 }
2205
2206 #define CODE                            1
2207 #  include "jit_sparc-cpu.c"
2208 #  include "jit_sparc-fpu.c"
2209 #  include "jit_fallback.c"
2210 #undef CODE
2211
2212 void
2213 jit_flush(void *fptr, void *tptr)
2214 {
2215 }
2216
2217 void
2218 _emit_ldxi(jit_state_t *_jit, jit_gpr_t r0, jit_gpr_t r1, jit_word_t i0)
2219 {
2220     ldxi(rn(r0), rn(r1), i0);
2221 }
2222
2223 void
2224 _emit_stxi(jit_state_t *_jit, jit_word_t i0, jit_gpr_t r0, jit_gpr_t r1)
2225 {
2226     stxi(i0, rn(r0), rn(r1));
2227 }
2228
2229 void
2230 _emit_ldxi_d(jit_state_t *_jit, jit_fpr_t r0, jit_gpr_t r1, jit_word_t i0)
2231 {
2232     ldxi_d(rn(r0), rn(r1), i0);
2233 }
2234
2235 void
2236 _emit_stxi_d(jit_state_t *_jit, jit_word_t i0, jit_gpr_t r0, jit_fpr_t r1)
2237 {
2238     stxi_d(i0, rn(r0), rn(r1));
2239 }
2240
2241 static void
2242 _patch(jit_state_t *_jit, jit_word_t instr, jit_node_t *node)
2243 {
2244     jit_int32_t         flag;
2245
2246     assert(node->flag & jit_flag_node);
2247     if (node->code == jit_code_movi)
2248         flag = node->v.n->flag;
2249     else
2250         flag = node->u.n->flag;
2251     assert(!(flag & jit_flag_patch));
2252     if (_jitc->patches.offset >= _jitc->patches.length) {
2253         jit_realloc((jit_pointer_t *)&_jitc->patches.ptr,
2254                     _jitc->patches.length * sizeof(jit_patch_t),
2255                     (_jitc->patches.length + 1024) * sizeof(jit_patch_t));
2256         memset(_jitc->patches.ptr + _jitc->patches.length, 0,
2257                1024 * sizeof(jit_patch_t));
2258         _jitc->patches.length += 1024;
2259     }
2260     _jitc->patches.ptr[_jitc->patches.offset].inst = instr;
2261     _jitc->patches.ptr[_jitc->patches.offset].node = node;
2262     ++_jitc->patches.offset;
2263 }