deps/lightning/lib/jit_alpha.c

   1 /*
   2  * Copyright (C) 2014-2019  Free Software Foundation, Inc.
   3  *
   4  * This file is part of GNU lightning.
   5  *
   6  * GNU lightning is free software; you can redistribute it and/or modify it
   7  * under the terms of the GNU Lesser General Public License as published
   8  * by the Free Software Foundation; either version 3, or (at your option)
   9  * any later version.
  10  *
  11  * GNU lightning is distributed in the hope that it will be useful, but
  12  * WITHOUT ANY WARRANTY; without even the implied warranty of MERCHANTABILITY
  13  * or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU Lesser General Public
  14  * License for more details.
  15  *
  16  * Authors:
  17  *      Paulo Cesar Pereira de Andrade
  18  */
  19
  20 #define jit_arg_reg_p(i)                ((i) >= 0 && (i) < 6)
  21 #define jit_arg_f_reg_p(i)              ((i) >= 0 && (i) < 6)
  22 #if __BYTE_ORDER == __LITTLE_ENDIAN
  23 #  define C_DISP                        0
  24 #  define S_DISP                        0
  25 #  define I_DISP                        0
  26 #  define F_DISP                        0
  27 #else
  28 #  define C_DISP                        8 - sizeof(jit_int8_t)
  29 #  define S_DISP                        8 - sizeof(jit_int16_t)
  30 #  define I_DISP                        8 - sizeof(jit_int32_t)
  31 #  define F_DISP                        8 - sizeof(jit_float32_t)
  32 #endif
  33
  34 /*
  35  * Types
  36  */
  37 /*
  38  * What I could understand from gcc/config/alpha/alpha.c:alpha_build_builtin_va_list()
  39  * and other helpers, as well as objdump of simple test programs; could not
  40  * get gdb working on the test system I had access...
  41  *
  42  * base-48 to base is where up to 6 float registers are saved.
  43  * base to base+48 is where up to 6 integer registers are saved.
  44  * base+48... is where varargs arguments are stored.
  45  *
  46  *      if (offset < 48) {
  47  *              if (type == double)
  48  *                      offset -= 48;
  49  *      }
  50  *      load(reg, base, offset);
  51  *      offset += 8;
  52  */
  53 typedef struct jit_va_list {
  54     jit_pointer_t       base;
  55     jit_word_t          offset;
  56 } jit_va_list_t;
  57
  58 /*
  59  * Prototypes
  60  */
  61 #define patch(instr, node)              _patch(_jit, instr, node)
  62 static void _patch(jit_state_t*,jit_word_t,jit_node_t*);
  63
  64 #define PROTO                           1
  65 #  include "jit_alpha-cpu.c"
  66 #  include "jit_alpha-fpu.c"
  67 #  include "jit_fallback.c"
  68 #undef PROTO
  69
  70 /*
  71  * Initialization
  72  */
  73 jit_register_t          _rvs[] = {
  74     { rc(gpr) | 0x1c,                   "at" },
  75     { rc(gpr) | 0x00,                   "v0" },
  76     { rc(gpr) | 0x01,                   "t0" },
  77     { rc(gpr) | 0x02,                   "t1" },
  78     { rc(gpr) | 0x03,                   "t2" },
  79     { rc(gpr) | 0x04,                   "t3" },
  80     { rc(gpr) | 0x05,                   "t4" },
  81     { rc(gpr) | 0x06,                   "t5" },
  82     { rc(gpr) | 0x07,                   "t6" },
  83     { rc(gpr) | 0x08,                   "t7" },
  84     { rc(gpr) | 0x16,                   "t8" },
  85     { rc(gpr) | 0x17,                   "t9" },
  86     { rc(gpr) | 0x18,                   "t10" },
  87     { rc(gpr) | 0x19,                   "t11" },
  88     { rc(sav) | rc(gpr) | 0x09,         "s0" },
  89     { rc(sav) | rc(gpr) | 0x0a,         "s1" },
  90     { rc(sav) | rc(gpr) | 0x0b,         "s2" },
  91     { rc(sav) | rc(gpr) | 0x0c,         "s3" },
  92     { rc(sav) | rc(gpr) | 0x0d,         "s4" },
  93     { rc(sav) | rc(gpr) | 0x0e,         "s5" },
  94     { 0x0f,                             "fp" },
  95     { rc(arg) | rc(gpr) | 0x15,         "a5" },
  96     { rc(arg) | rc(gpr) | 0x14,         "a4" },
  97     { rc(arg) | rc(gpr) | 0x13,         "a3" },
  98     { rc(arg) | rc(gpr) | 0x12,         "a2" },
  99     { rc(arg) | rc(gpr) | 0x11,         "a1" },
 100     { rc(arg) | rc(gpr) | 0x10,         "a0" },
 101     { 0x1a,                             "ra" },
 102     { 0x1b,                             "pv" },
 103     { 0x1d,                             "gp" },
 104     { 0x1e,                             "sp" },
 105     { 0x1f,                             "zero" },
 106     { rc(fpr) | 0x00,                   "$f0" },
 107     { rc(fpr) | 0x01,                   "$f1" },
 108     { rc(sav) | rc(fpr) | 0x02,         "$f2" },
 109     { rc(sav) | rc(fpr) | 0x03,         "$f3" },
 110     { rc(sav) | rc(fpr) | 0x04,         "$f4" },
 111     { rc(sav) | rc(fpr) | 0x05,         "$f5" },
 112     { rc(sav) | rc(fpr) | 0x06,         "$f6" },
 113     { rc(sav) | rc(fpr) | 0x07,         "$f7" },
 114     { rc(sav) | rc(fpr) | 0x08,         "$f8" },
 115     { rc(sav) | rc(fpr) | 0x09,         "$f9" },
 116     { rc(fpr) | 0x0a,                   "$f10" },
 117     { rc(fpr) | 0x0b,                   "$f11" },
 118     { rc(fpr) | 0x0c,                   "$f12" },
 119     { rc(fpr) | 0x0d,                   "$f13" },
 120     { rc(fpr) | 0x0e,                   "$f14" },
 121     { rc(fpr) | 0x0f,                   "$f15" },
 122     { rc(arg) | rc(fpr) | 0x15,         "$f21" },
 123     { rc(arg) | rc(fpr) | 0x14,         "$f20" },
 124     { rc(arg) | rc(fpr) | 0x13,         "$f19" },
 125     { rc(arg) | rc(fpr) | 0x12,         "$f18" },
 126     { rc(arg) | rc(fpr) | 0x11,         "$f17" },
 127     { rc(arg) | rc(fpr) | 0x10,         "$f16" },
 128     { rc(fpr) | 0x16,                   "$f22" },
 129     { rc(fpr) | 0x17,                   "$f23" },
 130     { rc(fpr) | 0x18,                   "$f24" },
 131     { rc(fpr) | 0x19,                   "$f25" },
 132     { rc(fpr) | 0x1a,                   "$f26" },
 133     { rc(fpr) | 0x1b,                   "$f27" },
 134     { rc(fpr) | 0x1c,                   "$f28" },
 135     { rc(fpr) | 0x1d,                   "$f29" },
 136     { rc(fpr) | 0x1e,                   "$f30" },
 137     { 0x1f,                             "$f31" },
 138     { _NOREG,                           "<none>" },
 139 };
 140
 141 /*
 142  * Implementation
 143  */
 144 void
 145 jit_get_cpu(void)
 146 {
 147 }
 148
 149 void
 150 _jit_init(jit_state_t *_jit)
 151 {
 152     _jitc->reglen = jit_size(_rvs) - 1;
 153     jit_carry = _NOREG;
 154 }
 155
 156 void
 157 _jit_prolog(jit_state_t *_jit)
 158 {
 159     jit_int32_t          offset;
 160
 161     if (_jitc->function)
 162         jit_epilog();
 163     assert(jit_regset_cmp_ui(&_jitc->regarg, 0) == 0);
 164     jit_regset_set_ui(&_jitc->regsav, 0);
 165     offset = _jitc->functions.offset;
 166     if (offset >= _jitc->functions.length) {
 167         jit_realloc((jit_pointer_t *)&_jitc->functions.ptr,
 168                     _jitc->functions.length * sizeof(jit_function_t),
 169                     (_jitc->functions.length + 16) * sizeof(jit_function_t));
 170         _jitc->functions.length += 16;
 171     }
 172     _jitc->function = _jitc->functions.ptr + _jitc->functions.offset++;
 173     _jitc->function->self.size = stack_framesize;
 174     _jitc->function->self.argi = _jitc->function->self.alen = 0;
 175     /* float conversion */
 176     _jitc->function->self.aoff = -8;
 177     _jitc->function->self.call = jit_call_default;
 178     jit_alloc((jit_pointer_t *)&_jitc->function->regoff,
 179               _jitc->reglen * sizeof(jit_int32_t));
 180
 181     /* _no_link here does not mean the jit_link() call can be removed
 182      * by rewriting as:
 183      * _jitc->function->prolog = jit_new_node(jit_code_prolog);
 184      */
 185     _jitc->function->prolog = jit_new_node_no_link(jit_code_prolog);
 186     jit_link(_jitc->function->prolog);
 187     _jitc->function->prolog->w.w = offset;
 188     _jitc->function->epilog = jit_new_node_no_link(jit_code_epilog);
 189     /*  u:      label value
 190      *  v:      offset in blocks vector
 191      *  w:      offset in functions vector
 192      */
 193     _jitc->function->epilog->w.w = offset;
 194
 195     jit_regset_new(&_jitc->function->regset);
 196 }
 197
 198 jit_int32_t
 199 _jit_allocai(jit_state_t *_jit, jit_int32_t length)
 200 {
 201     assert(_jitc->function != NULL);
 202     switch (length) {
 203         case 0: case 1:                                         break;
 204         case 2:         _jitc->function->self.aoff &= -2;       break;
 205         case 3: case 4: _jitc->function->self.aoff &= -4;       break;
 206         default:        _jitc->function->self.aoff &= -8;       break;
 207     }
 208     _jitc->function->self.aoff -= length;
 209     if (!_jitc->realize) {
 210         jit_inc_synth_ww(allocai, _jitc->function->self.aoff, length);
 211         jit_dec_synth();
 212     }
 213     return (_jitc->function->self.aoff);
 214 }
 215
 216 void
 217 _jit_allocar(jit_state_t *_jit, jit_int32_t u, jit_int32_t v)
 218 {
 219     jit_int32_t          reg;
 220     assert(_jitc->function != NULL);
 221     jit_inc_synth_ww(allocar, u, v);
 222     if (!_jitc->function->allocar) {
 223         _jitc->function->aoffoff = jit_allocai(sizeof(jit_int32_t));
 224         _jitc->function->allocar = 1;
 225     }
 226     reg = jit_get_reg(jit_class_gpr);
 227     jit_negr(reg, v);
 228     jit_andi(reg, reg, -8);
 229     jit_ldxi_i(u, JIT_FP, _jitc->function->aoffoff);
 230     jit_addr(u, u, reg);
 231     jit_addr(JIT_SP, JIT_SP, reg);
 232     jit_stxi_i(_jitc->function->aoffoff, JIT_FP, u);
 233     jit_unget_reg(reg);
 234     jit_dec_synth();
 235 }
 236
 237 void
 238 _jit_ret(jit_state_t *_jit)
 239 {
 240     jit_node_t          *instr;
 241     assert(_jitc->function != NULL);
 242     jit_inc_synth(ret);
 243     /* jump to epilog */
 244     instr = jit_jmpi();
 245     jit_patch_at(instr, _jitc->function->epilog);
 246     jit_dec_synth();
 247 }
 248
 249 void
 250 _jit_retr(jit_state_t *_jit, jit_int32_t u)
 251 {
 252     jit_inc_synth_w(retr, u);
 253     if (JIT_RET != u)
 254         jit_movr(JIT_RET, u);
 255     jit_live(JIT_RET);
 256     jit_ret();
 257     jit_dec_synth();
 258 }
 259
 260 void
 261 _jit_reti(jit_state_t *_jit, jit_word_t u)
 262 {
 263     jit_inc_synth_w(reti, u);
 264     jit_movi(JIT_RET, u);
 265     jit_ret();
 266     jit_dec_synth();
 267 }
 268
 269 void
 270 _jit_retr_f(jit_state_t *_jit, jit_int32_t u)
 271 {
 272     jit_inc_synth_w(retr_f, u);
 273     if (u != JIT_FRET)
 274         jit_movr_f(JIT_FRET, u);
 275     else
 276         jit_live(JIT_FRET);
 277     jit_ret();
 278     jit_dec_synth();
 279 }
 280
 281 void
 282 _jit_reti_f(jit_state_t *_jit, jit_float32_t u)
 283 {
 284     jit_inc_synth_f(reti_f, u);
 285     jit_movi_f(JIT_FRET, u);
 286     jit_ret();
 287     jit_dec_synth();
 288 }
 289
 290 void
 291 _jit_retr_d(jit_state_t *_jit, jit_int32_t u)
 292 {
 293     jit_inc_synth_w(retr_d, u);
 294     if (u != JIT_FRET)
 295         jit_movr_d(JIT_FRET, u);
 296     else
 297         jit_live(JIT_FRET);
 298     jit_ret();
 299     jit_dec_synth();
 300 }
 301
 302 void
 303 _jit_reti_d(jit_state_t *_jit, jit_float64_t u)
 304 {
 305     jit_inc_synth_d(reti_d, u);
 306     jit_movi_d(JIT_FRET, u);
 307     jit_ret();
 308     jit_dec_synth();
 309 }
 310
 311 void
 312 _jit_epilog(jit_state_t *_jit)
 313 {
 314     assert(_jitc->function != NULL);
 315     assert(_jitc->function->epilog->next == NULL);
 316     jit_link(_jitc->function->epilog);
 317     _jitc->function = NULL;
 318 }
 319
 320 jit_bool_t
 321 _jit_arg_register_p(jit_state_t *_jit, jit_node_t *u)
 322 {
 323     if (u->code == jit_code_arg)
 324         return (jit_arg_reg_p(u->u.w));
 325     assert(u->code == jit_code_arg_f || u->code == jit_code_arg_d);
 326     return (jit_arg_f_reg_p(u->u.w));
 327 }
 328
 329 void
 330 _jit_ellipsis(jit_state_t *_jit)
 331 {
 332     jit_inc_synth(ellipsis);
 333     if (_jitc->prepare) {
 334         jit_link_prepare();
 335         assert(!(_jitc->function->call.call & jit_call_varargs));
 336         _jitc->function->call.call |= jit_call_varargs;
 337     }
 338     else {
 339         jit_link_prolog();
 340         assert(!(_jitc->function->self.call & jit_call_varargs));
 341         _jitc->function->self.call |= jit_call_varargs;
 342
 343         /* Allocate va_list like object in the stack */
 344         _jitc->function->vaoff = jit_allocai(sizeof(jit_va_list_t));
 345         _jitc->function->vagp = _jitc->function->self.argi;
 346     }
 347     jit_dec_synth();
 348 }
 349
 350 void
 351 _jit_va_push(jit_state_t *_jit, jit_int32_t u)
 352 {
 353     jit_int32_t         reg;
 354     jit_inc_synth_w(va_push, u);
 355     reg = jit_get_reg(jit_class_gpr);
 356     jit_ldxi(reg, u, offsetof(jit_va_list_t, base));
 357     jit_pushargr(reg);
 358     jit_ldxi(reg, u, offsetof(jit_va_list_t, offset));
 359     jit_pushargr(reg);
 360     jit_unget_reg(reg);
 361     jit_dec_synth();
 362 }
 363
 364 jit_node_t *
 365 _jit_arg(jit_state_t *_jit)
 366 {
 367     jit_node_t          *node;
 368     jit_int32_t          offset;
 369     assert(_jitc->function != NULL);
 370     if (jit_arg_reg_p(_jitc->function->self.argi))
 371         offset = _jitc->function->self.argi++;
 372     else {
 373         offset = _jitc->function->self.size;
 374         _jitc->function->self.size += 8;
 375     }
 376     node = jit_new_node_ww(jit_code_arg, offset,
 377                            ++_jitc->function->self.argn);
 378     jit_link_prolog();
 379     return (node);
 380 }
 381
 382 jit_node_t *
 383 _jit_arg_f(jit_state_t *_jit)
 384 {
 385     jit_node_t          *node;
 386     jit_int32_t          offset;
 387     assert(_jitc->function != NULL);
 388     if (jit_arg_f_reg_p(_jitc->function->self.argi))
 389         offset = _jitc->function->self.argi++;
 390     else {
 391         offset = _jitc->function->self.size;
 392         _jitc->function->self.size += 8;
 393     }
 394     node = jit_new_node_ww(jit_code_arg_f, offset,
 395                            ++_jitc->function->self.argn);
 396     jit_link_prolog();
 397     return (node);
 398 }
 399
 400 jit_node_t *
 401 _jit_arg_d(jit_state_t *_jit)
 402 {
 403     jit_node_t          *node;
 404     jit_int32_t          offset;
 405     assert(_jitc->function != NULL);
 406     if (jit_arg_f_reg_p(_jitc->function->self.argi))
 407         offset = _jitc->function->self.argi++;
 408     else {
 409         offset = _jitc->function->self.size;
 410         _jitc->function->self.size += 8;
 411     }
 412     node = jit_new_node_ww(jit_code_arg_d, offset,
 413                            ++_jitc->function->self.argn);
 414     jit_link_prolog();
 415     return (node);
 416 }
 417
 418 void
 419 _jit_getarg_c(jit_state_t *_jit, jit_int32_t u, jit_node_t *v)
 420 {
 421     assert(v->code == jit_code_arg);
 422     jit_inc_synth_wp(getarg_c, u, v);
 423     if (jit_arg_reg_p(v->u.w))
 424         jit_extr_c(u, _A0 - v->u.w);
 425     else
 426         jit_ldxi_c(u, _FP, v->u.w + C_DISP);
 427     jit_dec_synth();
 428 }
 429
 430 void
 431 _jit_getarg_uc(jit_state_t *_jit, jit_int32_t u, jit_node_t *v)
 432 {
 433     assert(v->code == jit_code_arg);
 434     jit_inc_synth_wp(getarg_uc, u, v);
 435     if (jit_arg_reg_p(v->u.w))
 436         jit_extr_uc(u, _A0 - v->u.w);
 437     else
 438         jit_ldxi_uc(u, _FP, v->u.w + C_DISP);
 439     jit_dec_synth();
 440 }
 441
 442 void
 443 _jit_getarg_s(jit_state_t *_jit, jit_int32_t u, jit_node_t *v)
 444 {
 445     assert(v->code == jit_code_arg);
 446     jit_inc_synth_wp(getarg_s, u, v);
 447     if (jit_arg_reg_p(v->u.w))
 448         jit_extr_s(u, _A0 - v->u.w);
 449     else
 450         jit_ldxi_s(u, _FP, v->u.w + S_DISP);
 451     jit_dec_synth();
 452 }
 453
 454 void
 455 _jit_getarg_us(jit_state_t *_jit, jit_int32_t u, jit_node_t *v)
 456 {
 457     assert(v->code == jit_code_arg);
 458     jit_inc_synth_wp(getarg_us, u, v);
 459     if (jit_arg_reg_p(v->u.w))
 460         jit_extr_us(u, _A0 - v->u.w);
 461     else
 462         jit_ldxi_us(u, _FP, v->u.w + S_DISP);
 463     jit_dec_synth();
 464 }
 465
 466 void
 467 _jit_getarg_i(jit_state_t *_jit, jit_int32_t u, jit_node_t *v)
 468 {
 469     assert(v->code == jit_code_arg);
 470     jit_inc_synth_wp(getarg_i, u, v);
 471     if (jit_arg_reg_p(v->u.w))
 472         jit_extr_i(u, _A0 - v->u.w);
 473     else
 474         jit_ldxi_i(u, _FP, v->u.w + I_DISP);
 475     jit_dec_synth();
 476 }
 477
 478 void
 479 _jit_getarg_ui(jit_state_t *_jit, jit_int32_t u, jit_node_t *v)
 480 {
 481     assert(v->code == jit_code_arg);
 482     jit_inc_synth_wp(getarg_ui, u, v);
 483     if (jit_arg_reg_p(v->u.w))
 484         jit_extr_ui(u, _A0 - v->u.w);
 485     else
 486         jit_ldxi_ui(u, _FP, v->u.w + I_DISP);
 487     jit_dec_synth();
 488 }
 489
 490 void
 491 _jit_getarg_l(jit_state_t *_jit, jit_int32_t u, jit_node_t *v)
 492 {
 493     assert(v->code == jit_code_arg);
 494     jit_inc_synth_wp(getarg_l, u, v);
 495     if (jit_arg_reg_p(v->u.w))
 496         jit_movr(u, _A0 - v->u.w);
 497     else
 498         jit_ldxi_l(u, _FP, v->u.w);
 499     jit_dec_synth();
 500 }
 501
 502 void
 503 _jit_putargr(jit_state_t *_jit, jit_int32_t u, jit_node_t *v)
 504 {
 505     assert(v->code == jit_code_arg);
 506     jit_inc_synth_wp(putargr, u, v);
 507     if (jit_arg_reg_p(v->u.w))
 508         jit_movr(_A0 - v->u.w, u);
 509     else
 510         jit_stxi(v->u.w, _FP, u);
 511     jit_dec_synth();
 512 }
 513
 514 void
 515 _jit_putargi(jit_state_t *_jit, jit_word_t u, jit_node_t *v)
 516 {
 517     jit_int32_t         regno;
 518     assert(v->code == jit_code_arg);
 519     jit_inc_synth_wp(putargi, u, v);
 520     if (jit_arg_reg_p(v->u.w))
 521         jit_movi(_A0 - v->u.w, u);
 522     else {
 523         regno = jit_get_reg(jit_class_gpr);
 524         jit_movi(regno, u);
 525         jit_stxi(v->u.w, _FP, regno);
 526         jit_unget_reg(regno);
 527     }
 528     jit_dec_synth();
 529 }
 530
 531 void
 532 _jit_getarg_f(jit_state_t *_jit, jit_int32_t u, jit_node_t *v)
 533 {
 534     assert(v->code == jit_code_arg_f);
 535     jit_inc_synth_wp(getarg_f, u, v);
 536     if (jit_arg_f_reg_p(v->u.w))
 537         jit_movr_f(u, _F16 - v->u.w);
 538     else
 539         jit_ldxi_f(u, _FP, v->u.w + F_DISP);
 540     jit_dec_synth();
 541 }
 542
 543 void
 544 _jit_putargr_f(jit_state_t *_jit, jit_int32_t u, jit_node_t *v)
 545 {
 546     assert(v->code == jit_code_arg_f);
 547     jit_inc_synth_wp(putargr_f, u, v);
 548     if (jit_arg_f_reg_p(v->u.w))
 549         jit_movr_f(_F16 - v->u.w, u);
 550     else
 551         jit_stxi_f(v->u.w, _FP, u + F_DISP);
 552     jit_dec_synth();
 553 }
 554
 555 void
 556 _jit_putargi_f(jit_state_t *_jit, jit_float32_t u, jit_node_t *v)
 557 {
 558     jit_int32_t         regno;
 559     assert(v->code == jit_code_arg_f);
 560     jit_inc_synth_fp(putargi_f, u, v);
 561     if (jit_arg_f_reg_p(v->u.w))
 562         jit_movi_f(_F16 - v->u.w, u);
 563     else {
 564         regno = jit_get_reg(jit_class_fpr);
 565         jit_movi_f(regno, u);
 566         jit_stxi_f(v->u.w, _FP, regno + F_DISP);
 567         jit_unget_reg(regno);
 568     }
 569     jit_dec_synth();
 570 }
 571
 572 void
 573 _jit_getarg_d(jit_state_t *_jit, jit_int32_t u, jit_node_t *v)
 574 {
 575     assert(v->code == jit_code_arg_d);
 576     jit_inc_synth_wp(getarg_d, u, v);
 577     if (jit_arg_f_reg_p(v->u.w))
 578         jit_movr_d(u, _F16 - v->u.w);
 579     else
 580         jit_ldxi_d(u, _FP, v->u.w);
 581     jit_dec_synth();
 582 }
 583
 584 void
 585 _jit_putargr_d(jit_state_t *_jit, jit_int32_t u, jit_node_t *v)
 586 {
 587     assert(v->code == jit_code_arg_d);
 588     jit_inc_synth_wp(putargr_d, u, v);
 589     if (jit_arg_f_reg_p(v->u.w))
 590         jit_movr_d(_F16 - v->u.w, u);
 591     else
 592         jit_stxi_d(v->u.w, _FP, u);
 593     jit_dec_synth();
 594 }
 595
 596 void
 597 _jit_putargi_d(jit_state_t *_jit, jit_float64_t u, jit_node_t *v)
 598 {
 599     jit_int32_t         regno;
 600     assert(v->code == jit_code_arg_d);
 601     jit_inc_synth_dp(putargi_d, u, v);
 602     if (jit_arg_f_reg_p(v->u.w))
 603         jit_movi_d(_F16 - v->u.w, u);
 604     else {
 605         regno = jit_get_reg(jit_class_fpr);
 606         jit_movi_d(regno, u);
 607         jit_stxi_d(v->u.w, _FP, regno);
 608         jit_unget_reg(regno);
 609     }
 610     jit_dec_synth();
 611 }
 612
 613 void
 614 _jit_pushargr(jit_state_t *_jit, jit_int32_t u)
 615 {
 616     assert(_jitc->function != NULL);
 617     jit_inc_synth_w(pushargr, u);
 618     jit_link_prepare();
 619     if (jit_arg_reg_p(_jitc->function->call.argi)) {
 620         jit_movr(_A0 - _jitc->function->call.argi, u);
 621         ++_jitc->function->call.argi;
 622     }
 623     else {
 624         jit_stxi(_jitc->function->call.size, JIT_SP, u);
 625         _jitc->function->call.size += 8;
 626     }
 627     jit_dec_synth();
 628 }
 629
 630 void
 631 _jit_pushargi(jit_state_t *_jit, jit_int64_t u)
 632 {
 633     jit_int32_t         regno;
 634     assert(_jitc->function != NULL);
 635     jit_inc_synth_w(pushargi, u);
 636     jit_link_prepare();
 637     if (jit_arg_reg_p(_jitc->function->call.argi)) {
 638         jit_movi(_A0 - _jitc->function->call.argi, u);
 639         ++_jitc->function->call.argi;
 640     }
 641     else {
 642         regno = jit_get_reg(jit_class_gpr);
 643         jit_movi(regno, u);
 644         jit_stxi(_jitc->function->call.size, JIT_SP, regno);
 645         _jitc->function->call.size += 8;
 646         jit_unget_reg(regno);
 647     }
 648     jit_dec_synth();
 649 }
 650
 651 void
 652 _jit_pushargr_f(jit_state_t *_jit, jit_int32_t u)
 653 {
 654     assert(_jitc->function != NULL);
 655     jit_inc_synth_w(pushargr_f, u);
 656     jit_link_prepare();
 657     if (jit_arg_f_reg_p(_jitc->function->call.argi)) {
 658         jit_movr_f(_F16 - _jitc->function->call.argi, u);
 659         ++_jitc->function->call.argi;
 660     }
 661     else {
 662         jit_stxi_f(_jitc->function->call.size + F_DISP, JIT_SP, u);
 663         _jitc->function->call.size += 8;
 664     }
 665     jit_dec_synth();
 666 }
 667
 668 void
 669 _jit_pushargi_f(jit_state_t *_jit, jit_float32_t u)
 670 {
 671     jit_int32_t         regno;
 672     assert(_jitc->function != NULL);
 673     jit_inc_synth_f(pushargi_f, u);
 674     jit_link_prepare();
 675     if (jit_arg_f_reg_p(_jitc->function->call.argi)) {
 676         jit_movi_f(_F16 - _jitc->function->call.argi, u);
 677         ++_jitc->function->call.argi;
 678     }
 679     else {
 680         regno = jit_get_reg(jit_class_fpr);
 681         jit_movi_f(regno, u);
 682         jit_stxi_f(_jitc->function->call.size + F_DISP, JIT_SP, regno);
 683         _jitc->function->call.size += 8;
 684         jit_unget_reg(regno);
 685     }
 686     jit_dec_synth();
 687 }
 688
 689 void
 690 _jit_pushargr_d(jit_state_t *_jit, jit_int32_t u)
 691 {
 692     assert(_jitc->function != NULL);
 693     jit_inc_synth_w(pushargr_d, u);
 694     jit_link_prepare();
 695     if (jit_arg_f_reg_p(_jitc->function->call.argi)) {
 696         jit_movr_d(_F16 - _jitc->function->call.argi, u);
 697         ++_jitc->function->call.argi;
 698     }
 699     else {
 700         jit_stxi_d(_jitc->function->call.size, JIT_SP, u);
 701         _jitc->function->call.size += 8;
 702     }
 703     jit_dec_synth();
 704 }
 705
 706 void
 707 _jit_pushargi_d(jit_state_t *_jit, jit_float64_t u)
 708 {
 709     jit_int32_t         regno;
 710     assert(_jitc->function != NULL);
 711     jit_inc_synth_d(pushargi_d, u);
 712     jit_link_prepare();
 713     if (jit_arg_f_reg_p(_jitc->function->call.argi)) {
 714         jit_movi_d(_F16 - _jitc->function->call.argi, u);
 715         ++_jitc->function->call.argi;
 716     }
 717     else {
 718         regno = jit_get_reg(jit_class_fpr);
 719         jit_movi_d(regno, u);
 720         jit_stxi_d(_jitc->function->call.size, JIT_SP, regno);
 721         _jitc->function->call.size += 8;
 722         jit_unget_reg(regno);
 723     }
 724     jit_dec_synth();
 725 }
 726
 727 jit_bool_t
 728 _jit_regarg_p(jit_state_t *_jit, jit_node_t *node, jit_int32_t regno)
 729 {
 730     jit_int32_t         spec;
 731
 732     spec = jit_class(_rvs[regno].spec);
 733     if (spec & jit_class_arg) {
 734         if (spec & jit_class_gpr) {
 735             regno = _A0 - regno;
 736             if (regno >= 0 && regno < node->v.w)
 737                 return (1);
 738         }
 739         else if (spec & jit_class_fpr) {
 740             regno = _F16 - regno;
 741             if (regno >= 0 && regno < node->w.w)
 742                 return (1);
 743         }
 744     }
 745
 746     return (0);
 747 }
 748
 749 void
 750 _jit_finishr(jit_state_t *_jit, jit_int32_t r0)
 751 {
 752     jit_node_t          *call;
 753     assert(_jitc->function != NULL);
 754     jit_inc_synth_w(finishr, r0);
 755     if (_jitc->function->self.alen < _jitc->function->call.size)
 756         _jitc->function->self.alen = _jitc->function->call.size;
 757     call = jit_callr(r0);
 758     call->v.w = call->w.w = _jitc->function->self.argi;
 759     _jitc->function->call.argi = _jitc->function->call.size = 0;
 760     _jitc->prepare = 0;
 761     jit_dec_synth();
 762 }
 763
 764 jit_node_t *
 765 _jit_finishi(jit_state_t *_jit, jit_pointer_t i0)
 766 {
 767     jit_node_t          *call;
 768     assert(_jitc->function != NULL);
 769     jit_inc_synth_w(finishi, (jit_word_t)i0);
 770     if (_jitc->function->self.alen < _jitc->function->call.size)
 771         _jitc->function->self.alen = _jitc->function->call.size;
 772     call = jit_calli(i0);
 773     call->v.w = call->w.w = _jitc->function->self.argf;
 774     _jitc->function->call.argi = _jitc->function->call.size = 0;
 775     _jitc->prepare = 0;
 776     jit_dec_synth();
 777     return (call);
 778 }
 779
 780 void
 781 _jit_retval_c(jit_state_t *_jit, jit_int32_t r0)
 782 {
 783     jit_inc_synth_w(retval_c, r0);
 784     jit_extr_c(r0, JIT_RET);
 785     jit_dec_synth();
 786 }
 787
 788 void
 789 _jit_retval_uc(jit_state_t *_jit, jit_int32_t r0)
 790 {
 791     jit_inc_synth_w(retval_uc, r0);
 792     jit_extr_uc(r0, JIT_RET);
 793     jit_dec_synth();
 794 }
 795
 796 void
 797 _jit_retval_s(jit_state_t *_jit, jit_int32_t r0)
 798 {
 799     jit_inc_synth_w(retval_s, r0);
 800     jit_extr_s(r0, JIT_RET);
 801     jit_dec_synth();
 802 }
 803
 804 void
 805 _jit_retval_us(jit_state_t *_jit, jit_int32_t r0)
 806 {
 807     jit_inc_synth_w(retval_us, r0);
 808     jit_extr_us(r0, JIT_RET);
 809     jit_dec_synth();
 810 }
 811
 812 void
 813 _jit_retval_i(jit_state_t *_jit, jit_int32_t r0)
 814 {
 815     jit_inc_synth_w(retval_i, r0);
 816     jit_extr_i(r0, JIT_RET);
 817     jit_dec_synth();
 818 }
 819
 820 void
 821 _jit_retval_ui(jit_state_t *_jit, jit_int32_t r0)
 822 {
 823     jit_inc_synth_w(retval_ui, r0);
 824     jit_extr_ui(r0, JIT_RET);
 825     jit_dec_synth();
 826 }
 827
 828 void
 829 _jit_retval_l(jit_state_t *_jit, jit_int32_t r0)
 830 {
 831     jit_inc_synth_w(retval_l, r0);
 832     if (r0 != JIT_RET)
 833         jit_movr(r0, JIT_RET);
 834     jit_dec_synth();
 835 }
 836
 837 void
 838 _jit_retval_f(jit_state_t *_jit, jit_int32_t r0)
 839 {
 840     jit_inc_synth_w(retval_f, r0);
 841     if (r0 != JIT_FRET)
 842         jit_movr_f(r0, JIT_FRET);
 843     jit_dec_synth();
 844 }
 845
 846 void
 847 _jit_retval_d(jit_state_t *_jit, jit_int32_t r0)
 848 {
 849     jit_inc_synth_w(retval_d, r0);
 850     if (r0 != JIT_FRET)
 851         jit_movr_d(r0, JIT_FRET);
 852     jit_dec_synth();
 853 }
 854
 855 jit_pointer_t
 856 _emit_code(jit_state_t *_jit)
 857 {
 858     jit_node_t          *node;
 859     jit_node_t          *temp;
 860     jit_word_t           word;
 861     jit_word_t           value;
 862     jit_int32_t          offset;
 863     struct {
 864         jit_node_t      *node;
 865         jit_uint8_t     *data;
 866         jit_word_t       word;
 867 #if DEVEL_DISASSEMBLER
 868         jit_word_t       prevw;
 869 #endif
 870         jit_int32_t      const_offset;
 871         jit_int32_t      patch_offset;
 872     } undo;
 873 #if DEVEL_DISASSEMBLER
 874     jit_word_t           prevw;
 875 #endif
 876
 877     _jitc->function = NULL;
 878
 879     jit_reglive_setup();
 880
 881     undo.word = 0;
 882     undo.node = NULL;
 883     undo.const_offset = undo.patch_offset = 0;
 884 #define case_rr(name, type)                                             \
 885             case jit_code_##name##r##type:                              \
 886                 name##r##type(rn(node->u.w), rn(node->v.w));            \
 887                 break
 888 #define case_rw(name, type)                                             \
 889             case jit_code_##name##i##type:                              \
 890                 name##i##type(rn(node->u.w), node->v.w);                \
 891                 break
 892 #define case_wr(name, type)                                             \
 893             case jit_code_##name##i##type:                              \
 894                 name##i##type(node->u.w, rn(node->v.w));                \
 895                 break
 896 #define case_rrr(name, type)                                            \
 897             case jit_code_##name##r##type:                              \
 898                 name##r##type(rn(node->u.w),                            \
 899                               rn(node->v.w), rn(node->w.w));            \
 900                 break
 901 #define case_rrw(name, type)                                            \
 902             case jit_code_##name##i##type:                              \
 903                 name##i##type(rn(node->u.w), rn(node->v.w), node->w.w); \
 904                 break
 905 #define case_rrf(name, type, size)                                      \
 906             case jit_code_##name##i##type:                              \
 907                 assert(node->flag & jit_flag_data);                     \
 908                 name##i##type(rn(node->u.w), rn(node->v.w),             \
 909                               (jit_float##size##_t *)node->w.n->u.w);   \
 910                 break
 911 #define case_rrrr(name, type)                                           \
 912             case jit_code_##name##r##type:                              \
 913                 name##r##type(rn(node->u.q.l), rn(node->u.q.h),         \
 914                               rn(node->v.w), rn(node->w.w));            \
 915                 break
 916 #define case_rrrw(name, type)                                           \
 917             case jit_code_##name##i##type:                              \
 918                 name##i##type(rn(node->u.q.l), rn(node->u.q.h),         \
 919                               rn(node->v.w), node->w.w);                \
 920                 break
 921 #define case_wrr(name, type)                                            \
 922             case jit_code_##name##i##type:                              \
 923                 name##i##type(node->u.w, rn(node->v.w), rn(node->w.w)); \
 924                 break
 925 #define case_brr(name, type)                                            \
 926             case jit_code_##name##r##type:                              \
 927                 temp = node->u.n;                                       \
 928                 assert(temp->code == jit_code_label ||                  \
 929                        temp->code == jit_code_epilog);                  \
 930                 if (temp->flag & jit_flag_patch)                        \
 931                     name##r##type(temp->u.w, rn(node->v.w),             \
 932                                   rn(node->w.w));                       \
 933                 else {                                                  \
 934                     word = name##r##type(_jit->pc.w,                    \
 935                                          rn(node->v.w), rn(node->w.w)); \
 936                     patch(word, node);                                  \
 937                 }                                                       \
 938                 break
 939 #define case_brw(name, type)                                            \
 940             case jit_code_##name##i##type:                              \
 941                 temp = node->u.n;                                       \
 942                 assert(temp->code == jit_code_label ||                  \
 943                        temp->code == jit_code_epilog);                  \
 944                 if (temp->flag & jit_flag_patch)                        \
 945                     name##i##type(temp->u.w,                            \
 946                                   rn(node->v.w), node->w.w);            \
 947                 else {                                                  \
 948                     word = name##i##type(_jit->pc.w,                    \
 949                                          rn(node->v.w), node->w.w);     \
 950                     patch(word, node);                                  \
 951                 }                                                       \
 952                 break
 953 #define case_brf(name, type, size)                                      \
 954             case jit_code_##name##i##type:                              \
 955                 temp = node->u.n;                                       \
 956                 assert(temp->code == jit_code_label ||                  \
 957                        temp->code == jit_code_epilog);                  \
 958                 if (temp->flag & jit_flag_patch)                        \
 959                     name##i##type(temp->u.w, rn(node->v.w),             \
 960                                 (jit_float##size##_t *)node->w.n->u.w); \
 961                 else {                                                  \
 962                     word = name##i##type(_jit->pc.w, rn(node->v.w),     \
 963                                 (jit_float##size##_t *)node->w.n->u.w); \
 964                     patch(word, node);                                  \
 965                 }                                                       \
 966                 break
 967 #if DEVEL_DISASSEMBLER
 968     prevw = _jit->pc.w;
 969 #endif
 970     for (node = _jitc->head; node; node = node->next) {
 971         if (_jit->pc.uc >= _jitc->code.end)
 972             return (NULL);
 973
 974 #if DEVEL_DISASSEMBLER
 975         node->offset = (jit_uword_t)_jit->pc.w - (jit_uword_t)prevw;
 976         prevw = _jit->pc.w;
 977 #endif
 978         value = jit_classify(node->code);
 979         jit_regarg_set(node, value);
 980         switch (node->code) {
 981             case jit_code_align:
 982                 assert(!(node->u.w & (node->u.w - 1)) &&
 983                        node->u.w <= sizeof(jit_word_t));
 984                 if (node->u.w == sizeof(jit_word_t) &&
 985                     (word = _jit->pc.w & (sizeof(jit_word_t) - 1)))
 986                     nop(sizeof(jit_word_t) - word);
 987                 break;
 988             case jit_code_note:         case jit_code_name:
 989                 node->u.w = _jit->pc.w;
 990                 break;
 991             case jit_code_label:
 992                 /* remember label is defined */
 993                 node->flag |= jit_flag_patch;
 994                 node->u.w = _jit->pc.w;
 995                 break;
 996                 case_rrr(add,);
 997                 case_rrw(add,);
 998                 case_rrr(addc,);
 999                 case_rrw(addc,);
1000                 case_rrr(addx,);
1001                 case_rrw(addx,);
1002                 case_rrr(sub,);
1003                 case_rrw(sub,);
1004                 case_rrr(subc,);
1005                 case_rrw(subc,);
1006                 case_rrr(subx,);
1007                 case_rrw(subx,);
1008                 case_rrw(rsb,);
1009                 case_rrr(mul,);
1010                 case_rrw(mul,);
1011                 case_rrrr(qmul,);
1012                 case_rrrw(qmul,);
1013                 case_rrrr(qmul, _u);
1014                 case_rrrw(qmul, _u);
1015                 case_rrr(div,);
1016                 case_rrw(div,);
1017                 case_rrr(div, _u);
1018                 case_rrw(div, _u);
1019                 case_rrrr(qdiv,);
1020                 case_rrrw(qdiv,);
1021                 case_rrrr(qdiv, _u);
1022                 case_rrrw(qdiv, _u);
1023                 case_rrr(rem,);
1024                 case_rrw(rem,);
1025                 case_rrr(rem, _u);
1026                 case_rrw(rem, _u);
1027                 case_rrr(lsh,);
1028                 case_rrw(lsh,);
1029                 case_rrr(rsh,);
1030                 case_rrw(rsh,);
1031                 case_rrr(rsh, _u);
1032                 case_rrw(rsh, _u);
1033                 case_rrr(and,);
1034                 case_rrw(and,);
1035                 case_rrr(or,);
1036                 case_rrw(or,);
1037                 case_rrr(xor,);
1038                 case_rrw(xor,);
1039                 case_rr(trunc, _f_i);
1040                 case_rr(trunc, _d_i);
1041                 case_rr(trunc, _f_l);
1042                 case_rr(trunc, _d_l);
1043                 case_rr(ld, _c);
1044                 case_rw(ld, _c);
1045                 case_rr(ld, _uc);
1046                 case_rw(ld, _uc);
1047                 case_rr(ld, _s);
1048                 case_rw(ld, _s);
1049                 case_rr(ld, _us);
1050                 case_rw(ld, _us);
1051                 case_rr(ld, _i);
1052                 case_rw(ld, _i);
1053                 case_rr(ld, _ui);
1054                 case_rw(ld, _ui);
1055                 case_rr(ld, _l);
1056                 case_rw(ld, _l);
1057                 case_rrr(ldx, _c);
1058                 case_rrw(ldx, _c);
1059                 case_rrr(ldx, _uc);
1060                 case_rrw(ldx, _uc);
1061                 case_rrr(ldx, _s);
1062                 case_rrw(ldx, _s);
1063                 case_rrr(ldx, _us);
1064                 case_rrw(ldx, _us);
1065                 case_rrr(ldx, _i);
1066                 case_rrw(ldx, _i);
1067                 case_rrr(ldx, _ui);
1068                 case_rrw(ldx, _ui);
1069                 case_rrr(ldx, _l);
1070                 case_rrw(ldx, _l);
1071                 case_rr(st, _c);
1072                 case_wr(st, _c);
1073                 case_rr(st, _s);
1074                 case_wr(st, _s);
1075                 case_rr(st, _i);
1076                 case_wr(st, _i);
1077                 case_rr(st, _l);
1078                 case_wr(st, _l);
1079                 case_rrr(stx, _c);
1080                 case_wrr(stx, _c);
1081                 case_rrr(stx, _s);
1082                 case_wrr(stx, _s);
1083                 case_rrr(stx, _i);
1084                 case_wrr(stx, _i);
1085                 case_rrr(stx, _l);
1086                 case_wrr(stx, _l);
1087                 case_rr(hton, _us);
1088                 case_rr(hton, _ui);
1089                 case_rr(hton, _ul);
1090                 case_rr(bswap, _us);
1091                 case_rr(bswap, _ui);
1092                 case_rr(bswap, _ul);
1093                 case_rr(ext, _c);
1094                 case_rr(ext, _uc);
1095                 case_rr(ext, _s);
1096                 case_rr(ext, _us);
1097                 case_rr(ext, _i);
1098                 case_rr(ext, _ui);
1099             case jit_code_casr:
1100                 casr(rn(node->u.w), rn(node->v.w),
1101                      rn(node->w.q.l), rn(node->w.q.h));
1102                 break;
1103             case jit_code_casi:
1104                 casi(rn(node->u.w), node->v.w,
1105                      rn(node->w.q.l), rn(node->w.q.h));
1106                 break;
1107                 case_rrr(movn,);
1108                 case_rrr(movz,);
1109                 case_rr(mov,);
1110             case jit_code_movi:
1111                 if (node->flag & jit_flag_node) {
1112                     temp = node->v.n;
1113                     if (temp->code == jit_code_data ||
1114                         (temp->code == jit_code_label &&
1115                          (temp->flag & jit_flag_patch)))
1116                         movi(rn(node->u.w), temp->u.w);
1117                     else {
1118                         assert(temp->code == jit_code_label ||
1119                                temp->code == jit_code_epilog);
1120                         word = movi_p(rn(node->u.w), node->v.w);
1121                         patch(word, node);
1122                     }
1123                 }
1124                 else
1125                     movi(rn(node->u.w), node->v.w);
1126                 break;
1127                 case_rr(neg,);
1128                 case_rr(com,);
1129                 case_rrr(lt,);
1130                 case_rrw(lt,);
1131                 case_rrr(lt, _u);
1132                 case_rrw(lt, _u);
1133                 case_rrr(le,);
1134                 case_rrw(le,);
1135                 case_rrr(le, _u);
1136                 case_rrw(le, _u);
1137                 case_rrr(eq,);
1138                 case_rrw(eq,);
1139                 case_rrr(ge,);
1140                 case_rrw(ge,);
1141                 case_rrr(ge, _u);
1142                 case_rrw(ge, _u);
1143                 case_rrr(gt,);
1144                 case_rrw(gt,);
1145                 case_rrr(gt, _u);
1146                 case_rrw(gt, _u);
1147                 case_rrr(ne,);
1148                 case_rrw(ne,);
1149                 case_brr(blt,);
1150                 case_brw(blt,);
1151                 case_brr(blt, _u);
1152                 case_brw(blt, _u);
1153                 case_brr(ble,);
1154                 case_brw(ble,);
1155                 case_brr(ble, _u);
1156                 case_brw(ble, _u);
1157                 case_brr(beq,);
1158                 case_brw(beq,);
1159                 case_brr(bge,);
1160                 case_brw(bge,);
1161                 case_brr(bge, _u);
1162                 case_brw(bge, _u);
1163                 case_brr(bgt,);
1164                 case_brw(bgt,);
1165                 case_brr(bgt, _u);
1166                 case_brw(bgt, _u);
1167                 case_brr(bne,);
1168                 case_brw(bne,);
1169                 case_brr(boadd,);
1170                 case_brw(boadd,);
1171                 case_brr(boadd, _u);
1172                 case_brw(boadd, _u);
1173                 case_brr(bxadd,);
1174                 case_brw(bxadd,);
1175                 case_brr(bxadd, _u);
1176                 case_brw(bxadd, _u);
1177                 case_brr(bosub,);
1178                 case_brw(bosub,);
1179                 case_brr(bosub, _u);
1180                 case_brw(bosub, _u);
1181                 case_brr(bxsub,);
1182                 case_brw(bxsub,);
1183                 case_brr(bxsub, _u);
1184                 case_brw(bxsub, _u);
1185                 case_brr(bms,);
1186                 case_brw(bms,);
1187                 case_brr(bmc,);
1188                 case_brw(bmc,);
1189                 case_rrr(add, _f);
1190                 case_rrf(add, _f, 32);
1191                 case_rrr(sub, _f);
1192                 case_rrf(sub, _f, 32);
1193                 case_rrf(rsb, _f, 32);
1194                 case_rrr(mul, _f);
1195                 case_rrf(mul, _f, 32);
1196                 case_rrr(div, _f);
1197                 case_rrf(div, _f, 32);
1198                 case_rr(abs, _f);
1199                 case_rr(neg, _f);
1200                 case_rr(sqrt, _f);
1201                 case_rr(ext, _f);
1202                 case_rr(ld, _f);
1203                 case_rw(ld, _f);
1204                 case_rrr(ldx, _f);
1205                 case_rrw(ldx, _f);
1206                 case_rr(st, _f);
1207                 case_wr(st, _f);
1208                 case_rrr(stx, _f);
1209                 case_wrr(stx, _f);
1210                 case_rr(mov, _f);
1211             case jit_code_movi_f:
1212                 assert(node->flag & jit_flag_data);
1213                 movi_f(rn(node->u.w), (jit_float32_t *)node->v.n->u.w);
1214                 break;
1215                 case_rr(ext, _d_f);
1216                 case_rrr(lt, _f);
1217                 case_rrf(lt, _f, 32);
1218                 case_rrr(le, _f);
1219                 case_rrf(le, _f, 32);
1220                 case_rrr(eq, _f);
1221                 case_rrf(eq, _f, 32);
1222                 case_rrr(ge, _f);
1223                 case_rrf(ge, _f, 32);
1224                 case_rrr(gt, _f);
1225                 case_rrf(gt, _f, 32);
1226                 case_rrr(ne, _f);
1227                 case_rrf(ne, _f, 32);
1228                 case_rrr(unlt, _f);
1229                 case_rrf(unlt, _f, 32);
1230                 case_rrr(unle, _f);
1231                 case_rrf(unle, _f, 32);
1232                 case_rrr(uneq, _f);
1233                 case_rrf(uneq, _f, 32);
1234                 case_rrr(unge, _f);
1235                 case_rrf(unge, _f, 32);
1236                 case_rrr(ungt, _f);
1237                 case_rrf(ungt, _f, 32);
1238                 case_rrr(ltgt, _f);
1239                 case_rrf(ltgt, _f, 32);
1240                 case_rrr(ord, _f);
1241                 case_rrf(ord, _f, 32);
1242                 case_rrr(unord, _f);
1243                 case_rrf(unord, _f, 32);
1244                 case_brr(blt, _f);
1245                 case_brf(blt, _f, 32);
1246                 case_brr(ble, _f);
1247                 case_brf(ble, _f, 32);
1248                 case_brr(beq, _f);
1249                 case_brf(beq, _f, 32);
1250                 case_brr(bge, _f);
1251                 case_brf(bge, _f, 32);
1252                 case_brr(bgt, _f);
1253                 case_brf(bgt, _f, 32);
1254                 case_brr(bne, _f);
1255                 case_brf(bne, _f, 32);
1256                 case_brr(bunlt, _f);
1257                 case_brf(bunlt, _f, 32);
1258                 case_brr(bunle, _f);
1259                 case_brf(bunle, _f, 32);
1260                 case_brr(buneq, _f);
1261                 case_brf(buneq, _f, 32);
1262                 case_brr(bunge, _f);
1263                 case_brf(bunge, _f, 32);
1264                 case_brr(bungt, _f);
1265                 case_brf(bungt, _f, 32);
1266                 case_brr(bltgt, _f);
1267                 case_brf(bltgt, _f, 32);
1268                 case_brr(bord, _f);
1269                 case_brf(bord, _f, 32);
1270                 case_brr(bunord, _f);
1271                 case_brf(bunord, _f, 32);
1272                 case_rrr(add, _d);
1273                 case_rrf(add, _d, 64);
1274                 case_rrr(sub, _d);
1275                 case_rrf(sub, _d, 64);
1276                 case_rrf(rsb, _d, 64);
1277                 case_rrr(mul, _d);
1278                 case_rrf(mul, _d, 64);
1279                 case_rrr(div, _d);
1280                 case_rrf(div, _d, 64);
1281                 case_rr(abs, _d);
1282                 case_rr(neg, _d);
1283                 case_rr(sqrt, _d);
1284                 case_rr(ext, _d);
1285                 case_rr(ld, _d);
1286                 case_rw(ld, _d);
1287                 case_rrr(ldx, _d);
1288                 case_rrw(ldx, _d);
1289                 case_rr(st, _d);
1290                 case_wr(st, _d);
1291                 case_rrr(stx, _d);
1292                 case_wrr(stx, _d);
1293                 case_rr(mov, _d);
1294             case jit_code_movi_d:
1295                 assert(node->flag & jit_flag_data);
1296                 movi_d(rn(node->u.w), (jit_float64_t *)node->v.n->u.w);
1297                 break;
1298                 case_rr(ext, _f_d);
1299                 case_rrr(lt, _d);
1300                 case_rrf(lt, _d, 64);
1301                 case_rrr(le, _d);
1302                 case_rrf(le, _d, 64);
1303                 case_rrr(eq, _d);
1304                 case_rrf(eq, _d, 64);
1305                 case_rrr(ge, _d);
1306                 case_rrf(ge, _d, 64);
1307                 case_rrr(gt, _d);
1308                 case_rrf(gt, _d, 64);
1309                 case_rrr(ne, _d);
1310                 case_rrf(ne, _d, 64);
1311                 case_rrr(unlt, _d);
1312                 case_rrf(unlt, _d, 64);
1313                 case_rrr(unle, _d);
1314                 case_rrf(unle, _d, 64);
1315                 case_rrr(uneq, _d);
1316                 case_rrf(uneq, _d, 64);
1317                 case_rrr(unge, _d);
1318                 case_rrf(unge, _d, 64);
1319                 case_rrr(ungt, _d);
1320                 case_rrf(ungt, _d, 64);
1321                 case_rrr(ltgt, _d);
1322                 case_rrf(ltgt, _d, 64);
1323                 case_rrr(ord, _d);
1324                 case_rrf(ord, _d, 64);
1325                 case_rrr(unord, _d);
1326                 case_rrf(unord, _d, 64);
1327                 case_brr(blt, _d);
1328                 case_brf(blt, _d, 64);
1329                 case_brr(ble, _d);
1330                 case_brf(ble, _d, 64);
1331                 case_brr(beq, _d);
1332                 case_brf(beq, _d, 64);
1333                 case_brr(bge, _d);
1334                 case_brf(bge, _d, 64);
1335                 case_brr(bgt, _d);
1336                 case_brf(bgt, _d, 64);
1337                 case_brr(bne, _d);
1338                 case_brf(bne, _d, 64);
1339                 case_brr(bunlt, _d);
1340                 case_brf(bunlt, _d, 64);
1341                 case_brr(bunle, _d);
1342                 case_brf(bunle, _d, 64);
1343                 case_brr(buneq, _d);
1344                 case_brf(buneq, _d, 64);
1345                 case_brr(bunge, _d);
1346                 case_brf(bunge, _d, 64);
1347                 case_brr(bungt, _d);
1348                 case_brf(bungt, _d, 64);
1349                 case_brr(bltgt, _d);
1350                 case_brf(bltgt, _d, 64);
1351                 case_brr(bord, _d);
1352                 case_brf(bord, _d, 64);
1353                 case_brr(bunord, _d);
1354                 case_brf(bunord, _d, 64);
1355             case jit_code_jmpr:
1356                 jmpr(rn(node->u.w));
1357                 break;
1358             case jit_code_jmpi:
1359                 if (node->flag & jit_flag_node) {
1360                     temp = node->u.n;
1361                     assert(temp->code == jit_code_label ||
1362                            temp->code == jit_code_epilog);
1363                     if (temp->flag & jit_flag_patch)
1364                         jmpi(temp->u.w);
1365                     else {
1366                         word = jmpi_p(_jit->pc.w);
1367                         patch(word, node);
1368                     }
1369                 }
1370                 else
1371                     jmpi(node->u.w);
1372                 break;
1373             case jit_code_callr:
1374                 callr(rn(node->u.w));
1375                 break;
1376             case jit_code_calli:
1377                 if (node->flag & jit_flag_node) {
1378                     temp = node->u.n;
1379                     assert(temp->code == jit_code_label ||
1380                            temp->code == jit_code_epilog);
1381                     if (!(temp->flag & jit_flag_patch)) {
1382                         word = calli_p(temp->u.w);
1383                         patch(word, node);
1384                     }
1385                     else
1386                         calli(temp->u.w);
1387                 }
1388                 else
1389                     calli(node->u.w);
1390                 break;
1391             case jit_code_prolog:
1392                 _jitc->function = _jitc->functions.ptr + node->w.w;
1393                 undo.node = node;
1394                 undo.word = _jit->pc.w;
1395 #if DEVEL_DISASSEMBLER
1396                 undo.prevw = prevw;
1397 #endif
1398                 undo.patch_offset = _jitc->patches.offset;
1399             restart_function:
1400                 _jitc->again = 0;
1401                 prolog(node);
1402                 break;
1403             case jit_code_epilog:
1404                 assert(_jitc->function == _jitc->functions.ptr + node->w.w);
1405                 if (_jitc->again) {
1406                     for (temp = undo.node->next;
1407                          temp != node; temp = temp->next) {
1408                         if (temp->code == jit_code_label ||
1409                             temp->code == jit_code_epilog)
1410                             temp->flag &= ~jit_flag_patch;
1411                     }
1412                     temp->flag &= ~jit_flag_patch;
1413                     node = undo.node;
1414                     _jit->pc.w = undo.word;
1415 #if DEVEL_DISASSEMBLER
1416                     prevw = undo.prevw;
1417 #endif
1418                     _jitc->patches.offset = undo.patch_offset;
1419                     goto restart_function;
1420                 }
1421                 /* remember label is defined */
1422                 node->flag |= jit_flag_patch;
1423                 node->u.w = _jit->pc.w;
1424                 epilog(node);
1425                 _jitc->function = NULL;
1426                 break;
1427             case jit_code_va_start:
1428                 vastart(rn(node->u.w));
1429                 break;
1430             case jit_code_va_arg:
1431                 vaarg(rn(node->u.w), rn(node->v.w));
1432                 break;
1433             case jit_code_va_arg_d:
1434                 vaarg_d(rn(node->u.w), rn(node->v.w));
1435                 break;
1436             case jit_code_live:                 case jit_code_ellipsis:
1437             case jit_code_va_push:
1438             case jit_code_allocai:              case jit_code_allocar:
1439             case jit_code_arg:
1440             case jit_code_arg_f:                case jit_code_arg_d:
1441             case jit_code_va_end:
1442             case jit_code_ret:
1443             case jit_code_retr:                 case jit_code_reti:
1444             case jit_code_retr_f:               case jit_code_reti_f:
1445             case jit_code_retr_d:               case jit_code_reti_d:
1446             case jit_code_getarg_c:             case jit_code_getarg_uc:
1447             case jit_code_getarg_s:             case jit_code_getarg_us:
1448             case jit_code_getarg_i:             case jit_code_getarg_ui:
1449             case jit_code_getarg_l:
1450             case jit_code_getarg_f:             case jit_code_getarg_d:
1451             case jit_code_putargr:              case jit_code_putargi:
1452             case jit_code_putargr_f:            case jit_code_putargi_f:
1453             case jit_code_putargr_d:            case jit_code_putargi_d:
1454             case jit_code_pushargr:             case jit_code_pushargi:
1455             case jit_code_pushargr_f:           case jit_code_pushargi_f:
1456             case jit_code_pushargr_d:           case jit_code_pushargi_d:
1457             case jit_code_retval_c:             case jit_code_retval_uc:
1458             case jit_code_retval_s:             case jit_code_retval_us:
1459             case jit_code_retval_i:
1460             case jit_code_retval_ui:            case jit_code_retval_l:
1461             case jit_code_retval_f:             case jit_code_retval_d:
1462             case jit_code_prepare:
1463             case jit_code_finishr:              case jit_code_finishi:
1464                 break;
1465             default:
1466                 abort();
1467         }
1468         if (jit_carry != _NOREG) {
1469             switch (node->code) {
1470                 case jit_code_note:
1471                 case jit_code_addcr:            case jit_code_addci:
1472                 case jit_code_addxr:            case jit_code_addxi:
1473                 case jit_code_subcr:            case jit_code_subci:
1474                 case jit_code_subxr:            case jit_code_subxi:
1475                     break;
1476                 default:
1477                     jit_unget_reg(jit_carry);
1478                     jit_carry = _NOREG;
1479                     break;
1480             }
1481         }
1482         jit_regarg_clr(node, value);
1483         assert(_jitc->regarg == 0 ||
1484                (jit_carry != _NOREG && _jitc->regarg == (1 << jit_carry)));
1485         assert(_jitc->synth == 0);
1486         /* update register live state */
1487         jit_reglive(node);
1488     }
1489 #undef case_brf
1490 #undef case_brw
1491 #undef case_brr
1492 #undef case_wrr
1493 #undef case_rrrw
1494 #undef case_rrrr
1495 #undef case_rrf
1496 #undef case_rrw
1497 #undef case_rrr
1498 #undef case_wr
1499 #undef case_rw
1500 #undef case_rr
1501     for (offset = 0; offset < _jitc->patches.offset; offset++) {
1502         node = _jitc->patches.ptr[offset].node;
1503         word = node->code == jit_code_movi ? node->v.n->u.w : node->u.n->u.w;
1504         patch_at(_jitc->patches.ptr[offset].inst, word);
1505     }
1506
1507     jit_flush(_jit->code.ptr, _jit->pc.uc);
1508
1509     return (_jit->code.ptr);
1510 }
1511
1512 #define CODE                            1
1513 #  include "jit_alpha-cpu.c"
1514 #  include "jit_alpha-fpu.c"
1515 #  include "jit_fallback.c"
1516 #undef CODE
1517
1518 void
1519 jit_flush(void *fptr, void *tptr)
1520 {
1521 }
1522
1523 void
1524 _emit_ldxi(jit_state_t *_jit, jit_int32_t r0, jit_int32_t r1, jit_word_t i0)
1525 {
1526     ldxi(rn(r0), rn(r1), i0);
1527 }
1528
1529 void
1530 _emit_stxi(jit_state_t *_jit, jit_word_t i0, jit_int32_t r0, jit_int32_t r1)
1531 {
1532     stxi(i0, rn(r0), rn(r1));
1533 }
1534
1535 void
1536 _emit_ldxi_d(jit_state_t *_jit, jit_int32_t r0, jit_int32_t r1, jit_word_t i0)
1537 {
1538     ldxi_d(rn(r0), rn(r1), i0);
1539 }
1540
1541 void
1542 _emit_stxi_d(jit_state_t *_jit, jit_word_t i0, jit_int32_t r0, jit_int32_t r1)
1543 {
1544     stxi_d(i0, rn(r0), rn(r1));
1545 }
1546
1547 static void
1548 _patch(jit_state_t *_jit, jit_word_t instr, jit_node_t *node)
1549 {
1550     jit_int32_t          flag;
1551
1552     assert(node->flag & jit_flag_node);
1553     if (node->code == jit_code_movi)
1554         flag = node->v.n->flag;
1555     else
1556         flag = node->u.n->flag;
1557      assert(!(flag & jit_flag_patch));
1558     if (_jitc->patches.offset >= _jitc->patches.length) {
1559         jit_realloc((jit_pointer_t *)&_jitc->patches.ptr,
1560                     _jitc->patches.length * sizeof(jit_patch_t),
1561                     (_jitc->patches.length + 1024) * sizeof(jit_patch_t));
1562         _jitc->patches.length += 1024;
1563     }
1564     _jitc->patches.ptr[_jitc->patches.offset].inst = instr;
1565     _jitc->patches.ptr[_jitc->patches.offset].node = node;
1566     ++_jitc->patches.offset;
1567 }