deps/lightning/lib/jit_mips.c

   1 /*
   2  * Copyright (C) 2012-2019  Free Software Foundation, Inc.
   3  *
   4  * This file is part of GNU lightning.
   5  *
   6  * GNU lightning is free software; you can redistribute it and/or modify it
   7  * under the terms of the GNU Lesser General Public License as published
   8  * by the Free Software Foundation; either version 3, or (at your option)
   9  * any later version.
  10  *
  11  * GNU lightning is distributed in the hope that it will be useful, but
  12  * WITHOUT ANY WARRANTY; without even the implied warranty of MERCHANTABILITY
  13  * or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU Lesser General Public
  14  * License for more details.
  15  *
  16  * Authors:
  17  *      Paulo Cesar Pereira de Andrade
  18  */
  19
  20 #if defined(__linux__)
  21 #  include <sys/cachectl.h>
  22 #endif
  23
  24 #if NEW_ABI
  25 #  define NUM_WORD_ARGS                 8
  26 #  define STACK_SLOT                    8
  27 #  define STACK_SHIFT                   3
  28 #else
  29 #  define NUM_WORD_ARGS                 4
  30 #  define STACK_SLOT                    4
  31 #  define STACK_SHIFT                   2
  32 #endif
  33 #if NEW_ABI && __BYTE_ORDER == __BIG_ENDIAN && __WORDSIZE == 32
  34 #  define WORD_ADJUST                   4
  35 #else
  36 #  define WORD_ADJUST                   0
  37 #endif
  38 #define jit_arg_reg_p(i)                ((i) >= 0 && (i) < NUM_WORD_ARGS)
  39 #if __BYTE_ORDER == __LITTLE_ENDIAN
  40 #  define C_DISP                        0
  41 #  define S_DISP                        0
  42 #  define I_DISP                        0
  43 #else
  44 #  define C_DISP                        STACK_SLOT - sizeof(jit_int8_t)
  45 #  define S_DISP                        STACK_SLOT - sizeof(jit_int16_t)
  46 #  define I_DISP                        STACK_SLOT - sizeof(jit_int32_t)
  47 #endif
  48
  49 /*
  50  * Types
  51  */
  52 typedef struct jit_pointer_t jit_va_list_t;
  53
  54 /*
  55  * Prototypes
  56  */
  57 #define jit_make_arg(node)              _jit_make_arg(_jit,node)
  58 static jit_node_t *_jit_make_arg(jit_state_t*,jit_node_t*);
  59 #define jit_make_arg_f(node)            _jit_make_arg_f(_jit,node)
  60 static jit_node_t *_jit_make_arg_f(jit_state_t*,jit_node_t*);
  61 #define jit_make_arg_d(node)            _jit_make_arg_d(_jit,node)
  62 static jit_node_t *_jit_make_arg_d(jit_state_t*,jit_node_t*);
  63 #define patch(instr, node)              _patch(_jit, instr, node)
  64 static void _patch(jit_state_t*,jit_word_t,jit_node_t*);
  65
  66 #define PROTO                           1
  67 #  include "jit_rewind.c"
  68 #  include "jit_mips-cpu.c"
  69 #  include "jit_mips-fpu.c"
  70 #  include "jit_fallback.c"
  71 #undef PROTO
  72
  73 /*
  74  * Initialization
  75  */
  76 jit_register_t          _rvs[] = {
  77     { rc(gpr) | 0x01,                   "at" },
  78     { rc(gpr) | 0x02,                   "v0" },
  79     { rc(gpr) | 0x03,                   "v1" },
  80 #if !NEW_ABI
  81     { rc(gpr) | 0x08,                   "t0" },
  82     { rc(gpr) | 0x09,                   "t1" },
  83     { rc(gpr) | 0x0a,                   "t2" },
  84     { rc(gpr) | 0x0b,                   "t3" },
  85 #endif
  86     { rc(gpr) | 0x0c,                   "t4" },
  87     { rc(gpr) | 0x0d,                   "t5" },
  88     { rc(gpr) | 0x0e,                   "t6" },
  89     { rc(gpr) | 0x0f,                   "t7" },
  90     { rc(gpr) | 0x18,                   "t8" },
  91     { rc(gpr) | 0x19,                   "t9" },
  92     { rc(sav) | rc(gpr) | 0x10,         "s0" },
  93     { rc(sav) | rc(gpr) | 0x11,         "s1" },
  94     { rc(sav) | rc(gpr) | 0x12,         "s2" },
  95     { rc(sav) | rc(gpr) | 0x13,         "s3" },
  96     { rc(sav) | rc(gpr) | 0x14,         "s4" },
  97     { rc(sav) | rc(gpr) | 0x15,         "s5" },
  98     { rc(sav) | rc(gpr) | 0x16,         "s6" },
  99     { rc(sav) | rc(gpr) | 0x17,         "s7" },
 100     { 0x00,                             "zero" },
 101     { 0x1a,                             "k0" },
 102     { 0x1b,                             "k1" },
 103     { rc(sav) | 0x1f,                   "ra" },
 104     { rc(sav) | 0x1c,                   "gp" },
 105     { rc(sav) | 0x1d,                   "sp" },
 106     { rc(sav) | 0x1e,                   "fp" },
 107 #if NEW_ABI
 108     { rc(gpr) | 0x0b,                   "a7" },
 109     { rc(gpr) | 0x0a,                   "a6" },
 110     { rc(gpr) | 0x09,                   "a5" },
 111     { rc(gpr) | 0x08,                   "a4" },
 112 #endif
 113     { rc(arg) | rc(gpr) | 0x07,         "a3" },
 114     { rc(arg) | rc(gpr) | 0x06,         "a2" },
 115     { rc(arg) | rc(gpr) | 0x05,         "a1" },
 116     { rc(arg) | rc(gpr) | 0x04,         "a0" },
 117     { rc(fpr) | 0x00,                   "$f0" },
 118     { rc(fpr) | 0x02,                   "$f2" },
 119     { rc(fpr) | 0x04,                   "$f4" },
 120     { rc(fpr) | 0x06,                   "$f6" },
 121     { rc(fpr) | 0x08,                   "$f8" },
 122     { rc(fpr) | 0x0a,                   "$f10" },
 123 #if !NEW_ABI
 124     { rc(sav) | rc(fpr) | 0x10,         "$f16" },
 125     { rc(sav) | rc(fpr) | 0x12,         "$f18" },
 126 #endif
 127     { rc(sav) | rc(fpr) | 0x14,         "$f20" },
 128     { rc(sav) | rc(fpr) | 0x16,         "$f22" },
 129     { rc(sav) | rc(fpr) | 0x18,         "$f24" },
 130     { rc(sav) | rc(fpr) | 0x1a,         "$f26" },
 131     { rc(sav) | rc(fpr) | 0x1c,         "$f28" },
 132     { rc(sav) | rc(fpr) | 0x1e,         "$f30" },
 133 #if NEW_ABI
 134     { rc(arg) | rc(fpr) | 0x13,         "$f19" },
 135     { rc(arg) | rc(fpr) | 0x12,         "$f18" },
 136     { rc(arg) | rc(fpr) | 0x11,         "$f17" },
 137     { rc(arg) | rc(fpr) | 0x10,         "$f16" },
 138     { rc(arg) | rc(fpr) | 0x0f,         "$f15" },
 139     { rc(arg) | rc(fpr) | 0x0e,         "$f14" },
 140     { rc(arg) | rc(fpr) | 0x0d,         "$f13" },
 141     { rc(arg) | rc(fpr) | 0x0c,         "$f12" },
 142 #else
 143     { rc(arg) | rc(fpr) | 0x0e,         "$f14" },
 144     { rc(arg) | rc(fpr) | 0x0c,         "$f12" },
 145 #endif
 146     { _NOREG,                           "<none>" },
 147 };
 148
 149 /*
 150  * Implementation
 151  */
 152 void
 153 jit_get_cpu(void)
 154 {
 155 }
 156
 157 void
 158 _jit_init(jit_state_t *_jit)
 159 {
 160     _jitc->reglen = jit_size(_rvs) - 1;
 161 /* Could also:
 162  *      o reserve a register for carry  (overkill)
 163  *      o use MTLO/MFLO                 (performance hit)
 164  * So, keep a register allocated after setting carry, and implicitly
 165  * deallocate it if it can no longer be tracked
 166  */
 167     jit_carry = _NOREG;
 168 }
 169
 170 void
 171 _jit_prolog(jit_state_t *_jit)
 172 {
 173     jit_int32_t          offset;
 174
 175     if (_jitc->function)
 176         jit_epilog();
 177     assert(jit_regset_cmp_ui(&_jitc->regarg, 0) == 0);
 178     jit_regset_set_ui(&_jitc->regsav, 0);
 179     offset = _jitc->functions.offset;
 180     if (offset >= _jitc->functions.length) {
 181         jit_realloc((jit_pointer_t *)&_jitc->functions.ptr,
 182                     _jitc->functions.length * sizeof(jit_function_t),
 183                     (_jitc->functions.length + 16) * sizeof(jit_function_t));
 184         _jitc->functions.length += 16;
 185     }
 186     _jitc->function = _jitc->functions.ptr + _jitc->functions.offset++;
 187     _jitc->function->self.size = stack_framesize;
 188     _jitc->function->self.argi = _jitc->function->self.argf =
 189         _jitc->function->self.aoff = _jitc->function->self.alen = 0;
 190     _jitc->function->self.call = jit_call_default;
 191     jit_alloc((jit_pointer_t *)&_jitc->function->regoff,
 192               _jitc->reglen * sizeof(jit_int32_t));
 193
 194     /* _no_link here does not mean the jit_link() call can be removed
 195      * by rewriting as:
 196      * _jitc->function->prolog = jit_new_node(jit_code_prolog);
 197      */
 198     _jitc->function->prolog = jit_new_node_no_link(jit_code_prolog);
 199     jit_link(_jitc->function->prolog);
 200     _jitc->function->prolog->w.w = offset;
 201     _jitc->function->epilog = jit_new_node_no_link(jit_code_epilog);
 202     /*  u:      label value
 203      *  v:      offset in blocks vector
 204      *  w:      offset in functions vector
 205      */
 206     _jitc->function->epilog->w.w = offset;
 207
 208     jit_regset_new(&_jitc->function->regset);
 209 }
 210
 211 jit_int32_t
 212 _jit_allocai(jit_state_t *_jit, jit_int32_t length)
 213 {
 214     assert(_jitc->function);
 215     switch (length) {
 216         case 0: case 1:                                         break;
 217         case 2:         _jitc->function->self.aoff &= -2;       break;
 218         case 3: case 4: _jitc->function->self.aoff &= -4;       break;
 219         default:        _jitc->function->self.aoff &= -8;       break;
 220     }
 221     _jitc->function->self.aoff -= length;
 222     if (!_jitc->realize) {
 223         jit_inc_synth_ww(allocai, _jitc->function->self.aoff, length);
 224         jit_dec_synth();
 225     }
 226     return (_jitc->function->self.aoff);
 227 }
 228
 229 void
 230 _jit_allocar(jit_state_t *_jit, jit_int32_t u, jit_int32_t v)
 231 {
 232     jit_int32_t          reg;
 233     assert(_jitc->function);
 234     jit_inc_synth_ww(allocar, u, v);
 235     if (!_jitc->function->allocar) {
 236         _jitc->function->aoffoff = jit_allocai(sizeof(jit_int32_t));
 237         _jitc->function->allocar = 1;
 238     }
 239     reg = jit_get_reg(jit_class_gpr);
 240     jit_negr(reg, v);
 241     jit_andi(reg, reg, -8);
 242     jit_ldxi_i(u, JIT_FP, _jitc->function->aoffoff);
 243     jit_addr(u, u, reg);
 244     jit_addr(JIT_SP, JIT_SP, reg);
 245     jit_stxi_i(_jitc->function->aoffoff, JIT_FP, u);
 246     jit_unget_reg(reg);
 247     jit_dec_synth();
 248 }
 249
 250 void
 251 _jit_ret(jit_state_t *_jit)
 252 {
 253     jit_node_t          *instr;
 254     assert(_jitc->function);
 255     jit_inc_synth(ret);
 256     /* jump to epilog */
 257     instr = jit_jmpi();
 258     jit_patch_at(instr, _jitc->function->epilog);
 259     jit_dec_synth();
 260 }
 261
 262 void
 263 _jit_retr(jit_state_t *_jit, jit_int32_t u)
 264 {
 265     jit_inc_synth_w(retr, u);
 266     if (JIT_RET != u)
 267         jit_movr(JIT_RET, u);
 268     jit_live(JIT_RET);
 269     jit_ret();
 270     jit_dec_synth();
 271 }
 272
 273 void
 274 _jit_reti(jit_state_t *_jit, jit_word_t u)
 275 {
 276     jit_inc_synth_w(reti, u);
 277     jit_movi(JIT_RET, u);
 278     jit_ret();
 279     jit_dec_synth();
 280 }
 281
 282 void
 283 _jit_retr_f(jit_state_t *_jit, jit_int32_t u)
 284 {
 285     jit_inc_synth_w(retr_f, u);
 286     if (JIT_FRET != u)
 287         jit_movr_f(JIT_FRET, u);
 288     else
 289         jit_live(JIT_FRET);
 290     jit_ret();
 291     jit_dec_synth();
 292 }
 293
 294 void
 295 _jit_reti_f(jit_state_t *_jit, jit_float32_t u)
 296 {
 297     jit_inc_synth_f(reti_f, u);
 298     jit_movi_f(JIT_FRET, u);
 299     jit_ret();
 300     jit_dec_synth();
 301 }
 302
 303 void
 304 _jit_retr_d(jit_state_t *_jit, jit_int32_t u)
 305 {
 306     jit_inc_synth_w(retr_d, u);
 307     if (JIT_FRET != u)
 308         jit_movr_d(JIT_FRET, u);
 309     else
 310         jit_live(JIT_FRET);
 311     jit_ret();
 312     jit_dec_synth();
 313 }
 314
 315 void
 316 _jit_reti_d(jit_state_t *_jit, jit_float64_t u)
 317 {
 318     jit_inc_synth_d(reti_d, u);
 319     jit_movi_d(JIT_FRET, u);
 320     jit_ret();
 321     jit_dec_synth();
 322 }
 323
 324 void
 325 _jit_epilog(jit_state_t *_jit)
 326 {
 327     assert(_jitc->function);
 328     assert(_jitc->function->epilog->next == NULL);
 329     jit_link(_jitc->function->epilog);
 330     _jitc->function = NULL;
 331 }
 332
 333 jit_bool_t
 334 _jit_arg_register_p(jit_state_t *_jit, jit_node_t *u)
 335 {
 336     if (u->code == jit_code_arg)
 337         return (jit_arg_reg_p(u->u.w));
 338     assert(u->code == jit_code_arg_f || u->code == jit_code_arg_d);
 339 #if NEW_ABI
 340     return (jit_arg_reg_p(u->u.w));
 341 #else
 342     return (u->u.w < 8);
 343 #endif
 344 }
 345
 346 static jit_node_t *
 347 _jit_make_arg(jit_state_t *_jit, jit_node_t *node)
 348 {
 349     jit_int32_t          offset;
 350 #if NEW_ABI
 351     if (jit_arg_reg_p(_jitc->function->self.argi))
 352         offset = _jitc->function->self.argi++;
 353     else {
 354         offset = _jitc->function->self.size;
 355         _jitc->function->self.size += STACK_SLOT;
 356     }
 357 #else
 358     offset = (_jitc->function->self.size - stack_framesize) >> STACK_SHIFT;
 359     _jitc->function->self.argi = 1;
 360     if (offset >= 4)
 361         offset = _jitc->function->self.size;
 362     _jitc->function->self.size += STACK_SLOT;
 363 #endif
 364     if (node == (jit_node_t *)0)
 365         node = jit_new_node(jit_code_arg);
 366     else
 367         link_node(node);
 368     node->u.w = offset;
 369     node->v.w = ++_jitc->function->self.argn;
 370     jit_link_prolog();
 371     return (node);
 372 }
 373
 374 static jit_node_t *
 375 _jit_make_arg_f(jit_state_t *_jit, jit_node_t *node)
 376 {
 377     jit_int32_t          offset;
 378 #if NEW_ABI
 379     if (jit_arg_reg_p(_jitc->function->self.argi)) {
 380         offset = _jitc->function->self.argi++;
 381         if (_jitc->function->self.call & jit_call_varargs)
 382             offset += 8;
 383     }
 384     else {
 385         offset = _jitc->function->self.size;
 386         _jitc->function->self.size += STACK_SLOT;
 387     }
 388 #else
 389     offset = (_jitc->function->self.size - stack_framesize) >> STACK_SHIFT;
 390     if (offset < NUM_WORD_ARGS) {
 391         if (!_jitc->function->self.argi &&
 392             !(_jitc->function->self.call & jit_call_varargs)) {
 393             if (offset == 0)
 394                 offset = 4;
 395             else {
 396                 offset = 6;
 397                 _jitc->function->self.argi = 1;
 398             }
 399             /* Use as flag to rewind in case of varargs function */
 400             ++_jitc->function->self.argf;
 401         }
 402     }
 403     else
 404         offset = _jitc->function->self.size;
 405     _jitc->function->self.size += STACK_SLOT;
 406 #endif
 407     if (node == (jit_node_t *)0)
 408         node = jit_new_node(jit_code_arg_f);
 409     else
 410         link_node(node);
 411     node->u.w = offset;
 412     node->v.w = ++_jitc->function->self.argn;
 413     jit_link_prolog();
 414     return (node);
 415 }
 416
 417 static jit_node_t *
 418 _jit_make_arg_d(jit_state_t *_jit, jit_node_t *node)
 419 {
 420     jit_int32_t          offset;
 421 #if NEW_ABI
 422     if (jit_arg_reg_p(_jitc->function->self.argi)) {
 423         offset = _jitc->function->self.argi++;
 424         if (_jitc->function->self.call & jit_call_varargs)
 425             offset += 8;
 426     }
 427     else {
 428         offset = _jitc->function->self.size;
 429         _jitc->function->self.size += STACK_SLOT;
 430     }
 431 #else
 432     if (_jitc->function->self.size & 7) {
 433         _jitc->function->self.size += 4;
 434         _jitc->function->self.argi = 1;
 435     }
 436     offset = (_jitc->function->self.size - stack_framesize) >> STACK_SHIFT;
 437     if (offset < NUM_WORD_ARGS) {
 438         if (!_jitc->function->self.argi &&
 439             !(_jitc->function->self.call & jit_call_varargs)) {
 440             offset += 4;
 441             /* Use as flag to rewind in case of varargs function */
 442             ++_jitc->function->self.argf;
 443         }
 444     }
 445     else
 446         offset = _jitc->function->self.size;
 447     _jitc->function->self.size += sizeof(jit_float64_t);
 448 #endif
 449     if (node == (jit_node_t *)0)
 450         node = jit_new_node(jit_code_arg_d);
 451     else
 452         link_node(node);
 453     node->u.w = offset;
 454     node->v.w = ++_jitc->function->self.argn;
 455     jit_link_prolog();
 456     return (node);
 457 }
 458
 459 void
 460 _jit_ellipsis(jit_state_t *_jit)
 461 {
 462     if (_jitc->prepare) {
 463         assert(!(_jitc->function->call.call & jit_call_varargs));
 464         _jitc->function->call.call |= jit_call_varargs;
 465 #if !NEW_ABI
 466         if (_jitc->function->call.argf)
 467             rewind_prepare();
 468 #endif
 469     }
 470     else {
 471         assert(!(_jitc->function->self.call & jit_call_varargs));
 472 #if NEW_ABI
 473         /* If varargs start in a register, allocate extra 64 bytes. */
 474         if (jit_arg_reg_p(_jitc->function->self.argi))
 475             rewind_prolog();
 476         /* Do not set during possible rewind. */
 477         _jitc->function->self.call |= jit_call_varargs;
 478 #else
 479         _jitc->function->self.call |= jit_call_varargs;
 480         if (_jitc->function->self.argf)
 481             rewind_prolog();
 482 #endif
 483         _jitc->function->vagp = _jitc->function->self.argi;
 484     }
 485     jit_inc_synth(ellipsis);
 486     if (_jitc->prepare)
 487         jit_link_prepare();
 488     else
 489         jit_link_prolog();
 490     jit_dec_synth();
 491 }
 492
 493 void
 494 _jit_va_push(jit_state_t *_jit, jit_int32_t u)
 495 {
 496     jit_inc_synth_w(va_push, u);
 497     jit_pushargr(u);
 498     jit_dec_synth();
 499 }
 500
 501 jit_node_t *
 502 _jit_arg(jit_state_t *_jit)
 503 {
 504     assert(_jitc->function);
 505     return (jit_make_arg((jit_node_t*)0));
 506 }
 507
 508 jit_node_t *
 509 _jit_arg_f(jit_state_t *_jit)
 510 {
 511     assert(_jitc->function);
 512     return (jit_make_arg_f((jit_node_t*)0));
 513 }
 514
 515 jit_node_t *
 516 _jit_arg_d(jit_state_t *_jit)
 517 {
 518     assert(_jitc->function);
 519     return (jit_make_arg_d((jit_node_t*)0));
 520 }
 521
 522 void
 523 _jit_getarg_c(jit_state_t *_jit, jit_int32_t u, jit_node_t *v)
 524 {
 525     assert(v->code == jit_code_arg);
 526     jit_inc_synth_wp(getarg_c, u, v);
 527     if (jit_arg_reg_p(v->u.w))
 528         jit_extr_c(u, _A0 - v->u.w);
 529     else
 530         jit_ldxi_c(u, _FP, v->u.w + C_DISP);
 531     jit_dec_synth();
 532 }
 533
 534 void
 535 _jit_getarg_uc(jit_state_t *_jit, jit_int32_t u, jit_node_t *v)
 536 {
 537     assert(v->code == jit_code_arg);
 538     jit_inc_synth_wp(getarg_uc, u, v);
 539     if (jit_arg_reg_p(v->u.w))
 540         jit_extr_uc(u, _A0 - v->u.w);
 541     else
 542         jit_ldxi_uc(u, _FP, v->u.w + C_DISP);
 543     jit_dec_synth();
 544 }
 545
 546 void
 547 _jit_getarg_s(jit_state_t *_jit, jit_int32_t u, jit_node_t *v)
 548 {
 549     assert(v->code == jit_code_arg);
 550     jit_inc_synth_wp(getarg_s, u, v);
 551     if (jit_arg_reg_p(v->u.w))
 552         jit_extr_s(u, _A0 - v->u.w);
 553     else
 554         jit_ldxi_s(u, _FP, v->u.w + S_DISP);
 555     jit_dec_synth();
 556 }
 557
 558 void
 559 _jit_getarg_us(jit_state_t *_jit, jit_int32_t u, jit_node_t *v)
 560 {
 561     assert(v->code == jit_code_arg);
 562     jit_inc_synth_wp(getarg_us, u, v);
 563     if (jit_arg_reg_p(v->u.w))
 564         jit_extr_us(u, _A0 - v->u.w);
 565     else
 566         jit_ldxi_us(u, _FP, v->u.w + S_DISP);
 567     jit_dec_synth();
 568 }
 569
 570 void
 571 _jit_getarg_i(jit_state_t *_jit, jit_int32_t u, jit_node_t *v)
 572 {
 573     assert(v->code == jit_code_arg);
 574     jit_inc_synth_wp(getarg_i, u, v);
 575     if (jit_arg_reg_p(v->u.w)) {
 576 #if __WORDSIZE == 64
 577         jit_extr_i(u, _A0 - v->u.w);
 578 #else
 579         jit_movr(u, _A0 - v->u.w);
 580 #endif
 581     }
 582     else
 583         jit_ldxi_i(u, _FP, v->u.w + I_DISP);
 584     jit_dec_synth();
 585 }
 586
 587 #if __WORDSIZE == 64
 588 void
 589 _jit_getarg_ui(jit_state_t *_jit, jit_int32_t u, jit_node_t *v)
 590 {
 591     assert(v->code == jit_code_arg);
 592     jit_inc_synth_wp(getarg_ui, u, v);
 593     if (jit_arg_reg_p(v->u.w))
 594         jit_extr_ui(u, _A0 - v->u.w);
 595     else
 596         jit_ldxi_ui(u, _FP, v->u.w + I_DISP);
 597     jit_dec_synth();
 598 }
 599
 600 void
 601 _jit_getarg_l(jit_state_t *_jit, jit_int32_t u, jit_node_t *v)
 602 {
 603     assert(v->code == jit_code_arg);
 604     jit_inc_synth_wp(getarg_l, u, v);
 605     if (jit_arg_reg_p(v->u.w))
 606         jit_movr(u, _A0 - v->u.w);
 607     else
 608         jit_ldxi_l(u, _FP, v->u.w);
 609     jit_dec_synth();
 610 }
 611 #endif
 612
 613 void
 614 _jit_putargr(jit_state_t *_jit, jit_int32_t u, jit_node_t *v)
 615 {
 616     jit_inc_synth_wp(putargr, u, v);
 617     assert(v->code == jit_code_arg);
 618     if (jit_arg_reg_p(v->u.w))
 619         jit_movr(_A0 - v->u.w, u);
 620     else
 621         jit_stxi(v->u.w + WORD_ADJUST, _FP, u);
 622     jit_dec_synth();
 623 }
 624
 625 void
 626 _jit_putargi(jit_state_t *_jit, jit_word_t u, jit_node_t *v)
 627 {
 628     jit_int32_t         regno;
 629     assert(v->code == jit_code_arg);
 630     jit_inc_synth_wp(putargi, u, v);
 631     if (jit_arg_reg_p(v->u.w))
 632         jit_movi(_A0 - v->u.w, u);
 633     else {
 634         regno = jit_get_reg(jit_class_gpr);
 635         jit_movi(regno, u);
 636         jit_stxi(v->u.w + WORD_ADJUST, _FP, regno);
 637         jit_unget_reg(regno);
 638     }
 639     jit_dec_synth();
 640 }
 641
 642 void
 643 _jit_getarg_f(jit_state_t *_jit, jit_int32_t u, jit_node_t *v)
 644 {
 645     assert(v->code == jit_code_arg_f);
 646     jit_inc_synth_wp(getarg_f, u, v);
 647 #if NEW_ABI
 648     if (jit_arg_reg_p(v->u.w))
 649         jit_movr_f(u, _F12 - v->u.w);
 650     else if (jit_arg_reg_p(v->u.w - 8))
 651         jit_movr_w_f(u, _A0 - v->u.w - 8);
 652 #else
 653     if (v->u.w < 4)
 654         jit_movr_w_f(u, _A0 - v->u.w);
 655     else if (v->u.w < 8)
 656         jit_movr_f(u, _F12 - ((v->u.w - 4) >> 1));
 657 #endif
 658     else
 659         jit_ldxi_f(u, _FP, v->u.w);
 660     jit_dec_synth();
 661 }
 662
 663 void
 664 _jit_putargr_f(jit_state_t *_jit, jit_int32_t u, jit_node_t *v)
 665 {
 666     assert(v->code == jit_code_arg_f);
 667     jit_inc_synth_wp(putargr_f, u, v);
 668 #if NEW_ABI
 669     if (jit_arg_reg_p(v->u.w))
 670         jit_movr_f(_F12 - v->u.w, u);
 671     else if (jit_arg_reg_p(v->u.w - 8))
 672         jit_movr_f_w(_A0 - v->u.w - 8, u);
 673 #else
 674     if (v->u.w < 4)
 675         jit_movr_f_w(_A0 - v->u.w, u);
 676     else if (v->u.w < 8)
 677         jit_movr_f(_F12 - ((v->u.w - 4) >> 1), u);
 678 #endif
 679     else
 680         jit_stxi_f(v->u.w, _FP, u);
 681     jit_dec_synth();
 682 }
 683
 684 void
 685 _jit_putargi_f(jit_state_t *_jit, jit_float32_t u, jit_node_t *v)
 686 {
 687     jit_int32_t         regno;
 688     assert(v->code == jit_code_arg_f);
 689     jit_inc_synth_fp(putargi_f, u, v);
 690 #if NEW_ABI
 691     if (jit_arg_reg_p(v->u.w))
 692         jit_movi_f(_F12 - v->u.w, u);
 693     else if (jit_arg_reg_p(v->u.w - 8)) {
 694         regno = jit_get_reg(jit_class_fpr);
 695         jit_movi_f(regno, u);
 696         jit_movr_f_w(_A0 - v->u.w - 8, u);
 697         jit_unget_reg(regno);
 698     }
 699 #else
 700     if (v->u.w < 4) {
 701         regno = jit_get_reg(jit_class_fpr);
 702         jit_movi_f(regno, u);
 703         jit_movr_f_w(_A0 - ((v->u.w - 4) >> 1), regno);
 704         jit_unget_reg(regno);
 705     }
 706     else if (v->u.w < 8)
 707         jit_movi_f(_F12 - ((v->u.w - 4) >> 1), u);
 708 #endif
 709     else {
 710         regno = jit_get_reg(jit_class_fpr);
 711         jit_movi_f(regno, u);
 712         jit_stxi_f(v->u.w, _FP, regno);
 713         jit_unget_reg(regno);
 714     }
 715     jit_dec_synth();
 716 }
 717
 718 void
 719 _jit_getarg_d(jit_state_t *_jit, jit_int32_t u, jit_node_t *v)
 720 {
 721     assert(v->code == jit_code_arg_d);
 722     jit_inc_synth_wp(getarg_d, u, v);
 723 #if NEW_ABI
 724     if (jit_arg_reg_p(v->u.w))
 725         jit_movr_d(u, _F12 - v->u.w);
 726     else if (jit_arg_reg_p(v->u.w - 8))
 727         jit_movr_d_w(_A0 - v->u.w - 8, u);
 728 #else
 729     if (v->u.w < 4)
 730         jit_movr_ww_d(u, _A0 - v->u.w, _A0 - (v->u.w + 1));
 731     else if (v->u.w < 8)
 732         jit_movr_d(u, _F12 - ((v->u.w - 4) >> 1));
 733 #endif
 734     else
 735         jit_ldxi_d(u, _FP, v->u.w);
 736     jit_dec_synth();
 737 }
 738
 739 void
 740 _jit_putargr_d(jit_state_t *_jit, jit_int32_t u, jit_node_t *v)
 741 {
 742     assert(v->code == jit_code_arg_d);
 743     jit_inc_synth_wp(putargr_d, u, v);
 744 #if NEW_ABI
 745     if (jit_arg_reg_p(v->u.w))
 746         jit_movr_d(_F12 - v->u.w, u);
 747     else if (jit_arg_reg_p(v->u.w - 8))
 748         jit_movr_d_w(_A0 - v->u.w - 8, u);
 749 #else
 750     if (v->u.w < 4)
 751         jit_movr_d_ww(_A0 - v->u.w, _A0 - (v->u.w + 1), u);
 752     else if (v->u.w < 8)
 753         jit_movr_d(_F12 - ((v->u.w - 4) >> 1), u);
 754 #endif
 755     else
 756         jit_stxi_d(v->u.w, _FP, u);
 757     jit_dec_synth();
 758 }
 759
 760 void
 761 _jit_putargi_d(jit_state_t *_jit, jit_float64_t u, jit_node_t *v)
 762 {
 763     jit_int32_t         regno;
 764     assert(v->code == jit_code_arg_d);
 765     jit_inc_synth_dp(putargi_d, u, v);
 766 #if NEW_ABI
 767     if (jit_arg_reg_p(v->u.w))
 768         jit_movi_d(_F12 - v->u.w, u);
 769     else if (jit_arg_reg_p(v->u.w - 8)) {
 770         regno = jit_get_reg(jit_class_fpr);
 771         jit_movi_d(regno, u);
 772         jit_movr_d_w(_A0 - v->u.w - 8, u);
 773         jit_unget_reg(regno);
 774     }
 775 #else
 776     if (v->u.w < 4) {
 777         regno = jit_get_reg(jit_class_fpr);
 778         jit_movi_d(regno, u);
 779         jit_movr_d_ww(_A0 - v->u.w, _A0 - (v->u.w + 1), regno);
 780         jit_unget_reg(regno);
 781     }
 782     else if (v->u.w < 8)
 783         jit_movi_d(_F12 - ((v->u.w - 4) >> 1), u);
 784 #endif
 785     else {
 786         regno = jit_get_reg(jit_class_fpr);
 787         jit_movi_d(regno, u);
 788         jit_stxi_d(v->u.w, _FP, regno);
 789         jit_unget_reg(regno);
 790     }
 791     jit_dec_synth();
 792 }
 793
 794 void
 795 _jit_pushargr(jit_state_t *_jit, jit_int32_t u)
 796 {
 797     jit_inc_synth_w(pushargr, u);
 798     jit_link_prepare();
 799 #if NEW_ABI
 800     assert(_jitc->function);
 801     if (jit_arg_reg_p(_jitc->function->call.argi)) {
 802         jit_movr(_A0 - _jitc->function->call.argi, u);
 803         ++_jitc->function->call.argi;
 804     }
 805     else {
 806         jit_stxi(_jitc->function->call.size + WORD_ADJUST, JIT_SP, u);
 807         _jitc->function->call.size += STACK_SLOT;
 808     }
 809 #else
 810     jit_word_t          offset;
 811     assert(_jitc->function);
 812     offset = _jitc->function->call.size >> STACK_SHIFT;
 813     _jitc->function->call.argi = 1;
 814     if (jit_arg_reg_p(offset))
 815         jit_movr(_A0 - offset, u);
 816     else
 817         jit_stxi(_jitc->function->call.size, JIT_SP, u);
 818     _jitc->function->call.size += STACK_SLOT;
 819 #endif
 820     jit_dec_synth();
 821 }
 822
 823 void
 824 _jit_pushargi(jit_state_t *_jit, jit_word_t u)
 825 {
 826     jit_int32_t         regno;
 827 #if !NEW_ABI
 828     jit_word_t          offset;
 829 #endif
 830     assert(_jitc->function);
 831     jit_inc_synth_w(pushargi, u);
 832     jit_link_prepare();
 833 #if NEW_ABI
 834     if (jit_arg_reg_p(_jitc->function->call.argi)) {
 835         jit_movi(_A0 - _jitc->function->call.argi, u);
 836         ++_jitc->function->call.argi;
 837     }
 838     else {
 839         regno = jit_get_reg(jit_class_gpr);
 840         jit_movi(regno, u);
 841         jit_stxi(_jitc->function->call.size + WORD_ADJUST, JIT_SP, regno);
 842         _jitc->function->call.size += STACK_SLOT;
 843         jit_unget_reg(regno);
 844     }
 845 #else
 846     offset = _jitc->function->call.size >> STACK_SHIFT;
 847     ++_jitc->function->call.argi;
 848     if (jit_arg_reg_p(offset))
 849         jit_movi(_A0 - offset, u);
 850     else {
 851         regno = jit_get_reg(jit_class_gpr);
 852         jit_movi(regno, u);
 853         jit_stxi(_jitc->function->call.size, JIT_SP, regno);
 854         jit_unget_reg(regno);
 855     }
 856     _jitc->function->call.size += STACK_SLOT;
 857 #endif
 858     jit_dec_synth();
 859 }
 860
 861 void
 862 _jit_pushargr_f(jit_state_t *_jit, jit_int32_t u)
 863 {
 864 #if !NEW_ABI
 865     jit_word_t          offset;
 866 #endif
 867     assert(_jitc->function);
 868     jit_inc_synth_w(pushargr_f, u);
 869     jit_link_prepare();
 870 #if NEW_ABI
 871     if (jit_arg_reg_p(_jitc->function->call.argi)) {
 872         if (!(_jitc->function->call.call & jit_call_varargs))
 873             jit_movr_f(_F12 - _jitc->function->call.argi, u);
 874         else
 875             jit_movr_f_w(_A0 - _jitc->function->call.argi, u);
 876         ++_jitc->function->call.argi;
 877     }
 878     else {
 879         jit_stxi_f(_jitc->function->call.size, JIT_SP, u);
 880         _jitc->function->call.size += STACK_SLOT;
 881     }
 882 #else
 883     offset = _jitc->function->call.size >> STACK_SHIFT;
 884     if (offset < 2 && !_jitc->function->call.argi &&
 885         !(_jitc->function->call.call & jit_call_varargs)) {
 886         ++_jitc->function->call.argf;
 887         jit_movr_f(_F12 - offset, u);
 888     }
 889     else if (offset < 4) {
 890         ++_jitc->function->call.argi;
 891         jit_movr_f_w(_A0 - offset, u);
 892     }
 893     else
 894         jit_stxi_f(_jitc->function->call.size, JIT_SP, u);
 895     _jitc->function->call.size += STACK_SLOT;
 896 #endif
 897     jit_dec_synth();
 898 }
 899
 900 void
 901 _jit_pushargi_f(jit_state_t *_jit, jit_float32_t u)
 902 {
 903     jit_int32_t         regno;
 904 #if !NEW_ABI
 905     jit_word_t          offset;
 906 #endif
 907     assert(_jitc->function);
 908     jit_inc_synth_f(pushargi_f, u);
 909     jit_link_prepare();
 910 #if NEW_ABI
 911     if (jit_arg_reg_p(_jitc->function->call.argi)) {
 912         if (!(_jitc->function->call.call & jit_call_varargs))
 913             jit_movi_f(_F12 - _jitc->function->call.argi, u);
 914         else
 915             jit_movi_f_w(_A0 - _jitc->function->call.argi, u);
 916         ++_jitc->function->call.argi;
 917     }
 918     else {
 919         regno = jit_get_reg(jit_class_fpr);
 920         jit_movi_f(regno, u);
 921         jit_stxi_f(_jitc->function->call.size, JIT_SP, regno);
 922         _jitc->function->call.size += STACK_SLOT;
 923         jit_unget_reg(regno);
 924     }
 925 #else
 926     offset = _jitc->function->call.size >> STACK_SHIFT;
 927     if (offset < 2 && !_jitc->function->call.argi &&
 928         !(_jitc->function->call.call & jit_call_varargs)) {
 929         ++_jitc->function->call.argf;
 930         jit_movi_f(_F12 - offset, u);
 931     }
 932     else if (offset < 4) {
 933         ++_jitc->function->call.argi;
 934         jit_movi_f_w(_A0 - offset, u);
 935     }
 936     else {
 937         regno = jit_get_reg(jit_class_fpr);
 938         jit_movi_f(regno, u);
 939         jit_stxi_f(_jitc->function->call.size, JIT_SP, regno);
 940         jit_unget_reg(regno);
 941     }
 942     _jitc->function->call.size += STACK_SLOT;
 943 #endif
 944     jit_dec_synth();
 945 }
 946
 947 void
 948 _jit_pushargr_d(jit_state_t *_jit, jit_int32_t u)
 949 {
 950 #if !NEW_ABI
 951     jit_bool_t          adjust;
 952     jit_word_t          offset;
 953 #endif
 954     assert(_jitc->function);
 955     jit_inc_synth_w(pushargr_d, u);
 956     jit_link_prepare();
 957 #if NEW_ABI
 958     if (jit_arg_reg_p(_jitc->function->call.argi)) {
 959         if (!(_jitc->function->call.call & jit_call_varargs))
 960             jit_movr_d(_F12 - _jitc->function->call.argi, u);
 961         else
 962             jit_movr_d_w(_A0 - _jitc->function->call.argi, u);
 963         ++_jitc->function->call.argi;
 964     }
 965     else {
 966         jit_stxi_d(_jitc->function->call.size, JIT_SP, u);
 967         _jitc->function->call.size += STACK_SLOT;
 968     }
 969 #else
 970     adjust = !!_jitc->function->call.argi;
 971     if (_jitc->function->call.size & 7) {
 972         _jitc->function->call.size += 4;
 973         adjust = 1;
 974     }
 975     offset = _jitc->function->call.size >> STACK_SHIFT;
 976     if (offset < 3) {
 977         if (adjust || (_jitc->function->call.call & jit_call_varargs)) {
 978             jit_movr_d_ww(_A0 - offset, _A0 - (offset + 1), u);
 979             _jitc->function->call.argi += 2;
 980         }
 981         else {
 982             jit_movr_d(_F12 - (offset >> 1), u);
 983             ++_jitc->function->call.argf;
 984         }
 985     }
 986     else
 987         jit_stxi_d(_jitc->function->call.size, JIT_SP, u);
 988     _jitc->function->call.size += sizeof(jit_float64_t);
 989 #endif
 990     jit_dec_synth();
 991 }
 992
 993 void
 994 _jit_pushargi_d(jit_state_t *_jit, jit_float64_t u)
 995 {
 996     jit_int32_t         regno;
 997 #if !NEW_ABI
 998     jit_bool_t          adjust;
 999     jit_word_t          offset;
1000 #endif
1001     assert(_jitc->function);
1002     jit_inc_synth_d(pushargi_d, u);
1003     jit_link_prepare();
1004 #if NEW_ABI
1005     if (jit_arg_reg_p(_jitc->function->call.argi)) {
1006         if (!(_jitc->function->call.call & jit_call_varargs))
1007             jit_movi_d(_F12 - _jitc->function->call.argi, u);
1008         else
1009             jit_movi_d_w(_A0 - _jitc->function->call.argi, u);
1010         ++_jitc->function->call.argi;
1011     }
1012     else {
1013         regno = jit_get_reg(jit_class_fpr);
1014         jit_movi_d(regno, u);
1015         jit_stxi_d(_jitc->function->call.size, JIT_SP, regno);
1016         _jitc->function->call.size += STACK_SLOT;
1017         jit_unget_reg(regno);
1018     }
1019 #else
1020     adjust = !!_jitc->function->call.argi;
1021     if (_jitc->function->call.size & 7) {
1022         _jitc->function->call.size += 4;
1023         adjust = 1;
1024     }
1025     offset = _jitc->function->call.size >> STACK_SHIFT;
1026     if (offset < 3) {
1027         if (adjust || (_jitc->function->call.call & jit_call_varargs)) {
1028             jit_movi_d_ww(_A0 - offset, _A0 - (offset + 1), u);
1029             _jitc->function->call.argi += 2;
1030         }
1031         else {
1032             jit_movi_d(_F12 - (offset >> 1), u);
1033             ++_jitc->function->call.argf;
1034         }
1035     }
1036     else {
1037         regno = jit_get_reg(jit_class_fpr);
1038         jit_movi_d(regno, u);
1039         jit_stxi_d(_jitc->function->call.size, JIT_SP, regno);
1040         jit_unget_reg(regno);
1041     }
1042     _jitc->function->call.size += sizeof(jit_float64_t);
1043 #endif
1044     jit_dec_synth();
1045 }
1046
1047 jit_bool_t
1048 _jit_regarg_p(jit_state_t *_jit, jit_node_t *node, jit_int32_t regno)
1049 {
1050     jit_int32_t         spec;
1051
1052     spec = jit_class(_rvs[regno].spec);
1053     if (spec & jit_class_arg) {
1054         if (spec & jit_class_gpr) {
1055             regno = _A0 - regno;
1056             if (regno >= 0 && regno < node->v.w)
1057                 return (1);
1058         }
1059         else if (spec & jit_class_fpr) {
1060             regno = _F12 - regno;
1061             if (regno >= 0 && regno < node->w.w)
1062                 return (1);
1063         }
1064     }
1065
1066     return (0);
1067 }
1068
1069 void
1070 _jit_finishr(jit_state_t *_jit, jit_int32_t r0)
1071 {
1072     jit_node_t          *call;
1073     assert(_jitc->function);
1074     jit_inc_synth_w(finishr, r0);
1075     if (_jitc->function->self.alen < _jitc->function->call.size)
1076         _jitc->function->self.alen = _jitc->function->call.size;
1077     call = jit_callr(r0);
1078     call->v.w = _jitc->function->self.argi;
1079 #if NEW_ABI
1080     call->w.w = call->v.w;
1081 #else
1082     call->w.w = _jitc->function->self.argf;
1083 #endif
1084     _jitc->function->call.argi = _jitc->function->call.argf =
1085         _jitc->function->call.size = 0;
1086     _jitc->prepare = 0;
1087     jit_dec_synth();
1088 }
1089
1090 jit_node_t *
1091 _jit_finishi(jit_state_t *_jit, jit_pointer_t i0)
1092 {
1093     jit_node_t          *call;
1094     jit_node_t          *node;
1095     assert(_jitc->function);
1096     jit_inc_synth_w(finishi, (jit_word_t)i0);
1097     if (_jitc->function->self.alen < _jitc->function->call.size)
1098         _jitc->function->self.alen = _jitc->function->call.size;
1099     node = jit_movi(_T9, (jit_word_t)i0);
1100     call = jit_callr(_T9);
1101     call->v.w = _jitc->function->call.argi;
1102 #if NEW_ABI
1103     call->w.w = call->v.w;
1104 #else
1105     call->w.w = _jitc->function->call.argf;
1106 #endif
1107     _jitc->function->call.argi = _jitc->function->call.argf =
1108         _jitc->function->call.size = 0;
1109     _jitc->prepare = 0;
1110     jit_dec_synth();
1111     return (node);
1112 }
1113
1114 void
1115 _jit_retval_c(jit_state_t *_jit, jit_int32_t r0)
1116 {
1117     jit_extr_c(r0, JIT_RET);
1118 }
1119
1120 void
1121 _jit_retval_uc(jit_state_t *_jit, jit_int32_t r0)
1122 {
1123     jit_extr_uc(r0, JIT_RET);
1124 }
1125
1126 void
1127 _jit_retval_s(jit_state_t *_jit, jit_int32_t r0)
1128 {
1129     jit_extr_s(r0, JIT_RET);
1130 }
1131
1132 void
1133 _jit_retval_us(jit_state_t *_jit, jit_int32_t r0)
1134 {
1135     jit_extr_us(r0, JIT_RET);
1136 }
1137
1138 void
1139 _jit_retval_i(jit_state_t *_jit, jit_int32_t r0)
1140 {
1141 #if __WORDSIZE == 32
1142     if (r0 != JIT_RET)
1143         jit_movr(r0, JIT_RET);
1144 #else
1145     jit_extr_i(r0, JIT_RET);
1146 #endif
1147 }
1148
1149 #if __WORDSIZE == 64
1150 void
1151 _jit_retval_ui(jit_state_t *_jit, jit_int32_t r0)
1152 {
1153     jit_extr_ui(r0, JIT_RET);
1154 }
1155
1156 void
1157 _jit_retval_l(jit_state_t *_jit, jit_int32_t r0)
1158 {
1159     if (r0 != JIT_RET)
1160         jit_movr(r0, JIT_RET);
1161 }
1162 #endif
1163
1164 void
1165 _jit_retval_f(jit_state_t *_jit, jit_int32_t r0)
1166 {
1167     if (r0 != JIT_FRET)
1168         jit_movr_f(r0, JIT_FRET);
1169 }
1170
1171 void
1172 _jit_retval_d(jit_state_t *_jit, jit_int32_t r0)
1173 {
1174     if (r0 != JIT_FRET)
1175         jit_movr_d(r0, JIT_FRET);
1176 }
1177
1178 jit_pointer_t
1179 _emit_code(jit_state_t *_jit)
1180 {
1181     jit_node_t          *node;
1182     jit_node_t          *temp;
1183     jit_word_t           word;
1184     jit_int32_t          value;
1185     jit_int32_t          offset;
1186     struct {
1187         jit_node_t      *node;
1188         jit_word_t       word;
1189 #if DEVEL_DISASSEMBLER
1190         jit_word_t       prevw;
1191 #endif
1192         jit_int32_t      patch_offset;
1193     } undo;
1194 #if DEVEL_DISASSEMBLER
1195     jit_word_t           prevw;
1196 #endif
1197
1198     _jitc->function = NULL;
1199
1200     jit_reglive_setup();
1201
1202     undo.word = 0;
1203     undo.node = NULL;
1204     undo.patch_offset = 0;
1205 #define case_rr(name, type)                                             \
1206             case jit_code_##name##r##type:                              \
1207                 name##r##type(rn(node->u.w), rn(node->v.w));            \
1208                 break
1209 #define case_rw(name, type)                                             \
1210             case jit_code_##name##i##type:                              \
1211                 name##i##type(rn(node->u.w), node->v.w);                \
1212                 break
1213 #define case_wr(name, type)                                             \
1214             case jit_code_##name##i##type:                              \
1215                 name##i##type(node->u.w, rn(node->v.w));                \
1216                 break
1217 #define case_rrr(name, type)                                            \
1218             case jit_code_##name##r##type:                              \
1219                 name##r##type(rn(node->u.w),                            \
1220                               rn(node->v.w), rn(node->w.w));            \
1221                 break
1222 #define case_rrw(name, type)                                            \
1223             case jit_code_##name##i##type:                              \
1224                 name##i##type(rn(node->u.w), rn(node->v.w), node->w.w); \
1225                 break
1226 #define case_rrrr(name, type)                                           \
1227             case jit_code_##name##r##type:                              \
1228                 name##r##type(rn(node->u.q.l), rn(node->u.q.h),         \
1229                               rn(node->v.w), rn(node->w.w));            \
1230                 break
1231 #define case_rrrw(name, type)                                           \
1232             case jit_code_##name##i##type:                              \
1233                 name##i##type(rn(node->u.q.l), rn(node->u.q.h),         \
1234                               rn(node->v.w), node->w.w);                \
1235                 break
1236 #define case_rrf(name, type, size)                                      \
1237             case jit_code_##name##i##type:                              \
1238                 assert(node->flag & jit_flag_data);                     \
1239                 name##i##type(rn(node->u.w), rn(node->v.w),             \
1240                               (jit_float##size##_t *)node->w.n->u.w);   \
1241                 break
1242 #define case_wrr(name, type)                                            \
1243             case jit_code_##name##i##type:                              \
1244                 name##i##type(node->u.w, rn(node->v.w), rn(node->w.w)); \
1245                 break
1246 #define case_brr(name, type)                                            \
1247             case jit_code_##name##r##type:                              \
1248                 temp = node->u.n;                                       \
1249                 assert(temp->code == jit_code_label ||                  \
1250                        temp->code == jit_code_epilog);                  \
1251                 if (temp->flag & jit_flag_patch)                        \
1252                     name##r##type(temp->u.w, rn(node->v.w),             \
1253                                   rn(node->w.w));                       \
1254                 else {                                                  \
1255                     word = name##r##type(_jit->pc.w,                    \
1256                                          rn(node->v.w), rn(node->w.w)); \
1257                     patch(word, node);                                  \
1258                 }                                                       \
1259                 break
1260 #define case_brw(name, type)                                            \
1261             case jit_code_##name##i##type:                              \
1262                 temp = node->u.n;                                       \
1263                 assert(temp->code == jit_code_label ||                  \
1264                        temp->code == jit_code_epilog);                  \
1265                 if (temp->flag & jit_flag_patch)                        \
1266                     name##i##type(temp->u.w,                            \
1267                                   rn(node->v.w), node->w.w);            \
1268                 else {                                                  \
1269                     word = name##i##type(_jit->pc.w,                    \
1270                                          rn(node->v.w), node->w.w);     \
1271                     patch(word, node);                                  \
1272                 }                                                       \
1273                 break
1274 #define case_brf(name, type, size)                                      \
1275             case jit_code_##name##i##type:                              \
1276                 temp = node->u.n;                                       \
1277                 assert(temp->code == jit_code_label ||                  \
1278                        temp->code == jit_code_epilog);                  \
1279                 if (temp->flag & jit_flag_patch)                        \
1280                     name##i##type(temp->u.w, rn(node->v.w),             \
1281                                 (jit_float##size##_t *)node->w.n->u.w); \
1282                 else {                                                  \
1283                     word = name##i##type(_jit->pc.w, rn(node->v.w),     \
1284                                 (jit_float##size##_t *)node->w.n->u.w); \
1285                     patch(word, node);                                  \
1286                 }                                                       \
1287                 break
1288 #if DEVEL_DISASSEMBLER
1289     prevw = _jit->pc.w;
1290 #endif
1291     for (node = _jitc->head; node; node = node->next) {
1292         if (_jit->pc.uc >= _jitc->code.end)
1293             return (NULL);
1294
1295 #if DEVEL_DISASSEMBLER
1296         node->offset = (jit_uword_t)_jit->pc.w - (jit_uword_t)prevw;
1297         prevw = _jit->pc.w;
1298 #endif
1299         value = jit_classify(node->code);
1300         jit_regarg_set(node, value);
1301         switch (node->code) {
1302             case jit_code_align:
1303                 assert(!(node->u.w & (node->u.w - 1)) &&
1304                        node->u.w <= sizeof(jit_word_t));
1305                 if (node->u.w == sizeof(jit_word_t) &&
1306                     (word = _jit->pc.w & (sizeof(jit_word_t) - 1)))
1307                     nop(sizeof(jit_word_t) - word);
1308                 break;
1309             case jit_code_note:         case jit_code_name:
1310                 node->u.w = _jit->pc.w;
1311                 break;
1312             case jit_code_label:
1313                 /* remember label is defined */
1314                 node->flag |= jit_flag_patch;
1315                 node->u.w = _jit->pc.w;
1316                 break;
1317                 case_rrr(add,);
1318                 case_rrw(add,);
1319                 case_rrr(addc,);
1320                 case_rrw(addc,);
1321                 case_rrr(addx,);
1322                 case_rrw(addx,);
1323                 case_rrr(sub,);
1324                 case_rrw(sub,);
1325                 case_rrr(subc,);
1326                 case_rrw(subc,);
1327                 case_rrr(subx,);
1328                 case_rrw(subx,);
1329                 case_rrw(rsb,);
1330                 case_rrr(mul,);
1331                 case_rrw(mul,);
1332                 case_rrrr(qmul,);
1333                 case_rrrw(qmul,);
1334                 case_rrrr(qmul, _u);
1335                 case_rrrw(qmul, _u);
1336                 case_rrr(div,);
1337                 case_rrw(div,);
1338                 case_rrr(div, _u);
1339                 case_rrw(div, _u);
1340                 case_rrrr(qdiv,);
1341                 case_rrrw(qdiv,);
1342                 case_rrrr(qdiv, _u);
1343                 case_rrrw(qdiv, _u);
1344                 case_rrr(rem,);
1345                 case_rrw(rem,);
1346                 case_rrr(rem, _u);
1347                 case_rrw(rem, _u);
1348                 case_rrr(lsh,);
1349                 case_rrw(lsh,);
1350                 case_rrr(rsh,);
1351                 case_rrw(rsh,);
1352                 case_rrr(rsh, _u);
1353                 case_rrw(rsh, _u);
1354                 case_rrr(and,);
1355                 case_rrw(and,);
1356                 case_rrr(or,);
1357                 case_rrw(or,);
1358                 case_rrr(xor,);
1359                 case_rrw(xor,);
1360                 case_rr(trunc, _f_i);
1361                 case_rr(trunc, _d_i);
1362 #if __WORDSIZE == 64
1363                 case_rr(trunc, _f_l);
1364                 case_rr(trunc, _d_l);
1365 #endif
1366                 case_rr(ld, _c);
1367                 case_rw(ld, _c);
1368                 case_rr(ld, _uc);
1369                 case_rw(ld, _uc);
1370                 case_rr(ld, _s);
1371                 case_rw(ld, _s);
1372                 case_rr(ld, _us);
1373                 case_rw(ld, _us);
1374                 case_rr(ld, _i);
1375                 case_rw(ld, _i);
1376 #if __WORDSIZE == 64
1377                 case_rr(ld, _ui);
1378                 case_rw(ld, _ui);
1379                 case_rr(ld, _l);
1380                 case_rw(ld, _l);
1381 #endif
1382                 case_rrr(ldx, _c);
1383                 case_rrw(ldx, _c);
1384                 case_rrr(ldx, _uc);
1385                 case_rrw(ldx, _uc);
1386                 case_rrr(ldx, _s);
1387                 case_rrw(ldx, _s);
1388                 case_rrr(ldx, _us);
1389                 case_rrw(ldx, _us);
1390                 case_rrr(ldx, _i);
1391                 case_rrw(ldx, _i);
1392 #if __WORDSIZE == 64
1393                 case_rrr(ldx, _ui);
1394                 case_rrw(ldx, _ui);
1395                 case_rrr(ldx, _l);
1396                 case_rrw(ldx, _l);
1397 #endif
1398                 case_rr(st, _c);
1399                 case_wr(st, _c);
1400                 case_rr(st, _s);
1401                 case_wr(st, _s);
1402                 case_rr(st, _i);
1403                 case_wr(st, _i);
1404 #if __WORDSIZE == 64
1405                 case_rr(st, _l);
1406                 case_wr(st, _l);
1407 #endif
1408                 case_rrr(stx, _c);
1409                 case_wrr(stx, _c);
1410                 case_rrr(stx, _s);
1411                 case_wrr(stx, _s);
1412                 case_rrr(stx, _i);
1413                 case_wrr(stx, _i);
1414 #if __WORDSIZE == 64
1415                 case_rrr(stx, _l);
1416                 case_wrr(stx, _l);
1417 #endif
1418                 case_rr(hton, _us);
1419                 case_rr(hton, _ui);
1420 #if __WORDSIZE == 64
1421                 case_rr(hton, _ul);
1422 #endif
1423                 case_rr(bswap, _us);
1424                 case_rr(bswap, _ui);
1425 #if __WORDSIZE == 64
1426                 case_rr(bswap, _ul);
1427 #endif
1428                 case_rr(ext, _c);
1429                 case_rr(ext, _uc);
1430                 case_rr(ext, _s);
1431                 case_rr(ext, _us);
1432 #if __WORDSIZE == 64
1433                 case_rr(ext, _i);
1434                 case_rr(ext, _ui);
1435 #endif
1436             case jit_code_casr:
1437                 casr(rn(node->u.w), rn(node->v.w),
1438                      rn(node->w.q.l), rn(node->w.q.h));
1439                 break;
1440             case jit_code_casi:
1441                 casi(rn(node->u.w), node->v.w,
1442                      rn(node->w.q.l), rn(node->w.q.h));
1443                 break;
1444                 case_rrr(movn,);
1445                 case_rrr(movz,);
1446                 case_rr(mov,);
1447             case jit_code_movi:
1448                 if (node->flag & jit_flag_node) {
1449                     temp = node->v.n;
1450                     if (temp->code == jit_code_data ||
1451                         (temp->code == jit_code_label &&
1452                          (temp->flag & jit_flag_patch)))
1453                         movi(rn(node->u.w), temp->u.w);
1454                     else {
1455                         assert(temp->code == jit_code_label ||
1456                                temp->code == jit_code_epilog);
1457                         word = movi_p(rn(node->u.w), node->v.w);
1458                         patch(word, node);
1459                     }
1460                 }
1461                 else
1462                     movi(rn(node->u.w), node->v.w);
1463                 break;
1464                 case_rr(neg,);
1465                 case_rr(com,);
1466                 case_rrr(lt,);
1467                 case_rrw(lt,);
1468                 case_rrr(lt, _u);
1469                 case_rrw(lt, _u);
1470                 case_rrr(le,);
1471                 case_rrw(le,);
1472                 case_rrr(le, _u);
1473                 case_rrw(le, _u);
1474                 case_rrr(eq,);
1475                 case_rrw(eq,);
1476                 case_rrr(ge,);
1477                 case_rrw(ge,);
1478                 case_rrr(ge, _u);
1479                 case_rrw(ge, _u);
1480                 case_rrr(gt,);
1481                 case_rrw(gt,);
1482                 case_rrr(gt, _u);
1483                 case_rrw(gt, _u);
1484                 case_rrr(ne,);
1485                 case_rrw(ne,);
1486                 case_brr(blt,);
1487                 case_brw(blt,);
1488                 case_brr(blt, _u);
1489                 case_brw(blt, _u);
1490                 case_brr(ble,);
1491                 case_brw(ble,);
1492                 case_brr(ble, _u);
1493                 case_brw(ble, _u);
1494                 case_brr(beq,);
1495                 case_brw(beq,);
1496                 case_brr(bge,);
1497                 case_brw(bge,);
1498                 case_brr(bge, _u);
1499                 case_brw(bge, _u);
1500                 case_brr(bgt,);
1501                 case_brw(bgt,);
1502                 case_brr(bgt, _u);
1503                 case_brw(bgt, _u);
1504                 case_brr(bne,);
1505                 case_brw(bne,);
1506                 case_brr(boadd,);
1507                 case_brw(boadd,);
1508                 case_brr(boadd, _u);
1509                 case_brw(boadd, _u);
1510                 case_brr(bxadd,);
1511                 case_brw(bxadd,);
1512                 case_brr(bxadd, _u);
1513                 case_brw(bxadd, _u);
1514                 case_brr(bosub,);
1515                 case_brw(bosub,);
1516                 case_brr(bosub, _u);
1517                 case_brw(bosub, _u);
1518                 case_brr(bxsub,);
1519                 case_brw(bxsub,);
1520                 case_brr(bxsub, _u);
1521                 case_brw(bxsub, _u);
1522                 case_brr(bms,);
1523                 case_brw(bms,);
1524                 case_brr(bmc,);
1525                 case_brw(bmc,);
1526                 case_rrr(add, _f);
1527                 case_rrf(add, _f, 32);
1528                 case_rrr(sub, _f);
1529                 case_rrf(sub, _f, 32);
1530                 case_rrf(rsb, _f, 32);
1531                 case_rrr(mul, _f);
1532                 case_rrf(mul, _f, 32);
1533                 case_rrr(div, _f);
1534                 case_rrf(div, _f, 32);
1535                 case_rr(abs, _f);
1536                 case_rr(neg, _f);
1537                 case_rr(sqrt, _f);
1538                 case_rr(ext, _f);
1539                 case_rr(ld, _f);
1540                 case_rw(ld, _f);
1541                 case_rrr(ldx, _f);
1542                 case_rrw(ldx, _f);
1543                 case_rr(st, _f);
1544                 case_wr(st, _f);
1545                 case_rrr(stx, _f);
1546                 case_wrr(stx, _f);
1547                 case_rr(mov, _f);
1548             case jit_code_movi_f:
1549                 assert(node->flag & jit_flag_data);
1550                 movi_f(rn(node->u.w), (jit_float32_t *)node->v.n->u.w);
1551                 break;
1552                 case_rr(ext, _d_f);
1553                 case_rrr(lt, _f);
1554                 case_rrf(lt, _f, 32);
1555                 case_rrr(le, _f);
1556                 case_rrf(le, _f, 32);
1557                 case_rrr(eq, _f);
1558                 case_rrf(eq, _f, 32);
1559                 case_rrr(ge, _f);
1560                 case_rrf(ge, _f, 32);
1561                 case_rrr(gt, _f);
1562                 case_rrf(gt, _f, 32);
1563                 case_rrr(ne, _f);
1564                 case_rrf(ne, _f, 32);
1565                 case_rrr(unlt, _f);
1566                 case_rrf(unlt, _f, 32);
1567                 case_rrr(unle, _f);
1568                 case_rrf(unle, _f, 32);
1569                 case_rrr(uneq, _f);
1570                 case_rrf(uneq, _f, 32);
1571                 case_rrr(unge, _f);
1572                 case_rrf(unge, _f, 32);
1573                 case_rrr(ungt, _f);
1574                 case_rrf(ungt, _f, 32);
1575                 case_rrr(ltgt, _f);
1576                 case_rrf(ltgt, _f, 32);
1577                 case_rrr(ord, _f);
1578                 case_rrf(ord, _f, 32);
1579                 case_rrr(unord, _f);
1580                 case_rrf(unord, _f, 32);
1581                 case_brr(blt, _f);
1582                 case_brf(blt, _f, 32);
1583                 case_brr(ble, _f);
1584                 case_brf(ble, _f, 32);
1585                 case_brr(beq, _f);
1586                 case_brf(beq, _f, 32);
1587                 case_brr(bge, _f);
1588                 case_brf(bge, _f, 32);
1589                 case_brr(bgt, _f);
1590                 case_brf(bgt, _f, 32);
1591                 case_brr(bne, _f);
1592                 case_brf(bne, _f, 32);
1593                 case_brr(bunlt, _f);
1594                 case_brf(bunlt, _f, 32);
1595                 case_brr(bunle, _f);
1596                 case_brf(bunle, _f, 32);
1597                 case_brr(buneq, _f);
1598                 case_brf(buneq, _f, 32);
1599                 case_brr(bunge, _f);
1600                 case_brf(bunge, _f, 32);
1601                 case_brr(bungt, _f);
1602                 case_brf(bungt, _f, 32);
1603                 case_brr(bltgt, _f);
1604                 case_brf(bltgt, _f, 32);
1605                 case_brr(bord, _f);
1606                 case_brf(bord, _f, 32);
1607                 case_brr(bunord, _f);
1608                 case_brf(bunord, _f, 32);
1609                 case_rrr(add, _d);
1610                 case_rrf(add, _d, 64);
1611                 case_rrr(sub, _d);
1612                 case_rrf(sub, _d, 64);
1613                 case_rrf(rsb, _d, 64);
1614                 case_rrr(mul, _d);
1615                 case_rrf(mul, _d, 64);
1616                 case_rrr(div, _d);
1617                 case_rrf(div, _d, 64);
1618                 case_rr(abs, _d);
1619                 case_rr(neg, _d);
1620                 case_rr(sqrt, _d);
1621                 case_rr(ext, _d);
1622                 case_rr(ld, _d);
1623                 case_rw(ld, _d);
1624                 case_rrr(ldx, _d);
1625                 case_rrw(ldx, _d);
1626                 case_rr(st, _d);
1627                 case_wr(st, _d);
1628                 case_rrr(stx, _d);
1629                 case_wrr(stx, _d);
1630                 case_rr(mov, _d);
1631             case jit_code_movi_d:
1632                 assert(node->flag & jit_flag_data);
1633                 movi_d(rn(node->u.w), (jit_float64_t *)node->v.n->u.w);
1634                 break;
1635                 case_rr(ext, _f_d);
1636                 case_rrr(lt, _d);
1637                 case_rrf(lt, _d, 64);
1638                 case_rrr(le, _d);
1639                 case_rrf(le, _d, 64);
1640                 case_rrr(eq, _d);
1641                 case_rrf(eq, _d, 64);
1642                 case_rrr(ge, _d);
1643                 case_rrf(ge, _d, 64);
1644                 case_rrr(gt, _d);
1645                 case_rrf(gt, _d, 64);
1646                 case_rrr(ne, _d);
1647                 case_rrf(ne, _d, 64);
1648                 case_rrr(unlt, _d);
1649                 case_rrf(unlt, _d, 64);
1650                 case_rrr(unle, _d);
1651                 case_rrf(unle, _d, 64);
1652                 case_rrr(uneq, _d);
1653                 case_rrf(uneq, _d, 64);
1654                 case_rrr(unge, _d);
1655                 case_rrf(unge, _d, 64);
1656                 case_rrr(ungt, _d);
1657                 case_rrf(ungt, _d, 64);
1658                 case_rrr(ltgt, _d);
1659                 case_rrf(ltgt, _d, 64);
1660                 case_rrr(ord, _d);
1661                 case_rrf(ord, _d, 64);
1662                 case_rrr(unord, _d);
1663                 case_rrf(unord, _d, 64);
1664                 case_brr(blt, _d);
1665                 case_brf(blt, _d, 64);
1666                 case_brr(ble, _d);
1667                 case_brf(ble, _d, 64);
1668                 case_brr(beq, _d);
1669                 case_brf(beq, _d, 64);
1670                 case_brr(bge, _d);
1671                 case_brf(bge, _d, 64);
1672                 case_brr(bgt, _d);
1673                 case_brf(bgt, _d, 64);
1674                 case_brr(bne, _d);
1675                 case_brf(bne, _d, 64);
1676                 case_brr(bunlt, _d);
1677                 case_brf(bunlt, _d, 64);
1678                 case_brr(bunle, _d);
1679                 case_brf(bunle, _d, 64);
1680                 case_brr(buneq, _d);
1681                 case_brf(buneq, _d, 64);
1682                 case_brr(bunge, _d);
1683                 case_brf(bunge, _d, 64);
1684                 case_brr(bungt, _d);
1685                 case_brf(bungt, _d, 64);
1686                 case_brr(bltgt, _d);
1687                 case_brf(bltgt, _d, 64);
1688                 case_brr(bord, _d);
1689                 case_brf(bord, _d, 64);
1690                 case_brr(bunord, _d);
1691                 case_brf(bunord, _d, 64);
1692             case jit_code_jmpr:
1693                 jmpr(rn(node->u.w));
1694                 break;
1695             case jit_code_jmpi:
1696                 if (node->flag & jit_flag_node) {
1697                     temp = node->u.n;
1698                     assert(temp->code == jit_code_label ||
1699                            temp->code == jit_code_epilog);
1700                     if (temp->flag & jit_flag_patch)
1701                         jmpi(temp->u.w);
1702                     else {
1703                         word = jmpi(_jit->pc.w);
1704                         patch(word, node);
1705                     }
1706                 }
1707                 else
1708                     jmpi(node->u.w);
1709                 break;
1710             case jit_code_callr:
1711                 callr(rn(node->u.w));
1712                 break;
1713             case jit_code_calli:
1714                 if (node->flag & jit_flag_node) {
1715                     temp = node->u.n;
1716                     assert(temp->code == jit_code_label ||
1717                            temp->code == jit_code_epilog);
1718                     word = calli_p(temp->u.w);
1719                     if (!(temp->flag & jit_flag_patch))
1720                         patch(word, node);
1721                 }
1722                 else
1723                     calli(node->u.w);
1724                 break;
1725             case jit_code_prolog:
1726                 _jitc->function = _jitc->functions.ptr + node->w.w;
1727                 undo.node = node;
1728                 undo.word = _jit->pc.w;
1729 #if DEVEL_DISASSEMBLER
1730                 undo.prevw = prevw;
1731 #endif
1732                 undo.patch_offset = _jitc->patches.offset;
1733             restart_function:
1734                 _jitc->again = 0;
1735                 prolog(node);
1736                 break;
1737             case jit_code_epilog:
1738                 assert(_jitc->function == _jitc->functions.ptr + node->w.w);
1739                 if (_jitc->again) {
1740                     for (temp = undo.node->next;
1741                          temp != node; temp = temp->next) {
1742                         if (temp->code == jit_code_label ||
1743                             temp->code == jit_code_epilog)
1744                             temp->flag &= ~jit_flag_patch;
1745                     }
1746                     temp->flag &= ~jit_flag_patch;
1747                     node = undo.node;
1748                     _jit->pc.w = undo.word;
1749 #if DEVEL_DISASSEMBLER
1750                     prevw = undo.prevw;
1751 #endif
1752                     _jitc->patches.offset = undo.patch_offset;
1753                     goto restart_function;
1754                 }
1755                 /* remember label is defined */
1756                 node->flag |= jit_flag_patch;
1757                 node->u.w = _jit->pc.w;
1758                 epilog(node);
1759                 _jitc->function = NULL;
1760                 break;
1761 #if !NEW_ABI
1762             case jit_code_movr_w_f:
1763                 movr_w_f(rn(node->u.w), rn(node->v.w));
1764                 break;
1765 #endif
1766             case jit_code_movr_f_w:
1767                 movr_f_w(rn(node->u.w), rn(node->v.w));
1768                 break;
1769             case jit_code_movi_f_w:
1770                 assert(node->flag & jit_flag_data);
1771                 movi_f_w(rn(node->u.w), (jit_float32_t *)node->v.n->u.w);
1772                 break;
1773 #if NEW_ABI
1774             case jit_code_movr_d_w:
1775                 movr_d_w(rn(node->u.w), rn(node->v.w));
1776                 break;
1777             case jit_code_movi_d_w:
1778                 assert(node->flag & jit_flag_data);
1779                 movi_d_w(rn(node->u.w), (jit_float64_t *)node->v.n->u.w);
1780                 break;
1781 #else
1782             case jit_code_movr_ww_d:
1783                 movr_ww_d(rn(node->u.w), rn(node->v.w), rn(node->w.w));
1784                 break;
1785             case jit_code_movr_d_ww:
1786                 movr_d_ww(rn(node->u.w), rn(node->v.w), rn(node->w.w));
1787                 break;
1788             case jit_code_movi_d_ww:
1789                 assert(node->flag & jit_flag_data);
1790                 movi_d_ww(rn(node->u.w), rn(node->v.w),
1791                           (jit_float64_t *)node->w.n->u.w);
1792                 break;
1793 #endif
1794             case jit_code_va_start:
1795                 vastart(rn(node->u.w));
1796                 break;
1797             case jit_code_va_arg:
1798                 vaarg(rn(node->u.w), rn(node->v.w));
1799                 break;
1800             case jit_code_va_arg_d:
1801                 vaarg_d(rn(node->u.w), rn(node->v.w));
1802                 break;
1803             case jit_code_live:
1804             case jit_code_arg:                  case jit_code_ellipsis:
1805             case jit_code_va_push:
1806             case jit_code_allocai:              case jit_code_allocar:
1807             case jit_code_arg_f:                case jit_code_arg_d:
1808             case jit_code_va_end:
1809             case jit_code_ret:
1810             case jit_code_retr:                 case jit_code_reti:
1811             case jit_code_retr_f:               case jit_code_reti_f:
1812             case jit_code_retr_d:               case jit_code_reti_d:
1813             case jit_code_getarg_c:             case jit_code_getarg_uc:
1814             case jit_code_getarg_s:             case jit_code_getarg_us:
1815             case jit_code_getarg_i:
1816 #if __WORDSIZE == 64
1817             case jit_code_getarg_ui:            case jit_code_getarg_l:
1818 #endif
1819             case jit_code_getarg_f:             case jit_code_getarg_d:
1820             case jit_code_putargr:              case jit_code_putargi:
1821             case jit_code_putargr_f:            case jit_code_putargi_f:
1822             case jit_code_putargr_d:            case jit_code_putargi_d:
1823             case jit_code_pushargr:             case jit_code_pushargi:
1824             case jit_code_pushargr_f:           case jit_code_pushargi_f:
1825             case jit_code_pushargr_d:           case jit_code_pushargi_d:
1826             case jit_code_retval_c:             case jit_code_retval_uc:
1827             case jit_code_retval_s:             case jit_code_retval_us:
1828             case jit_code_retval_i:
1829 #if __WORDSIZE == 64
1830             case jit_code_retval_ui:            case jit_code_retval_l:
1831 #endif
1832             case jit_code_retval_f:             case jit_code_retval_d:
1833             case jit_code_prepare:
1834             case jit_code_finishr:              case jit_code_finishi:
1835                 break;
1836             default:
1837                 abort();
1838         }
1839         if (jit_carry != _NOREG) {
1840             switch (node->code) {
1841                 case jit_code_note:
1842                 case jit_code_addcr:            case jit_code_addci:
1843                 case jit_code_addxr:            case jit_code_addxi:
1844                 case jit_code_subcr:            case jit_code_subci:
1845                 case jit_code_subxr:            case jit_code_subxi:
1846                     break;
1847                 default:
1848                     jit_unget_reg(jit_carry);
1849                     jit_carry = _NOREG;
1850                     break;
1851             }
1852         }
1853         jit_regarg_clr(node, value);
1854         assert(_jitc->regarg == 0 ||
1855                (jit_carry != _NOREG && _jitc->regarg == (1 << jit_carry)));
1856         assert(_jitc->synth == 0);
1857         /* update register live state */
1858         jit_reglive(node);
1859     }
1860 #undef case_brf
1861 #undef case_brw
1862 #undef case_brr
1863 #undef case_wrr
1864 #undef case_rrf
1865 #undef case_rrw
1866 #undef case_rrr
1867 #undef case_wr
1868 #undef case_rw
1869 #undef case_rr
1870
1871     for (offset = 0; offset < _jitc->patches.offset; offset++) {
1872         node = _jitc->patches.ptr[offset].node;
1873         word = node->code == jit_code_movi ? node->v.n->u.w : node->u.n->u.w;
1874         patch_at(_jitc->patches.ptr[offset].inst, word);
1875     }
1876
1877     jit_flush(_jit->code.ptr, _jit->pc.uc);
1878
1879     return (_jit->code.ptr);
1880 }
1881
1882 #define CODE                            1
1883 #  include "jit_rewind.c"
1884 #  include "jit_mips-cpu.c"
1885 #  include "jit_mips-fpu.c"
1886 #  include "jit_fallback.c"
1887 #undef CODE
1888
1889 void
1890 jit_flush(void *fptr, void *tptr)
1891 {
1892 #if defined(__linux__)
1893     jit_word_t          f, t, s;
1894
1895     s = sysconf(_SC_PAGE_SIZE);
1896     f = (jit_word_t)fptr & -s;
1897     t = (((jit_word_t)tptr) + s - 1) & -s;
1898     _flush_cache((void *)f, t - f, ICACHE);
1899 #endif
1900 }
1901
1902 void
1903 _emit_ldxi(jit_state_t *_jit, jit_int32_t r0, jit_int32_t r1, jit_word_t i0)
1904 {
1905     ldxi(rn(r0), rn(r1), i0);
1906 }
1907
1908 void
1909 _emit_stxi(jit_state_t *_jit, jit_word_t i0, jit_int32_t r0, jit_int32_t r1)
1910 {
1911     stxi(i0, rn(r0), rn(r1));
1912 }
1913
1914 void
1915 _emit_ldxi_d(jit_state_t *_jit, jit_int32_t r0, jit_int32_t r1, jit_word_t i0)
1916 {
1917     ldxi_d(rn(r0), rn(r1), i0);
1918 }
1919
1920 void
1921 _emit_stxi_d(jit_state_t *_jit, jit_word_t i0, jit_int32_t r0, jit_int32_t r1)
1922 {
1923     stxi_d(i0, rn(r0), rn(r1));
1924 }
1925
1926 static void
1927 _patch(jit_state_t *_jit, jit_word_t instr, jit_node_t *node)
1928 {
1929     jit_int32_t          flag;
1930
1931     assert(node->flag & jit_flag_node);
1932     if (node->code == jit_code_movi)
1933         flag = node->v.n->flag;
1934     else
1935         flag = node->u.n->flag;
1936     assert(!(flag & jit_flag_patch));
1937     if (_jitc->patches.offset >= _jitc->patches.length) {
1938         jit_realloc((jit_pointer_t *)&_jitc->patches.ptr,
1939                     _jitc->patches.length * sizeof(jit_patch_t),
1940                     (_jitc->patches.length + 1024) * sizeof(jit_patch_t));
1941         _jitc->patches.length += 1024;
1942     }
1943     _jitc->patches.ptr[_jitc->patches.offset].inst = instr;
1944     _jitc->patches.ptr[_jitc->patches.offset].node = node;
1945     ++_jitc->patches.offset;
1946 }