deps/lightning/lib/jit_mips.c

   1 /*
   2  * Copyright (C) 2012-2022  Free Software Foundation, Inc.
   3  *
   4  * This file is part of GNU lightning.
   5  *
   6  * GNU lightning is free software; you can redistribute it and/or modify it
   7  * under the terms of the GNU Lesser General Public License as published
   8  * by the Free Software Foundation; either version 3, or (at your option)
   9  * any later version.
  10  *
  11  * GNU lightning is distributed in the hope that it will be useful, but
  12  * WITHOUT ANY WARRANTY; without even the implied warranty of MERCHANTABILITY
  13  * or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU Lesser General Public
  14  * License for more details.
  15  *
  16  * Authors:
  17  *      Paulo Cesar Pereira de Andrade
  18  */
  19
  20 #if defined(__linux__)
  21 #  include <sys/cachectl.h>
  22 #endif
  23
  24 #if NEW_ABI
  25 #  define NUM_WORD_ARGS                 8
  26 #  define STACK_SLOT                    8
  27 #  define STACK_SHIFT                   3
  28 #else
  29 #  define NUM_WORD_ARGS                 4
  30 #  define STACK_SLOT                    4
  31 #  define STACK_SHIFT                   2
  32 #endif
  33 #if NEW_ABI && __BYTE_ORDER == __BIG_ENDIAN && __WORDSIZE == 32
  34 #  define WORD_ADJUST                   4
  35 #else
  36 #  define WORD_ADJUST                   0
  37 #endif
  38 #define jit_arg_reg_p(i)                ((i) >= 0 && (i) < NUM_WORD_ARGS)
  39 #if __BYTE_ORDER == __LITTLE_ENDIAN
  40 #  define C_DISP                        0
  41 #  define S_DISP                        0
  42 #  define I_DISP                        0
  43 #else
  44 #  define C_DISP                        STACK_SLOT - sizeof(jit_int8_t)
  45 #  define S_DISP                        STACK_SLOT - sizeof(jit_int16_t)
  46 #  define I_DISP                        STACK_SLOT - sizeof(jit_int32_t)
  47 #endif
  48
  49 /*
  50  * Types
  51  */
  52 typedef struct jit_pointer_t jit_va_list_t;
  53
  54 /*
  55  * Prototypes
  56  */
  57 #define jit_make_arg(node)              _jit_make_arg(_jit,node)
  58 static jit_node_t *_jit_make_arg(jit_state_t*,jit_node_t*);
  59 #define jit_make_arg_f(node)            _jit_make_arg_f(_jit,node)
  60 static jit_node_t *_jit_make_arg_f(jit_state_t*,jit_node_t*);
  61 #define jit_make_arg_d(node)            _jit_make_arg_d(_jit,node)
  62 static jit_node_t *_jit_make_arg_d(jit_state_t*,jit_node_t*);
  63 #define patch(instr, node)              _patch(_jit, instr, node)
  64 static void _patch(jit_state_t*,jit_word_t,jit_node_t*);
  65
  66 #define PROTO                           1
  67 #  include "jit_rewind.c"
  68 #  include "jit_mips-cpu.c"
  69 #  include "jit_mips-fpu.c"
  70 #undef PROTO
  71
  72 /*
  73  * Initialization
  74  */
  75 jit_register_t          _rvs[] = {
  76     { rc(gpr) | 0x01,                   "at" },
  77     { rc(gpr) | 0x02,                   "v0" },
  78     { rc(gpr) | 0x03,                   "v1" },
  79 #if !NEW_ABI
  80     { rc(gpr) | 0x08,                   "t0" },
  81     { rc(gpr) | 0x09,                   "t1" },
  82     { rc(gpr) | 0x0a,                   "t2" },
  83     { rc(gpr) | 0x0b,                   "t3" },
  84 #endif
  85     { rc(gpr) | 0x0c,                   "t4" },
  86     { rc(gpr) | 0x0d,                   "t5" },
  87     { rc(gpr) | 0x0e,                   "t6" },
  88     { rc(gpr) | 0x0f,                   "t7" },
  89     { rc(gpr) | 0x18,                   "t8" },
  90     { rc(gpr) | 0x19,                   "t9" },
  91     { rc(sav) | rc(gpr) | 0x10,         "s0" },
  92     { rc(sav) | rc(gpr) | 0x11,         "s1" },
  93     { rc(sav) | rc(gpr) | 0x12,         "s2" },
  94     { rc(sav) | rc(gpr) | 0x13,         "s3" },
  95     { rc(sav) | rc(gpr) | 0x14,         "s4" },
  96     { rc(sav) | rc(gpr) | 0x15,         "s5" },
  97     { rc(sav) | rc(gpr) | 0x16,         "s6" },
  98     { rc(sav) | rc(gpr) | 0x17,         "s7" },
  99     { 0x00,                             "zero" },
 100     { 0x1a,                             "k0" },
 101     { 0x1b,                             "k1" },
 102     { rc(sav) | 0x1f,                   "ra" },
 103     { rc(sav) | 0x1c,                   "gp" },
 104     { rc(sav) | 0x1d,                   "sp" },
 105     { rc(sav) | 0x1e,                   "fp" },
 106 #if NEW_ABI
 107     { rc(gpr) | 0x0b,                   "a7" },
 108     { rc(gpr) | 0x0a,                   "a6" },
 109     { rc(gpr) | 0x09,                   "a5" },
 110     { rc(gpr) | 0x08,                   "a4" },
 111 #endif
 112     { rc(arg) | rc(gpr) | 0x07,         "a3" },
 113     { rc(arg) | rc(gpr) | 0x06,         "a2" },
 114     { rc(arg) | rc(gpr) | 0x05,         "a1" },
 115     { rc(arg) | rc(gpr) | 0x04,         "a0" },
 116     { rc(fpr) | 0x00,                   "$f0" },
 117     { rc(fpr) | 0x02,                   "$f2" },
 118     { rc(fpr) | 0x04,                   "$f4" },
 119     { rc(fpr) | 0x06,                   "$f6" },
 120     { rc(fpr) | 0x08,                   "$f8" },
 121     { rc(fpr) | 0x0a,                   "$f10" },
 122 #if !NEW_ABI
 123     { rc(sav) | rc(fpr) | 0x10,         "$f16" },
 124     { rc(sav) | rc(fpr) | 0x12,         "$f18" },
 125 #endif
 126     { rc(sav) | rc(fpr) | 0x14,         "$f20" },
 127     { rc(sav) | rc(fpr) | 0x16,         "$f22" },
 128     { rc(sav) | rc(fpr) | 0x18,         "$f24" },
 129     { rc(sav) | rc(fpr) | 0x1a,         "$f26" },
 130     { rc(sav) | rc(fpr) | 0x1c,         "$f28" },
 131     { rc(sav) | rc(fpr) | 0x1e,         "$f30" },
 132 #if NEW_ABI
 133     { rc(arg) | rc(fpr) | 0x13,         "$f19" },
 134     { rc(arg) | rc(fpr) | 0x12,         "$f18" },
 135     { rc(arg) | rc(fpr) | 0x11,         "$f17" },
 136     { rc(arg) | rc(fpr) | 0x10,         "$f16" },
 137     { rc(arg) | rc(fpr) | 0x0f,         "$f15" },
 138     { rc(arg) | rc(fpr) | 0x0e,         "$f14" },
 139     { rc(arg) | rc(fpr) | 0x0d,         "$f13" },
 140     { rc(arg) | rc(fpr) | 0x0c,         "$f12" },
 141 #else
 142     { rc(arg) | rc(fpr) | 0x0e,         "$f14" },
 143     { rc(arg) | rc(fpr) | 0x0c,         "$f12" },
 144 #endif
 145     { _NOREG,                           "<none>" },
 146 };
 147
 148 /*
 149  * Implementation
 150  */
 151 void
 152 jit_get_cpu(void)
 153 {
 154 }
 155
 156 void
 157 _jit_init(jit_state_t *_jit)
 158 {
 159     _jitc->reglen = jit_size(_rvs) - 1;
 160 /* Could also:
 161  *      o reserve a register for carry  (overkill)
 162  *      o use MTLO/MFLO                 (performance hit)
 163  * So, keep a register allocated after setting carry, and implicitly
 164  * deallocate it if it can no longer be tracked
 165  */
 166     jit_carry = _NOREG;
 167 }
 168
 169 void
 170 _jit_prolog(jit_state_t *_jit)
 171 {
 172     jit_int32_t          offset;
 173
 174     if (_jitc->function)
 175         jit_epilog();
 176     assert(jit_regset_cmp_ui(&_jitc->regarg, 0) == 0);
 177     jit_regset_set_ui(&_jitc->regsav, 0);
 178     offset = _jitc->functions.offset;
 179     if (offset >= _jitc->functions.length) {
 180         jit_realloc((jit_pointer_t *)&_jitc->functions.ptr,
 181                     _jitc->functions.length * sizeof(jit_function_t),
 182                     (_jitc->functions.length + 16) * sizeof(jit_function_t));
 183         _jitc->functions.length += 16;
 184     }
 185     _jitc->function = _jitc->functions.ptr + _jitc->functions.offset++;
 186     _jitc->function->self.size = stack_framesize;
 187     _jitc->function->self.argi = _jitc->function->self.argf =
 188         _jitc->function->self.aoff = _jitc->function->self.alen = 0;
 189     _jitc->function->self.call = jit_call_default;
 190     jit_alloc((jit_pointer_t *)&_jitc->function->regoff,
 191               _jitc->reglen * sizeof(jit_int32_t));
 192
 193     /* _no_link here does not mean the jit_link() call can be removed
 194      * by rewriting as:
 195      * _jitc->function->prolog = jit_new_node(jit_code_prolog);
 196      */
 197     _jitc->function->prolog = jit_new_node_no_link(jit_code_prolog);
 198     jit_link(_jitc->function->prolog);
 199     _jitc->function->prolog->w.w = offset;
 200     _jitc->function->epilog = jit_new_node_no_link(jit_code_epilog);
 201     /*  u:      label value
 202      *  v:      offset in blocks vector
 203      *  w:      offset in functions vector
 204      */
 205     _jitc->function->epilog->w.w = offset;
 206
 207     jit_regset_new(&_jitc->function->regset);
 208 }
 209
 210 jit_int32_t
 211 _jit_allocai(jit_state_t *_jit, jit_int32_t length)
 212 {
 213     assert(_jitc->function);
 214     switch (length) {
 215         case 0: case 1:                                         break;
 216         case 2:         _jitc->function->self.aoff &= -2;       break;
 217         case 3: case 4: _jitc->function->self.aoff &= -4;       break;
 218         default:        _jitc->function->self.aoff &= -8;       break;
 219     }
 220     _jitc->function->self.aoff -= length;
 221     if (!_jitc->realize) {
 222         jit_inc_synth_ww(allocai, _jitc->function->self.aoff, length);
 223         jit_dec_synth();
 224     }
 225     return (_jitc->function->self.aoff);
 226 }
 227
 228 void
 229 _jit_allocar(jit_state_t *_jit, jit_int32_t u, jit_int32_t v)
 230 {
 231     jit_int32_t          reg;
 232     assert(_jitc->function);
 233     jit_inc_synth_ww(allocar, u, v);
 234     if (!_jitc->function->allocar) {
 235         _jitc->function->aoffoff = jit_allocai(sizeof(jit_int32_t));
 236         _jitc->function->allocar = 1;
 237     }
 238     reg = jit_get_reg(jit_class_gpr);
 239     jit_negr(reg, v);
 240     jit_andi(reg, reg, -8);
 241     jit_ldxi_i(u, JIT_FP, _jitc->function->aoffoff);
 242     jit_addr(u, u, reg);
 243     jit_addr(JIT_SP, JIT_SP, reg);
 244     jit_stxi_i(_jitc->function->aoffoff, JIT_FP, u);
 245     jit_unget_reg(reg);
 246     jit_dec_synth();
 247 }
 248
 249 void
 250 _jit_ret(jit_state_t *_jit)
 251 {
 252     jit_node_t          *instr;
 253     assert(_jitc->function);
 254     jit_inc_synth(ret);
 255     /* jump to epilog */
 256     instr = jit_jmpi();
 257     jit_patch_at(instr, _jitc->function->epilog);
 258     jit_dec_synth();
 259 }
 260
 261 void
 262 _jit_retr(jit_state_t *_jit, jit_int32_t u)
 263 {
 264     jit_inc_synth_w(retr, u);
 265     if (JIT_RET != u)
 266         jit_movr(JIT_RET, u);
 267     jit_live(JIT_RET);
 268     jit_ret();
 269     jit_dec_synth();
 270 }
 271
 272 void
 273 _jit_reti(jit_state_t *_jit, jit_word_t u)
 274 {
 275     jit_inc_synth_w(reti, u);
 276     jit_movi(JIT_RET, u);
 277     jit_ret();
 278     jit_dec_synth();
 279 }
 280
 281 void
 282 _jit_retr_f(jit_state_t *_jit, jit_int32_t u)
 283 {
 284     jit_inc_synth_w(retr_f, u);
 285     if (JIT_FRET != u)
 286         jit_movr_f(JIT_FRET, u);
 287     else
 288         jit_live(JIT_FRET);
 289     jit_ret();
 290     jit_dec_synth();
 291 }
 292
 293 void
 294 _jit_reti_f(jit_state_t *_jit, jit_float32_t u)
 295 {
 296     jit_inc_synth_f(reti_f, u);
 297     jit_movi_f(JIT_FRET, u);
 298     jit_ret();
 299     jit_dec_synth();
 300 }
 301
 302 void
 303 _jit_retr_d(jit_state_t *_jit, jit_int32_t u)
 304 {
 305     jit_inc_synth_w(retr_d, u);
 306     if (JIT_FRET != u)
 307         jit_movr_d(JIT_FRET, u);
 308     else
 309         jit_live(JIT_FRET);
 310     jit_ret();
 311     jit_dec_synth();
 312 }
 313
 314 void
 315 _jit_reti_d(jit_state_t *_jit, jit_float64_t u)
 316 {
 317     jit_inc_synth_d(reti_d, u);
 318     jit_movi_d(JIT_FRET, u);
 319     jit_ret();
 320     jit_dec_synth();
 321 }
 322
 323 void
 324 _jit_epilog(jit_state_t *_jit)
 325 {
 326     assert(_jitc->function);
 327     assert(_jitc->function->epilog->next == NULL);
 328     jit_link(_jitc->function->epilog);
 329     _jitc->function = NULL;
 330 }
 331
 332 jit_bool_t
 333 _jit_arg_register_p(jit_state_t *_jit, jit_node_t *u)
 334 {
 335     if (u->code == jit_code_arg)
 336         return (jit_arg_reg_p(u->u.w));
 337     assert(u->code == jit_code_arg_f || u->code == jit_code_arg_d);
 338 #if NEW_ABI
 339     return (jit_arg_reg_p(u->u.w));
 340 #else
 341     return (u->u.w < 8);
 342 #endif
 343 }
 344
 345 static jit_node_t *
 346 _jit_make_arg(jit_state_t *_jit, jit_node_t *node)
 347 {
 348     jit_int32_t          offset;
 349 #if NEW_ABI
 350     if (jit_arg_reg_p(_jitc->function->self.argi))
 351         offset = _jitc->function->self.argi++;
 352     else {
 353         offset = _jitc->function->self.size;
 354         _jitc->function->self.size += STACK_SLOT;
 355     }
 356 #else
 357     offset = (_jitc->function->self.size - stack_framesize) >> STACK_SHIFT;
 358     _jitc->function->self.argi = 1;
 359     if (offset >= 4)
 360         offset = _jitc->function->self.size;
 361     _jitc->function->self.size += STACK_SLOT;
 362 #endif
 363     if (node == (jit_node_t *)0)
 364         node = jit_new_node(jit_code_arg);
 365     else
 366         link_node(node);
 367     node->u.w = offset;
 368     node->v.w = ++_jitc->function->self.argn;
 369     jit_link_prolog();
 370     return (node);
 371 }
 372
 373 static jit_node_t *
 374 _jit_make_arg_f(jit_state_t *_jit, jit_node_t *node)
 375 {
 376     jit_int32_t          offset;
 377 #if NEW_ABI
 378     if (jit_arg_reg_p(_jitc->function->self.argi)) {
 379         offset = _jitc->function->self.argi++;
 380         if (_jitc->function->self.call & jit_call_varargs)
 381             offset += 8;
 382     }
 383     else {
 384         offset = _jitc->function->self.size;
 385         _jitc->function->self.size += STACK_SLOT;
 386     }
 387 #else
 388     offset = (_jitc->function->self.size - stack_framesize) >> STACK_SHIFT;
 389     if (offset < NUM_WORD_ARGS) {
 390         if (!_jitc->function->self.argi &&
 391             !(_jitc->function->self.call & jit_call_varargs)) {
 392             if (offset == 0)
 393                 offset = 4;
 394             else {
 395                 offset = 6;
 396                 _jitc->function->self.argi = 1;
 397             }
 398             /* Use as flag to rewind in case of varargs function */
 399             ++_jitc->function->self.argf;
 400         }
 401     }
 402     else
 403         offset = _jitc->function->self.size;
 404     _jitc->function->self.size += STACK_SLOT;
 405 #endif
 406     if (node == (jit_node_t *)0)
 407         node = jit_new_node(jit_code_arg_f);
 408     else
 409         link_node(node);
 410     node->u.w = offset;
 411     node->v.w = ++_jitc->function->self.argn;
 412     jit_link_prolog();
 413     return (node);
 414 }
 415
 416 static jit_node_t *
 417 _jit_make_arg_d(jit_state_t *_jit, jit_node_t *node)
 418 {
 419     jit_int32_t          offset;
 420 #if NEW_ABI
 421     if (jit_arg_reg_p(_jitc->function->self.argi)) {
 422         offset = _jitc->function->self.argi++;
 423         if (_jitc->function->self.call & jit_call_varargs)
 424             offset += 8;
 425     }
 426     else {
 427         offset = _jitc->function->self.size;
 428         _jitc->function->self.size += STACK_SLOT;
 429     }
 430 #else
 431     if (_jitc->function->self.size & 7) {
 432         _jitc->function->self.size += 4;
 433         _jitc->function->self.argi = 1;
 434     }
 435     offset = (_jitc->function->self.size - stack_framesize) >> STACK_SHIFT;
 436     if (offset < NUM_WORD_ARGS) {
 437         if (!_jitc->function->self.argi &&
 438             !(_jitc->function->self.call & jit_call_varargs)) {
 439             offset += 4;
 440             /* Use as flag to rewind in case of varargs function */
 441             ++_jitc->function->self.argf;
 442         }
 443     }
 444     else
 445         offset = _jitc->function->self.size;
 446     _jitc->function->self.size += sizeof(jit_float64_t);
 447 #endif
 448     if (node == (jit_node_t *)0)
 449         node = jit_new_node(jit_code_arg_d);
 450     else
 451         link_node(node);
 452     node->u.w = offset;
 453     node->v.w = ++_jitc->function->self.argn;
 454     jit_link_prolog();
 455     return (node);
 456 }
 457
 458 void
 459 _jit_ellipsis(jit_state_t *_jit)
 460 {
 461     if (_jitc->prepare) {
 462         assert(!(_jitc->function->call.call & jit_call_varargs));
 463         _jitc->function->call.call |= jit_call_varargs;
 464 #if !NEW_ABI
 465         if (_jitc->function->call.argf)
 466             rewind_prepare();
 467 #endif
 468     }
 469     else {
 470         assert(!(_jitc->function->self.call & jit_call_varargs));
 471 #if NEW_ABI
 472         /* If varargs start in a register, allocate extra 64 bytes. */
 473         if (jit_arg_reg_p(_jitc->function->self.argi))
 474             rewind_prolog();
 475         /* Do not set during possible rewind. */
 476         _jitc->function->self.call |= jit_call_varargs;
 477 #else
 478         _jitc->function->self.call |= jit_call_varargs;
 479         if (_jitc->function->self.argf)
 480             rewind_prolog();
 481 #endif
 482         _jitc->function->vagp = _jitc->function->self.argi;
 483     }
 484     jit_inc_synth(ellipsis);
 485     if (_jitc->prepare)
 486         jit_link_prepare();
 487     else
 488         jit_link_prolog();
 489     jit_dec_synth();
 490 }
 491
 492 void
 493 _jit_va_push(jit_state_t *_jit, jit_int32_t u)
 494 {
 495     jit_inc_synth_w(va_push, u);
 496     jit_pushargr(u);
 497     jit_dec_synth();
 498 }
 499
 500 jit_node_t *
 501 _jit_arg(jit_state_t *_jit)
 502 {
 503     assert(_jitc->function);
 504     return (jit_make_arg((jit_node_t*)0));
 505 }
 506
 507 jit_node_t *
 508 _jit_arg_f(jit_state_t *_jit)
 509 {
 510     assert(_jitc->function);
 511     return (jit_make_arg_f((jit_node_t*)0));
 512 }
 513
 514 jit_node_t *
 515 _jit_arg_d(jit_state_t *_jit)
 516 {
 517     assert(_jitc->function);
 518     return (jit_make_arg_d((jit_node_t*)0));
 519 }
 520
 521 void
 522 _jit_getarg_c(jit_state_t *_jit, jit_int32_t u, jit_node_t *v)
 523 {
 524     assert(v->code == jit_code_arg);
 525     jit_inc_synth_wp(getarg_c, u, v);
 526     if (jit_arg_reg_p(v->u.w))
 527         jit_extr_c(u, _A0 - v->u.w);
 528     else
 529         jit_ldxi_c(u, _FP, v->u.w + C_DISP);
 530     jit_dec_synth();
 531 }
 532
 533 void
 534 _jit_getarg_uc(jit_state_t *_jit, jit_int32_t u, jit_node_t *v)
 535 {
 536     assert(v->code == jit_code_arg);
 537     jit_inc_synth_wp(getarg_uc, u, v);
 538     if (jit_arg_reg_p(v->u.w))
 539         jit_extr_uc(u, _A0 - v->u.w);
 540     else
 541         jit_ldxi_uc(u, _FP, v->u.w + C_DISP);
 542     jit_dec_synth();
 543 }
 544
 545 void
 546 _jit_getarg_s(jit_state_t *_jit, jit_int32_t u, jit_node_t *v)
 547 {
 548     assert(v->code == jit_code_arg);
 549     jit_inc_synth_wp(getarg_s, u, v);
 550     if (jit_arg_reg_p(v->u.w))
 551         jit_extr_s(u, _A0 - v->u.w);
 552     else
 553         jit_ldxi_s(u, _FP, v->u.w + S_DISP);
 554     jit_dec_synth();
 555 }
 556
 557 void
 558 _jit_getarg_us(jit_state_t *_jit, jit_int32_t u, jit_node_t *v)
 559 {
 560     assert(v->code == jit_code_arg);
 561     jit_inc_synth_wp(getarg_us, u, v);
 562     if (jit_arg_reg_p(v->u.w))
 563         jit_extr_us(u, _A0 - v->u.w);
 564     else
 565         jit_ldxi_us(u, _FP, v->u.w + S_DISP);
 566     jit_dec_synth();
 567 }
 568
 569 void
 570 _jit_getarg_i(jit_state_t *_jit, jit_int32_t u, jit_node_t *v)
 571 {
 572     assert(v->code == jit_code_arg);
 573     jit_inc_synth_wp(getarg_i, u, v);
 574     if (jit_arg_reg_p(v->u.w)) {
 575 #if __WORDSIZE == 64
 576         jit_extr_i(u, _A0 - v->u.w);
 577 #else
 578         jit_movr(u, _A0 - v->u.w);
 579 #endif
 580     }
 581     else
 582         jit_ldxi_i(u, _FP, v->u.w + I_DISP);
 583     jit_dec_synth();
 584 }
 585
 586 #if __WORDSIZE == 64
 587 void
 588 _jit_getarg_ui(jit_state_t *_jit, jit_int32_t u, jit_node_t *v)
 589 {
 590     assert(v->code == jit_code_arg);
 591     jit_inc_synth_wp(getarg_ui, u, v);
 592     if (jit_arg_reg_p(v->u.w))
 593         jit_extr_ui(u, _A0 - v->u.w);
 594     else
 595         jit_ldxi_ui(u, _FP, v->u.w + I_DISP);
 596     jit_dec_synth();
 597 }
 598
 599 void
 600 _jit_getarg_l(jit_state_t *_jit, jit_int32_t u, jit_node_t *v)
 601 {
 602     assert(v->code == jit_code_arg);
 603     jit_inc_synth_wp(getarg_l, u, v);
 604     if (jit_arg_reg_p(v->u.w))
 605         jit_movr(u, _A0 - v->u.w);
 606     else
 607         jit_ldxi_l(u, _FP, v->u.w);
 608     jit_dec_synth();
 609 }
 610 #endif
 611
 612 void
 613 _jit_putargr(jit_state_t *_jit, jit_int32_t u, jit_node_t *v)
 614 {
 615     jit_inc_synth_wp(putargr, u, v);
 616     assert(v->code == jit_code_arg);
 617     if (jit_arg_reg_p(v->u.w))
 618         jit_movr(_A0 - v->u.w, u);
 619     else
 620         jit_stxi(v->u.w + WORD_ADJUST, _FP, u);
 621     jit_dec_synth();
 622 }
 623
 624 void
 625 _jit_putargi(jit_state_t *_jit, jit_word_t u, jit_node_t *v)
 626 {
 627     jit_int32_t         regno;
 628     assert(v->code == jit_code_arg);
 629     jit_inc_synth_wp(putargi, u, v);
 630     if (jit_arg_reg_p(v->u.w))
 631         jit_movi(_A0 - v->u.w, u);
 632     else {
 633         regno = jit_get_reg(jit_class_gpr);
 634         jit_movi(regno, u);
 635         jit_stxi(v->u.w + WORD_ADJUST, _FP, regno);
 636         jit_unget_reg(regno);
 637     }
 638     jit_dec_synth();
 639 }
 640
 641 void
 642 _jit_getarg_f(jit_state_t *_jit, jit_int32_t u, jit_node_t *v)
 643 {
 644     assert(v->code == jit_code_arg_f);
 645     jit_inc_synth_wp(getarg_f, u, v);
 646 #if NEW_ABI
 647     if (jit_arg_reg_p(v->u.w))
 648         jit_movr_f(u, _F12 - v->u.w);
 649     else if (jit_arg_reg_p(v->u.w - 8))
 650         jit_movr_w_f(u, _A0 - v->u.w - 8);
 651 #else
 652     if (v->u.w < 4)
 653         jit_movr_w_f(u, _A0 - v->u.w);
 654     else if (v->u.w < 8)
 655         jit_movr_f(u, _F12 - ((v->u.w - 4) >> 1));
 656 #endif
 657     else
 658         jit_ldxi_f(u, _FP, v->u.w);
 659     jit_dec_synth();
 660 }
 661
 662 void
 663 _jit_putargr_f(jit_state_t *_jit, jit_int32_t u, jit_node_t *v)
 664 {
 665     assert(v->code == jit_code_arg_f);
 666     jit_inc_synth_wp(putargr_f, u, v);
 667 #if NEW_ABI
 668     if (jit_arg_reg_p(v->u.w))
 669         jit_movr_f(_F12 - v->u.w, u);
 670     else if (jit_arg_reg_p(v->u.w - 8))
 671         jit_movr_f_w(_A0 - v->u.w - 8, u);
 672 #else
 673     if (v->u.w < 4)
 674         jit_movr_f_w(_A0 - v->u.w, u);
 675     else if (v->u.w < 8)
 676         jit_movr_f(_F12 - ((v->u.w - 4) >> 1), u);
 677 #endif
 678     else
 679         jit_stxi_f(v->u.w, _FP, u);
 680     jit_dec_synth();
 681 }
 682
 683 void
 684 _jit_putargi_f(jit_state_t *_jit, jit_float32_t u, jit_node_t *v)
 685 {
 686     jit_int32_t         regno;
 687     assert(v->code == jit_code_arg_f);
 688     jit_inc_synth_fp(putargi_f, u, v);
 689 #if NEW_ABI
 690     if (jit_arg_reg_p(v->u.w))
 691         jit_movi_f(_F12 - v->u.w, u);
 692     else if (jit_arg_reg_p(v->u.w - 8)) {
 693         regno = jit_get_reg(jit_class_fpr);
 694         jit_movi_f(regno, u);
 695         jit_movr_f_w(_A0 - v->u.w - 8, u);
 696         jit_unget_reg(regno);
 697     }
 698 #else
 699     if (v->u.w < 4) {
 700         regno = jit_get_reg(jit_class_fpr);
 701         jit_movi_f(regno, u);
 702         jit_movr_f_w(_A0 - ((v->u.w - 4) >> 1), regno);
 703         jit_unget_reg(regno);
 704     }
 705     else if (v->u.w < 8)
 706         jit_movi_f(_F12 - ((v->u.w - 4) >> 1), u);
 707 #endif
 708     else {
 709         regno = jit_get_reg(jit_class_fpr);
 710         jit_movi_f(regno, u);
 711         jit_stxi_f(v->u.w, _FP, regno);
 712         jit_unget_reg(regno);
 713     }
 714     jit_dec_synth();
 715 }
 716
 717 void
 718 _jit_getarg_d(jit_state_t *_jit, jit_int32_t u, jit_node_t *v)
 719 {
 720     assert(v->code == jit_code_arg_d);
 721     jit_inc_synth_wp(getarg_d, u, v);
 722 #if NEW_ABI
 723     if (jit_arg_reg_p(v->u.w))
 724         jit_movr_d(u, _F12 - v->u.w);
 725     else if (jit_arg_reg_p(v->u.w - 8))
 726         jit_movr_d_w(_A0 - v->u.w - 8, u);
 727 #else
 728     if (v->u.w < 4)
 729         jit_movr_ww_d(u, _A0 - v->u.w, _A0 - (v->u.w + 1));
 730     else if (v->u.w < 8)
 731         jit_movr_d(u, _F12 - ((v->u.w - 4) >> 1));
 732 #endif
 733     else
 734         jit_ldxi_d(u, _FP, v->u.w);
 735     jit_dec_synth();
 736 }
 737
 738 void
 739 _jit_putargr_d(jit_state_t *_jit, jit_int32_t u, jit_node_t *v)
 740 {
 741     assert(v->code == jit_code_arg_d);
 742     jit_inc_synth_wp(putargr_d, u, v);
 743 #if NEW_ABI
 744     if (jit_arg_reg_p(v->u.w))
 745         jit_movr_d(_F12 - v->u.w, u);
 746     else if (jit_arg_reg_p(v->u.w - 8))
 747         jit_movr_d_w(_A0 - v->u.w - 8, u);
 748 #else
 749     if (v->u.w < 4)
 750         jit_movr_d_ww(_A0 - v->u.w, _A0 - (v->u.w + 1), u);
 751     else if (v->u.w < 8)
 752         jit_movr_d(_F12 - ((v->u.w - 4) >> 1), u);
 753 #endif
 754     else
 755         jit_stxi_d(v->u.w, _FP, u);
 756     jit_dec_synth();
 757 }
 758
 759 void
 760 _jit_putargi_d(jit_state_t *_jit, jit_float64_t u, jit_node_t *v)
 761 {
 762     jit_int32_t         regno;
 763     assert(v->code == jit_code_arg_d);
 764     jit_inc_synth_dp(putargi_d, u, v);
 765 #if NEW_ABI
 766     if (jit_arg_reg_p(v->u.w))
 767         jit_movi_d(_F12 - v->u.w, u);
 768     else if (jit_arg_reg_p(v->u.w - 8)) {
 769         regno = jit_get_reg(jit_class_fpr);
 770         jit_movi_d(regno, u);
 771         jit_movr_d_w(_A0 - v->u.w - 8, u);
 772         jit_unget_reg(regno);
 773     }
 774 #else
 775     if (v->u.w < 4) {
 776         regno = jit_get_reg(jit_class_fpr);
 777         jit_movi_d(regno, u);
 778         jit_movr_d_ww(_A0 - v->u.w, _A0 - (v->u.w + 1), regno);
 779         jit_unget_reg(regno);
 780     }
 781     else if (v->u.w < 8)
 782         jit_movi_d(_F12 - ((v->u.w - 4) >> 1), u);
 783 #endif
 784     else {
 785         regno = jit_get_reg(jit_class_fpr);
 786         jit_movi_d(regno, u);
 787         jit_stxi_d(v->u.w, _FP, regno);
 788         jit_unget_reg(regno);
 789     }
 790     jit_dec_synth();
 791 }
 792
 793 void
 794 _jit_pushargr(jit_state_t *_jit, jit_int32_t u)
 795 {
 796     jit_inc_synth_w(pushargr, u);
 797     jit_link_prepare();
 798 #if NEW_ABI
 799     assert(_jitc->function);
 800     if (jit_arg_reg_p(_jitc->function->call.argi)) {
 801         jit_movr(_A0 - _jitc->function->call.argi, u);
 802         ++_jitc->function->call.argi;
 803     }
 804     else {
 805         jit_stxi(_jitc->function->call.size + WORD_ADJUST, JIT_SP, u);
 806         _jitc->function->call.size += STACK_SLOT;
 807     }
 808 #else
 809     jit_word_t          offset;
 810     assert(_jitc->function);
 811     offset = _jitc->function->call.size >> STACK_SHIFT;
 812     _jitc->function->call.argi = 1;
 813     if (jit_arg_reg_p(offset))
 814         jit_movr(_A0 - offset, u);
 815     else
 816         jit_stxi(_jitc->function->call.size, JIT_SP, u);
 817     _jitc->function->call.size += STACK_SLOT;
 818 #endif
 819     jit_dec_synth();
 820 }
 821
 822 void
 823 _jit_pushargi(jit_state_t *_jit, jit_word_t u)
 824 {
 825     jit_int32_t         regno;
 826 #if !NEW_ABI
 827     jit_word_t          offset;
 828 #endif
 829     assert(_jitc->function);
 830     jit_inc_synth_w(pushargi, u);
 831     jit_link_prepare();
 832 #if NEW_ABI
 833     if (jit_arg_reg_p(_jitc->function->call.argi)) {
 834         jit_movi(_A0 - _jitc->function->call.argi, u);
 835         ++_jitc->function->call.argi;
 836     }
 837     else {
 838         regno = jit_get_reg(jit_class_gpr);
 839         jit_movi(regno, u);
 840         jit_stxi(_jitc->function->call.size + WORD_ADJUST, JIT_SP, regno);
 841         _jitc->function->call.size += STACK_SLOT;
 842         jit_unget_reg(regno);
 843     }
 844 #else
 845     offset = _jitc->function->call.size >> STACK_SHIFT;
 846     ++_jitc->function->call.argi;
 847     if (jit_arg_reg_p(offset))
 848         jit_movi(_A0 - offset, u);
 849     else {
 850         regno = jit_get_reg(jit_class_gpr);
 851         jit_movi(regno, u);
 852         jit_stxi(_jitc->function->call.size, JIT_SP, regno);
 853         jit_unget_reg(regno);
 854     }
 855     _jitc->function->call.size += STACK_SLOT;
 856 #endif
 857     jit_dec_synth();
 858 }
 859
 860 void
 861 _jit_pushargr_f(jit_state_t *_jit, jit_int32_t u)
 862 {
 863 #if !NEW_ABI
 864     jit_word_t          offset;
 865 #endif
 866     assert(_jitc->function);
 867     jit_inc_synth_w(pushargr_f, u);
 868     jit_link_prepare();
 869 #if NEW_ABI
 870     if (jit_arg_reg_p(_jitc->function->call.argi)) {
 871         if (!(_jitc->function->call.call & jit_call_varargs))
 872             jit_movr_f(_F12 - _jitc->function->call.argi, u);
 873         else
 874             jit_movr_f_w(_A0 - _jitc->function->call.argi, u);
 875         ++_jitc->function->call.argi;
 876     }
 877     else {
 878         jit_stxi_f(_jitc->function->call.size, JIT_SP, u);
 879         _jitc->function->call.size += STACK_SLOT;
 880     }
 881 #else
 882     offset = _jitc->function->call.size >> STACK_SHIFT;
 883     if (offset < 2 && !_jitc->function->call.argi &&
 884         !(_jitc->function->call.call & jit_call_varargs)) {
 885         ++_jitc->function->call.argf;
 886         jit_movr_f(_F12 - offset, u);
 887     }
 888     else if (offset < 4) {
 889         ++_jitc->function->call.argi;
 890         jit_movr_f_w(_A0 - offset, u);
 891     }
 892     else
 893         jit_stxi_f(_jitc->function->call.size, JIT_SP, u);
 894     _jitc->function->call.size += STACK_SLOT;
 895 #endif
 896     jit_dec_synth();
 897 }
 898
 899 void
 900 _jit_pushargi_f(jit_state_t *_jit, jit_float32_t u)
 901 {
 902     jit_int32_t         regno;
 903 #if !NEW_ABI
 904     jit_word_t          offset;
 905 #endif
 906     assert(_jitc->function);
 907     jit_inc_synth_f(pushargi_f, u);
 908     jit_link_prepare();
 909 #if NEW_ABI
 910     if (jit_arg_reg_p(_jitc->function->call.argi)) {
 911         if (!(_jitc->function->call.call & jit_call_varargs))
 912             jit_movi_f(_F12 - _jitc->function->call.argi, u);
 913         else
 914             jit_movi_f_w(_A0 - _jitc->function->call.argi, u);
 915         ++_jitc->function->call.argi;
 916     }
 917     else {
 918         regno = jit_get_reg(jit_class_fpr);
 919         jit_movi_f(regno, u);
 920         jit_stxi_f(_jitc->function->call.size, JIT_SP, regno);
 921         _jitc->function->call.size += STACK_SLOT;
 922         jit_unget_reg(regno);
 923     }
 924 #else
 925     offset = _jitc->function->call.size >> STACK_SHIFT;
 926     if (offset < 2 && !_jitc->function->call.argi &&
 927         !(_jitc->function->call.call & jit_call_varargs)) {
 928         ++_jitc->function->call.argf;
 929         jit_movi_f(_F12 - offset, u);
 930     }
 931     else if (offset < 4) {
 932         ++_jitc->function->call.argi;
 933         jit_movi_f_w(_A0 - offset, u);
 934     }
 935     else {
 936         regno = jit_get_reg(jit_class_fpr);
 937         jit_movi_f(regno, u);
 938         jit_stxi_f(_jitc->function->call.size, JIT_SP, regno);
 939         jit_unget_reg(regno);
 940     }
 941     _jitc->function->call.size += STACK_SLOT;
 942 #endif
 943     jit_dec_synth();
 944 }
 945
 946 void
 947 _jit_pushargr_d(jit_state_t *_jit, jit_int32_t u)
 948 {
 949 #if !NEW_ABI
 950     jit_bool_t          adjust;
 951     jit_word_t          offset;
 952 #endif
 953     assert(_jitc->function);
 954     jit_inc_synth_w(pushargr_d, u);
 955     jit_link_prepare();
 956 #if NEW_ABI
 957     if (jit_arg_reg_p(_jitc->function->call.argi)) {
 958         if (!(_jitc->function->call.call & jit_call_varargs))
 959             jit_movr_d(_F12 - _jitc->function->call.argi, u);
 960         else
 961             jit_movr_d_w(_A0 - _jitc->function->call.argi, u);
 962         ++_jitc->function->call.argi;
 963     }
 964     else {
 965         jit_stxi_d(_jitc->function->call.size, JIT_SP, u);
 966         _jitc->function->call.size += STACK_SLOT;
 967     }
 968 #else
 969     adjust = !!_jitc->function->call.argi;
 970     if (_jitc->function->call.size & 7) {
 971         _jitc->function->call.size += 4;
 972         adjust = 1;
 973     }
 974     offset = _jitc->function->call.size >> STACK_SHIFT;
 975     if (offset < 3) {
 976         if (adjust || (_jitc->function->call.call & jit_call_varargs)) {
 977             jit_movr_d_ww(_A0 - offset, _A0 - (offset + 1), u);
 978             _jitc->function->call.argi += 2;
 979         }
 980         else {
 981             jit_movr_d(_F12 - (offset >> 1), u);
 982             ++_jitc->function->call.argf;
 983         }
 984     }
 985     else
 986         jit_stxi_d(_jitc->function->call.size, JIT_SP, u);
 987     _jitc->function->call.size += sizeof(jit_float64_t);
 988 #endif
 989     jit_dec_synth();
 990 }
 991
 992 void
 993 _jit_pushargi_d(jit_state_t *_jit, jit_float64_t u)
 994 {
 995     jit_int32_t         regno;
 996 #if !NEW_ABI
 997     jit_bool_t          adjust;
 998     jit_word_t          offset;
 999 #endif
1000     assert(_jitc->function);
1001     jit_inc_synth_d(pushargi_d, u);
1002     jit_link_prepare();
1003 #if NEW_ABI
1004     if (jit_arg_reg_p(_jitc->function->call.argi)) {
1005         if (!(_jitc->function->call.call & jit_call_varargs))
1006             jit_movi_d(_F12 - _jitc->function->call.argi, u);
1007         else
1008             jit_movi_d_w(_A0 - _jitc->function->call.argi, u);
1009         ++_jitc->function->call.argi;
1010     }
1011     else {
1012         regno = jit_get_reg(jit_class_fpr);
1013         jit_movi_d(regno, u);
1014         jit_stxi_d(_jitc->function->call.size, JIT_SP, regno);
1015         _jitc->function->call.size += STACK_SLOT;
1016         jit_unget_reg(regno);
1017     }
1018 #else
1019     adjust = !!_jitc->function->call.argi;
1020     if (_jitc->function->call.size & 7) {
1021         _jitc->function->call.size += 4;
1022         adjust = 1;
1023     }
1024     offset = _jitc->function->call.size >> STACK_SHIFT;
1025     if (offset < 3) {
1026         if (adjust || (_jitc->function->call.call & jit_call_varargs)) {
1027             jit_movi_d_ww(_A0 - offset, _A0 - (offset + 1), u);
1028             _jitc->function->call.argi += 2;
1029         }
1030         else {
1031             jit_movi_d(_F12 - (offset >> 1), u);
1032             ++_jitc->function->call.argf;
1033         }
1034     }
1035     else {
1036         regno = jit_get_reg(jit_class_fpr);
1037         jit_movi_d(regno, u);
1038         jit_stxi_d(_jitc->function->call.size, JIT_SP, regno);
1039         jit_unget_reg(regno);
1040     }
1041     _jitc->function->call.size += sizeof(jit_float64_t);
1042 #endif
1043     jit_dec_synth();
1044 }
1045
1046 jit_bool_t
1047 _jit_regarg_p(jit_state_t *_jit, jit_node_t *node, jit_int32_t regno)
1048 {
1049     jit_int32_t         spec;
1050
1051     spec = jit_class(_rvs[regno].spec);
1052     if (spec & jit_class_arg) {
1053         if (spec & jit_class_gpr) {
1054             regno = _A0 - regno;
1055             if (regno >= 0 && regno < node->v.w)
1056                 return (1);
1057         }
1058         else if (spec & jit_class_fpr) {
1059             regno = _F12 - regno;
1060             if (regno >= 0 && regno < node->w.w)
1061                 return (1);
1062         }
1063     }
1064
1065     return (0);
1066 }
1067
1068 void
1069 _jit_finishr(jit_state_t *_jit, jit_int32_t r0)
1070 {
1071     jit_node_t          *call;
1072     assert(_jitc->function);
1073     jit_inc_synth_w(finishr, r0);
1074     if (_jitc->function->self.alen < _jitc->function->call.size)
1075         _jitc->function->self.alen = _jitc->function->call.size;
1076     call = jit_callr(r0);
1077     call->v.w = _jitc->function->self.argi;
1078 #if NEW_ABI
1079     call->w.w = call->v.w;
1080 #else
1081     call->w.w = _jitc->function->self.argf;
1082 #endif
1083     _jitc->function->call.argi = _jitc->function->call.argf =
1084         _jitc->function->call.size = 0;
1085     _jitc->prepare = 0;
1086     jit_dec_synth();
1087 }
1088
1089 jit_node_t *
1090 _jit_finishi(jit_state_t *_jit, jit_pointer_t i0)
1091 {
1092     jit_node_t          *call;
1093     jit_node_t          *node;
1094     assert(_jitc->function);
1095     jit_inc_synth_w(finishi, (jit_word_t)i0);
1096     if (_jitc->function->self.alen < _jitc->function->call.size)
1097         _jitc->function->self.alen = _jitc->function->call.size;
1098     node = jit_movi(_T9, (jit_word_t)i0);
1099     call = jit_callr(_T9);
1100     call->v.w = _jitc->function->call.argi;
1101 #if NEW_ABI
1102     call->w.w = call->v.w;
1103 #else
1104     call->w.w = _jitc->function->call.argf;
1105 #endif
1106     _jitc->function->call.argi = _jitc->function->call.argf =
1107         _jitc->function->call.size = 0;
1108     _jitc->prepare = 0;
1109     jit_dec_synth();
1110     return (node);
1111 }
1112
1113 void
1114 _jit_retval_c(jit_state_t *_jit, jit_int32_t r0)
1115 {
1116     jit_extr_c(r0, JIT_RET);
1117 }
1118
1119 void
1120 _jit_retval_uc(jit_state_t *_jit, jit_int32_t r0)
1121 {
1122     jit_extr_uc(r0, JIT_RET);
1123 }
1124
1125 void
1126 _jit_retval_s(jit_state_t *_jit, jit_int32_t r0)
1127 {
1128     jit_extr_s(r0, JIT_RET);
1129 }
1130
1131 void
1132 _jit_retval_us(jit_state_t *_jit, jit_int32_t r0)
1133 {
1134     jit_extr_us(r0, JIT_RET);
1135 }
1136
1137 void
1138 _jit_retval_i(jit_state_t *_jit, jit_int32_t r0)
1139 {
1140 #if __WORDSIZE == 32
1141     if (r0 != JIT_RET)
1142         jit_movr(r0, JIT_RET);
1143 #else
1144     jit_extr_i(r0, JIT_RET);
1145 #endif
1146 }
1147
1148 #if __WORDSIZE == 64
1149 void
1150 _jit_retval_ui(jit_state_t *_jit, jit_int32_t r0)
1151 {
1152     jit_extr_ui(r0, JIT_RET);
1153 }
1154
1155 void
1156 _jit_retval_l(jit_state_t *_jit, jit_int32_t r0)
1157 {
1158     if (r0 != JIT_RET)
1159         jit_movr(r0, JIT_RET);
1160 }
1161 #endif
1162
1163 void
1164 _jit_retval_f(jit_state_t *_jit, jit_int32_t r0)
1165 {
1166     if (r0 != JIT_FRET)
1167         jit_movr_f(r0, JIT_FRET);
1168 }
1169
1170 void
1171 _jit_retval_d(jit_state_t *_jit, jit_int32_t r0)
1172 {
1173     if (r0 != JIT_FRET)
1174         jit_movr_d(r0, JIT_FRET);
1175 }
1176
1177 jit_pointer_t
1178 _emit_code(jit_state_t *_jit)
1179 {
1180     jit_node_t          *node;
1181     jit_node_t          *temp;
1182     jit_word_t           word;
1183     jit_int32_t          value;
1184     jit_int32_t          offset;
1185     struct {
1186         jit_node_t      *node;
1187         jit_word_t       word;
1188 #if DEVEL_DISASSEMBLER
1189         jit_word_t       prevw;
1190 #endif
1191         jit_int32_t      patch_offset;
1192     } undo;
1193 #if DEVEL_DISASSEMBLER
1194     jit_word_t           prevw;
1195 #endif
1196
1197     _jitc->function = NULL;
1198
1199     jit_reglive_setup();
1200
1201     undo.word = 0;
1202     undo.node = NULL;
1203     undo.patch_offset = 0;
1204 #define case_rr(name, type)                                             \
1205             case jit_code_##name##r##type:                              \
1206                 name##r##type(rn(node->u.w), rn(node->v.w));            \
1207                 break
1208 #define case_rw(name, type)                                             \
1209             case jit_code_##name##i##type:                              \
1210                 name##i##type(rn(node->u.w), node->v.w);                \
1211                 break
1212 #define case_wr(name, type)                                             \
1213             case jit_code_##name##i##type:                              \
1214                 name##i##type(node->u.w, rn(node->v.w));                \
1215                 break
1216 #define case_rrr(name, type)                                            \
1217             case jit_code_##name##r##type:                              \
1218                 name##r##type(rn(node->u.w),                            \
1219                               rn(node->v.w), rn(node->w.w));            \
1220                 break
1221 #define case_rrw(name, type)                                            \
1222             case jit_code_##name##i##type:                              \
1223                 name##i##type(rn(node->u.w), rn(node->v.w), node->w.w); \
1224                 break
1225 #define case_rrrr(name, type)                                           \
1226             case jit_code_##name##r##type:                              \
1227                 name##r##type(rn(node->u.q.l), rn(node->u.q.h),         \
1228                               rn(node->v.w), rn(node->w.w));            \
1229                 break
1230 #define case_rrrw(name, type)                                           \
1231             case jit_code_##name##i##type:                              \
1232                 name##i##type(rn(node->u.q.l), rn(node->u.q.h),         \
1233                               rn(node->v.w), node->w.w);                \
1234                 break
1235 #define case_rrf(name, type, size)                                      \
1236             case jit_code_##name##i##type:                              \
1237                 assert(node->flag & jit_flag_data);                     \
1238                 name##i##type(rn(node->u.w), rn(node->v.w),             \
1239                               (jit_float##size##_t *)node->w.n->u.w);   \
1240                 break
1241 #define case_wrr(name, type)                                            \
1242             case jit_code_##name##i##type:                              \
1243                 name##i##type(node->u.w, rn(node->v.w), rn(node->w.w)); \
1244                 break
1245 #define case_brr(name, type)                                            \
1246             case jit_code_##name##r##type:                              \
1247                 temp = node->u.n;                                       \
1248                 assert(temp->code == jit_code_label ||                  \
1249                        temp->code == jit_code_epilog);                  \
1250                 if (temp->flag & jit_flag_patch)                        \
1251                     name##r##type(temp->u.w, rn(node->v.w),             \
1252                                   rn(node->w.w));                       \
1253                 else {                                                  \
1254                     word = name##r##type(_jit->pc.w,                    \
1255                                          rn(node->v.w), rn(node->w.w)); \
1256                     patch(word, node);                                  \
1257                 }                                                       \
1258                 break
1259 #define case_brw(name, type)                                            \
1260             case jit_code_##name##i##type:                              \
1261                 temp = node->u.n;                                       \
1262                 assert(temp->code == jit_code_label ||                  \
1263                        temp->code == jit_code_epilog);                  \
1264                 if (temp->flag & jit_flag_patch)                        \
1265                     name##i##type(temp->u.w,                            \
1266                                   rn(node->v.w), node->w.w);            \
1267                 else {                                                  \
1268                     word = name##i##type(_jit->pc.w,                    \
1269                                          rn(node->v.w), node->w.w);     \
1270                     patch(word, node);                                  \
1271                 }                                                       \
1272                 break
1273 #define case_brf(name, type, size)                                      \
1274             case jit_code_##name##i##type:                              \
1275                 temp = node->u.n;                                       \
1276                 assert(temp->code == jit_code_label ||                  \
1277                        temp->code == jit_code_epilog);                  \
1278                 if (temp->flag & jit_flag_patch)                        \
1279                     name##i##type(temp->u.w, rn(node->v.w),             \
1280                                 (jit_float##size##_t *)node->w.n->u.w); \
1281                 else {                                                  \
1282                     word = name##i##type(_jit->pc.w, rn(node->v.w),     \
1283                                 (jit_float##size##_t *)node->w.n->u.w); \
1284                     patch(word, node);                                  \
1285                 }                                                       \
1286                 break
1287 #if DEVEL_DISASSEMBLER
1288     prevw = _jit->pc.w;
1289 #endif
1290     for (node = _jitc->head; node; node = node->next) {
1291         if (_jit->pc.uc >= _jitc->code.end)
1292             return (NULL);
1293
1294 #if DEVEL_DISASSEMBLER
1295         node->offset = (jit_uword_t)_jit->pc.w - (jit_uword_t)prevw;
1296         prevw = _jit->pc.w;
1297 #endif
1298         value = jit_classify(node->code);
1299         jit_regarg_set(node, value);
1300         switch (node->code) {
1301             case jit_code_align:
1302                 /* Must align to a power of two */
1303                 assert(!(node->u.w & (node->u.w - 1)));
1304                 if ((word = _jit->pc.w & (node->u.w - 1)))
1305                     nop(node->u.w - word);
1306                 break;
1307             case jit_code_note:         case jit_code_name:
1308                 node->u.w = _jit->pc.w;
1309                 break;
1310             case jit_code_label:
1311                 /* remember label is defined */
1312                 node->flag |= jit_flag_patch;
1313                 node->u.w = _jit->pc.w;
1314                 break;
1315                 case_rrr(add,);
1316                 case_rrw(add,);
1317                 case_rrr(addc,);
1318                 case_rrw(addc,);
1319                 case_rrr(addx,);
1320                 case_rrw(addx,);
1321                 case_rrr(sub,);
1322                 case_rrw(sub,);
1323                 case_rrr(subc,);
1324                 case_rrw(subc,);
1325                 case_rrr(subx,);
1326                 case_rrw(subx,);
1327                 case_rrw(rsb,);
1328                 case_rrr(mul,);
1329                 case_rrw(mul,);
1330                 case_rrrr(qmul,);
1331                 case_rrrw(qmul,);
1332                 case_rrrr(qmul, _u);
1333                 case_rrrw(qmul, _u);
1334                 case_rrr(div,);
1335                 case_rrw(div,);
1336                 case_rrr(div, _u);
1337                 case_rrw(div, _u);
1338                 case_rrrr(qdiv,);
1339                 case_rrrw(qdiv,);
1340                 case_rrrr(qdiv, _u);
1341                 case_rrrw(qdiv, _u);
1342                 case_rrr(rem,);
1343                 case_rrw(rem,);
1344                 case_rrr(rem, _u);
1345                 case_rrw(rem, _u);
1346                 case_rrr(lsh,);
1347                 case_rrw(lsh,);
1348                 case_rrr(rsh,);
1349                 case_rrw(rsh,);
1350                 case_rrr(rsh, _u);
1351                 case_rrw(rsh, _u);
1352                 case_rrr(and,);
1353                 case_rrw(and,);
1354                 case_rrr(or,);
1355                 case_rrw(or,);
1356                 case_rrr(xor,);
1357                 case_rrw(xor,);
1358                 case_rr(trunc, _f_i);
1359                 case_rr(trunc, _d_i);
1360 #if __WORDSIZE == 64
1361                 case_rr(trunc, _f_l);
1362                 case_rr(trunc, _d_l);
1363 #endif
1364                 case_rr(ld, _c);
1365                 case_rw(ld, _c);
1366                 case_rr(ld, _uc);
1367                 case_rw(ld, _uc);
1368                 case_rr(ld, _s);
1369                 case_rw(ld, _s);
1370                 case_rr(ld, _us);
1371                 case_rw(ld, _us);
1372                 case_rr(ld, _i);
1373                 case_rw(ld, _i);
1374 #if __WORDSIZE == 64
1375                 case_rr(ld, _ui);
1376                 case_rw(ld, _ui);
1377                 case_rr(ld, _l);
1378                 case_rw(ld, _l);
1379 #endif
1380                 case_rrr(ldx, _c);
1381                 case_rrw(ldx, _c);
1382                 case_rrr(ldx, _uc);
1383                 case_rrw(ldx, _uc);
1384                 case_rrr(ldx, _s);
1385                 case_rrw(ldx, _s);
1386                 case_rrr(ldx, _us);
1387                 case_rrw(ldx, _us);
1388                 case_rrr(ldx, _i);
1389                 case_rrw(ldx, _i);
1390 #if __WORDSIZE == 64
1391                 case_rrr(ldx, _ui);
1392                 case_rrw(ldx, _ui);
1393                 case_rrr(ldx, _l);
1394                 case_rrw(ldx, _l);
1395 #endif
1396                 case_rr(st, _c);
1397                 case_wr(st, _c);
1398                 case_rr(st, _s);
1399                 case_wr(st, _s);
1400                 case_rr(st, _i);
1401                 case_wr(st, _i);
1402 #if __WORDSIZE == 64
1403                 case_rr(st, _l);
1404                 case_wr(st, _l);
1405 #endif
1406                 case_rrr(stx, _c);
1407                 case_wrr(stx, _c);
1408                 case_rrr(stx, _s);
1409                 case_wrr(stx, _s);
1410                 case_rrr(stx, _i);
1411                 case_wrr(stx, _i);
1412 #if __WORDSIZE == 64
1413                 case_rrr(stx, _l);
1414                 case_wrr(stx, _l);
1415 #endif
1416                 case_rr(hton, _us);
1417                 case_rr(hton, _ui);
1418 #if __WORDSIZE == 64
1419                 case_rr(hton, _ul);
1420 #endif
1421                 case_rr(bswap, _us);
1422                 case_rr(bswap, _ui);
1423 #if __WORDSIZE == 64
1424                 case_rr(bswap, _ul);
1425 #endif
1426                 case_rr(ext, _c);
1427                 case_rr(ext, _uc);
1428                 case_rr(ext, _s);
1429                 case_rr(ext, _us);
1430 #if __WORDSIZE == 64
1431                 case_rr(ext, _i);
1432                 case_rr(ext, _ui);
1433 #endif
1434             case jit_code_casr:
1435                 casr(rn(node->u.w), rn(node->v.w),
1436                      rn(node->w.q.l), rn(node->w.q.h));
1437                 break;
1438             case jit_code_casi:
1439                 casi(rn(node->u.w), node->v.w,
1440                      rn(node->w.q.l), rn(node->w.q.h));
1441                 break;
1442                 case_rrr(movn,);
1443                 case_rrr(movz,);
1444                 case_rr(mov,);
1445             case jit_code_movi:
1446                 if (node->flag & jit_flag_node) {
1447                     temp = node->v.n;
1448                     if (temp->code == jit_code_data ||
1449                         (temp->code == jit_code_label &&
1450                          (temp->flag & jit_flag_patch)))
1451                         movi(rn(node->u.w), temp->u.w);
1452                     else {
1453                         assert(temp->code == jit_code_label ||
1454                                temp->code == jit_code_epilog);
1455                         word = movi_p(rn(node->u.w), node->v.w);
1456                         patch(word, node);
1457                     }
1458                 }
1459                 else
1460                     movi(rn(node->u.w), node->v.w);
1461                 break;
1462                 case_rr(neg,);
1463                 case_rr(com,);
1464                 case_rrr(lt,);
1465                 case_rrw(lt,);
1466                 case_rrr(lt, _u);
1467                 case_rrw(lt, _u);
1468                 case_rrr(le,);
1469                 case_rrw(le,);
1470                 case_rrr(le, _u);
1471                 case_rrw(le, _u);
1472                 case_rrr(eq,);
1473                 case_rrw(eq,);
1474                 case_rrr(ge,);
1475                 case_rrw(ge,);
1476                 case_rrr(ge, _u);
1477                 case_rrw(ge, _u);
1478                 case_rrr(gt,);
1479                 case_rrw(gt,);
1480                 case_rrr(gt, _u);
1481                 case_rrw(gt, _u);
1482                 case_rrr(ne,);
1483                 case_rrw(ne,);
1484                 case_brr(blt,);
1485                 case_brw(blt,);
1486                 case_brr(blt, _u);
1487                 case_brw(blt, _u);
1488                 case_brr(ble,);
1489                 case_brw(ble,);
1490                 case_brr(ble, _u);
1491                 case_brw(ble, _u);
1492                 case_brr(beq,);
1493                 case_brw(beq,);
1494                 case_brr(bge,);
1495                 case_brw(bge,);
1496                 case_brr(bge, _u);
1497                 case_brw(bge, _u);
1498                 case_brr(bgt,);
1499                 case_brw(bgt,);
1500                 case_brr(bgt, _u);
1501                 case_brw(bgt, _u);
1502                 case_brr(bne,);
1503                 case_brw(bne,);
1504                 case_brr(boadd,);
1505                 case_brw(boadd,);
1506                 case_brr(boadd, _u);
1507                 case_brw(boadd, _u);
1508                 case_brr(bxadd,);
1509                 case_brw(bxadd,);
1510                 case_brr(bxadd, _u);
1511                 case_brw(bxadd, _u);
1512                 case_brr(bosub,);
1513                 case_brw(bosub,);
1514                 case_brr(bosub, _u);
1515                 case_brw(bosub, _u);
1516                 case_brr(bxsub,);
1517                 case_brw(bxsub,);
1518                 case_brr(bxsub, _u);
1519                 case_brw(bxsub, _u);
1520                 case_brr(bms,);
1521                 case_brw(bms,);
1522                 case_brr(bmc,);
1523                 case_brw(bmc,);
1524                 case_rrr(add, _f);
1525                 case_rrf(add, _f, 32);
1526                 case_rrr(sub, _f);
1527                 case_rrf(sub, _f, 32);
1528                 case_rrf(rsb, _f, 32);
1529                 case_rrr(mul, _f);
1530                 case_rrf(mul, _f, 32);
1531                 case_rrr(div, _f);
1532                 case_rrf(div, _f, 32);
1533                 case_rr(abs, _f);
1534                 case_rr(neg, _f);
1535                 case_rr(sqrt, _f);
1536                 case_rr(ext, _f);
1537                 case_rr(ld, _f);
1538                 case_rw(ld, _f);
1539                 case_rrr(ldx, _f);
1540                 case_rrw(ldx, _f);
1541                 case_rr(st, _f);
1542                 case_wr(st, _f);
1543                 case_rrr(stx, _f);
1544                 case_wrr(stx, _f);
1545                 case_rr(mov, _f);
1546             case jit_code_movi_f:
1547                 assert(node->flag & jit_flag_data);
1548                 movi_f(rn(node->u.w), (jit_float32_t *)node->v.n->u.w);
1549                 break;
1550                 case_rr(ext, _d_f);
1551                 case_rrr(lt, _f);
1552                 case_rrf(lt, _f, 32);
1553                 case_rrr(le, _f);
1554                 case_rrf(le, _f, 32);
1555                 case_rrr(eq, _f);
1556                 case_rrf(eq, _f, 32);
1557                 case_rrr(ge, _f);
1558                 case_rrf(ge, _f, 32);
1559                 case_rrr(gt, _f);
1560                 case_rrf(gt, _f, 32);
1561                 case_rrr(ne, _f);
1562                 case_rrf(ne, _f, 32);
1563                 case_rrr(unlt, _f);
1564                 case_rrf(unlt, _f, 32);
1565                 case_rrr(unle, _f);
1566                 case_rrf(unle, _f, 32);
1567                 case_rrr(uneq, _f);
1568                 case_rrf(uneq, _f, 32);
1569                 case_rrr(unge, _f);
1570                 case_rrf(unge, _f, 32);
1571                 case_rrr(ungt, _f);
1572                 case_rrf(ungt, _f, 32);
1573                 case_rrr(ltgt, _f);
1574                 case_rrf(ltgt, _f, 32);
1575                 case_rrr(ord, _f);
1576                 case_rrf(ord, _f, 32);
1577                 case_rrr(unord, _f);
1578                 case_rrf(unord, _f, 32);
1579                 case_brr(blt, _f);
1580                 case_brf(blt, _f, 32);
1581                 case_brr(ble, _f);
1582                 case_brf(ble, _f, 32);
1583                 case_brr(beq, _f);
1584                 case_brf(beq, _f, 32);
1585                 case_brr(bge, _f);
1586                 case_brf(bge, _f, 32);
1587                 case_brr(bgt, _f);
1588                 case_brf(bgt, _f, 32);
1589                 case_brr(bne, _f);
1590                 case_brf(bne, _f, 32);
1591                 case_brr(bunlt, _f);
1592                 case_brf(bunlt, _f, 32);
1593                 case_brr(bunle, _f);
1594                 case_brf(bunle, _f, 32);
1595                 case_brr(buneq, _f);
1596                 case_brf(buneq, _f, 32);
1597                 case_brr(bunge, _f);
1598                 case_brf(bunge, _f, 32);
1599                 case_brr(bungt, _f);
1600                 case_brf(bungt, _f, 32);
1601                 case_brr(bltgt, _f);
1602                 case_brf(bltgt, _f, 32);
1603                 case_brr(bord, _f);
1604                 case_brf(bord, _f, 32);
1605                 case_brr(bunord, _f);
1606                 case_brf(bunord, _f, 32);
1607                 case_rrr(add, _d);
1608                 case_rrf(add, _d, 64);
1609                 case_rrr(sub, _d);
1610                 case_rrf(sub, _d, 64);
1611                 case_rrf(rsb, _d, 64);
1612                 case_rrr(mul, _d);
1613                 case_rrf(mul, _d, 64);
1614                 case_rrr(div, _d);
1615                 case_rrf(div, _d, 64);
1616                 case_rr(abs, _d);
1617                 case_rr(neg, _d);
1618                 case_rr(sqrt, _d);
1619                 case_rr(ext, _d);
1620                 case_rr(ld, _d);
1621                 case_rw(ld, _d);
1622                 case_rrr(ldx, _d);
1623                 case_rrw(ldx, _d);
1624                 case_rr(st, _d);
1625                 case_wr(st, _d);
1626                 case_rrr(stx, _d);
1627                 case_wrr(stx, _d);
1628                 case_rr(mov, _d);
1629             case jit_code_movi_d:
1630                 assert(node->flag & jit_flag_data);
1631                 movi_d(rn(node->u.w), (jit_float64_t *)node->v.n->u.w);
1632                 break;
1633                 case_rr(ext, _f_d);
1634                 case_rrr(lt, _d);
1635                 case_rrf(lt, _d, 64);
1636                 case_rrr(le, _d);
1637                 case_rrf(le, _d, 64);
1638                 case_rrr(eq, _d);
1639                 case_rrf(eq, _d, 64);
1640                 case_rrr(ge, _d);
1641                 case_rrf(ge, _d, 64);
1642                 case_rrr(gt, _d);
1643                 case_rrf(gt, _d, 64);
1644                 case_rrr(ne, _d);
1645                 case_rrf(ne, _d, 64);
1646                 case_rrr(unlt, _d);
1647                 case_rrf(unlt, _d, 64);
1648                 case_rrr(unle, _d);
1649                 case_rrf(unle, _d, 64);
1650                 case_rrr(uneq, _d);
1651                 case_rrf(uneq, _d, 64);
1652                 case_rrr(unge, _d);
1653                 case_rrf(unge, _d, 64);
1654                 case_rrr(ungt, _d);
1655                 case_rrf(ungt, _d, 64);
1656                 case_rrr(ltgt, _d);
1657                 case_rrf(ltgt, _d, 64);
1658                 case_rrr(ord, _d);
1659                 case_rrf(ord, _d, 64);
1660                 case_rrr(unord, _d);
1661                 case_rrf(unord, _d, 64);
1662                 case_brr(blt, _d);
1663                 case_brf(blt, _d, 64);
1664                 case_brr(ble, _d);
1665                 case_brf(ble, _d, 64);
1666                 case_brr(beq, _d);
1667                 case_brf(beq, _d, 64);
1668                 case_brr(bge, _d);
1669                 case_brf(bge, _d, 64);
1670                 case_brr(bgt, _d);
1671                 case_brf(bgt, _d, 64);
1672                 case_brr(bne, _d);
1673                 case_brf(bne, _d, 64);
1674                 case_brr(bunlt, _d);
1675                 case_brf(bunlt, _d, 64);
1676                 case_brr(bunle, _d);
1677                 case_brf(bunle, _d, 64);
1678                 case_brr(buneq, _d);
1679                 case_brf(buneq, _d, 64);
1680                 case_brr(bunge, _d);
1681                 case_brf(bunge, _d, 64);
1682                 case_brr(bungt, _d);
1683                 case_brf(bungt, _d, 64);
1684                 case_brr(bltgt, _d);
1685                 case_brf(bltgt, _d, 64);
1686                 case_brr(bord, _d);
1687                 case_brf(bord, _d, 64);
1688                 case_brr(bunord, _d);
1689                 case_brf(bunord, _d, 64);
1690             case jit_code_jmpr:
1691                 jmpr(rn(node->u.w));
1692                 break;
1693             case jit_code_jmpi:
1694                 if (node->flag & jit_flag_node) {
1695                     temp = node->u.n;
1696                     assert(temp->code == jit_code_label ||
1697                            temp->code == jit_code_epilog);
1698                     if (temp->flag & jit_flag_patch)
1699                         jmpi(temp->u.w);
1700                     else {
1701                         word = jmpi(_jit->pc.w);
1702                         patch(word, node);
1703                     }
1704                 }
1705                 else
1706                     jmpi(node->u.w);
1707                 break;
1708             case jit_code_callr:
1709                 callr(rn(node->u.w));
1710                 break;
1711             case jit_code_calli:
1712                 if (node->flag & jit_flag_node) {
1713                     temp = node->u.n;
1714                     assert(temp->code == jit_code_label ||
1715                            temp->code == jit_code_epilog);
1716                     word = calli_p(temp->u.w);
1717                     if (!(temp->flag & jit_flag_patch))
1718                         patch(word, node);
1719                 }
1720                 else
1721                     calli(node->u.w);
1722                 break;
1723             case jit_code_prolog:
1724                 _jitc->function = _jitc->functions.ptr + node->w.w;
1725                 undo.node = node;
1726                 undo.word = _jit->pc.w;
1727 #if DEVEL_DISASSEMBLER
1728                 undo.prevw = prevw;
1729 #endif
1730                 undo.patch_offset = _jitc->patches.offset;
1731             restart_function:
1732                 _jitc->again = 0;
1733                 prolog(node);
1734                 break;
1735             case jit_code_epilog:
1736                 assert(_jitc->function == _jitc->functions.ptr + node->w.w);
1737                 if (_jitc->again) {
1738                     for (temp = undo.node->next;
1739                          temp != node; temp = temp->next) {
1740                         if (temp->code == jit_code_label ||
1741                             temp->code == jit_code_epilog)
1742                             temp->flag &= ~jit_flag_patch;
1743                     }
1744                     temp->flag &= ~jit_flag_patch;
1745                     node = undo.node;
1746                     _jit->pc.w = undo.word;
1747 #if DEVEL_DISASSEMBLER
1748                     prevw = undo.prevw;
1749 #endif
1750                     _jitc->patches.offset = undo.patch_offset;
1751                     goto restart_function;
1752                 }
1753                 /* remember label is defined */
1754                 node->flag |= jit_flag_patch;
1755                 node->u.w = _jit->pc.w;
1756                 epilog(node);
1757                 _jitc->function = NULL;
1758                 break;
1759 #if !NEW_ABI
1760             case jit_code_movr_w_f:
1761                 movr_w_f(rn(node->u.w), rn(node->v.w));
1762                 break;
1763 #endif
1764             case jit_code_movr_f_w:
1765                 movr_f_w(rn(node->u.w), rn(node->v.w));
1766                 break;
1767             case jit_code_movi_f_w:
1768                 assert(node->flag & jit_flag_data);
1769                 movi_f_w(rn(node->u.w), (jit_float32_t *)node->v.n->u.w);
1770                 break;
1771 #if NEW_ABI
1772             case jit_code_movr_d_w:
1773                 movr_d_w(rn(node->u.w), rn(node->v.w));
1774                 break;
1775             case jit_code_movi_d_w:
1776                 assert(node->flag & jit_flag_data);
1777                 movi_d_w(rn(node->u.w), (jit_float64_t *)node->v.n->u.w);
1778                 break;
1779 #else
1780             case jit_code_movr_ww_d:
1781                 movr_ww_d(rn(node->u.w), rn(node->v.w), rn(node->w.w));
1782                 break;
1783             case jit_code_movr_d_ww:
1784                 movr_d_ww(rn(node->u.w), rn(node->v.w), rn(node->w.w));
1785                 break;
1786             case jit_code_movi_d_ww:
1787                 assert(node->flag & jit_flag_data);
1788                 movi_d_ww(rn(node->u.w), rn(node->v.w),
1789                           (jit_float64_t *)node->w.n->u.w);
1790                 break;
1791 #endif
1792             case jit_code_va_start:
1793                 vastart(rn(node->u.w));
1794                 break;
1795             case jit_code_va_arg:
1796                 vaarg(rn(node->u.w), rn(node->v.w));
1797                 break;
1798             case jit_code_va_arg_d:
1799                 vaarg_d(rn(node->u.w), rn(node->v.w));
1800                 break;
1801             case jit_code_live:
1802             case jit_code_arg:                  case jit_code_ellipsis:
1803             case jit_code_va_push:
1804             case jit_code_allocai:              case jit_code_allocar:
1805             case jit_code_arg_f:                case jit_code_arg_d:
1806             case jit_code_va_end:
1807             case jit_code_ret:
1808             case jit_code_retr:                 case jit_code_reti:
1809             case jit_code_retr_f:               case jit_code_reti_f:
1810             case jit_code_retr_d:               case jit_code_reti_d:
1811             case jit_code_getarg_c:             case jit_code_getarg_uc:
1812             case jit_code_getarg_s:             case jit_code_getarg_us:
1813             case jit_code_getarg_i:
1814 #if __WORDSIZE == 64
1815             case jit_code_getarg_ui:            case jit_code_getarg_l:
1816 #endif
1817             case jit_code_getarg_f:             case jit_code_getarg_d:
1818             case jit_code_putargr:              case jit_code_putargi:
1819             case jit_code_putargr_f:            case jit_code_putargi_f:
1820             case jit_code_putargr_d:            case jit_code_putargi_d:
1821             case jit_code_pushargr:             case jit_code_pushargi:
1822             case jit_code_pushargr_f:           case jit_code_pushargi_f:
1823             case jit_code_pushargr_d:           case jit_code_pushargi_d:
1824             case jit_code_retval_c:             case jit_code_retval_uc:
1825             case jit_code_retval_s:             case jit_code_retval_us:
1826             case jit_code_retval_i:
1827 #if __WORDSIZE == 64
1828             case jit_code_retval_ui:            case jit_code_retval_l:
1829 #endif
1830             case jit_code_retval_f:             case jit_code_retval_d:
1831             case jit_code_prepare:
1832             case jit_code_finishr:              case jit_code_finishi:
1833                 break;
1834             default:
1835                 abort();
1836         }
1837         if (jit_carry != _NOREG) {
1838             switch (node->code) {
1839                 case jit_code_note:
1840                 case jit_code_addcr:            case jit_code_addci:
1841                 case jit_code_addxr:            case jit_code_addxi:
1842                 case jit_code_subcr:            case jit_code_subci:
1843                 case jit_code_subxr:            case jit_code_subxi:
1844                     break;
1845                 default:
1846                     jit_unget_reg(jit_carry);
1847                     jit_carry = _NOREG;
1848                     break;
1849             }
1850         }
1851         jit_regarg_clr(node, value);
1852         assert(_jitc->regarg == 0 ||
1853                (jit_carry != _NOREG && _jitc->regarg == (1 << jit_carry)));
1854         assert(_jitc->synth == 0);
1855         /* update register live state */
1856         jit_reglive(node);
1857     }
1858 #undef case_brf
1859 #undef case_brw
1860 #undef case_brr
1861 #undef case_wrr
1862 #undef case_rrf
1863 #undef case_rrw
1864 #undef case_rrr
1865 #undef case_wr
1866 #undef case_rw
1867 #undef case_rr
1868
1869     for (offset = 0; offset < _jitc->patches.offset; offset++) {
1870         node = _jitc->patches.ptr[offset].node;
1871         word = node->code == jit_code_movi ? node->v.n->u.w : node->u.n->u.w;
1872         patch_at(_jitc->patches.ptr[offset].inst, word);
1873     }
1874
1875     jit_flush(_jit->code.ptr, _jit->pc.uc);
1876
1877     return (_jit->code.ptr);
1878 }
1879
1880 #define CODE                            1
1881 #  include "jit_rewind.c"
1882 #  include "jit_mips-cpu.c"
1883 #  include "jit_mips-fpu.c"
1884 #undef CODE
1885
1886 void
1887 jit_flush(void *fptr, void *tptr)
1888 {
1889 #if defined(__linux__)
1890     jit_word_t          f, t, s;
1891
1892     s = sysconf(_SC_PAGE_SIZE);
1893     f = (jit_word_t)fptr & -s;
1894     t = (((jit_word_t)tptr) + s - 1) & -s;
1895     _flush_cache((void *)f, t - f, ICACHE);
1896 #endif
1897 }
1898
1899 void
1900 _emit_ldxi(jit_state_t *_jit, jit_int32_t r0, jit_int32_t r1, jit_word_t i0)
1901 {
1902     ldxi(rn(r0), rn(r1), i0);
1903 }
1904
1905 void
1906 _emit_stxi(jit_state_t *_jit, jit_word_t i0, jit_int32_t r0, jit_int32_t r1)
1907 {
1908     stxi(i0, rn(r0), rn(r1));
1909 }
1910
1911 void
1912 _emit_ldxi_d(jit_state_t *_jit, jit_int32_t r0, jit_int32_t r1, jit_word_t i0)
1913 {
1914     ldxi_d(rn(r0), rn(r1), i0);
1915 }
1916
1917 void
1918 _emit_stxi_d(jit_state_t *_jit, jit_word_t i0, jit_int32_t r0, jit_int32_t r1)
1919 {
1920     stxi_d(i0, rn(r0), rn(r1));
1921 }
1922
1923 static void
1924 _patch(jit_state_t *_jit, jit_word_t instr, jit_node_t *node)
1925 {
1926     jit_int32_t          flag;
1927
1928     assert(node->flag & jit_flag_node);
1929     if (node->code == jit_code_movi)
1930         flag = node->v.n->flag;
1931     else
1932         flag = node->u.n->flag;
1933     assert(!(flag & jit_flag_patch));
1934     if (_jitc->patches.offset >= _jitc->patches.length) {
1935         jit_realloc((jit_pointer_t *)&_jitc->patches.ptr,
1936                     _jitc->patches.length * sizeof(jit_patch_t),
1937                     (_jitc->patches.length + 1024) * sizeof(jit_patch_t));
1938         _jitc->patches.length += 1024;
1939     }
1940     _jitc->patches.ptr[_jitc->patches.offset].inst = instr;
1941     _jitc->patches.ptr[_jitc->patches.offset].node = node;
1942     ++_jitc->patches.offset;
1943 }