deps/lightning/lib/jit_mips.c

   1 /*
   2  * Copyright (C) 2012-2023  Free Software Foundation, Inc.
   3  *
   4  * This file is part of GNU lightning.
   5  *
   6  * GNU lightning is free software; you can redistribute it and/or modify it
   7  * under the terms of the GNU Lesser General Public License as published
   8  * by the Free Software Foundation; either version 3, or (at your option)
   9  * any later version.
  10  *
  11  * GNU lightning is distributed in the hope that it will be useful, but
  12  * WITHOUT ANY WARRANTY; without even the implied warranty of MERCHANTABILITY
  13  * or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU Lesser General Public
  14  * License for more details.
  15  *
  16  * Authors:
  17  *      Paulo Cesar Pereira de Andrade
  18  */
  19
  20 #if defined(__linux__)
  21 #  include <sys/cachectl.h>
  22 #endif
  23
  24 #if __mips_hard_float
  25 #  define __mips_soft_float             0
  26 #elif __mips_soft_float
  27 #  define __mips_hard_float             0
  28 #else
  29 /* Must have a floating point unit and cannot figure
  30  * if can attempt to work with software floats
  31  */
  32 #  define __mips_soft_float             0
  33 #  define __mips_hard_float             1
  34 #endif
  35
  36 #if NEW_ABI
  37 /*   callee save                                    + variadic arguments
  38  *   align16(ra+fp+s[0-7]++f20+f22+f24+f26+f28+f30) + align16(a[0-7]) */
  39 #  define stack_framesize               (128 + 64)
  40 #else
  41 /*   callee save
  42  *   align16(ra+fp+s[0-7]+f16+f18+f20+f22+f24+f26+f28+f30) */
  43 #  define stack_framesize               128
  44 #endif
  45
  46 #if NEW_ABI
  47 #  define NUM_WORD_ARGS                 8
  48 #  define STACK_SLOT                    8
  49 #  define STACK_SHIFT                   3
  50 #else
  51 #  define NUM_WORD_ARGS                 4
  52 #  define STACK_SLOT                    4
  53 #  define STACK_SHIFT                   2
  54 #endif
  55 #if NEW_ABI && __BYTE_ORDER == __BIG_ENDIAN && __WORDSIZE == 32
  56 #  define WORD_ADJUST                   4
  57 #else
  58 #  define WORD_ADJUST                   0
  59 #endif
  60 #define jit_arg_reg_p(i)                ((i) >= 0 && (i) < NUM_WORD_ARGS)
  61 #if __BYTE_ORDER == __LITTLE_ENDIAN
  62 #  define C_DISP                        0
  63 #  define S_DISP                        0
  64 #  define I_DISP                        0
  65 #else
  66 #  define C_DISP                        STACK_SLOT - sizeof(jit_int8_t)
  67 #  define S_DISP                        STACK_SLOT - sizeof(jit_int16_t)
  68 #  define I_DISP                        STACK_SLOT - sizeof(jit_int32_t)
  69 #endif
  70
  71 /*
  72  * Types
  73  */
  74 typedef struct jit_pointer_t jit_va_list_t;
  75
  76 /*
  77  * Prototypes
  78  */
  79 #define jit_make_arg(node,code)         _jit_make_arg(_jit,node,code)
  80 static jit_node_t *_jit_make_arg(jit_state_t*,jit_node_t*,jit_code_t);
  81 #define jit_make_arg_f(node)            _jit_make_arg_f(_jit,node)
  82 static jit_node_t *_jit_make_arg_f(jit_state_t*,jit_node_t*);
  83 #define jit_make_arg_d(node)            _jit_make_arg_d(_jit,node)
  84 static jit_node_t *_jit_make_arg_d(jit_state_t*,jit_node_t*);
  85 #define compute_framesize()             _compute_framesize(_jit)
  86 static void _compute_framesize(jit_state_t*);
  87 #define patch(instr, node)              _patch(_jit, instr, node)
  88 static void _patch(jit_state_t*,jit_word_t,jit_node_t*);
  89
  90 #define PROTO                           1
  91 #  include "jit_rewind.c"
  92 #  include "jit_mips-cpu.c"
  93 #  include "jit_mips-fpu.c"
  94 #  include "jit_fallback.c"
  95 #undef PROTO
  96
  97 /*
  98  * Initialization
  99  */
 100 jit_cpu_t               jit_cpu;
 101 jit_register_t          _rvs[] = {
 102     { rc(gpr) | 0x01,                   "at" },
 103     { rc(gpr) | 0x02,                   "v0" },
 104     { rc(gpr) | 0x03,                   "v1" },
 105 #if !NEW_ABI
 106     { rc(gpr) | 0x08,                   "t0" },
 107     { rc(gpr) | 0x09,                   "t1" },
 108     { rc(gpr) | 0x0a,                   "t2" },
 109     { rc(gpr) | 0x0b,                   "t3" },
 110 #endif
 111     { rc(gpr) | 0x0c,                   "t4" },
 112     { rc(gpr) | 0x0d,                   "t5" },
 113     { rc(gpr) | 0x0e,                   "t6" },
 114     { rc(gpr) | 0x0f,                   "t7" },
 115     { rc(gpr) | 0x18,                   "t8" },
 116     { rc(gpr) | 0x19,                   "t9" },
 117     { rc(sav) | rc(gpr) | 0x10,         "s0" },
 118     { rc(sav) | rc(gpr) | 0x11,         "s1" },
 119     { rc(sav) | rc(gpr) | 0x12,         "s2" },
 120     { rc(sav) | rc(gpr) | 0x13,         "s3" },
 121     { rc(sav) | rc(gpr) | 0x14,         "s4" },
 122     { rc(sav) | rc(gpr) | 0x15,         "s5" },
 123     { rc(sav) | rc(gpr) | 0x16,         "s6" },
 124     { rc(sav) | rc(gpr) | 0x17,         "s7" },
 125     { 0x00,                             "zero" },
 126     { 0x1a,                             "k0" },
 127     { 0x1b,                             "k1" },
 128     { rc(sav) | 0x1f,                   "ra" },
 129     { rc(sav) | 0x1c,                   "gp" },
 130     { rc(sav) | 0x1d,                   "sp" },
 131     { rc(sav) | 0x1e,                   "fp" },
 132 #if NEW_ABI
 133     { rc(gpr) | 0x0b,                   "a7" },
 134     { rc(gpr) | 0x0a,                   "a6" },
 135     { rc(gpr) | 0x09,                   "a5" },
 136     { rc(gpr) | 0x08,                   "a4" },
 137 #endif
 138     { rc(arg) | rc(gpr) | 0x07,         "a3" },
 139     { rc(arg) | rc(gpr) | 0x06,         "a2" },
 140     { rc(arg) | rc(gpr) | 0x05,         "a1" },
 141     { rc(arg) | rc(gpr) | 0x04,         "a0" },
 142     { rc(fpr) | 0x00,                   "$f0" },
 143     { rc(fpr) | 0x02,                   "$f2" },
 144     { rc(fpr) | 0x04,                   "$f4" },
 145     { rc(fpr) | 0x06,                   "$f6" },
 146     { rc(fpr) | 0x08,                   "$f8" },
 147     { rc(fpr) | 0x0a,                   "$f10" },
 148 #if !NEW_ABI
 149     { rc(sav) | rc(fpr) | 0x10,         "$f16" },
 150     { rc(sav) | rc(fpr) | 0x12,         "$f18" },
 151 #endif
 152     { rc(sav) | rc(fpr) | 0x14,         "$f20" },
 153     { rc(sav) | rc(fpr) | 0x16,         "$f22" },
 154     { rc(sav) | rc(fpr) | 0x18,         "$f24" },
 155     { rc(sav) | rc(fpr) | 0x1a,         "$f26" },
 156     { rc(sav) | rc(fpr) | 0x1c,         "$f28" },
 157     { rc(sav) | rc(fpr) | 0x1e,         "$f30" },
 158 #if NEW_ABI
 159     { rc(arg) | rc(fpr) | 0x13,         "$f19" },
 160     { rc(arg) | rc(fpr) | 0x12,         "$f18" },
 161     { rc(arg) | rc(fpr) | 0x11,         "$f17" },
 162     { rc(arg) | rc(fpr) | 0x10,         "$f16" },
 163     { rc(arg) | rc(fpr) | 0x0f,         "$f15" },
 164     { rc(arg) | rc(fpr) | 0x0e,         "$f14" },
 165     { rc(arg) | rc(fpr) | 0x0d,         "$f13" },
 166     { rc(arg) | rc(fpr) | 0x0c,         "$f12" },
 167 #else
 168     { rc(arg) | rc(fpr) | 0x0e,         "$f14" },
 169     { rc(arg) | rc(fpr) | 0x0c,         "$f12" },
 170 #endif
 171     { _NOREG,                           "<none>" },
 172 };
 173
 174 static jit_int32_t iregs[] = {
 175     _S0, _S1, _S2, _S3, _S4, _S5, _S6, _S7
 176 };
 177
 178 static jit_int32_t fregs[] = {
 179 #if !NEW_ABI
 180     _F16, _F18,
 181 #endif
 182     _F20, _F22, _F24, _F26, _F28, _F30
 183 };
 184
 185 /*
 186  * Implementation
 187  */
 188 void
 189 jit_get_cpu(void)
 190 {
 191     /* By default assume it works or have/need unaligned instructions. */
 192     jit_cpu.sll_delay = jit_cpu.cop1_delay = jit_cpu.lwl_lwr_delay =
 193         jit_cpu.unaligned = 1;
 194
 195 #if defined(__linux__)
 196     FILE        *fp;
 197     char        *ptr;
 198     char         buf[128];
 199
 200     if ((fp = fopen("/proc/cpuinfo", "r")) != NULL) {
 201         while (fgets(buf, sizeof(buf), fp)) {
 202             if (strncmp(buf, "isa\t\t\t: ", 8) == 0) {
 203                 if ((ptr = strstr(buf + 9, "mips64r")))
 204                     jit_cpu.release = strtoul(ptr + 7, NULL, 10);
 205                 break;
 206             }
 207             /* Just for some actual hardware tested. Below check
 208              * for mips 1 would disable these delays anyway. */
 209             if (strncmp(buf, "cpu model\t\t: ", 13) == 0) {
 210                 /* ICT Loongson-2 V0.3  FPU V0.1 */
 211                 if (strstr(buf + 13, "FPU V0.1"))
 212                   jit_cpu.sll_delay = jit_cpu.cop1_delay = 0;
 213                 /* Cavium Octeon III V0.2  FPU V0.0 */
 214                 else if (strstr(buf + 13, "FPU V0.0"))
 215                   jit_cpu.sll_delay = jit_cpu.cop1_delay = 0;
 216                 /* Cavium Octeon II V0.1 */
 217                 else if (strstr(buf + 13, " II "))
 218                   jit_cpu.sll_delay = jit_cpu.cop1_delay = 0;
 219                 break;
 220             }
 221         }
 222         fclose(fp);
 223     }
 224 #endif
 225 #if __mips_isa_rev
 226     if (!jit_cpu.release)
 227         jit_cpu.release = __mips_isa_rev;
 228 #elif defined _MIPS_ARCH
 229     if (!jit_cpu.release)
 230         jit_cpu.release = strtoul(&_MIPS_ARCH[4], NULL, 10);
 231 #elif defined(__mips) && __mips < 6
 232     if (!jit_cpu.release)
 233         jit_cpu.release = __mips;
 234 #endif
 235     /* Assume all mips 1 and 2, or detected as release 1 or 2 have this
 236      *  problem */
 237     /* Note that jit_cpu is global, and can be overriden, that is, add
 238      * the C code "jit_cpu.cop1_delay = 1;" after the call to init_jit()
 239      * if it is functional. */
 240     if (jit_cpu.cop1_delay && jit_cpu.release < 3)
 241         jit_cpu.cop1_delay = 0;
 242     if (jit_cpu.sll_delay && jit_cpu.release < 3)
 243         jit_cpu.sll_delay = 0;
 244     if (jit_cpu.lwl_lwr_delay && jit_cpu.release < 2)
 245         jit_cpu.lwl_lwr_delay = 0;
 246     if (jit_cpu.release >= 6)
 247         jit_cpu.unaligned = 0;
 248 }
 249
 250 void
 251 _jit_init(jit_state_t *_jit)
 252 {
 253     _jitc->reglen = jit_size(_rvs) - 1;
 254 /* Could also:
 255  *      o reserve a register for carry  (overkill)
 256  *      o use MTLO/MFLO                 (performance hit)
 257  * So, keep a register allocated after setting carry, and implicitly
 258  * deallocate it if it can no longer be tracked
 259  */
 260     jit_carry = _NOREG;
 261 }
 262
 263 void
 264 _jit_prolog(jit_state_t *_jit)
 265 {
 266     jit_int32_t          offset;
 267
 268     if (_jitc->function)
 269         jit_epilog();
 270     assert(jit_regset_cmp_ui(&_jitc->regarg, 0) == 0);
 271     jit_regset_set_ui(&_jitc->regsav, 0);
 272     offset = _jitc->functions.offset;
 273     if (offset >= _jitc->functions.length) {
 274         jit_realloc((jit_pointer_t *)&_jitc->functions.ptr,
 275                     _jitc->functions.length * sizeof(jit_function_t),
 276                     (_jitc->functions.length + 16) * sizeof(jit_function_t));
 277         _jitc->functions.length += 16;
 278     }
 279     _jitc->function = _jitc->functions.ptr + _jitc->functions.offset++;
 280     _jitc->function->self.size = stack_framesize;
 281     _jitc->function->self.argi = _jitc->function->self.argf =
 282         _jitc->function->self.aoff = _jitc->function->self.alen = 0;
 283     _jitc->function->self.call = jit_call_default;
 284     jit_alloc((jit_pointer_t *)&_jitc->function->regoff,
 285               _jitc->reglen * sizeof(jit_int32_t));
 286
 287     /* _no_link here does not mean the jit_link() call can be removed
 288      * by rewriting as:
 289      * _jitc->function->prolog = jit_new_node(jit_code_prolog);
 290      */
 291     _jitc->function->prolog = jit_new_node_no_link(jit_code_prolog);
 292     jit_link(_jitc->function->prolog);
 293     _jitc->function->prolog->w.w = offset;
 294     _jitc->function->epilog = jit_new_node_no_link(jit_code_epilog);
 295     /*  u:      label value
 296      *  v:      offset in blocks vector
 297      *  w:      offset in functions vector
 298      */
 299     _jitc->function->epilog->w.w = offset;
 300
 301     jit_regset_new(&_jitc->function->regset);
 302 }
 303
 304 jit_int32_t
 305 _jit_allocai(jit_state_t *_jit, jit_int32_t length)
 306 {
 307     assert(_jitc->function);
 308     jit_check_frame();
 309     switch (length) {
 310         case 0: case 1:                                         break;
 311         case 2:         _jitc->function->self.aoff &= -2;       break;
 312         case 3: case 4: _jitc->function->self.aoff &= -4;       break;
 313         default:        _jitc->function->self.aoff &= -8;       break;
 314     }
 315     _jitc->function->self.aoff -= length;
 316     if (!_jitc->realize) {
 317         jit_inc_synth_ww(allocai, _jitc->function->self.aoff, length);
 318         jit_dec_synth();
 319     }
 320     return (_jitc->function->self.aoff);
 321 }
 322
 323 void
 324 _jit_allocar(jit_state_t *_jit, jit_int32_t u, jit_int32_t v)
 325 {
 326     jit_int32_t          reg;
 327     assert(_jitc->function);
 328     jit_inc_synth_ww(allocar, u, v);
 329     if (!_jitc->function->allocar) {
 330         _jitc->function->aoffoff = jit_allocai(sizeof(jit_int32_t));
 331         _jitc->function->allocar = 1;
 332     }
 333     reg = jit_get_reg(jit_class_gpr);
 334     jit_negr(reg, v);
 335     jit_andi(reg, reg, -8);
 336     jit_ldxi_i(u, JIT_FP, _jitc->function->aoffoff);
 337     jit_addr(u, u, reg);
 338     jit_addr(JIT_SP, JIT_SP, reg);
 339     jit_stxi_i(_jitc->function->aoffoff, JIT_FP, u);
 340     jit_unget_reg(reg);
 341     jit_dec_synth();
 342 }
 343
 344 void
 345 _jit_ret(jit_state_t *_jit)
 346 {
 347     jit_node_t          *instr;
 348     assert(_jitc->function);
 349     jit_inc_synth(ret);
 350     /* jump to epilog */
 351     instr = jit_jmpi();
 352     jit_patch_at(instr, _jitc->function->epilog);
 353     jit_dec_synth();
 354 }
 355
 356 void
 357 _jit_retr(jit_state_t *_jit, jit_int32_t u, jit_code_t code)
 358 {
 359     jit_code_inc_synth_w(code, u);
 360     jit_movr(JIT_RET, u);
 361     jit_ret();
 362     jit_dec_synth();
 363 }
 364
 365 void
 366 _jit_reti(jit_state_t *_jit, jit_word_t u, jit_code_t code)
 367 {
 368     jit_code_inc_synth_w(code, u);
 369     jit_movi(JIT_RET, u);
 370     jit_ret();
 371     jit_dec_synth();
 372 }
 373
 374 void
 375 _jit_retr_f(jit_state_t *_jit, jit_int32_t u)
 376 {
 377     jit_inc_synth_w(retr_f, u);
 378 #if __mips_soft_float
 379 #  warning *** GNU Lightning will use hard float registers! ***
 380 #  warning *** Are you sure about -msoft-float usage?       ***
 381     jit_movr_f_w(JIT_RET, u);
 382 #else
 383     if (JIT_FRET != u)
 384         jit_movr_f(JIT_FRET, u);
 385     else
 386         jit_live(JIT_FRET);
 387 #endif
 388     jit_ret();
 389     jit_dec_synth();
 390 }
 391
 392 void
 393 _jit_reti_f(jit_state_t *_jit, jit_float32_t u)
 394 {
 395     jit_inc_synth_f(reti_f, u);
 396 #if __mips_soft_float
 397     jit_movi_f_w(JIT_RET, u);
 398 #else
 399     jit_movi_f(JIT_FRET, u);
 400 #endif
 401     jit_ret();
 402     jit_dec_synth();
 403 }
 404
 405 void
 406 _jit_retr_d(jit_state_t *_jit, jit_int32_t u)
 407 {
 408     jit_inc_synth_w(retr_d, u);
 409 #if __mips_soft_float
 410     jit_movr_d_w(JIT_RET, u);
 411 #else
 412     if (JIT_FRET != u)
 413         jit_movr_d(JIT_FRET, u);
 414     else
 415         jit_live(JIT_FRET);
 416 #endif
 417     jit_ret();
 418     jit_dec_synth();
 419 }
 420
 421 void
 422 _jit_reti_d(jit_state_t *_jit, jit_float64_t u)
 423 {
 424     jit_inc_synth_d(reti_d, u);
 425 #if __mips_soft_float
 426     jit_movi_d_w(JIT_RET, u);
 427 #else
 428     jit_movi_d(JIT_FRET, u);
 429 #endif
 430     jit_ret();
 431     jit_dec_synth();
 432 }
 433
 434 void
 435 _jit_epilog(jit_state_t *_jit)
 436 {
 437     assert(_jitc->function);
 438     assert(_jitc->function->epilog->next == NULL);
 439     jit_link(_jitc->function->epilog);
 440     _jitc->function = NULL;
 441 }
 442
 443 jit_bool_t
 444 _jit_arg_register_p(jit_state_t *_jit, jit_node_t *u)
 445 {
 446     if (u->code >= jit_code_arg_c && u->code <= jit_code_arg)
 447         return (jit_arg_reg_p(u->u.w));
 448     assert(u->code == jit_code_arg_f || u->code == jit_code_arg_d);
 449 #if NEW_ABI
 450     return (jit_arg_reg_p(u->u.w) || jit_arg_reg_p(u->u.w - 8));
 451 #else
 452     return (u->u.w < 8);
 453 #endif
 454 }
 455
 456 static jit_node_t *
 457 _jit_make_arg(jit_state_t *_jit, jit_node_t *node, jit_code_t code)
 458 {
 459     jit_int32_t          offset;
 460 #if NEW_ABI
 461     if (jit_arg_reg_p(_jitc->function->self.argi))
 462         offset = _jitc->function->self.argi++;
 463     else {
 464         offset = _jitc->function->self.size;
 465         _jitc->function->self.size += STACK_SLOT;
 466     }
 467 #else
 468     offset = (_jitc->function->self.size - stack_framesize) >> STACK_SHIFT;
 469     ++_jitc->function->self.argi;
 470     if (offset >= 4)
 471         offset = _jitc->function->self.size;
 472     _jitc->function->self.size += STACK_SLOT;
 473 #endif
 474     if (node == (jit_node_t *)0)
 475         node = jit_new_node(code);
 476     else
 477         link_node(node);
 478     node->u.w = offset;
 479     node->v.w = ++_jitc->function->self.argn;
 480     jit_link_prolog();
 481     return (node);
 482 }
 483
 484 static jit_node_t *
 485 _jit_make_arg_f(jit_state_t *_jit, jit_node_t *node)
 486 {
 487     jit_int32_t          offset;
 488 #if NEW_ABI
 489     if (jit_arg_reg_p(_jitc->function->self.argi)) {
 490         offset = _jitc->function->self.argi++;
 491         if (__mips_soft_float ||
 492             (_jitc->function->self.call & jit_call_varargs))
 493             offset += 8;
 494     }
 495     else {
 496         offset = _jitc->function->self.size;
 497         _jitc->function->self.size += STACK_SLOT;
 498     }
 499 #else
 500     offset = (_jitc->function->self.size - stack_framesize) >> STACK_SHIFT;
 501     if (offset < NUM_WORD_ARGS) {
 502         if (!_jitc->function->self.argi &&
 503             !(_jitc->function->self.call & jit_call_varargs)) {
 504             if (offset == 0)
 505                 offset = 4;
 506             else {
 507                 offset = 6;
 508                 _jitc->function->self.argi = 1;
 509             }
 510             /* Use as flag to rewind in case of varargs function */
 511             ++_jitc->function->self.argf;
 512         }
 513     }
 514     else
 515         offset = _jitc->function->self.size;
 516     _jitc->function->self.size += STACK_SLOT;
 517 #endif
 518     if (node == (jit_node_t *)0)
 519         node = jit_new_node(jit_code_arg_f);
 520     else
 521         link_node(node);
 522     node->u.w = offset;
 523     node->v.w = ++_jitc->function->self.argn;
 524     jit_link_prolog();
 525     return (node);
 526 }
 527
 528 static jit_node_t *
 529 _jit_make_arg_d(jit_state_t *_jit, jit_node_t *node)
 530 {
 531     jit_int32_t          offset;
 532 #if NEW_ABI
 533     if (jit_arg_reg_p(_jitc->function->self.argi)) {
 534         offset = _jitc->function->self.argi++;
 535         if (__mips_soft_float ||
 536             (_jitc->function->self.call & jit_call_varargs))
 537             offset += 8;
 538     }
 539     else {
 540         offset = _jitc->function->self.size;
 541         _jitc->function->self.size += STACK_SLOT;
 542     }
 543 #else
 544     if (_jitc->function->self.size & 7) {
 545         _jitc->function->self.size += 4;
 546         _jitc->function->self.argi = 1;
 547     }
 548     offset = (_jitc->function->self.size - stack_framesize) >> STACK_SHIFT;
 549     if (offset < NUM_WORD_ARGS) {
 550         if (!_jitc->function->self.argi &&
 551             !(_jitc->function->self.call & jit_call_varargs)) {
 552             offset += 4;
 553             /* Use as flag to rewind in case of varargs function */
 554             ++_jitc->function->self.argf;
 555         }
 556     }
 557     else
 558         offset = _jitc->function->self.size;
 559     _jitc->function->self.size += sizeof(jit_float64_t);
 560 #endif
 561     if (node == (jit_node_t *)0)
 562         node = jit_new_node(jit_code_arg_d);
 563     else
 564         link_node(node);
 565     node->u.w = offset;
 566     node->v.w = ++_jitc->function->self.argn;
 567     jit_link_prolog();
 568     return (node);
 569 }
 570
 571 void
 572 _jit_ellipsis(jit_state_t *_jit)
 573 {
 574     if (_jitc->prepare) {
 575         assert(!(_jitc->function->call.call & jit_call_varargs));
 576         _jitc->function->call.call |= jit_call_varargs;
 577 #if !NEW_ABI
 578         if (_jitc->function->call.argf)
 579             rewind_prepare();
 580 #endif
 581     }
 582     else {
 583         assert(!(_jitc->function->self.call & jit_call_varargs));
 584 #if NEW_ABI
 585         if (jit_arg_reg_p(_jitc->function->self.argi))
 586             rewind_prolog();
 587         /* Do not set during possible rewind. */
 588         _jitc->function->self.call |= jit_call_varargs;
 589 #else
 590         _jitc->function->self.call |= jit_call_varargs;
 591         if (_jitc->function->self.argf)
 592             rewind_prolog();
 593 #endif
 594         _jitc->function->vagp = _jitc->function->self.argi;
 595     }
 596     jit_inc_synth(ellipsis);
 597     jit_check_frame();
 598     if (_jitc->prepare)
 599         jit_link_prepare();
 600     else
 601         jit_link_prolog();
 602     jit_dec_synth();
 603 }
 604
 605 void
 606 _jit_va_push(jit_state_t *_jit, jit_int32_t u)
 607 {
 608     jit_inc_synth_w(va_push, u);
 609     jit_pushargr(u);
 610     jit_dec_synth();
 611 }
 612
 613 jit_node_t *
 614 _jit_arg(jit_state_t *_jit, jit_code_t code)
 615 {
 616     assert(_jitc->function);
 617     assert(!(_jitc->function->self.call & jit_call_varargs));
 618 #if STRONG_TYPE_CHECKING
 619     assert(code >= jit_code_arg_c && code <= jit_code_arg);
 620 #endif
 621     return (jit_make_arg((jit_node_t*)0, code));
 622 }
 623
 624 jit_node_t *
 625 _jit_arg_f(jit_state_t *_jit)
 626 {
 627     assert(_jitc->function);
 628     return (jit_make_arg_f((jit_node_t*)0));
 629 }
 630
 631 jit_node_t *
 632 _jit_arg_d(jit_state_t *_jit)
 633 {
 634     assert(_jitc->function);
 635     return (jit_make_arg_d((jit_node_t*)0));
 636 }
 637
 638 void
 639 _jit_getarg_c(jit_state_t *_jit, jit_int32_t u, jit_node_t *v)
 640 {
 641     assert_arg_type(v->code, jit_code_arg_c);
 642     jit_inc_synth_wp(getarg_c, u, v);
 643     if (jit_arg_reg_p(v->u.w))
 644         jit_extr_c(u, _A0 - v->u.w);
 645     else {
 646         jit_node_t      *node = jit_ldxi_c(u, _FP, v->u.w + C_DISP);
 647         jit_link_alist(node);
 648         jit_check_frame();
 649     }
 650     jit_dec_synth();
 651 }
 652
 653 void
 654 _jit_getarg_uc(jit_state_t *_jit, jit_int32_t u, jit_node_t *v)
 655 {
 656     assert_arg_type(v->code, jit_code_arg_c);
 657     jit_inc_synth_wp(getarg_uc, u, v);
 658     if (jit_arg_reg_p(v->u.w))
 659         jit_extr_uc(u, _A0 - v->u.w);
 660     else {
 661         jit_node_t      *node = jit_ldxi_uc(u, _FP, v->u.w + C_DISP);
 662         jit_link_alist(node);
 663         jit_check_frame();
 664     }
 665     jit_dec_synth();
 666 }
 667
 668 void
 669 _jit_getarg_s(jit_state_t *_jit, jit_int32_t u, jit_node_t *v)
 670 {
 671     assert_arg_type(v->code, jit_code_arg_s);
 672     jit_inc_synth_wp(getarg_s, u, v);
 673     if (jit_arg_reg_p(v->u.w))
 674         jit_extr_s(u, _A0 - v->u.w);
 675     else {
 676         jit_node_t      *node = jit_ldxi_s(u, _FP, v->u.w + S_DISP);
 677         jit_link_alist(node);
 678         jit_check_frame();
 679     }
 680     jit_dec_synth();
 681 }
 682
 683 void
 684 _jit_getarg_us(jit_state_t *_jit, jit_int32_t u, jit_node_t *v)
 685 {
 686     assert_arg_type(v->code, jit_code_arg_s);
 687     jit_inc_synth_wp(getarg_us, u, v);
 688     if (jit_arg_reg_p(v->u.w))
 689         jit_extr_us(u, _A0 - v->u.w);
 690     else {
 691         jit_node_t      *node = jit_ldxi_us(u, _FP, v->u.w + S_DISP);
 692         jit_link_alist(node);
 693         jit_check_frame();
 694     }
 695     jit_dec_synth();
 696 }
 697
 698 void
 699 _jit_getarg_i(jit_state_t *_jit, jit_int32_t u, jit_node_t *v)
 700 {
 701     assert_arg_type(v->code, jit_code_arg_i);
 702     jit_inc_synth_wp(getarg_i, u, v);
 703     if (jit_arg_reg_p(v->u.w)) {
 704 #if __WORDSIZE == 64
 705         jit_extr_i(u, _A0 - v->u.w);
 706 #else
 707         jit_movr(u, _A0 - v->u.w);
 708 #endif
 709     }
 710     else {
 711         jit_node_t      *node = jit_ldxi_i(u, _FP, v->u.w + I_DISP);
 712         jit_link_alist(node);
 713         jit_check_frame();
 714     }
 715     jit_dec_synth();
 716 }
 717
 718 #if __WORDSIZE == 64
 719 void
 720 _jit_getarg_ui(jit_state_t *_jit, jit_int32_t u, jit_node_t *v)
 721 {
 722     assert_arg_type(v->code, jit_code_arg_i);
 723     jit_inc_synth_wp(getarg_ui, u, v);
 724     if (jit_arg_reg_p(v->u.w))
 725         jit_extr_ui(u, _A0 - v->u.w);
 726     else {
 727         jit_node_t      *node = jit_ldxi_ui(u, _FP, v->u.w + I_DISP);
 728         jit_link_alist(node);
 729         jit_check_frame();
 730     }
 731     jit_dec_synth();
 732 }
 733
 734 void
 735 _jit_getarg_l(jit_state_t *_jit, jit_int32_t u, jit_node_t *v)
 736 {
 737     assert_arg_type(v->code, jit_code_arg_l);
 738     jit_inc_synth_wp(getarg_l, u, v);
 739     if (jit_arg_reg_p(v->u.w))
 740         jit_movr(u, _A0 - v->u.w);
 741     else {
 742         jit_node_t      *node = jit_ldxi_l(u, _FP, v->u.w);
 743         jit_link_alist(node);
 744         jit_check_frame();
 745     }
 746     jit_dec_synth();
 747 }
 748 #endif
 749
 750 void
 751 _jit_putargr(jit_state_t *_jit, jit_int32_t u, jit_node_t *v, jit_code_t code)
 752 {
 753     assert_putarg_type(code, v->code);
 754     jit_code_inc_synth_wp(code, u, v);
 755     if (jit_arg_reg_p(v->u.w))
 756         jit_movr(_A0 - v->u.w, u);
 757     else {
 758         jit_node_t      *node = jit_stxi(v->u.w + WORD_ADJUST, _FP, u);
 759         jit_link_alist(node);
 760         jit_check_frame();
 761     }
 762     jit_dec_synth();
 763 }
 764
 765 void
 766 _jit_putargi(jit_state_t *_jit, jit_word_t u, jit_node_t *v, jit_code_t code)
 767 {
 768     jit_int32_t         regno;
 769     assert_putarg_type(code, v->code);
 770     jit_code_inc_synth_wp(code, u, v);
 771     if (jit_arg_reg_p(v->u.w))
 772         jit_movi(_A0 - v->u.w, u);
 773     else {
 774         jit_node_t      *node;
 775         regno = jit_get_reg(jit_class_gpr);
 776         jit_movi(regno, u);
 777         node = jit_stxi(v->u.w + WORD_ADJUST, _FP, regno);
 778         jit_link_alist(node);
 779         jit_check_frame();
 780         jit_unget_reg(regno);
 781     }
 782     jit_dec_synth();
 783 }
 784
 785 void
 786 _jit_getarg_f(jit_state_t *_jit, jit_int32_t u, jit_node_t *v)
 787 {
 788     assert(v->code == jit_code_arg_f);
 789     jit_inc_synth_wp(getarg_f, u, v);
 790 #if NEW_ABI
 791     if (jit_arg_reg_p(v->u.w))
 792         jit_movr_f(u, _F12 - v->u.w);
 793     else if (jit_arg_reg_p(v->u.w - 8))
 794         jit_movr_w_f(u, _A0 - (v->u.w - 8));
 795 #else
 796     if (v->u.w < 4)
 797         jit_movr_w_f(u, _A0 - v->u.w);
 798     else if (v->u.w < 8)
 799         jit_movr_f(u, _F12 - ((v->u.w - 4) >> 1));
 800 #endif
 801     else {
 802         jit_node_t      *node = jit_ldxi_f(u, _FP, v->u.w);
 803         jit_link_alist(node);
 804         jit_check_frame();
 805     }
 806     jit_dec_synth();
 807 }
 808
 809 void
 810 _jit_putargr_f(jit_state_t *_jit, jit_int32_t u, jit_node_t *v)
 811 {
 812     assert(v->code == jit_code_arg_f);
 813     jit_inc_synth_wp(putargr_f, u, v);
 814 #if NEW_ABI
 815     if (jit_arg_reg_p(v->u.w))
 816         jit_movr_f(_F12 - v->u.w, u);
 817     else if (jit_arg_reg_p(v->u.w - 8))
 818         jit_movr_f_w(_A0 - (v->u.w - 8), u);
 819 #else
 820     if (v->u.w < 4)
 821         jit_movr_f_w(_A0 - v->u.w, u);
 822     else if (v->u.w < 8)
 823         jit_movr_f(_F12 - ((v->u.w - 4) >> 1), u);
 824 #endif
 825     else {
 826         jit_node_t      *node = jit_stxi_f(v->u.w, _FP, u);
 827         jit_link_alist(node);
 828         jit_check_frame();
 829     }
 830     jit_dec_synth();
 831 }
 832
 833 void
 834 _jit_putargi_f(jit_state_t *_jit, jit_float32_t u, jit_node_t *v)
 835 {
 836     jit_int32_t         regno;
 837     assert(v->code == jit_code_arg_f);
 838     jit_inc_synth_fp(putargi_f, u, v);
 839 #if NEW_ABI
 840     if (jit_arg_reg_p(v->u.w))
 841         jit_movi_f(_F12 - v->u.w, u);
 842     else if (jit_arg_reg_p(v->u.w - 8))
 843         jit_movi_f_w(_A0 - (v->u.w - 8), u);
 844 #else
 845     if (v->u.w < 4) {
 846         regno = jit_get_reg(jit_class_fpr);
 847         jit_movi_f(regno, u);
 848         jit_movr_f_w(_A0 - ((v->u.w - 4) >> 1), regno);
 849         jit_unget_reg(regno);
 850     }
 851     else if (v->u.w < 8)
 852         jit_movi_f(_F12 - ((v->u.w - 4) >> 1), u);
 853 #endif
 854     else {
 855         jit_node_t      *node;
 856         regno = jit_get_reg(jit_class_fpr);
 857         jit_movi_f(regno, u);
 858         node = jit_stxi_f(v->u.w, _FP, regno);
 859         jit_link_alist(node);
 860         jit_check_frame();
 861         jit_unget_reg(regno);
 862     }
 863     jit_dec_synth();
 864 }
 865
 866 void
 867 _jit_getarg_d(jit_state_t *_jit, jit_int32_t u, jit_node_t *v)
 868 {
 869     assert(v->code == jit_code_arg_d);
 870     jit_inc_synth_wp(getarg_d, u, v);
 871 #if NEW_ABI
 872     if (jit_arg_reg_p(v->u.w))
 873         jit_movr_d(u, _F12 - v->u.w);
 874     else if (jit_arg_reg_p(v->u.w - 8))
 875         jit_movr_w_d(u, _A0 - (v->u.w - 8));
 876 #else
 877     if (v->u.w < 4)
 878         jit_movr_ww_d(u, _A0 - v->u.w, _A0 - (v->u.w + 1));
 879     else if (v->u.w < 8)
 880         jit_movr_d(u, _F12 - ((v->u.w - 4) >> 1));
 881 #endif
 882     else {
 883         jit_node_t      *node = jit_ldxi_d(u, _FP, v->u.w);
 884         jit_link_alist(node);
 885         jit_check_frame();
 886     }
 887     jit_dec_synth();
 888 }
 889
 890 void
 891 _jit_putargr_d(jit_state_t *_jit, jit_int32_t u, jit_node_t *v)
 892 {
 893     assert(v->code == jit_code_arg_d);
 894     jit_inc_synth_wp(putargr_d, u, v);
 895 #if NEW_ABI
 896     if (jit_arg_reg_p(v->u.w))
 897         jit_movr_d(_F12 - v->u.w, u);
 898     else if (jit_arg_reg_p(v->u.w - 8))
 899         jit_movr_d_w(_A0 - (v->u.w - 8), u);
 900 #else
 901     if (v->u.w < 4)
 902         jit_movr_d_ww(_A0 - v->u.w, _A0 - (v->u.w + 1), u);
 903     else if (v->u.w < 8)
 904         jit_movr_d(_F12 - ((v->u.w - 4) >> 1), u);
 905 #endif
 906     else {
 907         jit_node_t      *node = jit_stxi_d(v->u.w, _FP, u);
 908         jit_link_alist(node);
 909         jit_check_frame();
 910     }
 911     jit_dec_synth();
 912 }
 913
 914 void
 915 _jit_putargi_d(jit_state_t *_jit, jit_float64_t u, jit_node_t *v)
 916 {
 917     jit_int32_t         regno;
 918     assert(v->code == jit_code_arg_d);
 919     jit_inc_synth_dp(putargi_d, u, v);
 920 #if NEW_ABI
 921     if (jit_arg_reg_p(v->u.w))
 922         jit_movi_d(_F12 - v->u.w, u);
 923     else if (jit_arg_reg_p(v->u.w - 8))
 924         jit_movi_d_w(_A0 - (v->u.w - 8), u);
 925 #else
 926     if (v->u.w < 4) {
 927         regno = jit_get_reg(jit_class_fpr);
 928         jit_movi_d(regno, u);
 929         jit_movr_d_ww(_A0 - v->u.w, _A0 - (v->u.w + 1), regno);
 930         jit_unget_reg(regno);
 931     }
 932     else if (v->u.w < 8)
 933         jit_movi_d(_F12 - ((v->u.w - 4) >> 1), u);
 934 #endif
 935     else {
 936         jit_node_t      *node;
 937         regno = jit_get_reg(jit_class_fpr);
 938         jit_movi_d(regno, u);
 939         node = jit_stxi_d(v->u.w, _FP, regno);
 940         jit_link_alist(node);
 941         jit_check_frame();
 942         jit_unget_reg(regno);
 943     }
 944     jit_dec_synth();
 945 }
 946
 947 void
 948 _jit_pushargr(jit_state_t *_jit, jit_int32_t u, jit_code_t code)
 949 {
 950     jit_code_inc_synth_w(code, u);
 951     jit_link_prepare();
 952 #if NEW_ABI
 953     assert(_jitc->function);
 954     if (jit_arg_reg_p(_jitc->function->call.argi)) {
 955         jit_movr(_A0 - _jitc->function->call.argi, u);
 956         ++_jitc->function->call.argi;
 957     }
 958     else {
 959         jit_check_frame();
 960         jit_stxi(_jitc->function->call.size + WORD_ADJUST, JIT_SP, u);
 961         _jitc->function->call.size += STACK_SLOT;
 962     }
 963 #else
 964     jit_word_t          offset;
 965     assert(_jitc->function);
 966     offset = _jitc->function->call.size >> STACK_SHIFT;
 967     ++_jitc->function->call.argi;
 968     if (jit_arg_reg_p(offset))
 969         jit_movr(_A0 - offset, u);
 970     else {
 971         jit_check_frame();
 972         jit_stxi(_jitc->function->call.size, JIT_SP, u);
 973     }
 974     _jitc->function->call.size += STACK_SLOT;
 975 #endif
 976     jit_dec_synth();
 977 }
 978
 979 void
 980 _jit_pushargi(jit_state_t *_jit, jit_word_t u, jit_code_t code)
 981 {
 982     jit_int32_t         regno;
 983 #if !NEW_ABI
 984     jit_word_t          offset;
 985 #endif
 986     assert(_jitc->function);
 987     jit_code_inc_synth_w(code, u);
 988     jit_link_prepare();
 989 #if NEW_ABI
 990     if (jit_arg_reg_p(_jitc->function->call.argi)) {
 991         jit_movi(_A0 - _jitc->function->call.argi, u);
 992         ++_jitc->function->call.argi;
 993     }
 994     else {
 995         jit_check_frame();
 996         regno = jit_get_reg(jit_class_gpr);
 997         jit_movi(regno, u);
 998         jit_stxi(_jitc->function->call.size + WORD_ADJUST, JIT_SP, regno);
 999         _jitc->function->call.size += STACK_SLOT;
1000         jit_unget_reg(regno);
1001     }
1002 #else
1003     offset = _jitc->function->call.size >> STACK_SHIFT;
1004     ++_jitc->function->call.argi;
1005     if (jit_arg_reg_p(offset))
1006         jit_movi(_A0 - offset, u);
1007     else {
1008         jit_check_frame();
1009         regno = jit_get_reg(jit_class_gpr);
1010         jit_movi(regno, u);
1011         jit_stxi(_jitc->function->call.size, JIT_SP, regno);
1012         jit_unget_reg(regno);
1013     }
1014     _jitc->function->call.size += STACK_SLOT;
1015 #endif
1016     jit_dec_synth();
1017 }
1018
1019 void
1020 _jit_pushargr_f(jit_state_t *_jit, jit_int32_t u)
1021 {
1022 #if !NEW_ABI
1023     jit_word_t          offset;
1024 #endif
1025     assert(_jitc->function);
1026     jit_inc_synth_w(pushargr_f, u);
1027     jit_link_prepare();
1028 #if NEW_ABI
1029     if (jit_arg_reg_p(_jitc->function->call.argi)) {
1030         if (__mips_hard_float &&
1031             !(_jitc->function->call.call & jit_call_varargs))
1032             jit_movr_f(_F12 - _jitc->function->call.argi, u);
1033         else
1034             jit_movr_f_w(_A0 - _jitc->function->call.argi, u);
1035         ++_jitc->function->call.argi;
1036     }
1037     else {
1038         jit_check_frame();
1039         jit_stxi_f(_jitc->function->call.size, JIT_SP, u);
1040         _jitc->function->call.size += STACK_SLOT;
1041     }
1042 #else
1043     offset = _jitc->function->call.size >> STACK_SHIFT;
1044     if (offset < 2 && !_jitc->function->call.argi &&
1045         !(_jitc->function->call.call & jit_call_varargs)) {
1046         ++_jitc->function->call.argf;
1047         jit_movr_f(_F12 - offset, u);
1048     }
1049     else if (offset < 4) {
1050         ++_jitc->function->call.argi;
1051         jit_movr_f_w(_A0 - offset, u);
1052     }
1053     else {
1054         jit_check_frame();
1055         jit_stxi_f(_jitc->function->call.size, JIT_SP, u);
1056     }
1057     _jitc->function->call.size += STACK_SLOT;
1058 #endif
1059     jit_dec_synth();
1060 }
1061
1062 void
1063 _jit_pushargi_f(jit_state_t *_jit, jit_float32_t u)
1064 {
1065     jit_int32_t         regno;
1066 #if !NEW_ABI
1067     jit_word_t          offset;
1068 #endif
1069     assert(_jitc->function);
1070     jit_inc_synth_f(pushargi_f, u);
1071     jit_link_prepare();
1072 #if NEW_ABI
1073     if (jit_arg_reg_p(_jitc->function->call.argi)) {
1074         if (__mips_hard_float &&
1075             !(_jitc->function->call.call & jit_call_varargs))
1076             jit_movi_f(_F12 - _jitc->function->call.argi, u);
1077         else
1078             jit_movi_f_w(_A0 - _jitc->function->call.argi, u);
1079         ++_jitc->function->call.argi;
1080     }
1081     else {
1082         jit_check_frame();
1083         regno = jit_get_reg(jit_class_fpr);
1084         jit_movi_f(regno, u);
1085         jit_stxi_f(_jitc->function->call.size, JIT_SP, regno);
1086         _jitc->function->call.size += STACK_SLOT;
1087         jit_unget_reg(regno);
1088     }
1089 #else
1090     offset = _jitc->function->call.size >> STACK_SHIFT;
1091     if (offset < 2 && !_jitc->function->call.argi &&
1092         !(_jitc->function->call.call & jit_call_varargs)) {
1093         ++_jitc->function->call.argf;
1094         jit_movi_f(_F12 - offset, u);
1095     }
1096     else if (offset < 4) {
1097         ++_jitc->function->call.argi;
1098         jit_movi_f_w(_A0 - offset, u);
1099     }
1100     else {
1101         jit_check_frame();
1102         regno = jit_get_reg(jit_class_fpr);
1103         jit_movi_f(regno, u);
1104         jit_stxi_f(_jitc->function->call.size, JIT_SP, regno);
1105         jit_unget_reg(regno);
1106     }
1107     _jitc->function->call.size += STACK_SLOT;
1108 #endif
1109     jit_dec_synth();
1110 }
1111
1112 void
1113 _jit_pushargr_d(jit_state_t *_jit, jit_int32_t u)
1114 {
1115 #if !NEW_ABI
1116     jit_bool_t          adjust;
1117     jit_word_t          offset;
1118 #endif
1119     assert(_jitc->function);
1120     jit_inc_synth_w(pushargr_d, u);
1121     jit_link_prepare();
1122 #if NEW_ABI
1123     if (jit_arg_reg_p(_jitc->function->call.argi)) {
1124         if (__mips_hard_float &&
1125             !(_jitc->function->call.call & jit_call_varargs))
1126             jit_movr_d(_F12 - _jitc->function->call.argi, u);
1127         else
1128             jit_movr_d_w(_A0 - _jitc->function->call.argi, u);
1129         ++_jitc->function->call.argi;
1130     }
1131     else {
1132         jit_check_frame();
1133         jit_stxi_d(_jitc->function->call.size, JIT_SP, u);
1134         _jitc->function->call.size += STACK_SLOT;
1135     }
1136 #else
1137     adjust = !!_jitc->function->call.argi;
1138     if (_jitc->function->call.size & 7) {
1139         _jitc->function->call.size += 4;
1140         adjust = 1;
1141     }
1142     offset = _jitc->function->call.size >> STACK_SHIFT;
1143     if (offset < 3) {
1144         if (adjust || (_jitc->function->call.call & jit_call_varargs)) {
1145             jit_movr_d_ww(_A0 - offset, _A0 - (offset + 1), u);
1146             _jitc->function->call.argi += 2;
1147         }
1148         else {
1149             jit_movr_d(_F12 - (offset >> 1), u);
1150             ++_jitc->function->call.argf;
1151         }
1152     }
1153     else {
1154         jit_check_frame();
1155         jit_stxi_d(_jitc->function->call.size, JIT_SP, u);
1156     }
1157     _jitc->function->call.size += sizeof(jit_float64_t);
1158 #endif
1159     jit_dec_synth();
1160 }
1161
1162 void
1163 _jit_pushargi_d(jit_state_t *_jit, jit_float64_t u)
1164 {
1165     jit_int32_t         regno;
1166 #if !NEW_ABI
1167     jit_bool_t          adjust;
1168     jit_word_t          offset;
1169 #endif
1170     assert(_jitc->function);
1171     jit_inc_synth_d(pushargi_d, u);
1172     jit_link_prepare();
1173 #if NEW_ABI
1174     if (jit_arg_reg_p(_jitc->function->call.argi)) {
1175         if (__mips_hard_float &&
1176             !(_jitc->function->call.call & jit_call_varargs))
1177             jit_movi_d(_F12 - _jitc->function->call.argi, u);
1178         else
1179             jit_movi_d_w(_A0 - _jitc->function->call.argi, u);
1180         ++_jitc->function->call.argi;
1181     }
1182     else {
1183         jit_check_frame();
1184         regno = jit_get_reg(jit_class_fpr);
1185         jit_movi_d(regno, u);
1186         jit_stxi_d(_jitc->function->call.size, JIT_SP, regno);
1187         _jitc->function->call.size += STACK_SLOT;
1188         jit_unget_reg(regno);
1189     }
1190 #else
1191     adjust = !!_jitc->function->call.argi;
1192     if (_jitc->function->call.size & 7) {
1193         _jitc->function->call.size += 4;
1194         adjust = 1;
1195     }
1196     offset = _jitc->function->call.size >> STACK_SHIFT;
1197     if (offset < 3) {
1198         if (adjust || (_jitc->function->call.call & jit_call_varargs)) {
1199             jit_movi_d_ww(_A0 - offset, _A0 - (offset + 1), u);
1200             _jitc->function->call.argi += 2;
1201         }
1202         else {
1203             jit_movi_d(_F12 - (offset >> 1), u);
1204             ++_jitc->function->call.argf;
1205         }
1206     }
1207     else {
1208         jit_check_frame();
1209         regno = jit_get_reg(jit_class_fpr);
1210         jit_movi_d(regno, u);
1211         jit_stxi_d(_jitc->function->call.size, JIT_SP, regno);
1212         jit_unget_reg(regno);
1213     }
1214     _jitc->function->call.size += sizeof(jit_float64_t);
1215 #endif
1216     jit_dec_synth();
1217 }
1218
1219 jit_bool_t
1220 _jit_regarg_p(jit_state_t *_jit, jit_node_t *node, jit_int32_t regno)
1221 {
1222     jit_int32_t         spec;
1223
1224     spec = jit_class(_rvs[regno].spec);
1225     if (spec & jit_class_arg) {
1226         if (spec & jit_class_gpr) {
1227             regno = _A0 - regno;
1228             if (regno >= 0 && regno < node->v.w)
1229                 return (1);
1230         }
1231         else if (spec & jit_class_fpr) {
1232             regno = _F12 - regno;
1233             if (regno >= 0 && regno < node->w.w)
1234                 return (1);
1235         }
1236     }
1237
1238     return (0);
1239 }
1240
1241 void
1242 _jit_finishr(jit_state_t *_jit, jit_int32_t r0)
1243 {
1244     jit_node_t          *call;
1245     assert(_jitc->function);
1246     jit_check_frame();
1247     jit_inc_synth_w(finishr, r0);
1248     if (_jitc->function->self.alen < _jitc->function->call.size)
1249         _jitc->function->self.alen = _jitc->function->call.size;
1250     call = jit_callr(r0);
1251     call->v.w = _jitc->function->self.argi;
1252 #if NEW_ABI
1253     call->w.w = call->v.w;
1254 #else
1255     call->w.w = _jitc->function->self.argf;
1256 #endif
1257     _jitc->function->call.argi = _jitc->function->call.argf =
1258         _jitc->function->call.size = 0;
1259     _jitc->prepare = 0;
1260     jit_dec_synth();
1261 }
1262
1263 jit_node_t *
1264 _jit_finishi(jit_state_t *_jit, jit_pointer_t i0)
1265 {
1266     jit_node_t          *call;
1267     assert(_jitc->function);
1268     jit_check_frame();
1269     jit_inc_synth_w(finishi, (jit_word_t)i0);
1270     if (_jitc->function->self.alen < _jitc->function->call.size)
1271         _jitc->function->self.alen = _jitc->function->call.size;
1272     call = jit_calli(i0);
1273     call->v.w = _jitc->function->call.argi;
1274 #if NEW_ABI
1275     call->w.w = call->v.w;
1276 #else
1277     call->w.w = _jitc->function->call.argf;
1278 #endif
1279     _jitc->function->call.argi = _jitc->function->call.argf =
1280         _jitc->function->call.size = 0;
1281     _jitc->prepare = 0;
1282     jit_dec_synth();
1283     return (call);
1284 }
1285
1286 void
1287 _jit_retval_c(jit_state_t *_jit, jit_int32_t r0)
1288 {
1289     jit_inc_synth_w(retval_c, r0);
1290     jit_extr_c(r0, JIT_RET);
1291     jit_dec_synth();
1292 }
1293
1294 void
1295 _jit_retval_uc(jit_state_t *_jit, jit_int32_t r0)
1296 {
1297     jit_inc_synth_w(retval_uc, r0);
1298     jit_extr_uc(r0, JIT_RET);
1299     jit_dec_synth();
1300 }
1301
1302 void
1303 _jit_retval_s(jit_state_t *_jit, jit_int32_t r0)
1304 {
1305     jit_inc_synth_w(retval_s, r0);
1306     jit_extr_s(r0, JIT_RET);
1307     jit_dec_synth();
1308 }
1309
1310 void
1311 _jit_retval_us(jit_state_t *_jit, jit_int32_t r0)
1312 {
1313     jit_inc_synth_w(retval_us, r0);
1314     jit_extr_us(r0, JIT_RET);
1315     jit_dec_synth();
1316 }
1317
1318 void
1319 _jit_retval_i(jit_state_t *_jit, jit_int32_t r0)
1320 {
1321     jit_inc_synth_w(retval_i, r0);
1322 #if __WORDSIZE == 32
1323     if (r0 != JIT_RET)
1324         jit_movr(r0, JIT_RET);
1325 #else
1326     jit_extr_i(r0, JIT_RET);
1327 #endif
1328     jit_dec_synth();
1329 }
1330
1331 #if __WORDSIZE == 64
1332 void
1333 _jit_retval_ui(jit_state_t *_jit, jit_int32_t r0)
1334 {
1335     jit_inc_synth_w(retval_ui, r0);
1336     jit_extr_ui(r0, JIT_RET);
1337     jit_dec_synth();
1338 }
1339
1340 void
1341 _jit_retval_l(jit_state_t *_jit, jit_int32_t r0)
1342 {
1343     jit_inc_synth_w(retval_l, r0);
1344     if (r0 != JIT_RET)
1345         jit_movr(r0, JIT_RET);
1346     jit_dec_synth();
1347 }
1348 #endif
1349
1350 void
1351 _jit_retval_f(jit_state_t *_jit, jit_int32_t r0)
1352 {
1353     jit_inc_synth_w(retval_f, r0);
1354 #if __mips_soft_float
1355     jit_movr_w_f(r0, JIT_RET);
1356 #else
1357     if (r0 != JIT_FRET)
1358         jit_movr_f(r0, JIT_FRET);
1359 #endif
1360     jit_dec_synth();
1361 }
1362
1363 void
1364 _jit_retval_d(jit_state_t *_jit, jit_int32_t r0)
1365 {
1366     jit_inc_synth_w(retval_d, r0);
1367 #if __mips_soft_float
1368     jit_movr_w_d(r0, JIT_RET);
1369 #else
1370     if (r0 != JIT_FRET)
1371         jit_movr_d(r0, JIT_FRET);
1372 #endif
1373     jit_dec_synth();
1374 }
1375
1376 jit_pointer_t
1377 _emit_code(jit_state_t *_jit)
1378 {
1379     jit_node_t          *node;
1380     jit_node_t          *temp;
1381     jit_word_t           word;
1382     jit_int32_t          value;
1383     jit_int32_t          offset;
1384
1385     struct {
1386         jit_node_t      *node;
1387         jit_word_t       word;
1388         jit_function_t   func;
1389 #if DEVEL_DISASSEMBLER
1390         jit_word_t       prevw;
1391 #endif
1392         jit_int32_t      patch_offset;
1393     } undo;
1394 #if DEVEL_DISASSEMBLER
1395     jit_word_t           prevw;
1396 #endif
1397
1398     _jitc->function = NULL;
1399     _jitc->inst.pend = 0;
1400
1401     jit_reglive_setup();
1402
1403     undo.word = 0;
1404     undo.node = NULL;
1405     undo.patch_offset = 0;
1406 #define case_rr(name, type)                                             \
1407             case jit_code_##name##r##type:                              \
1408                 name##r##type(rn(node->u.w), rn(node->v.w));            \
1409                 break
1410 #define case_rw(name, type)                                             \
1411             case jit_code_##name##i##type:                              \
1412                 name##i##type(rn(node->u.w), node->v.w);                \
1413                 break
1414 #define case_wr(name, type)                                             \
1415             case jit_code_##name##i##type:                              \
1416                 name##i##type(node->u.w, rn(node->v.w));                \
1417                 break
1418 #define case_rrr(name, type)                                            \
1419             case jit_code_##name##r##type:                              \
1420                 name##r##type(rn(node->u.w),                            \
1421                               rn(node->v.w), rn(node->w.w));            \
1422                 break
1423 #define case_rrw(name, type)                                            \
1424             case jit_code_##name##i##type:                              \
1425                 name##i##type(rn(node->u.w), rn(node->v.w), node->w.w); \
1426                 break
1427 #define case_rrrr(name, type)                                           \
1428             case jit_code_##name##r##type:                              \
1429                 name##r##type(rn(node->u.q.l), rn(node->u.q.h),         \
1430                               rn(node->v.w), rn(node->w.w));            \
1431                 break
1432 #define case_rrrw(name, type)                                           \
1433             case jit_code_##name##i##type:                              \
1434                 name##i##type(rn(node->u.q.l), rn(node->u.q.h),         \
1435                               rn(node->v.w), node->w.w);                \
1436                 break
1437 #define case_rqr(name, type)                                            \
1438             case jit_code_##name##r##type:                              \
1439                 name##r##type(rn(node->u.w), rn(node->v.q.l),           \
1440                               rn(node->v.q.h), rn(node->w.w));          \
1441             case jit_code_##name##i##type:                              \
1442                 break;
1443 #define case_rrf(name, type, size)                                      \
1444             case jit_code_##name##i##type:                              \
1445                 assert(node->flag & jit_flag_data);                     \
1446                 name##i##type(rn(node->u.w), rn(node->v.w),             \
1447                               (jit_float##size##_t *)node->w.n->u.w);   \
1448                 break
1449 #define case_wrr(name, type)                                            \
1450             case jit_code_##name##i##type:                              \
1451                 name##i##type(node->u.w, rn(node->v.w), rn(node->w.w)); \
1452                 break
1453 #define case_brr(name, type)                                            \
1454             case jit_code_##name##r##type:                              \
1455                 temp = node->u.n;                                       \
1456                 assert(temp->code == jit_code_label ||                  \
1457                        temp->code == jit_code_epilog);                  \
1458                 if (temp->flag & jit_flag_patch)                        \
1459                     name##r##type(temp->u.w, rn(node->v.w),             \
1460                                   rn(node->w.w));                       \
1461                 else {                                                  \
1462                     word = name##r##type(_jit->pc.w,                    \
1463                                          rn(node->v.w), rn(node->w.w)); \
1464                     patch(word, node);                                  \
1465                 }                                                       \
1466                 break
1467 #define case_brw(name, type)                                            \
1468             case jit_code_##name##i##type:                              \
1469                 temp = node->u.n;                                       \
1470                 assert(temp->code == jit_code_label ||                  \
1471                        temp->code == jit_code_epilog);                  \
1472                 if (temp->flag & jit_flag_patch)                        \
1473                     name##i##type(temp->u.w,                            \
1474                                   rn(node->v.w), node->w.w);            \
1475                 else {                                                  \
1476                     word = name##i##type(_jit->pc.w,                    \
1477                                          rn(node->v.w), node->w.w);     \
1478                     patch(word, node);                                  \
1479                 }                                                       \
1480                 break
1481 #define case_brf(name, type, size)                                      \
1482             case jit_code_##name##i##type:                              \
1483                 temp = node->u.n;                                       \
1484                 assert(temp->code == jit_code_label ||                  \
1485                        temp->code == jit_code_epilog);                  \
1486                 if (temp->flag & jit_flag_patch)                        \
1487                     name##i##type(temp->u.w, rn(node->v.w),             \
1488                                 (jit_float##size##_t *)node->w.n->u.w); \
1489                 else {                                                  \
1490                     word = name##i##type(_jit->pc.w, rn(node->v.w),     \
1491                                 (jit_float##size##_t *)node->w.n->u.w); \
1492                     patch(word, node);                                  \
1493                 }                                                       \
1494                 break
1495 #if DEVEL_DISASSEMBLER
1496     prevw = _jit->pc.w;
1497 #endif
1498     for (node = _jitc->head; node; node = node->next) {
1499         if (_jit->pc.uc >= _jitc->code.end)
1500             return (NULL);
1501
1502 #if DEVEL_DISASSEMBLER
1503         node->offset = (jit_uword_t)_jit->pc.w - (jit_uword_t)prevw;
1504         prevw = _jit->pc.w;
1505         if (_jitc->inst.pend) {
1506             node->offset += 4;
1507             prevw += 4;
1508         }
1509 #endif
1510         value = jit_classify(node->code);
1511 #if GET_JIT_SIZE
1512         flush();
1513 #endif
1514         jit_regarg_set(node, value);
1515         switch (node->code) {
1516             case jit_code_align:
1517                 /* Must align to a power of two */
1518                 assert(!(node->u.w & (node->u.w - 1)));
1519                 flush();
1520                 if ((word = _jit->pc.w & (node->u.w - 1)))
1521                     nop(node->u.w - word);
1522                 flush();
1523                 break;
1524             case jit_code_skip:
1525                 flush();
1526                 nop((node->u.w + 3) & ~3);
1527                 flush();
1528                 break;
1529             case jit_code_note:         case jit_code_name:
1530                 flush();
1531                 node->u.w = _jit->pc.w;
1532                 break;
1533             case jit_code_label:
1534                 flush();
1535                 /* remember label is defined */
1536                 node->flag |= jit_flag_patch;
1537                 node->u.w = _jit->pc.w;
1538                 break;
1539                 case_rrr(add,);
1540                 case_rrw(add,);
1541                 case_rrr(addc,);
1542                 case_rrw(addc,);
1543                 case_rrr(addx,);
1544                 case_rrw(addx,);
1545                 case_rrr(sub,);
1546                 case_rrw(sub,);
1547                 case_rrr(subc,);
1548                 case_rrw(subc,);
1549                 case_rrr(subx,);
1550                 case_rrw(subx,);
1551                 case_rrw(rsb,);
1552                 case_rrr(mul,);
1553                 case_rrw(mul,);
1554                 case_rrr(hmul,);
1555                 case_rrw(hmul,);
1556                 case_rrr(hmul, _u);
1557                 case_rrw(hmul, _u);
1558                 case_rrrr(qmul,);
1559                 case_rrrw(qmul,);
1560                 case_rrrr(qmul, _u);
1561                 case_rrrw(qmul, _u);
1562                 case_rrr(div,);
1563                 case_rrw(div,);
1564                 case_rrr(div, _u);
1565                 case_rrw(div, _u);
1566                 case_rrrr(qdiv,);
1567                 case_rrrw(qdiv,);
1568                 case_rrrr(qdiv, _u);
1569                 case_rrrw(qdiv, _u);
1570                 case_rrr(rem,);
1571                 case_rrw(rem,);
1572                 case_rrr(rem, _u);
1573                 case_rrw(rem, _u);
1574                 case_rrr(lsh,);
1575                 case_rrw(lsh,);
1576                 case_rrrr(qlsh,);
1577                 case_rrrw(qlsh,);
1578                 case_rrrr(qlsh, _u);
1579                 case_rrrw(qlsh, _u);
1580                 case_rrr(rsh,);
1581                 case_rrw(rsh,);
1582                 case_rrr(rsh, _u);
1583                 case_rrw(rsh, _u);
1584                 case_rrrr(qrsh,);
1585                 case_rrrw(qrsh,);
1586                 case_rrrr(qrsh, _u);
1587                 case_rrrw(qrsh, _u);
1588                 case_rrr(lrot,);
1589                 case_rrw(lrot,);
1590                 case_rrr(rrot,);
1591                 case_rrw(rrot,);
1592                 case_rrr(and,);
1593                 case_rrw(and,);
1594                 case_rrr(or,);
1595                 case_rrw(or,);
1596                 case_rrr(xor,);
1597                 case_rrw(xor,);
1598                 case_rr(trunc, _f_i);
1599                 case_rr(trunc, _d_i);
1600 #if __WORDSIZE == 64
1601                 case_rr(trunc, _f_l);
1602                 case_rr(trunc, _d_l);
1603 #endif
1604                 case_rr(ld, _c);
1605                 case_rw(ld, _c);
1606                 case_rr(ld, _uc);
1607                 case_rw(ld, _uc);
1608                 case_rr(ld, _s);
1609                 case_rw(ld, _s);
1610                 case_rr(ld, _us);
1611                 case_rw(ld, _us);
1612                 case_rr(ld, _i);
1613                 case_rw(ld, _i);
1614 #if __WORDSIZE == 64
1615                 case_rr(ld, _ui);
1616                 case_rw(ld, _ui);
1617                 case_rr(ld, _l);
1618                 case_rw(ld, _l);
1619 #endif
1620                 case_rrr(ldx, _c);
1621                 case_rrw(ldx, _c);
1622                 case_rrr(ldx, _uc);
1623                 case_rrw(ldx, _uc);
1624                 case_rrr(ldx, _s);
1625                 case_rrw(ldx, _s);
1626                 case_rrr(ldx, _us);
1627                 case_rrw(ldx, _us);
1628                 case_rrr(ldx, _i);
1629                 case_rrw(ldx, _i);
1630 #if __WORDSIZE == 64
1631                 case_rrr(ldx, _ui);
1632                 case_rrw(ldx, _ui);
1633                 case_rrr(ldx, _l);
1634                 case_rrw(ldx, _l);
1635 #endif
1636             case jit_code_unldr:
1637                 unldr(rn(node->u.w), rn(node->v.w), node->w.w);
1638                 break;
1639             case jit_code_unldi:
1640                 unldi(rn(node->u.w), node->v.w, node->w.w);
1641                 break;
1642             case jit_code_unldr_u:
1643                 unldr_u(rn(node->u.w), rn(node->v.w), node->w.w);
1644                 break;
1645             case jit_code_unldi_u:
1646                 unldi_u(rn(node->u.w), node->v.w, node->w.w);
1647                 break;
1648                 case_rr(st, _c);
1649                 case_wr(st, _c);
1650                 case_rr(st, _s);
1651                 case_wr(st, _s);
1652                 case_rr(st, _i);
1653                 case_wr(st, _i);
1654 #if __WORDSIZE == 64
1655                 case_rr(st, _l);
1656                 case_wr(st, _l);
1657 #endif
1658                 case_rrr(stx, _c);
1659                 case_wrr(stx, _c);
1660                 case_rrr(stx, _s);
1661                 case_wrr(stx, _s);
1662                 case_rrr(stx, _i);
1663                 case_wrr(stx, _i);
1664 #if __WORDSIZE == 64
1665                 case_rrr(stx, _l);
1666                 case_wrr(stx, _l);
1667 #endif
1668             case jit_code_unstr:
1669                 unstr(rn(node->u.w), rn(node->v.w), node->w.w);
1670                 break;
1671             case jit_code_unsti:
1672                 unsti(node->u.w, rn(node->v.w), node->w.w);
1673                 break;
1674                 case_rr(hton, _us);
1675                 case_rr(hton, _ui);
1676 #if __WORDSIZE == 64
1677                 case_rr(hton, _ul);
1678 #endif
1679                 case_rr(bswap, _us);
1680                 case_rr(bswap, _ui);
1681 #if __WORDSIZE == 64
1682                 case_rr(bswap, _ul);
1683 #endif
1684             case jit_code_extr:
1685                 extr(rn(node->u.w), rn(node->v.w), node->w.q.l, node->w.q.h);
1686                 break;
1687             case jit_code_extr_u:
1688                 extr_u(rn(node->u.w), rn(node->v.w), node->w.q.l, node->w.q.h);
1689                 break;
1690             case jit_code_depr:
1691                 depr(rn(node->u.w), rn(node->v.w), node->w.q.l, node->w.q.h);
1692                 break;
1693             case jit_code_depi:
1694                 depi(rn(node->u.w), node->v.w, node->w.q.l, node->w.q.h);
1695                 break;
1696                 case_rr(ext, _c);
1697                 case_rr(ext, _uc);
1698                 case_rr(ext, _s);
1699                 case_rr(ext, _us);
1700 #if __WORDSIZE == 64
1701                 case_rr(ext, _i);
1702                 case_rr(ext, _ui);
1703 #endif
1704             case jit_code_casr:
1705                 casr(rn(node->u.w), rn(node->v.w),
1706                      rn(node->w.q.l), rn(node->w.q.h));
1707                 break;
1708             case jit_code_casi:
1709                 casi(rn(node->u.w), node->v.w,
1710                      rn(node->w.q.l), rn(node->w.q.h));
1711                 break;
1712                 case_rrr(movn,);
1713                 case_rrr(movz,);
1714                 case_rr(mov,);
1715             case jit_code_movi:
1716                 if (node->flag & jit_flag_node) {
1717                     temp = node->v.n;
1718                     if (temp->code == jit_code_data ||
1719                         (temp->code == jit_code_label &&
1720                          (temp->flag & jit_flag_patch)))
1721                         movi(rn(node->u.w), temp->u.w);
1722                     else {
1723                         assert(temp->code == jit_code_label ||
1724                                temp->code == jit_code_epilog);
1725                         word = movi_p(rn(node->u.w), node->v.w);
1726                         patch(word, node);
1727                     }
1728                 }
1729                 else
1730                     movi(rn(node->u.w), node->v.w);
1731                 break;
1732                 case_rr(neg,);
1733                 case_rr(com,);
1734                 case_rr(clo,);
1735                 case_rr(clz,);
1736                 case_rr(cto,);
1737                 case_rr(ctz,);
1738                 case_rr(rbit,);
1739 #define popcntr(r0, r1) fallback_popcnt(r0, r1)
1740                 case_rr(popcnt,);
1741                 case_rrr(lt,);
1742                 case_rrw(lt,);
1743                 case_rrr(lt, _u);
1744                 case_rrw(lt, _u);
1745                 case_rrr(le,);
1746                 case_rrw(le,);
1747                 case_rrr(le, _u);
1748                 case_rrw(le, _u);
1749                 case_rrr(eq,);
1750                 case_rrw(eq,);
1751                 case_rrr(ge,);
1752                 case_rrw(ge,);
1753                 case_rrr(ge, _u);
1754                 case_rrw(ge, _u);
1755                 case_rrr(gt,);
1756                 case_rrw(gt,);
1757                 case_rrr(gt, _u);
1758                 case_rrw(gt, _u);
1759                 case_rrr(ne,);
1760                 case_rrw(ne,);
1761                 case_brr(blt,);
1762                 case_brw(blt,);
1763                 case_brr(blt, _u);
1764                 case_brw(blt, _u);
1765                 case_brr(ble,);
1766                 case_brw(ble,);
1767                 case_brr(ble, _u);
1768                 case_brw(ble, _u);
1769                 case_brr(beq,);
1770                 case_brw(beq,);
1771                 case_brr(bge,);
1772                 case_brw(bge,);
1773                 case_brr(bge, _u);
1774                 case_brw(bge, _u);
1775                 case_brr(bgt,);
1776                 case_brw(bgt,);
1777                 case_brr(bgt, _u);
1778                 case_brw(bgt, _u);
1779                 case_brr(bne,);
1780                 case_brw(bne,);
1781                 case_brr(boadd,);
1782                 case_brw(boadd,);
1783                 case_brr(boadd, _u);
1784                 case_brw(boadd, _u);
1785                 case_brr(bxadd,);
1786                 case_brw(bxadd,);
1787                 case_brr(bxadd, _u);
1788                 case_brw(bxadd, _u);
1789                 case_brr(bosub,);
1790                 case_brw(bosub,);
1791                 case_brr(bosub, _u);
1792                 case_brw(bosub, _u);
1793                 case_brr(bxsub,);
1794                 case_brw(bxsub,);
1795                 case_brr(bxsub, _u);
1796                 case_brw(bxsub, _u);
1797                 case_brr(bms,);
1798                 case_brw(bms,);
1799                 case_brr(bmc,);
1800                 case_brw(bmc,);
1801                 case_rrr(add, _f);
1802                 case_rrf(add, _f, 32);
1803                 case_rrr(sub, _f);
1804                 case_rrf(sub, _f, 32);
1805                 case_rrf(rsb, _f, 32);
1806                 case_rrr(mul, _f);
1807                 case_rrf(mul, _f, 32);
1808                 case_rrr(div, _f);
1809                 case_rrf(div, _f, 32);
1810                 case_rr(abs, _f);
1811                 case_rr(neg, _f);
1812                 case_rr(sqrt, _f);
1813                 case_rqr(fma, _f);
1814                 case_rqr(fms, _f);
1815                 case_rqr(fnma, _f);
1816                 case_rqr(fnms, _f);
1817                 case_rr(ext, _f);
1818                 case_rr(ld, _f);
1819                 case_rw(ld, _f);
1820                 case_rrr(ldx, _f);
1821                 case_rrw(ldx, _f);
1822             case jit_code_unldr_x:
1823                 unldr_x(rn(node->u.w), rn(node->v.w), node->w.w);
1824                 break;
1825             case jit_code_unldi_x:
1826                 unldi_x(rn(node->u.w), node->v.w, node->w.w);
1827                 break;
1828                 case_rr(st, _f);
1829                 case_wr(st, _f);
1830                 case_rrr(stx, _f);
1831                 case_wrr(stx, _f);
1832             case jit_code_unstr_x:
1833                 unstr_x(rn(node->u.w), rn(node->v.w), node->w.w);
1834                 break;
1835             case jit_code_unsti_x:
1836                 unsti_x(node->u.w, rn(node->v.w), node->w.w);
1837                 break;
1838                 case_rr(mov, _f);
1839             case jit_code_movi_f:
1840                 assert(node->flag & jit_flag_data);
1841                 movi_f(rn(node->u.w), (jit_float32_t *)node->v.n->u.w);
1842                 break;
1843                 case_rr(ext, _d_f);
1844                 case_rrr(lt, _f);
1845                 case_rrf(lt, _f, 32);
1846                 case_rrr(le, _f);
1847                 case_rrf(le, _f, 32);
1848                 case_rrr(eq, _f);
1849                 case_rrf(eq, _f, 32);
1850                 case_rrr(ge, _f);
1851                 case_rrf(ge, _f, 32);
1852                 case_rrr(gt, _f);
1853                 case_rrf(gt, _f, 32);
1854                 case_rrr(ne, _f);
1855                 case_rrf(ne, _f, 32);
1856                 case_rrr(unlt, _f);
1857                 case_rrf(unlt, _f, 32);
1858                 case_rrr(unle, _f);
1859                 case_rrf(unle, _f, 32);
1860                 case_rrr(uneq, _f);
1861                 case_rrf(uneq, _f, 32);
1862                 case_rrr(unge, _f);
1863                 case_rrf(unge, _f, 32);
1864                 case_rrr(ungt, _f);
1865                 case_rrf(ungt, _f, 32);
1866                 case_rrr(ltgt, _f);
1867                 case_rrf(ltgt, _f, 32);
1868                 case_rrr(ord, _f);
1869                 case_rrf(ord, _f, 32);
1870                 case_rrr(unord, _f);
1871                 case_rrf(unord, _f, 32);
1872                 case_brr(blt, _f);
1873                 case_brf(blt, _f, 32);
1874                 case_brr(ble, _f);
1875                 case_brf(ble, _f, 32);
1876                 case_brr(beq, _f);
1877                 case_brf(beq, _f, 32);
1878                 case_brr(bge, _f);
1879                 case_brf(bge, _f, 32);
1880                 case_brr(bgt, _f);
1881                 case_brf(bgt, _f, 32);
1882                 case_brr(bne, _f);
1883                 case_brf(bne, _f, 32);
1884                 case_brr(bunlt, _f);
1885                 case_brf(bunlt, _f, 32);
1886                 case_brr(bunle, _f);
1887                 case_brf(bunle, _f, 32);
1888                 case_brr(buneq, _f);
1889                 case_brf(buneq, _f, 32);
1890                 case_brr(bunge, _f);
1891                 case_brf(bunge, _f, 32);
1892                 case_brr(bungt, _f);
1893                 case_brf(bungt, _f, 32);
1894                 case_brr(bltgt, _f);
1895                 case_brf(bltgt, _f, 32);
1896                 case_brr(bord, _f);
1897                 case_brf(bord, _f, 32);
1898                 case_brr(bunord, _f);
1899                 case_brf(bunord, _f, 32);
1900                 case_rrr(add, _d);
1901                 case_rrf(add, _d, 64);
1902                 case_rrr(sub, _d);
1903                 case_rrf(sub, _d, 64);
1904                 case_rrf(rsb, _d, 64);
1905                 case_rrr(mul, _d);
1906                 case_rrf(mul, _d, 64);
1907                 case_rrr(div, _d);
1908                 case_rrf(div, _d, 64);
1909                 case_rr(abs, _d);
1910                 case_rr(neg, _d);
1911                 case_rr(sqrt, _d);
1912                 case_rqr(fma, _d);
1913                 case_rqr(fms, _d);
1914                 case_rqr(fnma, _d);
1915                 case_rqr(fnms, _d);
1916                 case_rr(ext, _d);
1917                 case_rr(ld, _d);
1918                 case_rw(ld, _d);
1919                 case_rrr(ldx, _d);
1920                 case_rrw(ldx, _d);
1921                 case_rr(st, _d);
1922                 case_wr(st, _d);
1923                 case_rrr(stx, _d);
1924                 case_wrr(stx, _d);
1925                 case_rr(mov, _d);
1926             case jit_code_movi_d:
1927                 assert(node->flag & jit_flag_data);
1928                 movi_d(rn(node->u.w), (jit_float64_t *)node->v.n->u.w);
1929                 break;
1930                 case_rr(ext, _f_d);
1931                 case_rrr(lt, _d);
1932                 case_rrf(lt, _d, 64);
1933                 case_rrr(le, _d);
1934                 case_rrf(le, _d, 64);
1935                 case_rrr(eq, _d);
1936                 case_rrf(eq, _d, 64);
1937                 case_rrr(ge, _d);
1938                 case_rrf(ge, _d, 64);
1939                 case_rrr(gt, _d);
1940                 case_rrf(gt, _d, 64);
1941                 case_rrr(ne, _d);
1942                 case_rrf(ne, _d, 64);
1943                 case_rrr(unlt, _d);
1944                 case_rrf(unlt, _d, 64);
1945                 case_rrr(unle, _d);
1946                 case_rrf(unle, _d, 64);
1947                 case_rrr(uneq, _d);
1948                 case_rrf(uneq, _d, 64);
1949                 case_rrr(unge, _d);
1950                 case_rrf(unge, _d, 64);
1951                 case_rrr(ungt, _d);
1952                 case_rrf(ungt, _d, 64);
1953                 case_rrr(ltgt, _d);
1954                 case_rrf(ltgt, _d, 64);
1955                 case_rrr(ord, _d);
1956                 case_rrf(ord, _d, 64);
1957                 case_rrr(unord, _d);
1958                 case_rrf(unord, _d, 64);
1959                 case_brr(blt, _d);
1960                 case_brf(blt, _d, 64);
1961                 case_brr(ble, _d);
1962                 case_brf(ble, _d, 64);
1963                 case_brr(beq, _d);
1964                 case_brf(beq, _d, 64);
1965                 case_brr(bge, _d);
1966                 case_brf(bge, _d, 64);
1967                 case_brr(bgt, _d);
1968                 case_brf(bgt, _d, 64);
1969                 case_brr(bne, _d);
1970                 case_brf(bne, _d, 64);
1971                 case_brr(bunlt, _d);
1972                 case_brf(bunlt, _d, 64);
1973                 case_brr(bunle, _d);
1974                 case_brf(bunle, _d, 64);
1975                 case_brr(buneq, _d);
1976                 case_brf(buneq, _d, 64);
1977                 case_brr(bunge, _d);
1978                 case_brf(bunge, _d, 64);
1979                 case_brr(bungt, _d);
1980                 case_brf(bungt, _d, 64);
1981                 case_brr(bltgt, _d);
1982                 case_brf(bltgt, _d, 64);
1983                 case_brr(bord, _d);
1984                 case_brf(bord, _d, 64);
1985                 case_brr(bunord, _d);
1986                 case_brf(bunord, _d, 64);
1987             case jit_code_jmpr:
1988                 jit_check_frame();
1989                 jmpr(rn(node->u.w));
1990                 break;
1991             case jit_code_jmpi:
1992                 if (node->flag & jit_flag_node) {
1993                     temp = node->u.n;
1994                     assert(temp->code == jit_code_label ||
1995                            temp->code == jit_code_epilog);
1996                     if (temp->flag & jit_flag_patch)
1997                         jmpi(temp->u.w, 0);
1998                     else {
1999                         word = _jit->code.length -
2000                             (_jit->pc.uc - _jit->code.ptr);
2001                         if ((jit_mips2_p() && can_relative_jump_p(word))
2002 #if !BALC_BROKEN
2003                             ||
2004                             (jit_mips6_p() && can_compact_jump_p(word))
2005 #endif
2006                             )
2007                             word = jmpi(_jit->pc.w, 1);
2008                         else
2009                             word = jmpi_p(_jit->pc.w);
2010                         patch(word, node);
2011                     }
2012                 }
2013                 else {
2014                     jit_check_frame();
2015                     jmpi(node->u.w, 0);
2016                 }
2017                 break;
2018             case jit_code_callr:
2019                 jit_check_frame();
2020                 callr(rn(node->u.w));
2021                 break;
2022             case jit_code_calli:
2023                 if (node->flag & jit_flag_node) {
2024                     temp = node->u.n;
2025                     assert(temp->code == jit_code_label ||
2026                            temp->code == jit_code_epilog);
2027                     if (temp->flag & jit_flag_patch)
2028                         calli(temp->u.w, 0);
2029                     else {
2030                         word = _jit->code.length -
2031                             (_jit->pc.uc - _jit->code.ptr);
2032                         if ((jit_mips2_p() && can_relative_jump_p(word))
2033 #if !BALC_BROKEN
2034                             ||
2035                             (jit_mips6_p() && can_compact_jump_p(word))
2036 #endif
2037                             )
2038                             word = calli(_jit->pc.w, 1);
2039                         else
2040                             word = calli_p(_jit->pc.w);
2041                         patch(word, node);
2042                     }
2043                 }
2044                 else {
2045                     jit_check_frame();
2046                     calli(node->u.w, 0);
2047                 }
2048                 break;
2049             case jit_code_prolog:
2050                 flush();
2051                 _jitc->function = _jitc->functions.ptr + node->w.w;
2052                 undo.node = node;
2053                 undo.word = _jit->pc.w;
2054                 memcpy(&undo.func, _jitc->function, sizeof(undo.func));
2055 #if DEVEL_DISASSEMBLER
2056                 undo.prevw = prevw;
2057 #endif
2058                 undo.patch_offset = _jitc->patches.offset;
2059             restart_function:
2060                 _jitc->again = 0;
2061                 compute_framesize();
2062                 patch_alist(0);
2063                 prolog(node);
2064                 break;
2065             case jit_code_epilog:
2066                 assert(_jitc->function == _jitc->functions.ptr + node->w.w);
2067                 if (_jitc->again) {
2068                     for (temp = undo.node->next;
2069                          temp != node; temp = temp->next) {
2070                         if (temp->code == jit_code_label ||
2071                             temp->code == jit_code_epilog)
2072                             temp->flag &= ~jit_flag_patch;
2073                     }
2074                     temp->flag &= ~jit_flag_patch;
2075                     node = undo.node;
2076                     _jit->pc.w = undo.word;
2077                     /* undo.func.self.aoff and undo.func.regset should not
2078                      * be undone, as they will be further updated, and are
2079                      * the reason of the undo. */
2080                     undo.func.self.aoff = _jitc->function->frame +
2081                         _jitc->function->self.aoff;
2082                     undo.func.need_frame = _jitc->function->need_frame;
2083                     jit_regset_set(&undo.func.regset, &_jitc->function->regset);
2084                     /* allocar information also does not need to be undone */
2085                     undo.func.aoffoff = _jitc->function->aoffoff;
2086                     undo.func.allocar = _jitc->function->allocar;
2087                     /* this will be recomputed but undo anyway to have it
2088                      * better self documented.*/
2089                     undo.func.need_stack = _jitc->function->need_stack;
2090                     memcpy(_jitc->function, &undo.func, sizeof(undo.func));
2091 #if DEVEL_DISASSEMBLER
2092                     prevw = undo.prevw;
2093 #endif
2094                     _jitc->patches.offset = undo.patch_offset;
2095                     patch_alist(1);
2096                     goto restart_function;
2097                 }
2098                 /* remember label is defined */
2099                 flush();
2100                 node->flag |= jit_flag_patch;
2101                 node->u.w = _jit->pc.w;
2102                 epilog(node);
2103                 _jitc->function = NULL;
2104                 break;
2105             case jit_code_movr_w_f:
2106                 movr_w_f(rn(node->u.w), rn(node->v.w));
2107                 break;
2108             case jit_code_movr_f_w:
2109                 movr_f_w(rn(node->u.w), rn(node->v.w));
2110                 break;
2111             case jit_code_movi_f_w:
2112                 assert(node->flag & jit_flag_data);
2113                 movi_f_w(rn(node->u.w), *(jit_float32_t *)node->v.n->u.w);
2114                 break;
2115             case jit_code_movi_w_f:
2116                 movi_w_f(rn(node->u.w), node->v.w);
2117                 break;
2118 #if __WORDSIZE == 64 || NEW_ABI
2119             case jit_code_movr_d_w:
2120                 movr_d_w(rn(node->u.w), rn(node->v.w));
2121                 break;
2122             case jit_code_movi_d_w:
2123                 assert(node->flag & jit_flag_data);
2124                 movi_d_w(rn(node->u.w), *(jit_float64_t *)node->v.n->u.w);
2125                 break;
2126             case jit_code_movr_w_d:
2127                 movr_w_d(rn(node->u.w), rn(node->v.w));
2128                 break;
2129             case jit_code_movi_w_d:
2130                 movi_w_d(rn(node->u.w), node->v.w);
2131                 break;
2132 #endif
2133 #if __WORDSIZE == 32
2134             case jit_code_movr_ww_d:
2135                 movr_ww_d(rn(node->u.w), rn(node->v.w), rn(node->w.w));
2136                 break;
2137             case jit_code_movr_d_ww:
2138                 movr_d_ww(rn(node->u.w), rn(node->v.w), rn(node->w.w));
2139                 break;
2140             case jit_code_movi_d_ww:
2141                 assert(node->flag & jit_flag_data);
2142                 movi_d_ww(rn(node->u.w), rn(node->v.w),
2143                           *(jit_float64_t *)node->w.n->u.w);
2144                 break;
2145             case jit_code_movi_ww_d:
2146                 movi_ww_d(rn(node->u.w), node->v.w, node->w.w);
2147                 break;
2148 #endif
2149             case jit_code_va_start:
2150                 vastart(rn(node->u.w));
2151                 break;
2152             case jit_code_va_arg:
2153                 vaarg(rn(node->u.w), rn(node->v.w));
2154                 break;
2155             case jit_code_va_arg_d:
2156                 vaarg_d(rn(node->u.w), rn(node->v.w));
2157                 break;
2158             case jit_code_live:                 case jit_code_ellipsis:
2159             case jit_code_va_push:
2160             case jit_code_allocai:              case jit_code_allocar:
2161             case jit_code_arg_c:                case jit_code_arg_s:
2162             case jit_code_arg_i:
2163 #  if __WORDSIZE == 64
2164             case jit_code_arg_l:
2165 #  endif
2166             case jit_code_arg_f:                case jit_code_arg_d:
2167             case jit_code_va_end:
2168             case jit_code_ret:
2169             case jit_code_retr_c:               case jit_code_reti_c:
2170             case jit_code_retr_uc:              case jit_code_reti_uc:
2171             case jit_code_retr_s:               case jit_code_reti_s:
2172             case jit_code_retr_us:              case jit_code_reti_us:
2173             case jit_code_retr_i:               case jit_code_reti_i:
2174 #if __WORDSIZE == 64
2175             case jit_code_retr_ui:              case jit_code_reti_ui:
2176             case jit_code_retr_l:               case jit_code_reti_l:
2177 #endif
2178             case jit_code_retr_f:               case jit_code_reti_f:
2179             case jit_code_retr_d:               case jit_code_reti_d:
2180             case jit_code_getarg_c:             case jit_code_getarg_uc:
2181             case jit_code_getarg_s:             case jit_code_getarg_us:
2182             case jit_code_getarg_i:
2183 #if __WORDSIZE == 64
2184             case jit_code_getarg_ui:            case jit_code_getarg_l:
2185 #endif
2186             case jit_code_getarg_f:             case jit_code_getarg_d:
2187             case jit_code_putargr_c:            case jit_code_putargi_c:
2188             case jit_code_putargr_uc:           case jit_code_putargi_uc:
2189             case jit_code_putargr_s:            case jit_code_putargi_s:
2190             case jit_code_putargr_us:           case jit_code_putargi_us:
2191             case jit_code_putargr_i:            case jit_code_putargi_i:
2192 #if __WORDSIZE == 64
2193             case jit_code_putargr_ui:           case jit_code_putargi_ui:
2194             case jit_code_putargr_l:            case jit_code_putargi_l:
2195 #endif
2196             case jit_code_putargr_f:            case jit_code_putargi_f:
2197             case jit_code_putargr_d:            case jit_code_putargi_d:
2198             case jit_code_pushargr_c:           case jit_code_pushargi_c:
2199             case jit_code_pushargr_uc:          case jit_code_pushargi_uc:
2200             case jit_code_pushargr_s:           case jit_code_pushargi_s:
2201             case jit_code_pushargr_us:          case jit_code_pushargi_us:
2202             case jit_code_pushargr_i:           case jit_code_pushargi_i:
2203 #if __WORDSIZE == 64
2204             case jit_code_pushargr_ui:          case jit_code_pushargi_ui:
2205             case jit_code_pushargr_l:           case jit_code_pushargi_l:
2206 #endif
2207             case jit_code_pushargr_f:           case jit_code_pushargi_f:
2208             case jit_code_pushargr_d:           case jit_code_pushargi_d:
2209             case jit_code_retval_c:             case jit_code_retval_uc:
2210             case jit_code_retval_s:             case jit_code_retval_us:
2211             case jit_code_retval_i:
2212 #if __WORDSIZE == 64
2213             case jit_code_retval_ui:            case jit_code_retval_l:
2214 #endif
2215             case jit_code_retval_f:             case jit_code_retval_d:
2216             case jit_code_prepare:
2217             case jit_code_finishr:              case jit_code_finishi:
2218             case jit_code_negi_f:               case jit_code_absi_f:
2219             case jit_code_sqrti_f:              case jit_code_negi_d:
2220             case jit_code_absi_d:               case jit_code_sqrti_d:
2221                 break;
2222             case jit_code_negi:
2223                 negi(rn(node->u.w), node->v.w);
2224                 break;
2225             case jit_code_comi:
2226                 comi(rn(node->u.w), node->v.w);
2227                 break;
2228             case jit_code_exti_c:
2229                 exti_c(rn(node->u.w), node->v.w);
2230                 break;
2231             case jit_code_exti_uc:
2232                 exti_uc(rn(node->u.w), node->v.w);
2233                 break;
2234             case jit_code_exti_s:
2235                 exti_s(rn(node->u.w), node->v.w);
2236                 break;
2237             case jit_code_exti_us:
2238                 exti_us(rn(node->u.w), node->v.w);
2239                 break;
2240             case jit_code_bswapi_us:
2241                 bswapi_us(rn(node->u.w), node->v.w);
2242                 break;
2243             case jit_code_bswapi_ui:
2244                 bswapi_ui(rn(node->u.w), node->v.w);
2245                 break;
2246             case jit_code_htoni_us:
2247                 htoni_us(rn(node->u.w), node->v.w);
2248                 break;
2249             case jit_code_htoni_ui:
2250                 htoni_ui(rn(node->u.w), node->v.w);
2251                 break;
2252 #if __WORDSIZE == 64
2253             case jit_code_exti_i:
2254                 exti_i(rn(node->u.w), node->v.w);
2255                 break;
2256             case jit_code_exti_ui:
2257                 exti_ui(rn(node->u.w), node->v.w);
2258                 break;
2259             case jit_code_bswapi_ul:
2260                 bswapi_ul(rn(node->u.w), node->v.w);
2261                 break;
2262             case jit_code_htoni_ul:
2263                 htoni_ul(rn(node->u.w), node->v.w);
2264                 break;
2265 #endif
2266             case jit_code_cloi:
2267                 cloi(rn(node->u.w), node->v.w);
2268                 break;
2269             case jit_code_clzi:
2270                 clzi(rn(node->u.w), node->v.w);
2271                 break;
2272             case jit_code_ctoi:
2273                 ctoi(rn(node->u.w), node->v.w);
2274                 break;
2275             case jit_code_ctzi:
2276                 ctzi(rn(node->u.w), node->v.w);
2277                 break;
2278             case jit_code_rbiti:
2279                 rbiti(rn(node->u.w), node->v.w);
2280                 break;
2281             case jit_code_popcnti:
2282                 popcnti(rn(node->u.w), node->v.w);
2283                 break;
2284             case jit_code_exti:
2285                 exti(rn(node->u.w), node->v.w, node->w.q.l, node->w.q.h);
2286                 break;
2287             case jit_code_exti_u:
2288                 exti_u(rn(node->u.w), node->v.w, node->w.q.l, node->w.q.h);
2289                 break;
2290             default:
2291                 abort();
2292         }
2293         if (jit_carry != _NOREG) {
2294             switch (node->code) {
2295                 case jit_code_note:
2296                 case jit_code_addcr:            case jit_code_addci:
2297                 case jit_code_addxr:            case jit_code_addxi:
2298                 case jit_code_subcr:            case jit_code_subci:
2299                 case jit_code_subxr:            case jit_code_subxi:
2300                     break;
2301                 default:
2302                     jit_unget_reg(jit_carry);
2303                     jit_carry = _NOREG;
2304                     break;
2305             }
2306         }
2307 #if GET_JIT_SIZE
2308         flush();
2309 #endif
2310         jit_regarg_clr(node, value);
2311         assert(_jitc->regarg == 0 ||
2312                (jit_carry != _NOREG && _jitc->regarg == (1 << jit_carry)));
2313         assert(_jitc->synth == 0);
2314         /* update register live state */
2315         jit_reglive(node);
2316     }
2317     flush();
2318 #undef case_brf
2319 #undef case_brw
2320 #undef case_brr
2321 #undef case_wrr
2322 #undef case_rrf
2323 #undef case_rrw
2324 #undef case_rrr
2325 #undef case_wr
2326 #undef case_rw
2327 #undef case_rr
2328
2329     for (offset = 0; offset < _jitc->patches.offset; offset++) {
2330         node = _jitc->patches.ptr[offset].node;
2331         word = node->code == jit_code_movi ? node->v.n->u.w : node->u.n->u.w;
2332         patch_at(_jitc->patches.ptr[offset].inst, word);
2333     }
2334
2335     jit_flush(_jit->code.ptr, _jit->pc.uc);
2336
2337     return (_jit->code.ptr);
2338 }
2339
2340 #define CODE                            1
2341 #  include "jit_rewind.c"
2342 #  include "jit_mips-cpu.c"
2343 #  include "jit_mips-fpu.c"
2344 #  include "jit_fallback.c"
2345 #undef CODE
2346
2347 void
2348 jit_flush(void *fptr, void *tptr)
2349 {
2350 #if defined(__linux__)
2351     jit_word_t          f, t, s;
2352
2353     s = sysconf(_SC_PAGE_SIZE);
2354     f = (jit_word_t)fptr & -s;
2355     t = (((jit_word_t)tptr) + s - 1) & -s;
2356     _flush_cache((void *)f, t - f, ICACHE);
2357 #endif
2358 }
2359
2360 void
2361 _emit_ldxi(jit_state_t *_jit, jit_int32_t r0, jit_int32_t r1, jit_word_t i0)
2362 {
2363     ldxi(rn(r0), rn(r1), i0);
2364 }
2365
2366 void
2367 _emit_stxi(jit_state_t *_jit, jit_word_t i0, jit_int32_t r0, jit_int32_t r1)
2368 {
2369     stxi(i0, rn(r0), rn(r1));
2370 }
2371
2372 void
2373 _emit_ldxi_d(jit_state_t *_jit, jit_int32_t r0, jit_int32_t r1, jit_word_t i0)
2374 {
2375     ldxi_d(rn(r0), rn(r1), i0);
2376 }
2377
2378 void
2379 _emit_stxi_d(jit_state_t *_jit, jit_word_t i0, jit_int32_t r0, jit_int32_t r1)
2380 {
2381     stxi_d(i0, rn(r0), rn(r1));
2382 }
2383
2384 static void
2385 _compute_framesize(jit_state_t *_jit)
2386 {
2387     jit_int32_t         reg;
2388     _jitc->framesize = STACK_SLOT << 1; /* ra+fp */
2389     for (reg = 0; reg < jit_size(iregs); reg++)
2390         if (jit_regset_tstbit(&_jitc->function->regset, iregs[reg]))
2391             _jitc->framesize += STACK_SLOT;
2392
2393     for (reg = 0; reg < jit_size(fregs); reg++)
2394         if (jit_regset_tstbit(&_jitc->function->regset, fregs[reg]))
2395             _jitc->framesize += sizeof(jit_float64_t);
2396
2397 #if NEW_ABI
2398     /* Space to store variadic arguments */
2399     if (_jitc->function->self.call & jit_call_varargs)
2400         _jitc->framesize += (NUM_WORD_ARGS - _jitc->function->vagp) * STACK_SLOT;
2401 #endif
2402
2403     /* Make sure functions called have a 16 byte aligned stack */
2404     _jitc->framesize = (_jitc->framesize + 15) & -16;
2405 }
2406
2407 static void
2408 _patch(jit_state_t *_jit, jit_word_t instr, jit_node_t *node)
2409 {
2410     jit_int32_t          flag;
2411
2412     assert(node->flag & jit_flag_node);
2413     if (node->code == jit_code_movi)
2414         flag = node->v.n->flag;
2415     else
2416         flag = node->u.n->flag;
2417     assert(!(flag & jit_flag_patch));
2418     if (_jitc->patches.offset >= _jitc->patches.length) {
2419         jit_realloc((jit_pointer_t *)&_jitc->patches.ptr,
2420                     _jitc->patches.length * sizeof(jit_patch_t),
2421                     (_jitc->patches.length + 1024) * sizeof(jit_patch_t));
2422         _jitc->patches.length += 1024;
2423     }
2424     _jitc->patches.ptr[_jitc->patches.offset].inst = instr;
2425     _jitc->patches.ptr[_jitc->patches.offset].node = node;
2426     ++_jitc->patches.offset;
2427 }