deps/lightning/lib/jit_riscv.c

   1 /*
   2  * Copyright (C) 2019-2022  Free Software Foundation, Inc.
   3  *
   4  * This file is part of GNU lightning.
   5  *
   6  * GNU lightning is free software; you can redistribute it and/or modify it
   7  * under the terms of the GNU Lesser General Public License as published
   8  * by the Free Software Foundation; either version 3, or (at your option)
   9  * any later version.
  10  *
  11  * GNU lightning is distributed in the hope that it will be useful, but
  12  * WITHOUT ANY WARRANTY; without even the implied warranty of MERCHANTABILITY
  13  * or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU Lesser General Public
  14  * License for more details.
  15  *
  16  * Authors:
  17  *      Paulo Cesar Pereira de Andrade
  18  */
  19
  20 #define jit_arg_reg_p(i)                ((i) >= 0 && (i) < 8)
  21 #define jit_arg_f_reg_p(i)              ((i) >= 0 && (i) < 8)
  22
  23 /*
  24  * Types
  25  */
  26 typedef jit_pointer_t jit_va_list_t;
  27
  28 /*
  29  * Prototypes
  30  */
  31 #if __WORDSIZE == 64
  32 #  define load_const(r0, i0)            _load_const(_jit, r0, i0)
  33 static void _load_const(jit_state_t*, jit_int32_t, jit_word_t);
  34 static jit_word_t hash_const(jit_word_t);
  35 #  define put_const(i0)                 _put_const(_jit, i0)
  36 static void _put_const(jit_state_t*, jit_word_t);
  37 #  define get_const(i0)                 _get_const(_jit, i0)
  38 static jit_word_t _get_const(jit_state_t*, jit_word_t);
  39 #endif
  40 #define patch(instr, node)              _patch(_jit, instr, node)
  41 static void _patch(jit_state_t*,jit_word_t,jit_node_t*);
  42
  43 #define PROTO                           1
  44 #  include "jit_riscv-cpu.c"
  45 #  include "jit_riscv-fpu.c"
  46 #undef PROTO
  47
  48 /*
  49  * Initialization
  50  */
  51 jit_register_t          _rvs[] = {
  52     { 0x00,                             "zero" },
  53     { 0x01,                             "ra" },
  54     { 0x02,                             "sp" },
  55     { 0x03,                             "gp" },
  56 #if 0           /* Pretend it does not exist, so _NOREG can be used in
  57                  * a 64 bit bitmask */
  58     { 0x04,                             "tp" },
  59 #endif
  60     { rc(gpr) | 0x05,                   "t0" },
  61     { rc(gpr) | 0x06,                   "t1" },
  62     { rc(gpr) | 0x07,                   "t2" },
  63     { rc(gpr) | 0x1c,                   "t3" },
  64     { rc(gpr) | 0x1d,                   "t4" },
  65     { rc(gpr) | 0x1e,                   "t5" },
  66     { rc(gpr) | 0x1f,                   "t6" },
  67     { 0x08,                             "fp" },
  68     { rc(sav) | rc(gpr) | 0x09,         "s1" },
  69     { rc(sav) | rc(gpr) | 0x12,         "s2" },
  70     { rc(sav) | rc(gpr) | 0x13,         "s3" },
  71     { rc(sav) | rc(gpr) | 0x14,         "s4" },
  72     { rc(sav) | rc(gpr) | 0x15,         "s5" },
  73     { rc(sav) | rc(gpr) | 0x16,         "s6" },
  74     { rc(sav) | rc(gpr) | 0x17,         "s7" },
  75     { rc(sav) | rc(gpr) | 0x18,         "s8" },
  76     { rc(sav) | rc(gpr) | 0x19,         "s9" },
  77     { rc(sav) | rc(gpr) | 0x1a,         "s10" },
  78     { rc(sav) | rc(gpr) | 0x1b,         "s11" },
  79     { rc(arg) | rc(gpr) | 0x11,         "a7" },
  80     { rc(arg) | rc(gpr) | 0x10,         "a6" },
  81     { rc(arg) | rc(gpr) | 0x0f,         "a5" },
  82     { rc(arg) | rc(gpr) | 0x0e,         "a4" },
  83     { rc(arg) | rc(gpr) | 0x0d,         "a3" },
  84     { rc(arg) | rc(gpr) | 0x0c,         "a2" },
  85     { rc(arg) | rc(gpr) | 0x0b,         "a1" },
  86     { rc(arg) | rc(gpr) | 0x0a,         "a0" },
  87     { rc(fpr) | 0x00,                   "ft0" },
  88     { rc(fpr) | 0x01,                   "ft1" },
  89     { rc(fpr) | 0x02,                   "ft2" },
  90     { rc(fpr) | 0x03,                   "ft3" },
  91     { rc(fpr) | 0x04,                   "ft4" },
  92     { rc(fpr) | 0x05,                   "ft5" },
  93     { rc(fpr) | 0x06,                   "ft6" },
  94     { rc(fpr) | 0x07,                   "ft7" },
  95     { rc(fpr) | 0x1c,                   "ft8" },
  96     { rc(fpr) | 0x1d,                   "ft9" },
  97     { rc(fpr) | 0x1e,                   "ft10" },
  98     { rc(fpr) | 0x1f,                   "ft11" },
  99     { rc(sav) | rc(fpr) | 0x08,         "fs0" },
 100     { rc(sav) | rc(fpr) | 0x09,         "fs1" },
 101     { rc(sav) | rc(fpr) | 0x12,         "fs2" },
 102     { rc(sav) | rc(fpr) | 0x13,         "fs3" },
 103     { rc(sav) | rc(fpr) | 0x14,         "fs4" },
 104     { rc(sav) | rc(fpr) | 0x15,         "fs5" },
 105     { rc(sav) | rc(fpr) | 0x16,         "fs6" },
 106     { rc(sav) | rc(fpr) | 0x17,         "fs7" },
 107     { rc(sav) | rc(fpr) | 0x18,         "fs8" },
 108     { rc(sav) | rc(fpr) | 0x19,         "fs9" },
 109     { rc(sav) | rc(fpr) | 0x1a,         "fs10" },
 110     { rc(sav) | rc(fpr) | 0x1b,         "fs11" },
 111     { rc(arg) | rc(fpr) | 0x11,         "fa7" },
 112     { rc(arg) | rc(fpr) | 0x10,         "fa6" },
 113     { rc(arg) | rc(fpr) | 0x0f,         "fa5" },
 114     { rc(arg) | rc(fpr) | 0x0e,         "fa4" },
 115     { rc(arg) | rc(fpr) | 0x0d,         "fa3" },
 116     { rc(arg) | rc(fpr) | 0x0c,         "fa2" },
 117     { rc(arg) | rc(fpr) | 0x0b,         "fa1" },
 118     { rc(arg) | rc(fpr) | 0x0a,         "fa0" },
 119     { _NOREG,                           "<none>" },
 120 };
 121
 122 /*
 123  * Implementation
 124  */
 125 void
 126 jit_get_cpu(void)
 127 {
 128 }
 129
 130 void
 131 _jit_init(jit_state_t *_jit)
 132 {
 133     _jitc->reglen = jit_size(_rvs) - 1;
 134     jit_carry = _NOREG;
 135 }
 136
 137 void
 138 _jit_prolog(jit_state_t *_jit)
 139 {
 140     jit_int32_t          offset;
 141
 142     if (_jitc->function)
 143         jit_epilog();
 144     assert(jit_regset_cmp_ui(&_jitc->regarg, 0) == 0);
 145     jit_regset_set_ui(&_jitc->regsav, 0);
 146     offset = _jitc->functions.offset;
 147     if (offset >= _jitc->functions.length) {
 148         jit_realloc((jit_pointer_t *)&_jitc->functions.ptr,
 149                     _jitc->functions.length * sizeof(jit_function_t),
 150                     (_jitc->functions.length + 16) * sizeof(jit_function_t));
 151         _jitc->functions.length += 16;
 152     }
 153     _jitc->function = _jitc->functions.ptr + _jitc->functions.offset++;
 154     _jitc->function->self.size = stack_framesize;
 155     _jitc->function->self.argi = _jitc->function->self.argf =
 156         _jitc->function->self.alen = 0;
 157     _jitc->function->self.aoff = 0;
 158     _jitc->function->self.call = jit_call_default;
 159     jit_alloc((jit_pointer_t *)&_jitc->function->regoff,
 160               _jitc->reglen * sizeof(jit_int32_t));
 161
 162     /* _no_link here does not mean the jit_link() call can be removed
 163      * by rewriting as:
 164      * _jitc->function->prolog = jit_new_node(jit_code_prolog);
 165      */
 166     _jitc->function->prolog = jit_new_node_no_link(jit_code_prolog);
 167     jit_link(_jitc->function->prolog);
 168     _jitc->function->prolog->w.w = offset;
 169     _jitc->function->epilog = jit_new_node_no_link(jit_code_epilog);
 170     /*  u:      label value
 171      *  v:      offset in blocks vector
 172      *  w:      offset in functions vector
 173      */
 174     _jitc->function->epilog->w.w = offset;
 175
 176     jit_regset_new(&_jitc->function->regset);
 177 }
 178
 179 jit_int32_t
 180 _jit_allocai(jit_state_t *_jit, jit_int32_t length)
 181 {
 182     assert(_jitc->function);
 183     switch (length) {
 184         case 0: case 1:                                         break;
 185         case 2:         _jitc->function->self.aoff &= -2;       break;
 186         case 3: case 4: _jitc->function->self.aoff &= -4;       break;
 187         default:        _jitc->function->self.aoff &= -8;       break;
 188     }
 189     _jitc->function->self.aoff -= length;
 190     if (!_jitc->realize) {
 191         jit_inc_synth_ww(allocai, _jitc->function->self.aoff, length);
 192         jit_dec_synth();
 193     }
 194     return (_jitc->function->self.aoff);
 195 }
 196
 197 void
 198 _jit_allocar(jit_state_t *_jit, jit_int32_t u, jit_int32_t v)
 199 {
 200     jit_int32_t          r0;
 201     assert(_jitc->function);
 202     jit_inc_synth_ww(allocar, u, v);
 203     if (!_jitc->function->allocar) {
 204         _jitc->function->aoffoff = jit_allocai(sizeof(jit_int32_t));
 205         _jitc->function->allocar = 1;
 206     }
 207     r0 = jit_get_reg(jit_class_gpr);
 208     jit_negr(r0, v);
 209     jit_andi(r0, r0, -16);
 210     jit_ldxi_i(u, JIT_FP, _jitc->function->aoffoff);
 211     jit_addr(u, u, r0);
 212     jit_addr(JIT_SP, JIT_SP, r0);
 213     jit_stxi_i(_jitc->function->aoffoff, JIT_FP, u);
 214     jit_unget_reg(r0);
 215     jit_dec_synth();
 216 }
 217
 218 void
 219 _jit_ret(jit_state_t *_jit)
 220 {
 221     jit_node_t          *instr;
 222     assert(_jitc->function);
 223     jit_inc_synth(ret);
 224     /* jump to epilog */
 225     instr = jit_jmpi();
 226     jit_patch_at(instr, _jitc->function->epilog);
 227     jit_dec_synth();
 228 }
 229
 230 void
 231 _jit_retr(jit_state_t *_jit, jit_int32_t u)
 232 {
 233     jit_inc_synth_w(retr, u);
 234     if (JIT_RET != u)
 235         jit_movr(JIT_RET, u);
 236     jit_live(JIT_RET);
 237     jit_ret();
 238     jit_dec_synth();
 239 }
 240
 241 void
 242 _jit_reti(jit_state_t *_jit, jit_word_t u)
 243 {
 244     jit_inc_synth_w(reti, u);
 245     jit_movi(JIT_RET, u);
 246     jit_ret();
 247     jit_dec_synth();
 248 }
 249
 250 void
 251 _jit_retr_f(jit_state_t *_jit, jit_int32_t u)
 252 {
 253     jit_inc_synth_w(retr_f, u);
 254     if (u != JIT_FRET)
 255         jit_movr_f(JIT_FRET, u);
 256     else
 257         jit_live(JIT_FRET);
 258     jit_ret();
 259     jit_dec_synth();
 260 }
 261
 262 void
 263 _jit_reti_f(jit_state_t *_jit, jit_float32_t u)
 264 {
 265     jit_inc_synth_f(reti_f, u);
 266     jit_movi_f(JIT_FRET, u);
 267     jit_ret();
 268     jit_dec_synth();
 269 }
 270
 271 void
 272 _jit_retr_d(jit_state_t *_jit, jit_int32_t u)
 273 {
 274     jit_inc_synth_w(retr_d, u);
 275     if (u != JIT_FRET)
 276         jit_movr_d(JIT_FRET, u);
 277     else
 278         jit_live(JIT_FRET);
 279     jit_ret();
 280     jit_dec_synth();
 281 }
 282
 283 void
 284 _jit_reti_d(jit_state_t *_jit, jit_float64_t u)
 285 {
 286     jit_inc_synth_d(reti_d, u);
 287     jit_movi_d(JIT_FRET, u);
 288     jit_ret();
 289     jit_dec_synth();
 290 }
 291
 292 void
 293 _jit_epilog(jit_state_t *_jit)
 294 {
 295     assert(_jitc->function);
 296     assert(_jitc->function->epilog->next == NULL);
 297     jit_link(_jitc->function->epilog);
 298     _jitc->function = NULL;
 299 }
 300
 301 jit_bool_t
 302 _jit_arg_register_p(jit_state_t *_jit, jit_node_t *u)
 303 {
 304     if (u->code == jit_code_arg)
 305         return (jit_arg_reg_p(u->u.w));
 306     assert(u->code == jit_code_arg_f || u->code == jit_code_arg_d);
 307     return (jit_arg_f_reg_p(u->u.w));
 308 }
 309
 310 void
 311 _jit_ellipsis(jit_state_t *_jit)
 312 {
 313     jit_inc_synth(ellipsis);
 314     if (_jitc->prepare) {
 315         jit_link_prepare();
 316         assert(!(_jitc->function->call.call & jit_call_varargs));
 317         _jitc->function->call.call |= jit_call_varargs;
 318     }
 319     else {
 320         jit_link_prolog();
 321         assert(!(_jitc->function->self.call & jit_call_varargs));
 322         _jitc->function->self.call |= jit_call_varargs;
 323         _jitc->function->vagp = _jitc->function->self.argi;
 324     }
 325     jit_dec_synth();
 326 }
 327
 328 void
 329 _jit_va_push(jit_state_t *_jit, jit_int32_t u)
 330 {
 331     jit_inc_synth_w(va_push, u);
 332     jit_pushargr(u);
 333     jit_dec_synth();
 334 }
 335
 336 jit_node_t *
 337 _jit_arg(jit_state_t *_jit)
 338 {
 339     jit_node_t          *node;
 340     jit_int32_t          offset;
 341     assert(_jitc->function);
 342     assert(!(_jitc->function->self.call & jit_call_varargs));
 343     if (jit_arg_reg_p(_jitc->function->self.argi))
 344         offset = _jitc->function->self.argi++;
 345     else {
 346         offset = _jitc->function->self.size;
 347         _jitc->function->self.size += sizeof(jit_word_t);
 348     }
 349     node = jit_new_node_ww(jit_code_arg, offset,
 350                            ++_jitc->function->self.argn);
 351     jit_link_prolog();
 352     return (node);
 353 }
 354
 355 jit_node_t *
 356 _jit_arg_f(jit_state_t *_jit)
 357 {
 358     jit_node_t          *node;
 359     jit_int32_t          offset;
 360     assert(_jitc->function);
 361     assert(!(_jitc->function->self.call & jit_call_varargs));
 362     if (jit_arg_f_reg_p(_jitc->function->self.argf))
 363         offset = _jitc->function->self.argf++;
 364     else if (jit_arg_reg_p(_jitc->function->self.argi)) {
 365         offset = _jitc->function->self.argi++;
 366         offset += 8;
 367     }
 368     else {
 369         offset = _jitc->function->self.size;
 370         _jitc->function->self.size += sizeof(jit_word_t);
 371     }
 372     node = jit_new_node_ww(jit_code_arg_f, offset,
 373                            ++_jitc->function->self.argn);
 374     jit_link_prolog();
 375     return (node);
 376 }
 377
 378 jit_node_t *
 379 _jit_arg_d(jit_state_t *_jit)
 380 {
 381     jit_node_t          *node;
 382     jit_int32_t          offset;
 383     assert(_jitc->function);
 384     assert(!(_jitc->function->self.call & jit_call_varargs));
 385     if (jit_arg_f_reg_p(_jitc->function->self.argf))
 386         offset = _jitc->function->self.argf++;
 387     else if (jit_arg_reg_p(_jitc->function->self.argi)) {
 388         offset = _jitc->function->self.argi++;
 389         offset += 8;
 390     }
 391     else {
 392         offset = _jitc->function->self.size;
 393         _jitc->function->self.size += sizeof(jit_word_t);
 394     }
 395     node = jit_new_node_ww(jit_code_arg_d, offset,
 396                            ++_jitc->function->self.argn);
 397     jit_link_prolog();
 398     return (node);
 399 }
 400
 401 void
 402 _jit_getarg_c(jit_state_t *_jit, jit_int32_t u, jit_node_t *v)
 403 {
 404     assert(v->code == jit_code_arg);
 405     jit_inc_synth_wp(getarg_c, u, v);
 406     if (jit_arg_reg_p(v->u.w))
 407         jit_extr_c(u, JIT_RA0 - v->u.w);
 408     else
 409         jit_ldxi_c(u, JIT_FP, v->u.w);
 410     jit_dec_synth();
 411 }
 412
 413 void
 414 _jit_getarg_uc(jit_state_t *_jit, jit_int32_t u, jit_node_t *v)
 415 {
 416     assert(v->code == jit_code_arg);
 417     jit_inc_synth_wp(getarg_uc, u, v);
 418     if (jit_arg_reg_p(v->u.w))
 419         jit_extr_uc(u, JIT_RA0 - v->u.w);
 420     else
 421         jit_ldxi_uc(u, JIT_FP, v->u.w);
 422     jit_dec_synth();
 423 }
 424
 425 void
 426 _jit_getarg_s(jit_state_t *_jit, jit_int32_t u, jit_node_t *v)
 427 {
 428     assert(v->code == jit_code_arg);
 429     jit_inc_synth_wp(getarg_s, u, v);
 430     if (jit_arg_reg_p(v->u.w))
 431         jit_extr_s(u, JIT_RA0 - v->u.w);
 432     else
 433         jit_ldxi_s(u, JIT_FP, v->u.w);
 434     jit_dec_synth();
 435 }
 436
 437 void
 438 _jit_getarg_us(jit_state_t *_jit, jit_int32_t u, jit_node_t *v)
 439 {
 440     assert(v->code == jit_code_arg);
 441     jit_inc_synth_wp(getarg_us, u, v);
 442     if (jit_arg_reg_p(v->u.w))
 443         jit_extr_us(u, JIT_RA0 - v->u.w);
 444     else
 445         jit_ldxi_us(u, JIT_FP, v->u.w);
 446     jit_dec_synth();
 447 }
 448
 449 void
 450 _jit_getarg_i(jit_state_t *_jit, jit_int32_t u, jit_node_t *v)
 451 {
 452     assert(v->code == jit_code_arg);
 453     jit_inc_synth_wp(getarg_i, u, v);
 454     if (jit_arg_reg_p(v->u.w))
 455         jit_extr_i(u, JIT_RA0 - v->u.w);
 456     else
 457         jit_ldxi_i(u, JIT_FP, v->u.w);
 458     jit_dec_synth();
 459 }
 460
 461 void
 462 _jit_getarg_ui(jit_state_t *_jit, jit_int32_t u, jit_node_t *v)
 463 {
 464     assert(v->code == jit_code_arg);
 465     jit_inc_synth_wp(getarg_ui, u, v);
 466     if (jit_arg_reg_p(v->u.w))
 467         jit_extr_ui(u, JIT_RA0 - v->u.w);
 468     else
 469         jit_ldxi_ui(u, JIT_FP, v->u.w);
 470     jit_dec_synth();
 471 }
 472
 473 void
 474 _jit_getarg_l(jit_state_t *_jit, jit_int32_t u, jit_node_t *v)
 475 {
 476     assert(v->code == jit_code_arg);
 477     jit_inc_synth_wp(getarg_l, u, v);
 478     if (jit_arg_reg_p(v->u.w))
 479         jit_movr(u, JIT_RA0 - v->u.w);
 480     else
 481         jit_ldxi_l(u, JIT_FP, v->u.w);
 482     jit_dec_synth();
 483 }
 484
 485 void
 486 _jit_putargr(jit_state_t *_jit, jit_int32_t u, jit_node_t *v)
 487 {
 488     assert(v->code == jit_code_arg);
 489     jit_inc_synth_wp(putargr, u, v);
 490     if (jit_arg_reg_p(v->u.w))
 491         jit_movr(JIT_RA0 - v->u.w, u);
 492     else
 493         jit_stxi(v->u.w, JIT_FP, u);
 494     jit_dec_synth();
 495 }
 496
 497 void
 498 _jit_putargi(jit_state_t *_jit, jit_word_t u, jit_node_t *v)
 499 {
 500     jit_int32_t         regno;
 501     assert(v->code == jit_code_arg);
 502     jit_inc_synth_wp(putargi, u, v);
 503     if (jit_arg_reg_p(v->u.w))
 504         jit_movi(JIT_RA0 - v->u.w, u);
 505     else {
 506         regno = jit_get_reg(jit_class_gpr);
 507         jit_movi(regno, u);
 508         jit_stxi(v->u.w, JIT_FP, regno);
 509         jit_unget_reg(regno);
 510     }
 511     jit_dec_synth();
 512 }
 513
 514 void
 515 _jit_getarg_f(jit_state_t *_jit, jit_int32_t u, jit_node_t *v)
 516 {
 517     assert(v->code == jit_code_arg_f);
 518     jit_inc_synth_wp(getarg_f, u, v);
 519     if (jit_arg_f_reg_p(v->u.w))
 520         jit_movr_f(u, JIT_FA0 - v->u.w);
 521     else if (jit_arg_reg_p(v->u.w - 8))
 522         jit_movr_w_f(u, JIT_RA0 - (v->u.w - 8));
 523     else
 524         jit_ldxi_f(u, JIT_FP, v->u.w);
 525     jit_dec_synth();
 526 }
 527
 528 void
 529 _jit_putargr_f(jit_state_t *_jit, jit_int32_t u, jit_node_t *v)
 530 {
 531     assert(v->code == jit_code_arg_f);
 532     jit_inc_synth_wp(putargr_f, u, v);
 533     if (jit_arg_f_reg_p(v->u.w))
 534         jit_movr_f(JIT_FA0 - v->u.w, u);
 535     else if (jit_arg_reg_p(v->u.w - 8))
 536         jit_movr_f_w(JIT_RA0 - (v->u.w - 8), u);
 537     else
 538         jit_stxi_f(v->u.w, JIT_FP, u);
 539     jit_dec_synth();
 540 }
 541
 542 void
 543 _jit_putargi_f(jit_state_t *_jit, jit_float32_t u, jit_node_t *v)
 544 {
 545     jit_int32_t         regno;
 546     assert(v->code == jit_code_arg_f);
 547     jit_inc_synth_fp(putargi_f, u, v);
 548     if (jit_arg_f_reg_p(v->u.w))
 549         jit_movi_f(JIT_FA0 - v->u.w, u);
 550     else if (jit_arg_reg_p(v->u.w - 8)) {
 551         union {
 552             jit_float32_t       f;
 553             jit_int32_t         i;
 554         } uu;
 555         uu.f = u;
 556         jit_movi(JIT_RA0 - (v->u.w - 8), uu.i);
 557     }
 558     else {
 559         regno = jit_get_reg(jit_class_fpr);
 560         jit_movi_f(regno, u);
 561         jit_stxi_f(v->u.w, JIT_FP, regno);
 562         jit_unget_reg(regno);
 563     }
 564     jit_dec_synth();
 565 }
 566
 567 void
 568 _jit_getarg_d(jit_state_t *_jit, jit_int32_t u, jit_node_t *v)
 569 {
 570     assert(v->code == jit_code_arg_d);
 571     jit_inc_synth_wp(getarg_d, u, v);
 572     if (jit_arg_f_reg_p(v->u.w))
 573         jit_movr_d(u, JIT_FA0 - v->u.w);
 574     else if (jit_arg_reg_p(v->u.w - 8))
 575         jit_movr_w_d(u, JIT_RA0 - (v->u.w - 8));
 576     else
 577         jit_ldxi_d(u, JIT_FP, v->u.w);
 578     jit_dec_synth();
 579 }
 580
 581 void
 582 _jit_putargr_d(jit_state_t *_jit, jit_int32_t u, jit_node_t *v)
 583 {
 584     assert(v->code == jit_code_arg_d);
 585     jit_inc_synth_wp(putargr_d, u, v);
 586     if (jit_arg_reg_p(v->u.w))
 587         jit_movr_d(JIT_FA0 - v->u.w, u);
 588     else if (jit_arg_reg_p(v->u.w - 8))
 589         jit_movr_d_w(JIT_RA0 - (v->u.w - 8), u);
 590     else
 591         jit_stxi_d(v->u.w, JIT_FP, u);
 592     jit_dec_synth();
 593 }
 594
 595 void
 596 _jit_putargi_d(jit_state_t *_jit, jit_float64_t u, jit_node_t *v)
 597 {
 598     jit_int32_t         regno;
 599     assert(v->code == jit_code_arg_d);
 600     jit_inc_synth_dp(putargi_d, u, v);
 601     if (jit_arg_reg_p(v->u.w))
 602         jit_movi_d(JIT_FA0 - v->u.w, u);
 603     else if (jit_arg_reg_p(v->u.w - 8)) {
 604         union {
 605             jit_float64_t       d;
 606             jit_int64_t         w;
 607         } uu;
 608         uu.d = u;
 609         jit_movi(JIT_RA0 - (v->u.w - 8), uu.w);
 610     }
 611     else {
 612         regno = jit_get_reg(jit_class_fpr);
 613         jit_movi_d(regno, u);
 614         jit_stxi_d(v->u.w, JIT_FP, regno);
 615         jit_unget_reg(regno);
 616     }
 617     jit_dec_synth();
 618 }
 619
 620 void
 621 _jit_pushargr(jit_state_t *_jit, jit_int32_t u)
 622 {
 623     assert(_jitc->function);
 624     jit_inc_synth_w(pushargr, u);
 625     jit_link_prepare();
 626     if (jit_arg_reg_p(_jitc->function->call.argi)) {
 627         jit_movr(JIT_RA0 - _jitc->function->call.argi, u);
 628         ++_jitc->function->call.argi;
 629     }
 630     else {
 631         jit_stxi(_jitc->function->call.size, JIT_SP, u);
 632         _jitc->function->call.size += sizeof(jit_word_t);
 633     }
 634     jit_dec_synth();
 635 }
 636
 637 void
 638 _jit_pushargi(jit_state_t *_jit, jit_word_t u)
 639 {
 640     jit_int32_t          regno;
 641     assert(_jitc->function);
 642     jit_inc_synth_w(pushargi, u);
 643     jit_link_prepare();
 644     if (jit_arg_reg_p(_jitc->function->call.argi)) {
 645         jit_movi(JIT_RA0 - _jitc->function->call.argi, u);
 646         ++_jitc->function->call.argi;
 647     }
 648     else {
 649         regno = jit_get_reg(jit_class_gpr);
 650         jit_movi(regno, u);
 651         jit_stxi(_jitc->function->call.size, JIT_SP, regno);
 652         jit_unget_reg(regno);
 653         _jitc->function->call.size += sizeof(jit_word_t);
 654     }
 655     jit_dec_synth();
 656 }
 657
 658 void
 659 _jit_pushargr_f(jit_state_t *_jit, jit_int32_t u)
 660 {
 661     assert(_jitc->function);
 662     jit_inc_synth_w(pushargr_f, u);
 663     jit_link_prepare();
 664     if (jit_arg_f_reg_p(_jitc->function->call.argf) &&
 665         !(_jitc->function->call.call & jit_call_varargs)) {
 666         jit_movr_f(JIT_FA0 - _jitc->function->call.argf, u);
 667         ++_jitc->function->call.argf;
 668     }
 669     else if (jit_arg_reg_p(_jitc->function->call.argi)) {
 670         jit_movr_f_w(JIT_RA0 - _jitc->function->call.argi, u);
 671         ++_jitc->function->call.argi;
 672     }
 673     else {
 674         jit_stxi_f(_jitc->function->call.size, JIT_SP, u);
 675         _jitc->function->call.size += sizeof(jit_word_t);
 676     }
 677     jit_dec_synth();
 678 }
 679
 680 void
 681 _jit_pushargi_f(jit_state_t *_jit, jit_float32_t u)
 682 {
 683     jit_int32_t         regno;
 684     assert(_jitc->function);
 685     jit_inc_synth_f(pushargi_f, u);
 686     jit_link_prepare();
 687     if (jit_arg_f_reg_p(_jitc->function->call.argf) &&
 688         !(_jitc->function->call.call & jit_call_varargs)) {
 689         jit_movi_f(JIT_FA0 - _jitc->function->call.argf, u);
 690         ++_jitc->function->call.argf;
 691     }
 692     else if (jit_arg_reg_p(_jitc->function->call.argi)) {
 693         jit_movi_f_w(JIT_RA0 - _jitc->function->call.argi, u);
 694         ++_jitc->function->call.argi;
 695     }
 696     else {
 697         regno = jit_get_reg(jit_class_fpr);
 698         jit_movi_f(regno, u);
 699         jit_stxi_f(_jitc->function->call.size, JIT_SP, regno);
 700         jit_unget_reg(regno);
 701         _jitc->function->call.size += sizeof(jit_word_t);
 702     }
 703     jit_dec_synth();
 704 }
 705
 706 void
 707 _jit_pushargr_d(jit_state_t *_jit, jit_int32_t u)
 708 {
 709     assert(_jitc->function);
 710     jit_inc_synth_w(pushargr_d, u);
 711     jit_link_prepare();
 712     if (jit_arg_f_reg_p(_jitc->function->call.argf) &&
 713         !(_jitc->function->call.call & jit_call_varargs)) {
 714         jit_movr_d(JIT_FA0 - _jitc->function->call.argf, u);
 715         ++_jitc->function->call.argf;
 716     }
 717     else if (jit_arg_reg_p(_jitc->function->call.argi)) {
 718         jit_movr_d_w(JIT_RA0 - _jitc->function->call.argi, u);
 719         ++_jitc->function->call.argi;
 720     }
 721     else {
 722         jit_stxi_d(_jitc->function->call.size, JIT_SP, u);
 723         _jitc->function->call.size += sizeof(jit_word_t);
 724     }
 725     jit_dec_synth();
 726 }
 727
 728 void
 729 _jit_pushargi_d(jit_state_t *_jit, jit_float64_t u)
 730 {
 731     jit_int32_t         regno;
 732     assert(_jitc->function);
 733     jit_inc_synth_d(pushargi_d, u);
 734     jit_link_prepare();
 735     if (jit_arg_f_reg_p(_jitc->function->call.argf) &&
 736         !(_jitc->function->call.call & jit_call_varargs)) {
 737         jit_movi_d(JIT_FA0 - _jitc->function->call.argf, u);
 738         ++_jitc->function->call.argf;
 739     }
 740     else if (jit_arg_reg_p(_jitc->function->call.argi)) {
 741         jit_movi_d_w(JIT_RA0 - _jitc->function->call.argi, u);
 742         ++_jitc->function->call.argi;
 743     }
 744     else {
 745         regno = jit_get_reg(jit_class_fpr);
 746         jit_movi_d(regno, u);
 747         jit_stxi_d(_jitc->function->call.size, JIT_SP, regno);
 748         jit_unget_reg(regno);
 749         _jitc->function->call.size += sizeof(jit_word_t);
 750     }
 751     jit_dec_synth();
 752 }
 753
 754 jit_bool_t
 755 _jit_regarg_p(jit_state_t *_jit, jit_node_t *node, jit_int32_t regno)
 756 {
 757     jit_int32_t         spec;
 758     spec = jit_class(_rvs[regno].spec);
 759     if (spec & jit_class_arg) {
 760         regno = JIT_RA0 - regno;
 761         if (regno >= 0 && regno < node->v.w)
 762             return (1);
 763         if (spec & jit_class_fpr) {
 764             regno = JIT_FA0 - regno;
 765             if (regno >= 0 && regno < node->w.w)
 766                 return (1);
 767         }
 768     }
 769
 770     return (0);
 771 }
 772
 773 void
 774 _jit_finishr(jit_state_t *_jit, jit_int32_t r0)
 775 {
 776     jit_node_t          *node;
 777     assert(_jitc->function);
 778     jit_inc_synth_w(finishr, r0);
 779     if (_jitc->function->self.alen < _jitc->function->call.size)
 780         _jitc->function->self.alen = _jitc->function->call.size;
 781     node = jit_callr(r0);
 782     node->v.w = _jitc->function->self.argi;
 783     node->w.w = _jitc->function->call.argf;
 784     _jitc->function->call.argi = _jitc->function->call.argf =
 785         _jitc->function->call.size = 0;
 786     _jitc->prepare = 0;
 787     jit_dec_synth();
 788 }
 789
 790 jit_node_t *
 791 _jit_finishi(jit_state_t *_jit, jit_pointer_t i0)
 792 {
 793     jit_node_t          *node;
 794     assert(_jitc->function);
 795     jit_inc_synth_w(finishi, (jit_word_t)i0);
 796     if (_jitc->function->self.alen < _jitc->function->call.size)
 797         _jitc->function->self.alen = _jitc->function->call.size;
 798     node = jit_calli(i0);
 799     node->v.w = _jitc->function->call.argi;
 800     node->w.w = _jitc->function->call.argf;
 801     _jitc->function->call.argi = _jitc->function->call.argf =
 802         _jitc->function->call.size = 0;
 803     _jitc->prepare = 0;
 804     jit_dec_synth();
 805     return (node);
 806 }
 807
 808 void
 809 _jit_retval_c(jit_state_t *_jit, jit_int32_t r0)
 810 {
 811     jit_inc_synth_w(retval_c, r0);
 812     jit_extr_c(r0, JIT_RET);
 813     jit_dec_synth();
 814 }
 815
 816 void
 817 _jit_retval_uc(jit_state_t *_jit, jit_int32_t r0)
 818 {
 819     jit_inc_synth_w(retval_uc, r0);
 820     jit_extr_uc(r0, JIT_RET);
 821     jit_dec_synth();
 822 }
 823
 824 void
 825 _jit_retval_s(jit_state_t *_jit, jit_int32_t r0)
 826 {
 827     jit_inc_synth_w(retval_s, r0);
 828     jit_extr_s(r0, JIT_RET);
 829     jit_dec_synth();
 830 }
 831
 832 void
 833 _jit_retval_us(jit_state_t *_jit, jit_int32_t r0)
 834 {
 835     jit_inc_synth_w(retval_us, r0);
 836     jit_extr_us(r0, JIT_RET);
 837     jit_dec_synth();
 838 }
 839
 840 void
 841 _jit_retval_i(jit_state_t *_jit, jit_int32_t r0)
 842 {
 843     jit_inc_synth_w(retval_i, r0);
 844     jit_extr_i(r0, JIT_RET);
 845     jit_dec_synth();
 846 }
 847
 848 void
 849 _jit_retval_ui(jit_state_t *_jit, jit_int32_t r0)
 850 {
 851     jit_inc_synth_w(retval_ui, r0);
 852     jit_extr_ui(r0, JIT_RET);
 853     jit_dec_synth();
 854 }
 855
 856 void
 857 _jit_retval_l(jit_state_t *_jit, jit_int32_t r0)
 858 {
 859     jit_inc_synth_w(retval_l, r0);
 860     if (r0 != JIT_RET)
 861         jit_movr(r0, JIT_RET);
 862     jit_dec_synth();
 863 }
 864
 865 void
 866 _jit_retval_f(jit_state_t *_jit, jit_int32_t r0)
 867 {
 868     jit_inc_synth_w(retval_f, r0);
 869     if (r0 != JIT_FRET)
 870         jit_movr_f(r0, JIT_FRET);
 871     jit_dec_synth();
 872 }
 873
 874 void
 875 _jit_retval_d(jit_state_t *_jit, jit_int32_t r0)
 876 {
 877     jit_inc_synth_w(retval_d, r0);
 878     if (r0 != JIT_FRET)
 879         jit_movr_d(r0, JIT_FRET);
 880     jit_dec_synth();
 881 }
 882
 883 jit_pointer_t
 884 _emit_code(jit_state_t *_jit)
 885 {
 886     jit_node_t          *node;
 887     jit_node_t          *temp;
 888     jit_word_t           word;
 889     jit_word_t           value;
 890     jit_int32_t          offset;
 891     struct {
 892         jit_node_t      *node;
 893         jit_uint8_t     *data;
 894         jit_word_t       word;
 895 #if DEVEL_DISASSEMBLER
 896         jit_word_t       prevw;
 897 #endif
 898         jit_int32_t      const_offset;
 899         jit_int32_t      patch_offset;
 900     } undo;
 901 #if DEVEL_DISASSEMBLER
 902     jit_word_t           prevw;
 903 #endif
 904
 905 #if __WORDSIZE == 64
 906     if (!_jitc->consts.hash.table) {
 907         jit_alloc((jit_pointer_t *)&_jitc->consts.hash.table,
 908                   16 * sizeof(jit_const_t *));
 909         _jitc->consts.hash.size = 16;
 910         jit_alloc((jit_pointer_t *)&_jitc->consts.pool.ptr,
 911                   sizeof(jit_const_t *));
 912         jit_alloc((jit_pointer_t *)_jitc->consts.pool.ptr,
 913                   1024 * sizeof(jit_const_t));
 914         _jitc->consts.pool.length = 1;
 915     }
 916     /* Reset table if starting over jit generation */
 917     else
 918         memset(_jitc->consts.hash.table, 0,
 919                _jitc->consts.hash.size * sizeof(jit_word_t));
 920     for (offset = 0; offset < _jitc->consts.pool.length; offset++) {
 921         jit_int32_t      i;
 922         jit_const_t     *list = _jitc->consts.pool.ptr[offset];
 923         for (i = 0; i < 1023; ++i, ++list)
 924             list->next = list + 1;
 925         if (offset + 1 < _jitc->consts.pool.length)
 926             list->next = _jitc->consts.pool.ptr[offset + 1];
 927         else
 928             list->next = NULL;
 929     }
 930     _jitc->consts.pool.list = _jitc->consts.pool.ptr[0];
 931     _jitc->consts.hash.count = 0;
 932     if (!_jitc->consts.vector.instrs) {
 933         jit_alloc((jit_pointer_t *)&_jitc->consts.vector.instrs,
 934                   16 * sizeof(jit_word_t));
 935         jit_alloc((jit_pointer_t *)&_jitc->consts.vector.values,
 936                   16 * sizeof(jit_word_t));
 937         _jitc->consts.vector.length = 16;
 938     }
 939     _jitc->consts.vector.offset = 0;
 940 #endif
 941
 942     _jitc->function = NULL;
 943
 944     jit_reglive_setup();
 945
 946     undo.word = 0;
 947     undo.node = NULL;
 948     undo.const_offset = undo.patch_offset = 0;
 949 #  define assert_data(node)             /**/
 950 #define case_rr(name, type)                                             \
 951             case jit_code_##name##r##type:                              \
 952                 name##r##type(rn(node->u.w), rn(node->v.w));            \
 953                 break
 954 #define case_rw(name, type)                                             \
 955             case jit_code_##name##i##type:                              \
 956                 name##i##type(rn(node->u.w), node->v.w);                \
 957                 break
 958 #define case_wr(name, type)                                             \
 959             case jit_code_##name##i##type:                              \
 960                 name##i##type(node->u.w, rn(node->v.w));                \
 961                 break
 962 #define case_rrr(name, type)                                            \
 963             case jit_code_##name##r##type:                              \
 964                 name##r##type(rn(node->u.w),                            \
 965                               rn(node->v.w), rn(node->w.w));            \
 966                 break
 967 #define case_rrrr(name, type)                                           \
 968             case jit_code_##name##r##type:                              \
 969                 name##r##type(rn(node->u.q.l), rn(node->u.q.h),         \
 970                               rn(node->v.w), rn(node->w.w));            \
 971                 break
 972 #define case_rrw(name, type)                                            \
 973             case jit_code_##name##i##type:                              \
 974                 name##i##type(rn(node->u.w), rn(node->v.w), node->w.w); \
 975                 break
 976 #define case_rrrw(name, type)                                           \
 977             case jit_code_##name##i##type:                              \
 978                 name##i##type(rn(node->u.q.l), rn(node->u.q.h),         \
 979                               rn(node->v.w), node->w.w);                \
 980                 break
 981 #define case_rrf(name)                                                  \
 982             case jit_code_##name##i_f:                                  \
 983                 assert_data(node);                                      \
 984                 name##i_f(rn(node->u.w), rn(node->v.w), node->w.f);     \
 985                 break
 986 #define case_rrd(name)                                                  \
 987             case jit_code_##name##i_d:                                  \
 988                 assert_data(node);                                      \
 989                 name##i_d(rn(node->u.w), rn(node->v.w), node->w.d);     \
 990                 break
 991 #define case_wrr(name, type)                                            \
 992             case jit_code_##name##i##type:                              \
 993                 name##i##type(node->u.w, rn(node->v.w), rn(node->w.w)); \
 994                 break
 995 #define case_brr(name, type)                                            \
 996             case jit_code_##name##r##type:                              \
 997                 temp = node->u.n;                                       \
 998                 assert(temp->code == jit_code_label ||                  \
 999                        temp->code == jit_code_epilog);                  \
1000                 if (temp->flag & jit_flag_patch)                        \
1001                     name##r##type(temp->u.w, rn(node->v.w),             \
1002                                   rn(node->w.w));                       \
1003                 else {                                                  \
1004                     word = name##r##type(_jit->pc.w,                    \
1005                                          rn(node->v.w), rn(node->w.w)); \
1006                     patch(word, node);                                  \
1007                 }                                                       \
1008                 break
1009 #define case_brw(name, type)                                            \
1010             case jit_code_##name##i##type:                              \
1011                 temp = node->u.n;                                       \
1012                 assert(temp->code == jit_code_label ||                  \
1013                        temp->code == jit_code_epilog);                  \
1014                 if (temp->flag & jit_flag_patch)                        \
1015                     name##i##type(temp->u.w,                            \
1016                                   rn(node->v.w), node->w.w);            \
1017                 else {                                                  \
1018                     word = name##i##type(_jit->pc.w,                    \
1019                                          rn(node->v.w), node->w.w);     \
1020                     patch(word, node);                                  \
1021                 }                                                       \
1022                 break;
1023 #define case_brf(name)                                                  \
1024             case jit_code_##name##i_f:                                  \
1025                 temp = node->u.n;                                       \
1026                 assert(temp->code == jit_code_label ||                  \
1027                        temp->code == jit_code_epilog);                  \
1028                 if (temp->flag & jit_flag_patch)                        \
1029                     name##i_f(temp->u.w, rn(node->v.w), node->w.f);     \
1030                 else {                                                  \
1031                     word = name##i_f(_jit->pc.w, rn(node->v.w),         \
1032                                 node->w.f);                             \
1033                     patch(word, node);                                  \
1034                 }                                                       \
1035                 break
1036 #define case_brd(name)                                                  \
1037             case jit_code_##name##i_d:                                  \
1038                 temp = node->u.n;                                       \
1039                 assert(temp->code == jit_code_label ||                  \
1040                        temp->code == jit_code_epilog);                  \
1041                 if (temp->flag & jit_flag_patch)                        \
1042                     name##i_d(temp->u.w, rn(node->v.w), node->w.d);     \
1043                 else {                                                  \
1044                     word = name##i_d(_jit->pc.w, rn(node->v.w),         \
1045                                 node->w.d);                             \
1046                     patch(word, node);                                  \
1047                 }                                                       \
1048                 break
1049 #if DEVEL_DISASSEMBLER
1050     prevw = _jit->pc.w;
1051 #endif
1052     for (node = _jitc->head; node; node = node->next) {
1053         if (_jit->pc.uc >= _jitc->code.end)
1054             return (NULL);
1055
1056 #if DEVEL_DISASSEMBLER
1057         node->offset = (jit_uword_t)_jit->pc.w - (jit_uword_t)prevw;
1058         prevw = _jit->pc.w;
1059 #endif
1060         value = jit_classify(node->code);
1061         jit_regarg_set(node, value);
1062         switch (node->code) {
1063             case jit_code_align:
1064                 /* Must align to a power of two */
1065                 assert(!(node->u.w & (node->u.w - 1)));
1066                 if ((word = _jit->pc.w & (node->u.w - 1)))
1067                     nop(node->u.w - word);
1068                 break;
1069             case jit_code_note:         case jit_code_name:
1070                 node->u.w = _jit->pc.w;
1071                 break;
1072             case jit_code_label:
1073                 /* remember label is defined */
1074                 node->flag |= jit_flag_patch;
1075                 node->u.w = _jit->pc.w;
1076                 break;
1077                 case_rrr(add,);
1078                 case_rrw(add,);
1079                 case_rrr(addc,);
1080                 case_rrw(addc,);
1081                 case_rrr(addx,);
1082                 case_rrw(addx,);
1083                 case_rrr(sub,);
1084                 case_rrw(sub,);
1085                 case_rrr(subc,);
1086                 case_rrw(subc,);
1087                 case_rrr(subx,);
1088                 case_rrw(subx,);
1089                 case_rrw(rsb,);
1090                 case_rrr(mul,);
1091                 case_rrw(mul,);
1092                 case_rrrr(qmul,);
1093                 case_rrrw(qmul,);
1094                 case_rrrr(qmul, _u);
1095                 case_rrrw(qmul, _u);
1096                 case_rrr(div,);
1097                 case_rrw(div,);
1098                 case_rrr(div, _u);
1099                 case_rrw(div, _u);
1100                 case_rrrr(qdiv,);
1101                 case_rrrw(qdiv,);
1102                 case_rrrr(qdiv, _u);
1103                 case_rrrw(qdiv, _u);
1104                 case_rrr(rem,);
1105                 case_rrw(rem,);
1106                 case_rrr(rem, _u);
1107                 case_rrw(rem, _u);
1108                 case_rrr(lsh,);
1109                 case_rrw(lsh,);
1110                 case_rrr(rsh,);
1111                 case_rrw(rsh,);
1112                 case_rrr(rsh, _u);
1113                 case_rrw(rsh, _u);
1114                 case_rr(neg,);
1115                 case_rr(com,);
1116                 case_rrr(and,);
1117                 case_rrw(and,);
1118                 case_rrr(or,);
1119                 case_rrw(or,);
1120                 case_rrr(xor,);
1121                 case_rrw(xor,);
1122                 case_rr(trunc, _f_i);
1123                 case_rr(trunc, _d_i);
1124                 case_rr(trunc, _f_l);
1125                 case_rr(trunc, _d_l);
1126                 case_rr(ld, _c);
1127                 case_rw(ld, _c);
1128                 case_rr(ld, _uc);
1129                 case_rw(ld, _uc);
1130                 case_rr(ld, _s);
1131                 case_rw(ld, _s);
1132                 case_rr(ld, _us);
1133                 case_rw(ld, _us);
1134                 case_rr(ld, _i);
1135                 case_rw(ld, _i);
1136                 case_rr(ld, _ui);
1137                 case_rw(ld, _ui);
1138                 case_rr(ld, _l);
1139                 case_rw(ld, _l);
1140                 case_rrr(ldx, _c);
1141                 case_rrw(ldx, _c);
1142                 case_rrr(ldx, _uc);
1143                 case_rrw(ldx, _uc);
1144                 case_rrr(ldx, _s);
1145                 case_rrw(ldx, _s);
1146                 case_rrr(ldx, _us);
1147                 case_rrw(ldx, _us);
1148                 case_rrr(ldx, _i);
1149                 case_rrw(ldx, _i);
1150                 case_rrr(ldx, _ui);
1151                 case_rrw(ldx, _ui);
1152                 case_rrr(ldx, _l);
1153                 case_rrw(ldx, _l);
1154                 case_rr(st, _c);
1155                 case_wr(st, _c);
1156                 case_rr(st, _s);
1157                 case_wr(st, _s);
1158                 case_rr(st, _i);
1159                 case_wr(st, _i);
1160                 case_rr(st, _l);
1161                 case_wr(st, _l);
1162                 case_rrr(stx, _c);
1163                 case_wrr(stx, _c);
1164                 case_rrr(stx, _s);
1165                 case_wrr(stx, _s);
1166                 case_rrr(stx, _i);
1167                 case_wrr(stx, _i);
1168                 case_rrr(stx, _l);
1169                 case_wrr(stx, _l);
1170                 case_rr(hton, _us);
1171                 case_rr(hton, _ui);
1172                 case_rr(hton, _ul);
1173                 case_rr(bswap, _us);
1174                 case_rr(bswap, _ui);
1175                 case_rr(bswap, _ul);
1176                 case_rr(ext, _c);
1177                 case_rr(ext, _uc);
1178                 case_rr(ext, _s);
1179                 case_rr(ext, _us);
1180                 case_rr(ext, _i);
1181                 case_rr(ext, _ui);
1182             case jit_code_casr:
1183                 casr(rn(node->u.w), rn(node->v.w),
1184                      rn(node->w.q.l), rn(node->w.q.h));
1185                 break;
1186             case jit_code_casi:
1187                 casi(rn(node->u.w), node->v.w,
1188                      rn(node->w.q.l), rn(node->w.q.h));
1189                 break;
1190                 case_rrr(movn,);
1191                 case_rrr(movz,);
1192                 case_rr(mov,);
1193             case jit_code_movi:
1194                 if (node->flag & jit_flag_node) {
1195                     temp = node->v.n;
1196                     if (temp->code == jit_code_data ||
1197                         (temp->code == jit_code_label &&
1198                          (temp->flag & jit_flag_patch)))
1199                         movi(rn(node->u.w), temp->u.w);
1200                     else {
1201                         assert(temp->code == jit_code_label ||
1202                                temp->code == jit_code_epilog);
1203                         word = movi_p(rn(node->u.w), temp->u.w);
1204                         patch(word, node);
1205                     }
1206                 }
1207                 else
1208                     movi(rn(node->u.w), node->v.w);
1209                 break;
1210                 case_rrr(lt,);
1211                 case_rrw(lt,);
1212                 case_rrr(lt, _u);
1213                 case_rrw(lt, _u);
1214                 case_rrr(le,);
1215                 case_rrw(le,);
1216                 case_rrr(le, _u);
1217                 case_rrw(le, _u);
1218                 case_rrr(eq,);
1219                 case_rrw(eq,);
1220                 case_rrr(ge,);
1221                 case_rrw(ge,);
1222                 case_rrr(ge, _u);
1223                 case_rrw(ge, _u);
1224                 case_rrr(gt,);
1225                 case_rrw(gt,);
1226                 case_rrr(gt, _u);
1227                 case_rrw(gt, _u);
1228                 case_rrr(ne,);
1229                 case_rrw(ne,);
1230                 case_brr(blt,);
1231                 case_brw(blt,);
1232                 case_brr(blt, _u);
1233                 case_brw(blt, _u);
1234                 case_brr(ble,);
1235                 case_brw(ble,);
1236                 case_brr(ble, _u);
1237                 case_brw(ble, _u);
1238                 case_brr(beq,);
1239                 case_brw(beq,);
1240                 case_brr(bge,);
1241                 case_brw(bge,);
1242                 case_brr(bge, _u);
1243                 case_brw(bge, _u);
1244                 case_brr(bgt,);
1245                 case_brw(bgt,);
1246                 case_brr(bgt, _u);
1247                 case_brw(bgt, _u);
1248                 case_brr(bne,);
1249                 case_brw(bne,);
1250                 case_brr(boadd,);
1251                 case_brw(boadd,);
1252                 case_brr(boadd, _u);
1253                 case_brw(boadd, _u);
1254                 case_brr(bxadd,);
1255                 case_brw(bxadd,);
1256                 case_brr(bxadd, _u);
1257                 case_brw(bxadd, _u);
1258                 case_brr(bosub,);
1259                 case_brw(bosub,);
1260                 case_brr(bosub, _u);
1261                 case_brw(bosub, _u);
1262                 case_brr(bxsub,);
1263                 case_brw(bxsub,);
1264                 case_brr(bxsub, _u);
1265                 case_brw(bxsub, _u);
1266                 case_brr(bms,);
1267                 case_brw(bms,);
1268                 case_brr(bmc,);
1269                 case_brw(bmc,);
1270                 case_rrr(add, _f);
1271                 case_rrf(add);
1272                 case_rrr(sub, _f);
1273                 case_rrf(sub);
1274                 case_rrf(rsb);
1275                 case_rrr(mul, _f);
1276                 case_rrf(mul);
1277                 case_rrr(div, _f);
1278                 case_rrf(div);
1279                 case_rr(abs, _f);
1280                 case_rr(neg, _f);
1281                 case_rr(sqrt, _f);
1282                 case_rr(ext, _f);
1283                 case_rr(ld, _f);
1284                 case_rw(ld, _f);
1285                 case_rrr(ldx, _f);
1286                 case_rrw(ldx, _f);
1287                 case_rr(st, _f);
1288                 case_wr(st, _f);
1289                 case_rrr(stx, _f);
1290                 case_wrr(stx, _f);
1291                 case_rr(mov, _f);
1292             case jit_code_movi_f:
1293                 assert_data(node);
1294                 movi_f(rn(node->u.w), node->v.f);
1295                 break;
1296                 case_rr(ext, _d_f);
1297                 case_rrr(lt, _f);
1298                 case_rrf(lt);
1299                 case_rrr(le, _f);
1300                 case_rrf(le);
1301                 case_rrr(eq, _f);
1302                 case_rrf(eq);
1303                 case_rrr(ge, _f);
1304                 case_rrf(ge);
1305                 case_rrr(gt, _f);
1306                 case_rrf(gt);
1307                 case_rrr(ne, _f);
1308                 case_rrf(ne);
1309                 case_rrr(unlt, _f);
1310                 case_rrf(unlt);
1311                 case_rrr(unle, _f);
1312                 case_rrf(unle);
1313                 case_rrr(uneq, _f);
1314                 case_rrf(uneq);
1315                 case_rrr(unge, _f);
1316                 case_rrf(unge);
1317                 case_rrr(ungt, _f);
1318                 case_rrf(ungt);
1319                 case_rrr(ltgt, _f);
1320                 case_rrf(ltgt);
1321                 case_rrr(ord, _f);
1322                 case_rrf(ord);
1323                 case_rrr(unord, _f);
1324                 case_rrf(unord);
1325                 case_brr(blt, _f);
1326                 case_brf(blt);
1327                 case_brr(ble, _f);
1328                 case_brf(ble);
1329                 case_brr(beq, _f);
1330                 case_brf(beq);
1331                 case_brr(bge, _f);
1332                 case_brf(bge);
1333                 case_brr(bgt, _f);
1334                 case_brf(bgt);
1335                 case_brr(bne, _f);
1336                 case_brf(bne);
1337                 case_brr(bunlt, _f);
1338                 case_brf(bunlt);
1339                 case_brr(bunle, _f);
1340                 case_brf(bunle);
1341                 case_brr(buneq, _f);
1342                 case_brf(buneq);
1343                 case_brr(bunge, _f);
1344                 case_brf(bunge);
1345                 case_brr(bungt, _f);
1346                 case_brf(bungt);
1347                 case_brr(bltgt, _f);
1348                 case_brf(bltgt);
1349                 case_brr(bord, _f);
1350                 case_brf(bord);
1351                 case_brr(bunord, _f);
1352                 case_brf(bunord);
1353                 case_rrr(add, _d);
1354                 case_rrd(add);
1355                 case_rrr(sub, _d);
1356                 case_rrd(sub);
1357                 case_rrd(rsb);
1358                 case_rrr(mul, _d);
1359                 case_rrd(mul);
1360                 case_rrr(div, _d);
1361                 case_rrd(div);
1362                 case_rr(abs, _d);
1363                 case_rr(neg, _d);
1364                 case_rr(sqrt, _d);
1365                 case_rr(ext, _d);
1366                 case_rr(ld, _d);
1367                 case_rw(ld, _d);
1368                 case_rrr(ldx, _d);
1369                 case_rrw(ldx, _d);
1370                 case_rr(st, _d);
1371                 case_wr(st, _d);
1372                 case_rrr(stx, _d);
1373                 case_wrr(stx, _d);
1374                 case_rr(mov, _d);
1375             case jit_code_movi_d:
1376                 assert_data(node);
1377                 movi_d(rn(node->u.w), node->v.d);
1378                 break;
1379                 case_rr(ext, _f_d);
1380                 case_rrr(lt, _d);
1381                 case_rrd(lt);
1382                 case_rrr(le, _d);
1383                 case_rrd(le);
1384                 case_rrr(eq, _d);
1385                 case_rrd(eq);
1386                 case_rrr(ge, _d);
1387                 case_rrd(ge);
1388                 case_rrr(gt, _d);
1389                 case_rrd(gt);
1390                 case_rrr(ne, _d);
1391                 case_rrd(ne);
1392                 case_rrr(unlt, _d);
1393                 case_rrd(unlt);
1394                 case_rrr(unle, _d);
1395                 case_rrd(unle);
1396                 case_rrr(uneq, _d);
1397                 case_rrd(uneq);
1398                 case_rrr(unge, _d);
1399                 case_rrd(unge);
1400                 case_rrr(ungt, _d);
1401                 case_rrd(ungt);
1402                 case_rrr(ltgt, _d);
1403                 case_rrd(ltgt);
1404                 case_rrr(ord, _d);
1405                 case_rrd(ord);
1406                 case_rrr(unord, _d);
1407                 case_rrd(unord);
1408                 case_brr(blt, _d);
1409                 case_brd(blt);
1410                 case_brr(ble, _d);
1411                 case_brd(ble);
1412                 case_brr(beq, _d);
1413                 case_brd(beq);
1414                 case_brr(bge, _d);
1415                 case_brd(bge);
1416                 case_brr(bgt, _d);
1417                 case_brd(bgt);
1418                 case_brr(bne, _d);
1419                 case_brd(bne);
1420                 case_brr(bunlt, _d);
1421                 case_brd(bunlt);
1422                 case_brr(bunle, _d);
1423                 case_brd(bunle);
1424                 case_brr(buneq, _d);
1425                 case_brd(buneq);
1426                 case_brr(bunge, _d);
1427                 case_brd(bunge);
1428                 case_brr(bungt, _d);
1429                 case_brd(bungt);
1430                 case_brr(bltgt, _d);
1431                 case_brd(bltgt);
1432                 case_brr(bord, _d);
1433                 case_brd(bord);
1434                 case_brr(bunord, _d);
1435                 case_brd(bunord);
1436             case jit_code_jmpr:
1437                 jmpr(rn(node->u.w));
1438                 break;
1439             case jit_code_jmpi:
1440                 if (node->flag & jit_flag_node) {
1441                     temp = node->u.n;
1442                     assert(temp->code == jit_code_label ||
1443                            temp->code == jit_code_epilog);
1444                     if (temp->flag & jit_flag_patch)
1445                         jmpi(temp->u.w);
1446                     else {
1447                         word = jmpi_p(_jit->pc.w);
1448                         patch(word, node);
1449                     }
1450                 }
1451                 else
1452                     jmpi(node->u.w);
1453                 break;
1454             case jit_code_callr:
1455                 callr(rn(node->u.w));
1456                 break;
1457             case jit_code_calli:
1458                 if (node->flag & jit_flag_node) {
1459                     temp = node->u.n;
1460                     assert(temp->code == jit_code_label ||
1461                            temp->code == jit_code_epilog);
1462                     if (temp->flag & jit_flag_patch)
1463                         calli(temp->u.w);
1464                     else {
1465                         word = calli_p(_jit->pc.w);
1466                         patch(word, node);
1467                     }
1468                 }
1469                 else
1470                     calli(node->u.w);
1471                 break;
1472             case jit_code_prolog:
1473                 _jitc->function = _jitc->functions.ptr + node->w.w;
1474                 undo.node = node;
1475                 undo.word = _jit->pc.w;
1476 #if DEVEL_DISASSEMBLER
1477                 undo.prevw = prevw;
1478 #endif
1479                 undo.patch_offset = _jitc->patches.offset;
1480             restart_function:
1481                 _jitc->again = 0;
1482                 prolog(node);
1483                 break;
1484             case jit_code_epilog:
1485                 assert(_jitc->function == _jitc->functions.ptr + node->w.w);
1486                 if (_jitc->again) {
1487                     for (temp = undo.node->next;
1488                          temp != node; temp = temp->next) {
1489                         if (temp->code == jit_code_label ||
1490                             temp->code == jit_code_epilog)
1491                             temp->flag &= ~jit_flag_patch;
1492                     }
1493                     temp->flag &= ~jit_flag_patch;
1494                     node = undo.node;
1495                     _jit->pc.w = undo.word;
1496 #if DEVEL_DISASSEMBLER
1497                     prevw = undo.prevw;
1498 #endif
1499                     _jitc->patches.offset = undo.patch_offset;
1500                     goto restart_function;
1501                 }
1502                 /* remember label is defined */
1503                 node->flag |= jit_flag_patch;
1504                 node->u.w = _jit->pc.w;
1505                 epilog(node);
1506                 _jitc->function = NULL;
1507                 break;
1508             case jit_code_movr_w_f:
1509                 movr_w_f(rn(node->u.w), rn(node->v.w));
1510                 break;
1511             case jit_code_movr_f_w:
1512                 movr_f_w(rn(node->u.w), rn(node->v.w));
1513                 break;
1514             case jit_code_movi_f_w:
1515                 assert_data(node);
1516                 movi_f_w(rn(node->u.w), node->v.f);
1517                 break;
1518             case jit_code_movr_w_d:
1519                 movr_w_d(rn(node->u.w), rn(node->v.w));
1520                 break;
1521             case jit_code_movr_d_w:
1522                 movr_d_w(rn(node->u.w), rn(node->v.w));
1523                 break;
1524             case jit_code_movi_d_w:
1525                 assert_data(node);
1526                 movi_d_w(rn(node->u.w), node->v.d);
1527                 break;
1528             case jit_code_va_start:
1529                 vastart(rn(node->u.w));
1530                 break;
1531             case jit_code_va_arg:
1532                 vaarg(rn(node->u.w), rn(node->v.w));
1533                 break;
1534             case jit_code_va_arg_d:
1535                 vaarg_d(rn(node->u.w), rn(node->v.w));
1536                 break;
1537             case jit_code_live:                 case jit_code_ellipsis:
1538             case jit_code_va_push:
1539             case jit_code_allocai:              case jit_code_allocar:
1540             case jit_code_arg:
1541             case jit_code_arg_f:                case jit_code_arg_d:
1542             case jit_code_va_end:
1543             case jit_code_ret:
1544             case jit_code_retr:                 case jit_code_reti:
1545             case jit_code_retr_f:               case jit_code_reti_f:
1546             case jit_code_retr_d:               case jit_code_reti_d:
1547             case jit_code_getarg_c:             case jit_code_getarg_uc:
1548             case jit_code_getarg_s:             case jit_code_getarg_us:
1549             case jit_code_getarg_i:             case jit_code_getarg_ui:
1550             case jit_code_getarg_l:
1551             case jit_code_getarg_f:             case jit_code_getarg_d:
1552             case jit_code_putargr:              case jit_code_putargi:
1553             case jit_code_putargr_f:            case jit_code_putargi_f:
1554             case jit_code_putargr_d:            case jit_code_putargi_d:
1555             case jit_code_pushargr:             case jit_code_pushargi:
1556             case jit_code_pushargr_f:           case jit_code_pushargi_f:
1557             case jit_code_pushargr_d:           case jit_code_pushargi_d:
1558             case jit_code_retval_c:             case jit_code_retval_uc:
1559             case jit_code_retval_s:             case jit_code_retval_us:
1560             case jit_code_retval_i:
1561             case jit_code_retval_ui:            case jit_code_retval_l:
1562             case jit_code_retval_f:             case jit_code_retval_d:
1563             case jit_code_prepare:
1564             case jit_code_finishr:              case jit_code_finishi:
1565                 break;
1566             default:
1567                 abort();
1568         }
1569         if (jit_carry != _NOREG) {
1570             switch (node->code) {
1571                 case jit_code_note:
1572                 case jit_code_addcr:            case jit_code_addci:
1573                 case jit_code_addxr:            case jit_code_addxi:
1574                 case jit_code_subcr:            case jit_code_subci:
1575                 case jit_code_subxr:            case jit_code_subxi:
1576                     break;
1577                 default:
1578                     jit_unget_reg(jit_carry);
1579                     jit_carry = _NOREG;
1580                     break;
1581             }
1582         }
1583         jit_regarg_clr(node, value);
1584         assert(_jitc->regarg == 0 ||
1585                (jit_carry != _NOREG && _jitc->regarg == (1 << jit_carry)));
1586         assert(_jitc->synth == 0);
1587         /* update register live state */
1588         jit_reglive(node);
1589     }
1590 #undef case_brw
1591 #undef case_brr
1592 #undef case_wrr
1593 #undef case_rrw
1594 #undef case_rrr
1595 #undef case_wr
1596 #undef case_rw
1597 #undef case_rr
1598
1599 #if __WORDSIZE == 64
1600     /* Record all constants to be patched */
1601     for (offset = 0; offset < _jitc->patches.offset; offset++) {
1602         node = _jitc->patches.ptr[offset].node;
1603         value = node->code == jit_code_movi ? node->v.n->u.w : node->u.n->u.w;
1604         put_const(value);
1605     }
1606     /* Record all direct constants */
1607     for (offset = 0; offset < _jitc->consts.vector.offset; offset++)
1608         put_const(_jitc->consts.vector.values[offset]);
1609     /* Now actually inject constants at the end of code buffer */
1610     if (_jitc->consts.hash.count) {
1611         jit_const_t     *entry;
1612         /* Insert nop if aligned at 4 bytes */
1613         if (_jit->pc.w % sizeof(jit_word_t))
1614             nop(_jit->pc.w % sizeof(jit_word_t));
1615         for (offset = 0; offset < _jitc->consts.hash.size; offset++) {
1616             entry = _jitc->consts.hash.table[offset];
1617             for (; entry; entry = entry->next) {
1618                 /* Make sure to not write out of bounds */
1619                 if (_jit->pc.uc >= _jitc->code.end)
1620                     return (NULL);
1621                 entry->address = _jit->pc.w;
1622                 *_jit->pc.ul++ = entry->value;
1623             }
1624         }
1625     }
1626 #endif
1627
1628     for (offset = 0; offset < _jitc->patches.offset; offset++) {
1629         node = _jitc->patches.ptr[offset].node;
1630         word = _jitc->patches.ptr[offset].inst;
1631         value = node->code == jit_code_movi ? node->v.n->u.w : node->u.n->u.w;
1632         patch_at(word, value);
1633     }
1634
1635 #if __WORDSIZE == 64
1636     /* Patch direct complex constants */
1637     if (_jitc->consts.vector.instrs) {
1638         for (offset = 0; offset < _jitc->consts.vector.offset; offset++)
1639             patch_at(_jitc->consts.vector.instrs[offset],
1640                      _jitc->consts.vector.values[offset]);
1641         jit_free((jit_pointer_t *)&_jitc->consts.vector.instrs);
1642         jit_free((jit_pointer_t *)&_jitc->consts.vector.values);
1643     }
1644
1645     /* Hash table no longer need */
1646     if (_jitc->consts.hash.table) {
1647         jit_free((jit_pointer_t *)&_jitc->consts.hash.table);
1648         for (offset = 0; offset < _jitc->consts.pool.length; offset++)
1649             jit_free((jit_pointer_t *)_jitc->consts.pool.ptr + offset);
1650         jit_free((jit_pointer_t *)&_jitc->consts.pool.ptr);
1651     }
1652 #endif
1653
1654     jit_flush(_jit->code.ptr, _jit->pc.uc);
1655
1656     return (_jit->code.ptr);
1657 }
1658
1659 #define CODE                            1
1660 #  include "jit_riscv-cpu.c"
1661 #  include "jit_riscv-fpu.c"
1662 #undef CODE
1663
1664 static void
1665 _load_const(jit_state_t *_jit, jit_int32_t reg, jit_word_t value)
1666 {
1667     if (_jitc->consts.vector.offset >= _jitc->consts.vector.length) {
1668         jit_word_t      new_size = _jitc->consts.vector.length *
1669                                    2 * sizeof(jit_word_t);
1670         jit_realloc((jit_pointer_t *)&_jitc->consts.vector.instrs,
1671                     _jitc->consts.vector.length * sizeof(jit_word_t), new_size);
1672         jit_realloc((jit_pointer_t *)&_jitc->consts.vector.values,
1673                     _jitc->consts.vector.length * sizeof(jit_word_t), new_size);
1674         _jitc->consts.vector.length *= 2;
1675     }
1676     _jitc->consts.vector.instrs[_jitc->consts.vector.offset] = _jit->pc.w;
1677     _jitc->consts.vector.values[_jitc->consts.vector.offset] = value;
1678     ++_jitc->consts.vector.offset;
1679     /* Resolve later the pc relative address */
1680     put_const(value);
1681     AUIPC(reg, 0);
1682     ADDI(reg, reg, 0);
1683     LD(reg, reg, 0);
1684 }
1685
1686 static jit_word_t
1687 hash_const(jit_word_t value)
1688 {
1689     const jit_uint8_t   *ptr;
1690     jit_word_t           i, key;
1691     for (i = key = 0, ptr = (jit_uint8_t *)&value; i < 4; ++i)
1692         key = (key << (key & 1)) ^ ptr[i];
1693     return (key);
1694
1695 }
1696
1697 static void
1698 _put_const(jit_state_t *_jit, jit_word_t value)
1699 {
1700     jit_word_t           key;
1701     jit_const_t         *entry;
1702
1703     /* Check if already inserted in table */
1704     key = hash_const(value) % _jitc->consts.hash.size;
1705     for (entry = _jitc->consts.hash.table[key]; entry; entry = entry->next) {
1706         if (entry->value == value)
1707             return;
1708     }
1709
1710     /* Check if need to increase pool size */
1711     if (_jitc->consts.pool.list->next == NULL) {
1712         jit_const_t     *list;
1713         jit_word_t       offset;
1714         jit_word_t       new_size = (_jitc->consts.pool.length + 1) *
1715                                     sizeof(jit_const_t*);
1716         jit_realloc((jit_pointer_t *)&_jitc->consts.pool.ptr,
1717                     _jitc->consts.pool.length * sizeof(jit_const_t*), new_size);
1718         jit_alloc((jit_pointer_t *)
1719                   _jitc->consts.pool.ptr + _jitc->consts.pool.length,
1720                   1024 * sizeof(jit_const_t));
1721         list = _jitc->consts.pool.ptr[_jitc->consts.pool.length];
1722         _jitc->consts.pool.list->next = list;
1723         for (offset = 0; offset < 1023; ++offset, ++list)
1724             list->next = list + 1;
1725         list->next = NULL;
1726         ++_jitc->consts.pool.length;
1727     }
1728
1729     /* Rehash if more than 75% used table */
1730     if (_jitc->consts.hash.count > (_jitc->consts.hash.size / 4) * 3) {
1731         jit_word_t        i, k;
1732         jit_const_t      *next;
1733         jit_const_t     **table;
1734         jit_alloc((jit_pointer_t *)&table,
1735                   _jitc->consts.hash.size * 2 * sizeof(jit_const_t *));
1736         for (i = 0; i < _jitc->consts.hash.size; ++i) {
1737             for (entry = _jitc->consts.hash.table[i]; entry; entry = next) {
1738                 next = entry->next;
1739                 k = hash_const(entry->value) % (_jitc->consts.hash.size * 2);
1740                 entry->next = table[k];
1741                 table[k] = entry;
1742             }
1743         }
1744         jit_free((jit_pointer_t *)&_jitc->consts.hash.table);
1745         _jitc->consts.hash.size *= 2;
1746         _jitc->consts.hash.table = table;
1747     }
1748
1749     /* Insert in hash */
1750     entry = _jitc->consts.pool.list;
1751     _jitc->consts.pool.list =  entry->next;
1752     ++_jitc->consts.hash.count;
1753     entry->value = value;
1754     entry->next = _jitc->consts.hash.table[key];
1755     _jitc->consts.hash.table[key] = entry;
1756 }
1757
1758 static jit_word_t
1759 _get_const(jit_state_t *_jit, jit_word_t value)
1760 {
1761     jit_word_t           key;
1762     jit_const_t         *entry;
1763     key = hash_const(value) % _jitc->consts.hash.size;
1764     for (entry = _jitc->consts.hash.table[key]; entry; entry = entry->next) {
1765         if (entry->value == value)
1766             return (entry->address);
1767     }
1768     /* Only the final patch should call get_const() */
1769     abort();
1770 }
1771
1772 void
1773 jit_flush(void *fptr, void *tptr)
1774 {
1775 #if defined(__GNUC__)
1776     jit_word_t          f, t, s;
1777
1778     s = sysconf(_SC_PAGE_SIZE);
1779     f = (jit_word_t)fptr & -s;
1780     t = (((jit_word_t)tptr) + s - 1) & -s;
1781     __clear_cache((void *)f, (void *)t);
1782 #endif
1783 }
1784
1785 void
1786 _emit_ldxi(jit_state_t *_jit, jit_int32_t r0, jit_int32_t r1, jit_word_t i0)
1787 {
1788     ldxi(rn(r0), rn(r1), i0);
1789 }
1790
1791 void
1792 _emit_stxi(jit_state_t *_jit, jit_word_t i0, jit_int32_t r0, jit_int32_t r1)
1793 {
1794     stxi(i0, rn(r0), rn(r1));
1795 }
1796
1797 void
1798 _emit_ldxi_d(jit_state_t *_jit, jit_int32_t r0, jit_int32_t r1, jit_word_t i0)
1799 {
1800     ldxi_d(rn(r0), rn(r1), i0);
1801 }
1802
1803 void
1804 _emit_stxi_d(jit_state_t *_jit, jit_word_t i0, jit_int32_t r0, jit_int32_t r1)
1805 {
1806     stxi_d(i0, rn(r0), rn(r1));
1807 }
1808
1809 static void
1810 _patch(jit_state_t *_jit, jit_word_t instr, jit_node_t *node)
1811 {
1812     jit_int32_t          flag;
1813
1814     assert(node->flag & jit_flag_node);
1815     if (node->code == jit_code_movi)
1816         flag = node->v.n->flag;
1817     else
1818         flag = node->u.n->flag;
1819     assert(!(flag & jit_flag_patch));
1820     if (_jitc->patches.offset >= _jitc->patches.length) {
1821         jit_realloc((jit_pointer_t *)&_jitc->patches.ptr,
1822                     _jitc->patches.length * sizeof(jit_patch_t),
1823                     (_jitc->patches.length + 1024) * sizeof(jit_patch_t));
1824         _jitc->patches.length += 1024;
1825     }
1826     _jitc->patches.ptr[_jitc->patches.offset].inst = instr;
1827     _jitc->patches.ptr[_jitc->patches.offset].node = node;
1828     ++_jitc->patches.offset;
1829 }