deps/lightning/lib/jit_riscv.c

   1 /*
   2  * Copyright (C) 2019  Free Software Foundation, Inc.
   3  *
   4  * This file is part of GNU lightning.
   5  *
   6  * GNU lightning is free software; you can redistribute it and/or modify it
   7  * under the terms of the GNU Lesser General Public License as published
   8  * by the Free Software Foundation; either version 3, or (at your option)
   9  * any later version.
  10  *
  11  * GNU lightning is distributed in the hope that it will be useful, but
  12  * WITHOUT ANY WARRANTY; without even the implied warranty of MERCHANTABILITY
  13  * or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU Lesser General Public
  14  * License for more details.
  15  *
  16  * Authors:
  17  *      Paulo Cesar Pereira de Andrade
  18  */
  19
  20 #define jit_arg_reg_p(i)                ((i) >= 0 && (i) < 8)
  21 #define jit_arg_f_reg_p(i)              ((i) >= 0 && (i) < 8)
  22
  23 /*
  24  * Types
  25  */
  26 typedef jit_pointer_t jit_va_list_t;
  27
  28 /*
  29  * Prototypes
  30  */
  31 #define patch(instr, node)              _patch(_jit, instr, node)
  32 static void _patch(jit_state_t*,jit_word_t,jit_node_t*);
  33
  34 #define PROTO                           1
  35 #  include "jit_riscv-cpu.c"
  36 #  include "jit_riscv-fpu.c"
  37 #undef PROTO
  38
  39 /*
  40  * Initialization
  41  */
  42 jit_register_t          _rvs[] = {
  43     { 0x00,                             "zero" },
  44     { 0x01,                             "ra" },
  45     { 0x02,                             "sp" },
  46     { 0x03,                             "gp" },
  47 #if 0           /* Pretend it does not exist, so _NOREG can be used in
  48                  * a 64 bit bitmask */
  49     { 0x04,                             "tp" },
  50 #endif
  51     { rc(gpr) | 0x05,                   "t0" },
  52     { rc(gpr) | 0x06,                   "t1" },
  53     { rc(gpr) | 0x07,                   "t2" },
  54     { rc(gpr) | 0x1c,                   "t3" },
  55     { rc(gpr) | 0x1d,                   "t4" },
  56     { rc(gpr) | 0x1e,                   "t5" },
  57     { rc(gpr) | 0x1f,                   "t6" },
  58     { 0x08,                             "fp" },
  59     { rc(sav) | rc(gpr) | 0x09,         "s1" },
  60     { rc(sav) | rc(gpr) | 0x12,         "s2" },
  61     { rc(sav) | rc(gpr) | 0x13,         "s3" },
  62     { rc(sav) | rc(gpr) | 0x14,         "s4" },
  63     { rc(sav) | rc(gpr) | 0x15,         "s5" },
  64     { rc(sav) | rc(gpr) | 0x16,         "s6" },
  65     { rc(sav) | rc(gpr) | 0x17,         "s7" },
  66     { rc(sav) | rc(gpr) | 0x18,         "s8" },
  67     { rc(sav) | rc(gpr) | 0x19,         "s9" },
  68     { rc(sav) | rc(gpr) | 0x1a,         "s10" },
  69     { rc(sav) | rc(gpr) | 0x1b,         "s11" },
  70     { rc(arg) | rc(gpr) | 0x11,         "a7" },
  71     { rc(arg) | rc(gpr) | 0x10,         "a6" },
  72     { rc(arg) | rc(gpr) | 0x0f,         "a5" },
  73     { rc(arg) | rc(gpr) | 0x0e,         "a4" },
  74     { rc(arg) | rc(gpr) | 0x0d,         "a3" },
  75     { rc(arg) | rc(gpr) | 0x0c,         "a2" },
  76     { rc(arg) | rc(gpr) | 0x0b,         "a1" },
  77     { rc(arg) | rc(gpr) | 0x0a,         "a0" },
  78     { rc(fpr) | 0x00,                   "ft0" },
  79     { rc(fpr) | 0x01,                   "ft1" },
  80     { rc(fpr) | 0x02,                   "ft2" },
  81     { rc(fpr) | 0x03,                   "ft3" },
  82     { rc(fpr) | 0x04,                   "ft4" },
  83     { rc(fpr) | 0x05,                   "ft5" },
  84     { rc(fpr) | 0x06,                   "ft6" },
  85     { rc(fpr) | 0x07,                   "ft7" },
  86     { rc(fpr) | 0x1c,                   "ft8" },
  87     { rc(fpr) | 0x1d,                   "ft9" },
  88     { rc(fpr) | 0x1e,                   "ft10" },
  89     { rc(fpr) | 0x1f,                   "ft11" },
  90     { rc(sav) | rc(fpr) | 0x08,         "fs0" },
  91     { rc(sav) | rc(fpr) | 0x09,         "fs1" },
  92     { rc(sav) | rc(fpr) | 0x12,         "fs2" },
  93     { rc(sav) | rc(fpr) | 0x13,         "fs3" },
  94     { rc(sav) | rc(fpr) | 0x14,         "fs4" },
  95     { rc(sav) | rc(fpr) | 0x15,         "fs5" },
  96     { rc(sav) | rc(fpr) | 0x16,         "fs6" },
  97     { rc(sav) | rc(fpr) | 0x17,         "fs7" },
  98     { rc(sav) | rc(fpr) | 0x18,         "fs8" },
  99     { rc(sav) | rc(fpr) | 0x19,         "fs9" },
 100     { rc(sav) | rc(fpr) | 0x1a,         "fs10" },
 101     { rc(sav) | rc(fpr) | 0x1b,         "fs11" },
 102     { rc(arg) | rc(fpr) | 0x11,         "fa7" },
 103     { rc(arg) | rc(fpr) | 0x10,         "fa6" },
 104     { rc(arg) | rc(fpr) | 0x0f,         "fa5" },
 105     { rc(arg) | rc(fpr) | 0x0e,         "fa4" },
 106     { rc(arg) | rc(fpr) | 0x0d,         "fa3" },
 107     { rc(arg) | rc(fpr) | 0x0c,         "fa2" },
 108     { rc(arg) | rc(fpr) | 0x0b,         "fa1" },
 109     { rc(arg) | rc(fpr) | 0x0a,         "fa0" },
 110     { _NOREG,                           "<none>" },
 111 };
 112
 113 /*
 114  * Implementation
 115  */
 116 void
 117 jit_get_cpu(void)
 118 {
 119 }
 120
 121 void
 122 _jit_init(jit_state_t *_jit)
 123 {
 124     _jitc->reglen = jit_size(_rvs) - 1;
 125     jit_carry = _NOREG;
 126 }
 127
 128 void
 129 _jit_prolog(jit_state_t *_jit)
 130 {
 131     jit_int32_t          offset;
 132
 133     if (_jitc->function)
 134         jit_epilog();
 135     assert(jit_regset_cmp_ui(&_jitc->regarg, 0) == 0);
 136     jit_regset_set_ui(&_jitc->regsav, 0);
 137     offset = _jitc->functions.offset;
 138     if (offset >= _jitc->functions.length) {
 139         jit_realloc((jit_pointer_t *)&_jitc->functions.ptr,
 140                     _jitc->functions.length * sizeof(jit_function_t),
 141                     (_jitc->functions.length + 16) * sizeof(jit_function_t));
 142         _jitc->functions.length += 16;
 143     }
 144     _jitc->function = _jitc->functions.ptr + _jitc->functions.offset++;
 145     _jitc->function->self.size = stack_framesize;
 146     _jitc->function->self.argi = _jitc->function->self.argf =
 147         _jitc->function->self.alen = 0;
 148     _jitc->function->self.aoff = 0;
 149     _jitc->function->self.call = jit_call_default;
 150     jit_alloc((jit_pointer_t *)&_jitc->function->regoff,
 151               _jitc->reglen * sizeof(jit_int32_t));
 152
 153     /* _no_link here does not mean the jit_link() call can be removed
 154      * by rewriting as:
 155      * _jitc->function->prolog = jit_new_node(jit_code_prolog);
 156      */
 157     _jitc->function->prolog = jit_new_node_no_link(jit_code_prolog);
 158     jit_link(_jitc->function->prolog);
 159     _jitc->function->prolog->w.w = offset;
 160     _jitc->function->epilog = jit_new_node_no_link(jit_code_epilog);
 161     /*  u:      label value
 162      *  v:      offset in blocks vector
 163      *  w:      offset in functions vector
 164      */
 165     _jitc->function->epilog->w.w = offset;
 166
 167     jit_regset_new(&_jitc->function->regset);
 168 }
 169
 170 jit_int32_t
 171 _jit_allocai(jit_state_t *_jit, jit_int32_t length)
 172 {
 173     assert(_jitc->function);
 174     switch (length) {
 175         case 0: case 1:                                         break;
 176         case 2:         _jitc->function->self.aoff &= -2;       break;
 177         case 3: case 4: _jitc->function->self.aoff &= -4;       break;
 178         default:        _jitc->function->self.aoff &= -8;       break;
 179     }
 180     _jitc->function->self.aoff -= length;
 181     if (!_jitc->realize) {
 182         jit_inc_synth_ww(allocai, _jitc->function->self.aoff, length);
 183         jit_dec_synth();
 184     }
 185     return (_jitc->function->self.aoff);
 186 }
 187
 188 void
 189 _jit_allocar(jit_state_t *_jit, jit_int32_t u, jit_int32_t v)
 190 {
 191     jit_int32_t          r0;
 192     assert(_jitc->function);
 193     jit_inc_synth_ww(allocar, u, v);
 194     if (!_jitc->function->allocar) {
 195         _jitc->function->aoffoff = jit_allocai(sizeof(jit_int32_t));
 196         _jitc->function->allocar = 1;
 197     }
 198     r0 = jit_get_reg(jit_class_gpr);
 199     jit_negr(r0, v);
 200     jit_andi(r0, r0, -16);
 201     jit_ldxi_i(u, JIT_FP, _jitc->function->aoffoff);
 202     jit_addr(u, u, r0);
 203     jit_addr(JIT_SP, JIT_SP, r0);
 204     jit_stxi_i(_jitc->function->aoffoff, JIT_FP, u);
 205     jit_unget_reg(r0);
 206     jit_dec_synth();
 207 }
 208
 209 void
 210 _jit_ret(jit_state_t *_jit)
 211 {
 212     jit_node_t          *instr;
 213     assert(_jitc->function);
 214     jit_inc_synth(ret);
 215     /* jump to epilog */
 216     instr = jit_jmpi();
 217     jit_patch_at(instr, _jitc->function->epilog);
 218     jit_dec_synth();
 219 }
 220
 221 void
 222 _jit_retr(jit_state_t *_jit, jit_int32_t u)
 223 {
 224     jit_inc_synth_w(retr, u);
 225     if (JIT_RET != u)
 226         jit_movr(JIT_RET, u);
 227     jit_live(JIT_RET);
 228     jit_ret();
 229     jit_dec_synth();
 230 }
 231
 232 void
 233 _jit_reti(jit_state_t *_jit, jit_word_t u)
 234 {
 235     jit_inc_synth_w(reti, u);
 236     jit_movi(JIT_RET, u);
 237     jit_ret();
 238     jit_dec_synth();
 239 }
 240
 241 void
 242 _jit_retr_f(jit_state_t *_jit, jit_int32_t u)
 243 {
 244     jit_inc_synth_w(retr_f, u);
 245     if (u != JIT_FRET)
 246         jit_movr_f(JIT_FRET, u);
 247     else
 248         jit_live(JIT_FRET);
 249     jit_ret();
 250     jit_dec_synth();
 251 }
 252
 253 void
 254 _jit_reti_f(jit_state_t *_jit, jit_float32_t u)
 255 {
 256     jit_inc_synth_f(reti_f, u);
 257     jit_movi_f(JIT_FRET, u);
 258     jit_ret();
 259     jit_dec_synth();
 260 }
 261
 262 void
 263 _jit_retr_d(jit_state_t *_jit, jit_int32_t u)
 264 {
 265     jit_inc_synth_w(retr_d, u);
 266     if (u != JIT_FRET)
 267         jit_movr_d(JIT_FRET, u);
 268     else
 269         jit_live(JIT_FRET);
 270     jit_ret();
 271     jit_dec_synth();
 272 }
 273
 274 void
 275 _jit_reti_d(jit_state_t *_jit, jit_float64_t u)
 276 {
 277     jit_inc_synth_d(reti_d, u);
 278     jit_movi_d(JIT_FRET, u);
 279     jit_ret();
 280     jit_dec_synth();
 281 }
 282
 283 void
 284 _jit_epilog(jit_state_t *_jit)
 285 {
 286     assert(_jitc->function);
 287     assert(_jitc->function->epilog->next == NULL);
 288     jit_link(_jitc->function->epilog);
 289     _jitc->function = NULL;
 290 }
 291
 292 jit_bool_t
 293 _jit_arg_register_p(jit_state_t *_jit, jit_node_t *u)
 294 {
 295     if (u->code == jit_code_arg)
 296         return (jit_arg_reg_p(u->u.w));
 297     assert(u->code == jit_code_arg_f || u->code == jit_code_arg_d);
 298     return (jit_arg_f_reg_p(u->u.w));
 299 }
 300
 301 void
 302 _jit_ellipsis(jit_state_t *_jit)
 303 {
 304     jit_inc_synth(ellipsis);
 305     if (_jitc->prepare) {
 306         jit_link_prepare();
 307         assert(!(_jitc->function->call.call & jit_call_varargs));
 308         _jitc->function->call.call |= jit_call_varargs;
 309     }
 310     else {
 311         jit_link_prolog();
 312         assert(!(_jitc->function->self.call & jit_call_varargs));
 313         _jitc->function->self.call |= jit_call_varargs;
 314         _jitc->function->vagp = _jitc->function->self.argi;
 315     }
 316     jit_dec_synth();
 317 }
 318
 319 void
 320 _jit_va_push(jit_state_t *_jit, jit_int32_t u)
 321 {
 322     jit_inc_synth_w(va_push, u);
 323     jit_pushargr(u);
 324     jit_dec_synth();
 325 }
 326
 327 jit_node_t *
 328 _jit_arg(jit_state_t *_jit)
 329 {
 330     jit_node_t          *node;
 331     jit_int32_t          offset;
 332     assert(_jitc->function);
 333     assert(!(_jitc->function->self.call & jit_call_varargs));
 334     if (jit_arg_reg_p(_jitc->function->self.argi))
 335         offset = _jitc->function->self.argi++;
 336     else {
 337         offset = _jitc->function->self.size;
 338         _jitc->function->self.size += sizeof(jit_word_t);
 339     }
 340     node = jit_new_node_ww(jit_code_arg, offset,
 341                            ++_jitc->function->self.argn);
 342     jit_link_prolog();
 343     return (node);
 344 }
 345
 346 jit_node_t *
 347 _jit_arg_f(jit_state_t *_jit)
 348 {
 349     jit_node_t          *node;
 350     jit_int32_t          offset;
 351     assert(_jitc->function);
 352     assert(!(_jitc->function->self.call & jit_call_varargs));
 353     if (jit_arg_f_reg_p(_jitc->function->self.argf))
 354         offset = _jitc->function->self.argf++;
 355     else if (jit_arg_reg_p(_jitc->function->self.argi)) {
 356         offset = _jitc->function->self.argi++;
 357         offset += 8;
 358     }
 359     else {
 360         offset = _jitc->function->self.size;
 361         _jitc->function->self.size += sizeof(jit_word_t);
 362     }
 363     node = jit_new_node_ww(jit_code_arg_f, offset,
 364                            ++_jitc->function->self.argn);
 365     jit_link_prolog();
 366     return (node);
 367 }
 368
 369 jit_node_t *
 370 _jit_arg_d(jit_state_t *_jit)
 371 {
 372     jit_node_t          *node;
 373     jit_int32_t          offset;
 374     assert(_jitc->function);
 375     assert(!(_jitc->function->self.call & jit_call_varargs));
 376     if (jit_arg_f_reg_p(_jitc->function->self.argf))
 377         offset = _jitc->function->self.argf++;
 378     else if (jit_arg_reg_p(_jitc->function->self.argi)) {
 379         offset = _jitc->function->self.argi++;
 380         offset += 8;
 381     }
 382     else {
 383         offset = _jitc->function->self.size;
 384         _jitc->function->self.size += sizeof(jit_word_t);
 385     }
 386     node = jit_new_node_ww(jit_code_arg_d, offset,
 387                            ++_jitc->function->self.argn);
 388     jit_link_prolog();
 389     return (node);
 390 }
 391
 392 void
 393 _jit_getarg_c(jit_state_t *_jit, jit_int32_t u, jit_node_t *v)
 394 {
 395     assert(v->code == jit_code_arg);
 396     jit_inc_synth_wp(getarg_c, u, v);
 397     if (jit_arg_reg_p(v->u.w))
 398         jit_extr_c(u, JIT_RA0 - v->u.w);
 399     else
 400         jit_ldxi_c(u, JIT_FP, v->u.w);
 401     jit_dec_synth();
 402 }
 403
 404 void
 405 _jit_getarg_uc(jit_state_t *_jit, jit_int32_t u, jit_node_t *v)
 406 {
 407     assert(v->code == jit_code_arg);
 408     jit_inc_synth_wp(getarg_uc, u, v);
 409     if (jit_arg_reg_p(v->u.w))
 410         jit_extr_uc(u, JIT_RA0 - v->u.w);
 411     else
 412         jit_ldxi_uc(u, JIT_FP, v->u.w);
 413     jit_dec_synth();
 414 }
 415
 416 void
 417 _jit_getarg_s(jit_state_t *_jit, jit_int32_t u, jit_node_t *v)
 418 {
 419     assert(v->code == jit_code_arg);
 420     jit_inc_synth_wp(getarg_s, u, v);
 421     if (jit_arg_reg_p(v->u.w))
 422         jit_extr_s(u, JIT_RA0 - v->u.w);
 423     else
 424         jit_ldxi_s(u, JIT_FP, v->u.w);
 425     jit_dec_synth();
 426 }
 427
 428 void
 429 _jit_getarg_us(jit_state_t *_jit, jit_int32_t u, jit_node_t *v)
 430 {
 431     assert(v->code == jit_code_arg);
 432     jit_inc_synth_wp(getarg_us, u, v);
 433     if (jit_arg_reg_p(v->u.w))
 434         jit_extr_us(u, JIT_RA0 - v->u.w);
 435     else
 436         jit_ldxi_us(u, JIT_FP, v->u.w);
 437     jit_dec_synth();
 438 }
 439
 440 void
 441 _jit_getarg_i(jit_state_t *_jit, jit_int32_t u, jit_node_t *v)
 442 {
 443     assert(v->code == jit_code_arg);
 444     jit_inc_synth_wp(getarg_i, u, v);
 445     if (jit_arg_reg_p(v->u.w))
 446         jit_extr_i(u, JIT_RA0 - v->u.w);
 447     else
 448         jit_ldxi_i(u, JIT_FP, v->u.w);
 449     jit_dec_synth();
 450 }
 451
 452 void
 453 _jit_getarg_ui(jit_state_t *_jit, jit_int32_t u, jit_node_t *v)
 454 {
 455     assert(v->code == jit_code_arg);
 456     jit_inc_synth_wp(getarg_ui, u, v);
 457     if (jit_arg_reg_p(v->u.w))
 458         jit_extr_ui(u, JIT_RA0 - v->u.w);
 459     else
 460         jit_ldxi_ui(u, JIT_FP, v->u.w);
 461     jit_dec_synth();
 462 }
 463
 464 void
 465 _jit_getarg_l(jit_state_t *_jit, jit_int32_t u, jit_node_t *v)
 466 {
 467     assert(v->code == jit_code_arg);
 468     jit_inc_synth_wp(getarg_l, u, v);
 469     if (jit_arg_reg_p(v->u.w))
 470         jit_movr(u, JIT_RA0 - v->u.w);
 471     else
 472         jit_ldxi_l(u, JIT_FP, v->u.w);
 473     jit_dec_synth();
 474 }
 475
 476 void
 477 _jit_putargr(jit_state_t *_jit, jit_int32_t u, jit_node_t *v)
 478 {
 479     assert(v->code == jit_code_arg);
 480     jit_inc_synth_wp(putargr, u, v);
 481     if (jit_arg_reg_p(v->u.w))
 482         jit_movr(JIT_RA0 - v->u.w, u);
 483     else
 484         jit_stxi(v->u.w, JIT_FP, u);
 485     jit_dec_synth();
 486 }
 487
 488 void
 489 _jit_putargi(jit_state_t *_jit, jit_word_t u, jit_node_t *v)
 490 {
 491     jit_int32_t         regno;
 492     assert(v->code == jit_code_arg);
 493     jit_inc_synth_wp(putargi, u, v);
 494     if (jit_arg_reg_p(v->u.w))
 495         jit_movi(JIT_RA0 - v->u.w, u);
 496     else {
 497         regno = jit_get_reg(jit_class_gpr);
 498         jit_movi(regno, u);
 499         jit_stxi(v->u.w, JIT_FP, regno);
 500         jit_unget_reg(regno);
 501     }
 502     jit_dec_synth();
 503 }
 504
 505 void
 506 _jit_getarg_f(jit_state_t *_jit, jit_int32_t u, jit_node_t *v)
 507 {
 508     assert(v->code == jit_code_arg_f);
 509     jit_inc_synth_wp(getarg_f, u, v);
 510     if (jit_arg_f_reg_p(v->u.w))
 511         jit_movr_f(u, JIT_FA0 - v->u.w);
 512     else if (jit_arg_reg_p(v->u.w - 8))
 513         jit_movr_w_f(u, JIT_RA0 - (v->u.w - 8));
 514     else
 515         jit_ldxi_f(u, JIT_FP, v->u.w);
 516     jit_dec_synth();
 517 }
 518
 519 void
 520 _jit_putargr_f(jit_state_t *_jit, jit_int32_t u, jit_node_t *v)
 521 {
 522     assert(v->code == jit_code_arg_f);
 523     jit_inc_synth_wp(putargr_f, u, v);
 524     if (jit_arg_f_reg_p(v->u.w))
 525         jit_movr_f(JIT_FA0 - v->u.w, u);
 526     else if (jit_arg_reg_p(v->u.w - 8))
 527         jit_movr_f_w(JIT_RA0 - (v->u.w - 8), u);
 528     else
 529         jit_stxi_f(v->u.w, JIT_FP, u);
 530     jit_dec_synth();
 531 }
 532
 533 void
 534 _jit_putargi_f(jit_state_t *_jit, jit_float32_t u, jit_node_t *v)
 535 {
 536     jit_int32_t         regno;
 537     assert(v->code == jit_code_arg_f);
 538     jit_inc_synth_fp(putargi_f, u, v);
 539     if (jit_arg_f_reg_p(v->u.w))
 540         jit_movi_f(JIT_FA0 - v->u.w, u);
 541     else if (jit_arg_reg_p(v->u.w - 8)) {
 542         union {
 543             jit_float32_t       f;
 544             jit_int32_t         i;
 545         } uu;
 546         uu.f = u;
 547         jit_movi(JIT_RA0 - (v->u.w - 8), uu.i);
 548     }
 549     else {
 550         regno = jit_get_reg(jit_class_fpr);
 551         jit_movi_f(regno, u);
 552         jit_stxi_f(v->u.w, JIT_FP, regno);
 553         jit_unget_reg(regno);
 554     }
 555     jit_dec_synth();
 556 }
 557
 558 void
 559 _jit_getarg_d(jit_state_t *_jit, jit_int32_t u, jit_node_t *v)
 560 {
 561     assert(v->code == jit_code_arg_d);
 562     jit_inc_synth_wp(getarg_d, u, v);
 563     if (jit_arg_f_reg_p(v->u.w))
 564         jit_movr_d(u, JIT_FA0 - v->u.w);
 565     else if (jit_arg_reg_p(v->u.w - 8))
 566         jit_movr_w_d(u, JIT_RA0 - (v->u.w - 8));
 567     else
 568         jit_ldxi_d(u, JIT_FP, v->u.w);
 569     jit_dec_synth();
 570 }
 571
 572 void
 573 _jit_putargr_d(jit_state_t *_jit, jit_int32_t u, jit_node_t *v)
 574 {
 575     assert(v->code == jit_code_arg_d);
 576     jit_inc_synth_wp(putargr_d, u, v);
 577     if (jit_arg_reg_p(v->u.w))
 578         jit_movr_d(JIT_FA0 - v->u.w, u);
 579     else if (jit_arg_reg_p(v->u.w - 8))
 580         jit_movr_d_w(JIT_RA0 - (v->u.w - 8), u);
 581     else
 582         jit_stxi_d(v->u.w, JIT_FP, u);
 583     jit_dec_synth();
 584 }
 585
 586 void
 587 _jit_putargi_d(jit_state_t *_jit, jit_float64_t u, jit_node_t *v)
 588 {
 589     jit_int32_t         regno;
 590     assert(v->code == jit_code_arg_d);
 591     jit_inc_synth_dp(putargi_d, u, v);
 592     if (jit_arg_reg_p(v->u.w))
 593         jit_movi_d(JIT_FA0 - v->u.w, u);
 594     else if (jit_arg_reg_p(v->u.w - 8)) {
 595         union {
 596             jit_float64_t       d;
 597             jit_int64_t         w;
 598         } uu;
 599         uu.d = u;
 600         jit_movi(JIT_RA0 - (v->u.w - 8), uu.w);
 601     }
 602     else {
 603         regno = jit_get_reg(jit_class_fpr);
 604         jit_movi_d(regno, u);
 605         jit_stxi_d(v->u.w, JIT_FP, regno);
 606         jit_unget_reg(regno);
 607     }
 608     jit_dec_synth();
 609 }
 610
 611 void
 612 _jit_pushargr(jit_state_t *_jit, jit_int32_t u)
 613 {
 614     assert(_jitc->function);
 615     jit_inc_synth_w(pushargr, u);
 616     jit_link_prepare();
 617     if (jit_arg_reg_p(_jitc->function->call.argi)) {
 618         jit_movr(JIT_RA0 - _jitc->function->call.argi, u);
 619         ++_jitc->function->call.argi;
 620     }
 621     else {
 622         jit_stxi(_jitc->function->call.size, JIT_SP, u);
 623         _jitc->function->call.size += sizeof(jit_word_t);
 624     }
 625     jit_dec_synth();
 626 }
 627
 628 void
 629 _jit_pushargi(jit_state_t *_jit, jit_word_t u)
 630 {
 631     jit_int32_t          regno;
 632     assert(_jitc->function);
 633     jit_inc_synth_w(pushargi, u);
 634     jit_link_prepare();
 635     if (jit_arg_reg_p(_jitc->function->call.argi)) {
 636         jit_movi(JIT_RA0 - _jitc->function->call.argi, u);
 637         ++_jitc->function->call.argi;
 638     }
 639     else {
 640         regno = jit_get_reg(jit_class_gpr);
 641         jit_movi(regno, u);
 642         jit_stxi(_jitc->function->call.size, JIT_SP, regno);
 643         jit_unget_reg(regno);
 644         _jitc->function->call.size += sizeof(jit_word_t);
 645     }
 646     jit_dec_synth();
 647 }
 648
 649 void
 650 _jit_pushargr_f(jit_state_t *_jit, jit_int32_t u)
 651 {
 652     assert(_jitc->function);
 653     jit_inc_synth_w(pushargr_f, u);
 654     jit_link_prepare();
 655     if (jit_arg_f_reg_p(_jitc->function->call.argf) &&
 656         !(_jitc->function->call.call & jit_call_varargs)) {
 657         jit_movr_f(JIT_FA0 - _jitc->function->call.argf, u);
 658         ++_jitc->function->call.argf;
 659     }
 660     else if (jit_arg_reg_p(_jitc->function->call.argi)) {
 661         jit_movr_f_w(JIT_RA0 - _jitc->function->call.argi, u);
 662         ++_jitc->function->call.argi;
 663     }
 664     else {
 665         jit_stxi_f(_jitc->function->call.size, JIT_SP, u);
 666         _jitc->function->call.size += sizeof(jit_word_t);
 667     }
 668     jit_dec_synth();
 669 }
 670
 671 void
 672 _jit_pushargi_f(jit_state_t *_jit, jit_float32_t u)
 673 {
 674     jit_int32_t         regno;
 675     assert(_jitc->function);
 676     jit_inc_synth_f(pushargi_f, u);
 677     jit_link_prepare();
 678     if (jit_arg_f_reg_p(_jitc->function->call.argf) &&
 679         !(_jitc->function->call.call & jit_call_varargs)) {
 680         jit_movi_f(JIT_FA0 - _jitc->function->call.argf, u);
 681         ++_jitc->function->call.argf;
 682     }
 683     else if (jit_arg_reg_p(_jitc->function->call.argi)) {
 684         jit_movi_f_w(JIT_RA0 - _jitc->function->call.argi, u);
 685         ++_jitc->function->call.argi;
 686     }
 687     else {
 688         regno = jit_get_reg(jit_class_fpr);
 689         jit_movi_f(regno, u);
 690         jit_stxi_f(_jitc->function->call.size, JIT_SP, regno);
 691         jit_unget_reg(regno);
 692         _jitc->function->call.size += sizeof(jit_word_t);
 693     }
 694     jit_dec_synth();
 695 }
 696
 697 void
 698 _jit_pushargr_d(jit_state_t *_jit, jit_int32_t u)
 699 {
 700     assert(_jitc->function);
 701     jit_inc_synth_w(pushargr_d, u);
 702     jit_link_prepare();
 703     if (jit_arg_f_reg_p(_jitc->function->call.argf) &&
 704         !(_jitc->function->call.call & jit_call_varargs)) {
 705         jit_movr_d(JIT_FA0 - _jitc->function->call.argf, u);
 706         ++_jitc->function->call.argf;
 707     }
 708     else if (jit_arg_reg_p(_jitc->function->call.argi)) {
 709         jit_movr_d_w(JIT_RA0 - _jitc->function->call.argi, u);
 710         ++_jitc->function->call.argi;
 711     }
 712     else {
 713         jit_stxi_d(_jitc->function->call.size, JIT_SP, u);
 714         _jitc->function->call.size += sizeof(jit_word_t);
 715     }
 716     jit_dec_synth();
 717 }
 718
 719 void
 720 _jit_pushargi_d(jit_state_t *_jit, jit_float64_t u)
 721 {
 722     jit_int32_t         regno;
 723     assert(_jitc->function);
 724     jit_inc_synth_d(pushargi_d, u);
 725     jit_link_prepare();
 726     if (jit_arg_f_reg_p(_jitc->function->call.argf) &&
 727         !(_jitc->function->call.call & jit_call_varargs)) {
 728         jit_movi_d(JIT_FA0 - _jitc->function->call.argf, u);
 729         ++_jitc->function->call.argf;
 730     }
 731     else if (jit_arg_reg_p(_jitc->function->call.argi)) {
 732         jit_movi_d_w(JIT_RA0 - _jitc->function->call.argi, u);
 733         ++_jitc->function->call.argi;
 734     }
 735     else {
 736         regno = jit_get_reg(jit_class_fpr);
 737         jit_movi_d(regno, u);
 738         jit_stxi_d(_jitc->function->call.size, JIT_SP, regno);
 739         jit_unget_reg(regno);
 740         _jitc->function->call.size += sizeof(jit_word_t);
 741     }
 742     jit_dec_synth();
 743 }
 744
 745 jit_bool_t
 746 _jit_regarg_p(jit_state_t *_jit, jit_node_t *node, jit_int32_t regno)
 747 {
 748     jit_int32_t         spec;
 749     spec = jit_class(_rvs[regno].spec);
 750     if (spec & jit_class_arg) {
 751         regno = JIT_RA0 - regno;
 752         if (regno >= 0 && regno < node->v.w)
 753             return (1);
 754         if (spec & jit_class_fpr) {
 755             regno = JIT_FA0 - regno;
 756             if (regno >= 0 && regno < node->w.w)
 757                 return (1);
 758         }
 759     }
 760
 761     return (0);
 762 }
 763
 764 void
 765 _jit_finishr(jit_state_t *_jit, jit_int32_t r0)
 766 {
 767     jit_node_t          *node;
 768     assert(_jitc->function);
 769     jit_inc_synth_w(finishr, r0);
 770     if (_jitc->function->self.alen < _jitc->function->call.size)
 771         _jitc->function->self.alen = _jitc->function->call.size;
 772     node = jit_callr(r0);
 773     node->v.w = _jitc->function->self.argi;
 774     node->w.w = _jitc->function->call.argf;
 775     _jitc->function->call.argi = _jitc->function->call.argf =
 776         _jitc->function->call.size = 0;
 777     _jitc->prepare = 0;
 778     jit_dec_synth();
 779 }
 780
 781 jit_node_t *
 782 _jit_finishi(jit_state_t *_jit, jit_pointer_t i0)
 783 {
 784     jit_node_t          *node;
 785     assert(_jitc->function);
 786     jit_inc_synth_w(finishi, (jit_word_t)i0);
 787     if (_jitc->function->self.alen < _jitc->function->call.size)
 788         _jitc->function->self.alen = _jitc->function->call.size;
 789     node = jit_calli(i0);
 790     node->v.w = _jitc->function->call.argi;
 791     node->w.w = _jitc->function->call.argf;
 792     _jitc->function->call.argi = _jitc->function->call.argf =
 793         _jitc->function->call.size = 0;
 794     _jitc->prepare = 0;
 795     jit_dec_synth();
 796     return (node);
 797 }
 798
 799 void
 800 _jit_retval_c(jit_state_t *_jit, jit_int32_t r0)
 801 {
 802     jit_inc_synth_w(retval_c, r0);
 803     jit_extr_c(r0, JIT_RET);
 804     jit_dec_synth();
 805 }
 806
 807 void
 808 _jit_retval_uc(jit_state_t *_jit, jit_int32_t r0)
 809 {
 810     jit_inc_synth_w(retval_uc, r0);
 811     jit_extr_uc(r0, JIT_RET);
 812     jit_dec_synth();
 813 }
 814
 815 void
 816 _jit_retval_s(jit_state_t *_jit, jit_int32_t r0)
 817 {
 818     jit_inc_synth_w(retval_s, r0);
 819     jit_extr_s(r0, JIT_RET);
 820     jit_dec_synth();
 821 }
 822
 823 void
 824 _jit_retval_us(jit_state_t *_jit, jit_int32_t r0)
 825 {
 826     jit_inc_synth_w(retval_us, r0);
 827     jit_extr_us(r0, JIT_RET);
 828     jit_dec_synth();
 829 }
 830
 831 void
 832 _jit_retval_i(jit_state_t *_jit, jit_int32_t r0)
 833 {
 834     jit_inc_synth_w(retval_i, r0);
 835     jit_extr_i(r0, JIT_RET);
 836     jit_dec_synth();
 837 }
 838
 839 void
 840 _jit_retval_ui(jit_state_t *_jit, jit_int32_t r0)
 841 {
 842     jit_inc_synth_w(retval_ui, r0);
 843     jit_extr_ui(r0, JIT_RET);
 844     jit_dec_synth();
 845 }
 846
 847 void
 848 _jit_retval_l(jit_state_t *_jit, jit_int32_t r0)
 849 {
 850     jit_inc_synth_w(retval_l, r0);
 851     if (r0 != JIT_RET)
 852         jit_movr(r0, JIT_RET);
 853     jit_dec_synth();
 854 }
 855
 856 void
 857 _jit_retval_f(jit_state_t *_jit, jit_int32_t r0)
 858 {
 859     jit_inc_synth_w(retval_f, r0);
 860     if (r0 != JIT_FRET)
 861         jit_movr_f(r0, JIT_FRET);
 862     jit_dec_synth();
 863 }
 864
 865 void
 866 _jit_retval_d(jit_state_t *_jit, jit_int32_t r0)
 867 {
 868     jit_inc_synth_w(retval_d, r0);
 869     if (r0 != JIT_FRET)
 870         jit_movr_d(r0, JIT_FRET);
 871     jit_dec_synth();
 872 }
 873
 874 jit_pointer_t
 875 _emit_code(jit_state_t *_jit)
 876 {
 877     jit_node_t          *node;
 878     jit_node_t          *temp;
 879     jit_word_t           word;
 880     jit_word_t           value;
 881     jit_int32_t          offset;
 882     struct {
 883         jit_node_t      *node;
 884         jit_uint8_t     *data;
 885         jit_word_t       word;
 886 #if DEVEL_DISASSEMBLER
 887         jit_word_t       prevw;
 888 #endif
 889         jit_int32_t      const_offset;
 890         jit_int32_t      patch_offset;
 891     } undo;
 892 #if DEVEL_DISASSEMBLER
 893     jit_word_t           prevw;
 894 #endif
 895
 896     _jitc->function = NULL;
 897
 898     jit_reglive_setup();
 899
 900     undo.word = 0;
 901     undo.node = NULL;
 902     undo.const_offset = undo.patch_offset = 0;
 903 #  define assert_data(node)             /**/
 904 #define case_rr(name, type)                                             \
 905             case jit_code_##name##r##type:                              \
 906                 name##r##type(rn(node->u.w), rn(node->v.w));            \
 907                 break
 908 #define case_rw(name, type)                                             \
 909             case jit_code_##name##i##type:                              \
 910                 name##i##type(rn(node->u.w), node->v.w);                \
 911                 break
 912 #define case_wr(name, type)                                             \
 913             case jit_code_##name##i##type:                              \
 914                 name##i##type(node->u.w, rn(node->v.w));                \
 915                 break
 916 #define case_rrr(name, type)                                            \
 917             case jit_code_##name##r##type:                              \
 918                 name##r##type(rn(node->u.w),                            \
 919                               rn(node->v.w), rn(node->w.w));            \
 920                 break
 921 #define case_rrrr(name, type)                                           \
 922             case jit_code_##name##r##type:                              \
 923                 name##r##type(rn(node->u.q.l), rn(node->u.q.h),         \
 924                               rn(node->v.w), rn(node->w.w));            \
 925                 break
 926 #define case_rrw(name, type)                                            \
 927             case jit_code_##name##i##type:                              \
 928                 name##i##type(rn(node->u.w), rn(node->v.w), node->w.w); \
 929                 break
 930 #define case_rrrw(name, type)                                           \
 931             case jit_code_##name##i##type:                              \
 932                 name##i##type(rn(node->u.q.l), rn(node->u.q.h),         \
 933                               rn(node->v.w), node->w.w);                \
 934                 break
 935 #define case_rrf(name)                                                  \
 936             case jit_code_##name##i_f:                                  \
 937                 assert_data(node);                                      \
 938                 name##i_f(rn(node->u.w), rn(node->v.w), node->w.f);     \
 939                 break
 940 #define case_rrd(name)                                                  \
 941             case jit_code_##name##i_d:                                  \
 942                 assert_data(node);                                      \
 943                 name##i_d(rn(node->u.w), rn(node->v.w), node->w.d);     \
 944                 break
 945 #define case_wrr(name, type)                                            \
 946             case jit_code_##name##i##type:                              \
 947                 name##i##type(node->u.w, rn(node->v.w), rn(node->w.w)); \
 948                 break
 949 #define case_brr(name, type)                                            \
 950             case jit_code_##name##r##type:                              \
 951                 temp = node->u.n;                                       \
 952                 assert(temp->code == jit_code_label ||                  \
 953                        temp->code == jit_code_epilog);                  \
 954                 if (temp->flag & jit_flag_patch)                        \
 955                     name##r##type(temp->u.w, rn(node->v.w),             \
 956                                   rn(node->w.w));                       \
 957                 else {                                                  \
 958                     word = name##r##type(_jit->pc.w,                    \
 959                                          rn(node->v.w), rn(node->w.w)); \
 960                     patch(word, node);                                  \
 961                 }                                                       \
 962                 break
 963 #define case_brw(name, type)                                            \
 964             case jit_code_##name##i##type:                              \
 965                 temp = node->u.n;                                       \
 966                 assert(temp->code == jit_code_label ||                  \
 967                        temp->code == jit_code_epilog);                  \
 968                 if (temp->flag & jit_flag_patch)                        \
 969                     name##i##type(temp->u.w,                            \
 970                                   rn(node->v.w), node->w.w);            \
 971                 else {                                                  \
 972                     word = name##i##type(_jit->pc.w,                    \
 973                                          rn(node->v.w), node->w.w);     \
 974                     patch(word, node);                                  \
 975                 }                                                       \
 976                 break;
 977 #define case_brf(name)                                                  \
 978             case jit_code_##name##i_f:                                  \
 979                 temp = node->u.n;                                       \
 980                 assert(temp->code == jit_code_label ||                  \
 981                        temp->code == jit_code_epilog);                  \
 982                 if (temp->flag & jit_flag_patch)                        \
 983                     name##i_f(temp->u.w, rn(node->v.w), node->w.f);     \
 984                 else {                                                  \
 985                     word = name##i_f(_jit->pc.w, rn(node->v.w),         \
 986                                 node->w.f);                             \
 987                     patch(word, node);                                  \
 988                 }                                                       \
 989                 break
 990 #define case_brd(name)                                                  \
 991             case jit_code_##name##i_d:                                  \
 992                 temp = node->u.n;                                       \
 993                 assert(temp->code == jit_code_label ||                  \
 994                        temp->code == jit_code_epilog);                  \
 995                 if (temp->flag & jit_flag_patch)                        \
 996                     name##i_d(temp->u.w, rn(node->v.w), node->w.d);     \
 997                 else {                                                  \
 998                     word = name##i_d(_jit->pc.w, rn(node->v.w),         \
 999                                 node->w.d);                             \
1000                     patch(word, node);                                  \
1001                 }                                                       \
1002                 break
1003 #if DEVEL_DISASSEMBLER
1004     prevw = _jit->pc.w;
1005 #endif
1006     for (node = _jitc->head; node; node = node->next) {
1007         if (_jit->pc.uc >= _jitc->code.end)
1008             return (NULL);
1009
1010 #if DEVEL_DISASSEMBLER
1011         node->offset = (jit_uword_t)_jit->pc.w - (jit_uword_t)prevw;
1012         prevw = _jit->pc.w;
1013 #endif
1014         value = jit_classify(node->code);
1015         jit_regarg_set(node, value);
1016         switch (node->code) {
1017             case jit_code_align:
1018                 assert(!(node->u.w & (node->u.w - 1)) &&
1019                        node->u.w <= sizeof(jit_word_t));
1020                 if (node->u.w == sizeof(jit_word_t) &&
1021                     (word = _jit->pc.w & (sizeof(jit_word_t) - 1)))
1022                     nop(sizeof(jit_word_t) - word);
1023                 break;
1024             case jit_code_note:         case jit_code_name:
1025                 node->u.w = _jit->pc.w;
1026                 break;
1027             case jit_code_label:
1028                 /* remember label is defined */
1029                 node->flag |= jit_flag_patch;
1030                 node->u.w = _jit->pc.w;
1031                 break;
1032                 case_rrr(add,);
1033                 case_rrw(add,);
1034                 case_rrr(addc,);
1035                 case_rrw(addc,);
1036                 case_rrr(addx,);
1037                 case_rrw(addx,);
1038                 case_rrr(sub,);
1039                 case_rrw(sub,);
1040                 case_rrr(subc,);
1041                 case_rrw(subc,);
1042                 case_rrr(subx,);
1043                 case_rrw(subx,);
1044                 case_rrw(rsb,);
1045                 case_rrr(mul,);
1046                 case_rrw(mul,);
1047                 case_rrrr(qmul,);
1048                 case_rrrw(qmul,);
1049                 case_rrrr(qmul, _u);
1050                 case_rrrw(qmul, _u);
1051                 case_rrr(div,);
1052                 case_rrw(div,);
1053                 case_rrr(div, _u);
1054                 case_rrw(div, _u);
1055                 case_rrrr(qdiv,);
1056                 case_rrrw(qdiv,);
1057                 case_rrrr(qdiv, _u);
1058                 case_rrrw(qdiv, _u);
1059                 case_rrr(rem,);
1060                 case_rrw(rem,);
1061                 case_rrr(rem, _u);
1062                 case_rrw(rem, _u);
1063                 case_rrr(lsh,);
1064                 case_rrw(lsh,);
1065                 case_rrr(rsh,);
1066                 case_rrw(rsh,);
1067                 case_rrr(rsh, _u);
1068                 case_rrw(rsh, _u);
1069                 case_rr(neg,);
1070                 case_rr(com,);
1071                 case_rrr(and,);
1072                 case_rrw(and,);
1073                 case_rrr(or,);
1074                 case_rrw(or,);
1075                 case_rrr(xor,);
1076                 case_rrw(xor,);
1077                 case_rr(trunc, _f_i);
1078                 case_rr(trunc, _d_i);
1079                 case_rr(trunc, _f_l);
1080                 case_rr(trunc, _d_l);
1081                 case_rr(ld, _c);
1082                 case_rw(ld, _c);
1083                 case_rr(ld, _uc);
1084                 case_rw(ld, _uc);
1085                 case_rr(ld, _s);
1086                 case_rw(ld, _s);
1087                 case_rr(ld, _us);
1088                 case_rw(ld, _us);
1089                 case_rr(ld, _i);
1090                 case_rw(ld, _i);
1091                 case_rr(ld, _ui);
1092                 case_rw(ld, _ui);
1093                 case_rr(ld, _l);
1094                 case_rw(ld, _l);
1095                 case_rrr(ldx, _c);
1096                 case_rrw(ldx, _c);
1097                 case_rrr(ldx, _uc);
1098                 case_rrw(ldx, _uc);
1099                 case_rrr(ldx, _s);
1100                 case_rrw(ldx, _s);
1101                 case_rrr(ldx, _us);
1102                 case_rrw(ldx, _us);
1103                 case_rrr(ldx, _i);
1104                 case_rrw(ldx, _i);
1105                 case_rrr(ldx, _ui);
1106                 case_rrw(ldx, _ui);
1107                 case_rrr(ldx, _l);
1108                 case_rrw(ldx, _l);
1109                 case_rr(st, _c);
1110                 case_wr(st, _c);
1111                 case_rr(st, _s);
1112                 case_wr(st, _s);
1113                 case_rr(st, _i);
1114                 case_wr(st, _i);
1115                 case_rr(st, _l);
1116                 case_wr(st, _l);
1117                 case_rrr(stx, _c);
1118                 case_wrr(stx, _c);
1119                 case_rrr(stx, _s);
1120                 case_wrr(stx, _s);
1121                 case_rrr(stx, _i);
1122                 case_wrr(stx, _i);
1123                 case_rrr(stx, _l);
1124                 case_wrr(stx, _l);
1125                 case_rr(hton, _us);
1126                 case_rr(hton, _ui);
1127                 case_rr(hton, _ul);
1128                 case_rr(bswap, _us);
1129                 case_rr(bswap, _ui);
1130                 case_rr(bswap, _ul);
1131                 case_rr(ext, _c);
1132                 case_rr(ext, _uc);
1133                 case_rr(ext, _s);
1134                 case_rr(ext, _us);
1135                 case_rr(ext, _i);
1136                 case_rr(ext, _ui);
1137                 case_rrr(movn,);
1138                 case_rrr(movz,);
1139                 case_rr(mov,);
1140             case jit_code_movi:
1141                 if (node->flag & jit_flag_node) {
1142                     temp = node->v.n;
1143                     if (temp->code == jit_code_data ||
1144                         (temp->code == jit_code_label &&
1145                          (temp->flag & jit_flag_patch)))
1146                         movi(rn(node->u.w), temp->u.w);
1147                     else {
1148                         assert(temp->code == jit_code_label ||
1149                                temp->code == jit_code_epilog);
1150                         word = movi_p(rn(node->u.w), temp->u.w);
1151                         patch(word, node);
1152                     }
1153                 }
1154                 else
1155                     movi(rn(node->u.w), node->v.w);
1156                 break;
1157                 case_rrr(lt,);
1158                 case_rrw(lt,);
1159                 case_rrr(lt, _u);
1160                 case_rrw(lt, _u);
1161                 case_rrr(le,);
1162                 case_rrw(le,);
1163                 case_rrr(le, _u);
1164                 case_rrw(le, _u);
1165                 case_rrr(eq,);
1166                 case_rrw(eq,);
1167                 case_rrr(ge,);
1168                 case_rrw(ge,);
1169                 case_rrr(ge, _u);
1170                 case_rrw(ge, _u);
1171                 case_rrr(gt,);
1172                 case_rrw(gt,);
1173                 case_rrr(gt, _u);
1174                 case_rrw(gt, _u);
1175                 case_rrr(ne,);
1176                 case_rrw(ne,);
1177                 case_brr(blt,);
1178                 case_brw(blt,);
1179                 case_brr(blt, _u);
1180                 case_brw(blt, _u);
1181                 case_brr(ble,);
1182                 case_brw(ble,);
1183                 case_brr(ble, _u);
1184                 case_brw(ble, _u);
1185                 case_brr(beq,);
1186                 case_brw(beq,);
1187                 case_brr(bge,);
1188                 case_brw(bge,);
1189                 case_brr(bge, _u);
1190                 case_brw(bge, _u);
1191                 case_brr(bgt,);
1192                 case_brw(bgt,);
1193                 case_brr(bgt, _u);
1194                 case_brw(bgt, _u);
1195                 case_brr(bne,);
1196                 case_brw(bne,);
1197                 case_brr(boadd,);
1198                 case_brw(boadd,);
1199                 case_brr(boadd, _u);
1200                 case_brw(boadd, _u);
1201                 case_brr(bxadd,);
1202                 case_brw(bxadd,);
1203                 case_brr(bxadd, _u);
1204                 case_brw(bxadd, _u);
1205                 case_brr(bosub,);
1206                 case_brw(bosub,);
1207                 case_brr(bosub, _u);
1208                 case_brw(bosub, _u);
1209                 case_brr(bxsub,);
1210                 case_brw(bxsub,);
1211                 case_brr(bxsub, _u);
1212                 case_brw(bxsub, _u);
1213                 case_brr(bms,);
1214                 case_brw(bms,);
1215                 case_brr(bmc,);
1216                 case_brw(bmc,);
1217                 case_rrr(add, _f);
1218                 case_rrf(add);
1219                 case_rrr(sub, _f);
1220                 case_rrf(sub);
1221                 case_rrf(rsb);
1222                 case_rrr(mul, _f);
1223                 case_rrf(mul);
1224                 case_rrr(div, _f);
1225                 case_rrf(div);
1226                 case_rr(abs, _f);
1227                 case_rr(neg, _f);
1228                 case_rr(sqrt, _f);
1229                 case_rr(ext, _f);
1230                 case_rr(ld, _f);
1231                 case_rw(ld, _f);
1232                 case_rrr(ldx, _f);
1233                 case_rrw(ldx, _f);
1234                 case_rr(st, _f);
1235                 case_wr(st, _f);
1236                 case_rrr(stx, _f);
1237                 case_wrr(stx, _f);
1238                 case_rr(mov, _f);
1239             case jit_code_movi_f:
1240                 assert_data(node);
1241                 movi_f(rn(node->u.w), node->v.f);
1242                 break;
1243                 case_rr(ext, _d_f);
1244                 case_rrr(lt, _f);
1245                 case_rrf(lt);
1246                 case_rrr(le, _f);
1247                 case_rrf(le);
1248                 case_rrr(eq, _f);
1249                 case_rrf(eq);
1250                 case_rrr(ge, _f);
1251                 case_rrf(ge);
1252                 case_rrr(gt, _f);
1253                 case_rrf(gt);
1254                 case_rrr(ne, _f);
1255                 case_rrf(ne);
1256                 case_rrr(unlt, _f);
1257                 case_rrf(unlt);
1258                 case_rrr(unle, _f);
1259                 case_rrf(unle);
1260                 case_rrr(uneq, _f);
1261                 case_rrf(uneq);
1262                 case_rrr(unge, _f);
1263                 case_rrf(unge);
1264                 case_rrr(ungt, _f);
1265                 case_rrf(ungt);
1266                 case_rrr(ltgt, _f);
1267                 case_rrf(ltgt);
1268                 case_rrr(ord, _f);
1269                 case_rrf(ord);
1270                 case_rrr(unord, _f);
1271                 case_rrf(unord);
1272                 case_brr(blt, _f);
1273                 case_brf(blt);
1274                 case_brr(ble, _f);
1275                 case_brf(ble);
1276                 case_brr(beq, _f);
1277                 case_brf(beq);
1278                 case_brr(bge, _f);
1279                 case_brf(bge);
1280                 case_brr(bgt, _f);
1281                 case_brf(bgt);
1282                 case_brr(bne, _f);
1283                 case_brf(bne);
1284                 case_brr(bunlt, _f);
1285                 case_brf(bunlt);
1286                 case_brr(bunle, _f);
1287                 case_brf(bunle);
1288                 case_brr(buneq, _f);
1289                 case_brf(buneq);
1290                 case_brr(bunge, _f);
1291                 case_brf(bunge);
1292                 case_brr(bungt, _f);
1293                 case_brf(bungt);
1294                 case_brr(bltgt, _f);
1295                 case_brf(bltgt);
1296                 case_brr(bord, _f);
1297                 case_brf(bord);
1298                 case_brr(bunord, _f);
1299                 case_brf(bunord);
1300                 case_rrr(add, _d);
1301                 case_rrd(add);
1302                 case_rrr(sub, _d);
1303                 case_rrd(sub);
1304                 case_rrd(rsb);
1305                 case_rrr(mul, _d);
1306                 case_rrd(mul);
1307                 case_rrr(div, _d);
1308                 case_rrd(div);
1309                 case_rr(abs, _d);
1310                 case_rr(neg, _d);
1311                 case_rr(sqrt, _d);
1312                 case_rr(ext, _d);
1313                 case_rr(ld, _d);
1314                 case_rw(ld, _d);
1315                 case_rrr(ldx, _d);
1316                 case_rrw(ldx, _d);
1317                 case_rr(st, _d);
1318                 case_wr(st, _d);
1319                 case_rrr(stx, _d);
1320                 case_wrr(stx, _d);
1321                 case_rr(mov, _d);
1322             case jit_code_movi_d:
1323                 assert_data(node);
1324                 movi_d(rn(node->u.w), node->v.d);
1325                 break;
1326                 case_rr(ext, _f_d);
1327                 case_rrr(lt, _d);
1328                 case_rrd(lt);
1329                 case_rrr(le, _d);
1330                 case_rrd(le);
1331                 case_rrr(eq, _d);
1332                 case_rrd(eq);
1333                 case_rrr(ge, _d);
1334                 case_rrd(ge);
1335                 case_rrr(gt, _d);
1336                 case_rrd(gt);
1337                 case_rrr(ne, _d);
1338                 case_rrd(ne);
1339                 case_rrr(unlt, _d);
1340                 case_rrd(unlt);
1341                 case_rrr(unle, _d);
1342                 case_rrd(unle);
1343                 case_rrr(uneq, _d);
1344                 case_rrd(uneq);
1345                 case_rrr(unge, _d);
1346                 case_rrd(unge);
1347                 case_rrr(ungt, _d);
1348                 case_rrd(ungt);
1349                 case_rrr(ltgt, _d);
1350                 case_rrd(ltgt);
1351                 case_rrr(ord, _d);
1352                 case_rrd(ord);
1353                 case_rrr(unord, _d);
1354                 case_rrd(unord);
1355                 case_brr(blt, _d);
1356                 case_brd(blt);
1357                 case_brr(ble, _d);
1358                 case_brd(ble);
1359                 case_brr(beq, _d);
1360                 case_brd(beq);
1361                 case_brr(bge, _d);
1362                 case_brd(bge);
1363                 case_brr(bgt, _d);
1364                 case_brd(bgt);
1365                 case_brr(bne, _d);
1366                 case_brd(bne);
1367                 case_brr(bunlt, _d);
1368                 case_brd(bunlt);
1369                 case_brr(bunle, _d);
1370                 case_brd(bunle);
1371                 case_brr(buneq, _d);
1372                 case_brd(buneq);
1373                 case_brr(bunge, _d);
1374                 case_brd(bunge);
1375                 case_brr(bungt, _d);
1376                 case_brd(bungt);
1377                 case_brr(bltgt, _d);
1378                 case_brd(bltgt);
1379                 case_brr(bord, _d);
1380                 case_brd(bord);
1381                 case_brr(bunord, _d);
1382                 case_brd(bunord);
1383             case jit_code_jmpr:
1384                 jmpr(rn(node->u.w));
1385                 break;
1386             case jit_code_jmpi:
1387                 if (node->flag & jit_flag_node) {
1388                     temp = node->u.n;
1389                     assert(temp->code == jit_code_label ||
1390                            temp->code == jit_code_epilog);
1391                     if (temp->flag & jit_flag_patch)
1392                         jmpi(temp->u.w);
1393                     else {
1394                         word = jmpi_p(_jit->pc.w);
1395                         patch(word, node);
1396                     }
1397                 }
1398                 else
1399                     jmpi(node->u.w);
1400                 break;
1401             case jit_code_callr:
1402                 callr(rn(node->u.w));
1403                 break;
1404             case jit_code_calli:
1405                 if (node->flag & jit_flag_node) {
1406                     temp = node->u.n;
1407                     assert(temp->code == jit_code_label ||
1408                            temp->code == jit_code_epilog);
1409                     if (temp->flag & jit_flag_patch)
1410                         calli(temp->u.w);
1411                     else {
1412                         word = calli_p(_jit->pc.w);
1413                         patch(word, node);
1414                     }
1415                 }
1416                 else
1417                     calli(node->u.w);
1418                 break;
1419             case jit_code_prolog:
1420                 _jitc->function = _jitc->functions.ptr + node->w.w;
1421                 undo.node = node;
1422                 undo.word = _jit->pc.w;
1423 #if DEVEL_DISASSEMBLER
1424                 undo.prevw = prevw;
1425 #endif
1426                 undo.patch_offset = _jitc->patches.offset;
1427             restart_function:
1428                 _jitc->again = 0;
1429                 prolog(node);
1430                 break;
1431             case jit_code_epilog:
1432                 assert(_jitc->function == _jitc->functions.ptr + node->w.w);
1433                 if (_jitc->again) {
1434                     for (temp = undo.node->next;
1435                          temp != node; temp = temp->next) {
1436                         if (temp->code == jit_code_label ||
1437                             temp->code == jit_code_epilog)
1438                             temp->flag &= ~jit_flag_patch;
1439                     }
1440                     temp->flag &= ~jit_flag_patch;
1441                     node = undo.node;
1442                     _jit->pc.w = undo.word;
1443 #if DEVEL_DISASSEMBLER
1444                     prevw = undo.prevw;
1445 #endif
1446                     _jitc->patches.offset = undo.patch_offset;
1447                     goto restart_function;
1448                 }
1449                 /* remember label is defined */
1450                 node->flag |= jit_flag_patch;
1451                 node->u.w = _jit->pc.w;
1452                 epilog(node);
1453                 _jitc->function = NULL;
1454                 break;
1455             case jit_code_movr_w_f:
1456                 movr_w_f(rn(node->u.w), rn(node->v.w));
1457                 break;
1458             case jit_code_movr_f_w:
1459                 movr_f_w(rn(node->u.w), rn(node->v.w));
1460                 break;
1461             case jit_code_movi_f_w:
1462                 assert_data(node);
1463                 movi_f_w(rn(node->u.w), node->v.f);
1464                 break;
1465             case jit_code_movr_w_d:
1466                 movr_w_d(rn(node->u.w), rn(node->v.w));
1467                 break;
1468             case jit_code_movr_d_w:
1469                 movr_d_w(rn(node->u.w), rn(node->v.w));
1470                 break;
1471             case jit_code_movi_d_w:
1472                 assert_data(node);
1473                 movi_d_w(rn(node->u.w), node->v.d);
1474                 break;
1475             case jit_code_va_start:
1476                 vastart(rn(node->u.w));
1477                 break;
1478             case jit_code_va_arg:
1479                 vaarg(rn(node->u.w), rn(node->v.w));
1480                 break;
1481             case jit_code_va_arg_d:
1482                 vaarg_d(rn(node->u.w), rn(node->v.w));
1483                 break;
1484             case jit_code_live:                 case jit_code_ellipsis:
1485             case jit_code_va_push:
1486             case jit_code_allocai:              case jit_code_allocar:
1487             case jit_code_arg:
1488             case jit_code_arg_f:                case jit_code_arg_d:
1489             case jit_code_va_end:
1490             case jit_code_ret:
1491             case jit_code_retr:                 case jit_code_reti:
1492             case jit_code_retr_f:               case jit_code_reti_f:
1493             case jit_code_retr_d:               case jit_code_reti_d:
1494             case jit_code_getarg_c:             case jit_code_getarg_uc:
1495             case jit_code_getarg_s:             case jit_code_getarg_us:
1496             case jit_code_getarg_i:             case jit_code_getarg_ui:
1497             case jit_code_getarg_l:
1498             case jit_code_getarg_f:             case jit_code_getarg_d:
1499             case jit_code_putargr:              case jit_code_putargi:
1500             case jit_code_putargr_f:            case jit_code_putargi_f:
1501             case jit_code_putargr_d:            case jit_code_putargi_d:
1502             case jit_code_pushargr:             case jit_code_pushargi:
1503             case jit_code_pushargr_f:           case jit_code_pushargi_f:
1504             case jit_code_pushargr_d:           case jit_code_pushargi_d:
1505             case jit_code_retval_c:             case jit_code_retval_uc:
1506             case jit_code_retval_s:             case jit_code_retval_us:
1507             case jit_code_retval_i:
1508             case jit_code_retval_ui:            case jit_code_retval_l:
1509             case jit_code_retval_f:             case jit_code_retval_d:
1510             case jit_code_prepare:
1511             case jit_code_finishr:              case jit_code_finishi:
1512                 break;
1513             default:
1514                 abort();
1515         }
1516         if (jit_carry != _NOREG) {
1517             switch (node->code) {
1518                 case jit_code_note:
1519                 case jit_code_addcr:            case jit_code_addci:
1520                 case jit_code_addxr:            case jit_code_addxi:
1521                 case jit_code_subcr:            case jit_code_subci:
1522                 case jit_code_subxr:            case jit_code_subxi:
1523                     break;
1524                 default:
1525                     jit_unget_reg(jit_carry);
1526                     jit_carry = _NOREG;
1527                     break;
1528             }
1529         }
1530         jit_regarg_clr(node, value);
1531         assert(_jitc->regarg == 0 ||
1532                (jit_carry != _NOREG && _jitc->regarg == (1 << jit_carry)));
1533         assert(_jitc->synth == 0);
1534         /* update register live state */
1535         jit_reglive(node);
1536     }
1537 #undef case_brw
1538 #undef case_brr
1539 #undef case_wrr
1540 #undef case_rrw
1541 #undef case_rrr
1542 #undef case_wr
1543 #undef case_rw
1544 #undef case_rr
1545
1546     for (offset = 0; offset < _jitc->patches.offset; offset++) {
1547         node = _jitc->patches.ptr[offset].node;
1548         word = _jitc->patches.ptr[offset].inst;
1549         value = node->code == jit_code_movi ? node->v.n->u.w : node->u.n->u.w;
1550         patch_at(word, value);
1551     }
1552
1553     jit_flush(_jit->code.ptr, _jit->pc.uc);
1554
1555     return (_jit->code.ptr);
1556 }
1557
1558 #define CODE                            1
1559 #  include "jit_riscv-cpu.c"
1560 #  include "jit_riscv-fpu.c"
1561 #undef CODE
1562
1563 void
1564 jit_flush(void *fptr, void *tptr)
1565 {
1566 #if defined(__GNUC__)
1567     jit_word_t          f, t, s;
1568
1569     s = sysconf(_SC_PAGE_SIZE);
1570     f = (jit_word_t)fptr & -s;
1571     t = (((jit_word_t)tptr) + s - 1) & -s;
1572     __clear_cache((void *)f, (void *)t);
1573 #endif
1574 }
1575
1576 void
1577 _emit_ldxi(jit_state_t *_jit, jit_int32_t r0, jit_int32_t r1, jit_word_t i0)
1578 {
1579     ldxi(rn(r0), rn(r1), i0);
1580 }
1581
1582 void
1583 _emit_stxi(jit_state_t *_jit, jit_word_t i0, jit_int32_t r0, jit_int32_t r1)
1584 {
1585     stxi(i0, rn(r0), rn(r1));
1586 }
1587
1588 void
1589 _emit_ldxi_d(jit_state_t *_jit, jit_int32_t r0, jit_int32_t r1, jit_word_t i0)
1590 {
1591     ldxi_d(rn(r0), rn(r1), i0);
1592 }
1593
1594 void
1595 _emit_stxi_d(jit_state_t *_jit, jit_word_t i0, jit_int32_t r0, jit_int32_t r1)
1596 {
1597     stxi_d(i0, rn(r0), rn(r1));
1598 }
1599
1600 static void
1601 _patch(jit_state_t *_jit, jit_word_t instr, jit_node_t *node)
1602 {
1603     jit_int32_t          flag;
1604
1605     assert(node->flag & jit_flag_node);
1606     if (node->code == jit_code_movi)
1607         flag = node->v.n->flag;
1608     else
1609         flag = node->u.n->flag;
1610     assert(!(flag & jit_flag_patch));
1611     if (_jitc->patches.offset >= _jitc->patches.length) {
1612         jit_realloc((jit_pointer_t *)&_jitc->patches.ptr,
1613                     _jitc->patches.length * sizeof(jit_patch_t),
1614                     (_jitc->patches.length + 1024) * sizeof(jit_patch_t));
1615         _jitc->patches.length += 1024;
1616     }
1617     _jitc->patches.ptr[_jitc->patches.offset].inst = instr;
1618     _jitc->patches.ptr[_jitc->patches.offset].node = node;
1619     ++_jitc->patches.offset;
1620 }