deps/lightning/lib/jit_riscv.c

   1 /*
   2  * Copyright (C) 2019  Free Software Foundation, Inc.
   3  *
   4  * This file is part of GNU lightning.
   5  *
   6  * GNU lightning is free software; you can redistribute it and/or modify it
   7  * under the terms of the GNU Lesser General Public License as published
   8  * by the Free Software Foundation; either version 3, or (at your option)
   9  * any later version.
  10  *
  11  * GNU lightning is distributed in the hope that it will be useful, but
  12  * WITHOUT ANY WARRANTY; without even the implied warranty of MERCHANTABILITY
  13  * or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU Lesser General Public
  14  * License for more details.
  15  *
  16  * Authors:
  17  *      Paulo Cesar Pereira de Andrade
  18  */
  19
  20 #define jit_arg_reg_p(i)                ((i) >= 0 && (i) < 8)
  21 #define jit_arg_f_reg_p(i)              ((i) >= 0 && (i) < 8)
  22
  23 /*
  24  * Types
  25  */
  26 typedef jit_pointer_t jit_va_list_t;
  27
  28 /*
  29  * Prototypes
  30  */
  31 #define patch(instr, node)              _patch(_jit, instr, node)
  32 static void _patch(jit_state_t*,jit_word_t,jit_node_t*);
  33
  34 #define PROTO                           1
  35 #  include "jit_riscv-cpu.c"
  36 #  include "jit_riscv-fpu.c"
  37 #  include "jit_fallback.c"
  38 #undef PROTO
  39
  40 /*
  41  * Initialization
  42  */
  43 jit_register_t          _rvs[] = {
  44     { 0x00,                             "zero" },
  45     { 0x01,                             "ra" },
  46     { 0x02,                             "sp" },
  47     { 0x03,                             "gp" },
  48 #if 0           /* Pretend it does not exist, so _NOREG can be used in
  49                  * a 64 bit bitmask */
  50     { 0x04,                             "tp" },
  51 #endif
  52     { rc(gpr) | 0x05,                   "t0" },
  53     { rc(gpr) | 0x06,                   "t1" },
  54     { rc(gpr) | 0x07,                   "t2" },
  55     { rc(gpr) | 0x1c,                   "t3" },
  56     { rc(gpr) | 0x1d,                   "t4" },
  57     { rc(gpr) | 0x1e,                   "t5" },
  58     { rc(gpr) | 0x1f,                   "t6" },
  59     { 0x08,                             "fp" },
  60     { rc(sav) | rc(gpr) | 0x09,         "s1" },
  61     { rc(sav) | rc(gpr) | 0x12,         "s2" },
  62     { rc(sav) | rc(gpr) | 0x13,         "s3" },
  63     { rc(sav) | rc(gpr) | 0x14,         "s4" },
  64     { rc(sav) | rc(gpr) | 0x15,         "s5" },
  65     { rc(sav) | rc(gpr) | 0x16,         "s6" },
  66     { rc(sav) | rc(gpr) | 0x17,         "s7" },
  67     { rc(sav) | rc(gpr) | 0x18,         "s8" },
  68     { rc(sav) | rc(gpr) | 0x19,         "s9" },
  69     { rc(sav) | rc(gpr) | 0x1a,         "s10" },
  70     { rc(sav) | rc(gpr) | 0x1b,         "s11" },
  71     { rc(arg) | rc(gpr) | 0x11,         "a7" },
  72     { rc(arg) | rc(gpr) | 0x10,         "a6" },
  73     { rc(arg) | rc(gpr) | 0x0f,         "a5" },
  74     { rc(arg) | rc(gpr) | 0x0e,         "a4" },
  75     { rc(arg) | rc(gpr) | 0x0d,         "a3" },
  76     { rc(arg) | rc(gpr) | 0x0c,         "a2" },
  77     { rc(arg) | rc(gpr) | 0x0b,         "a1" },
  78     { rc(arg) | rc(gpr) | 0x0a,         "a0" },
  79     { rc(fpr) | 0x00,                   "ft0" },
  80     { rc(fpr) | 0x01,                   "ft1" },
  81     { rc(fpr) | 0x02,                   "ft2" },
  82     { rc(fpr) | 0x03,                   "ft3" },
  83     { rc(fpr) | 0x04,                   "ft4" },
  84     { rc(fpr) | 0x05,                   "ft5" },
  85     { rc(fpr) | 0x06,                   "ft6" },
  86     { rc(fpr) | 0x07,                   "ft7" },
  87     { rc(fpr) | 0x1c,                   "ft8" },
  88     { rc(fpr) | 0x1d,                   "ft9" },
  89     { rc(fpr) | 0x1e,                   "ft10" },
  90     { rc(fpr) | 0x1f,                   "ft11" },
  91     { rc(sav) | rc(fpr) | 0x08,         "fs0" },
  92     { rc(sav) | rc(fpr) | 0x09,         "fs1" },
  93     { rc(sav) | rc(fpr) | 0x12,         "fs2" },
  94     { rc(sav) | rc(fpr) | 0x13,         "fs3" },
  95     { rc(sav) | rc(fpr) | 0x14,         "fs4" },
  96     { rc(sav) | rc(fpr) | 0x15,         "fs5" },
  97     { rc(sav) | rc(fpr) | 0x16,         "fs6" },
  98     { rc(sav) | rc(fpr) | 0x17,         "fs7" },
  99     { rc(sav) | rc(fpr) | 0x18,         "fs8" },
 100     { rc(sav) | rc(fpr) | 0x19,         "fs9" },
 101     { rc(sav) | rc(fpr) | 0x1a,         "fs10" },
 102     { rc(sav) | rc(fpr) | 0x1b,         "fs11" },
 103     { rc(arg) | rc(fpr) | 0x11,         "fa7" },
 104     { rc(arg) | rc(fpr) | 0x10,         "fa6" },
 105     { rc(arg) | rc(fpr) | 0x0f,         "fa5" },
 106     { rc(arg) | rc(fpr) | 0x0e,         "fa4" },
 107     { rc(arg) | rc(fpr) | 0x0d,         "fa3" },
 108     { rc(arg) | rc(fpr) | 0x0c,         "fa2" },
 109     { rc(arg) | rc(fpr) | 0x0b,         "fa1" },
 110     { rc(arg) | rc(fpr) | 0x0a,         "fa0" },
 111     { _NOREG,                           "<none>" },
 112 };
 113
 114 /*
 115  * Implementation
 116  */
 117 void
 118 jit_get_cpu(void)
 119 {
 120 }
 121
 122 void
 123 _jit_init(jit_state_t *_jit)
 124 {
 125     _jitc->reglen = jit_size(_rvs) - 1;
 126     jit_carry = _NOREG;
 127 }
 128
 129 void
 130 _jit_prolog(jit_state_t *_jit)
 131 {
 132     jit_int32_t          offset;
 133
 134     if (_jitc->function)
 135         jit_epilog();
 136     assert(jit_regset_cmp_ui(&_jitc->regarg, 0) == 0);
 137     jit_regset_set_ui(&_jitc->regsav, 0);
 138     offset = _jitc->functions.offset;
 139     if (offset >= _jitc->functions.length) {
 140         jit_realloc((jit_pointer_t *)&_jitc->functions.ptr,
 141                     _jitc->functions.length * sizeof(jit_function_t),
 142                     (_jitc->functions.length + 16) * sizeof(jit_function_t));
 143         _jitc->functions.length += 16;
 144     }
 145     _jitc->function = _jitc->functions.ptr + _jitc->functions.offset++;
 146     _jitc->function->self.size = stack_framesize;
 147     _jitc->function->self.argi = _jitc->function->self.argf =
 148         _jitc->function->self.alen = 0;
 149     _jitc->function->self.aoff = 0;
 150     _jitc->function->self.call = jit_call_default;
 151     jit_alloc((jit_pointer_t *)&_jitc->function->regoff,
 152               _jitc->reglen * sizeof(jit_int32_t));
 153
 154     /* _no_link here does not mean the jit_link() call can be removed
 155      * by rewriting as:
 156      * _jitc->function->prolog = jit_new_node(jit_code_prolog);
 157      */
 158     _jitc->function->prolog = jit_new_node_no_link(jit_code_prolog);
 159     jit_link(_jitc->function->prolog);
 160     _jitc->function->prolog->w.w = offset;
 161     _jitc->function->epilog = jit_new_node_no_link(jit_code_epilog);
 162     /*  u:      label value
 163      *  v:      offset in blocks vector
 164      *  w:      offset in functions vector
 165      */
 166     _jitc->function->epilog->w.w = offset;
 167
 168     jit_regset_new(&_jitc->function->regset);
 169 }
 170
 171 jit_int32_t
 172 _jit_allocai(jit_state_t *_jit, jit_int32_t length)
 173 {
 174     assert(_jitc->function);
 175     switch (length) {
 176         case 0: case 1:                                         break;
 177         case 2:         _jitc->function->self.aoff &= -2;       break;
 178         case 3: case 4: _jitc->function->self.aoff &= -4;       break;
 179         default:        _jitc->function->self.aoff &= -8;       break;
 180     }
 181     _jitc->function->self.aoff -= length;
 182     if (!_jitc->realize) {
 183         jit_inc_synth_ww(allocai, _jitc->function->self.aoff, length);
 184         jit_dec_synth();
 185     }
 186     return (_jitc->function->self.aoff);
 187 }
 188
 189 void
 190 _jit_allocar(jit_state_t *_jit, jit_int32_t u, jit_int32_t v)
 191 {
 192     jit_int32_t          r0;
 193     assert(_jitc->function);
 194     jit_inc_synth_ww(allocar, u, v);
 195     if (!_jitc->function->allocar) {
 196         _jitc->function->aoffoff = jit_allocai(sizeof(jit_int32_t));
 197         _jitc->function->allocar = 1;
 198     }
 199     r0 = jit_get_reg(jit_class_gpr);
 200     jit_negr(r0, v);
 201     jit_andi(r0, r0, -16);
 202     jit_ldxi_i(u, JIT_FP, _jitc->function->aoffoff);
 203     jit_addr(u, u, r0);
 204     jit_addr(JIT_SP, JIT_SP, r0);
 205     jit_stxi_i(_jitc->function->aoffoff, JIT_FP, u);
 206     jit_unget_reg(r0);
 207     jit_dec_synth();
 208 }
 209
 210 void
 211 _jit_ret(jit_state_t *_jit)
 212 {
 213     jit_node_t          *instr;
 214     assert(_jitc->function);
 215     jit_inc_synth(ret);
 216     /* jump to epilog */
 217     instr = jit_jmpi();
 218     jit_patch_at(instr, _jitc->function->epilog);
 219     jit_dec_synth();
 220 }
 221
 222 void
 223 _jit_retr(jit_state_t *_jit, jit_int32_t u)
 224 {
 225     jit_inc_synth_w(retr, u);
 226     if (JIT_RET != u)
 227         jit_movr(JIT_RET, u);
 228     jit_live(JIT_RET);
 229     jit_ret();
 230     jit_dec_synth();
 231 }
 232
 233 void
 234 _jit_reti(jit_state_t *_jit, jit_word_t u)
 235 {
 236     jit_inc_synth_w(reti, u);
 237     jit_movi(JIT_RET, u);
 238     jit_ret();
 239     jit_dec_synth();
 240 }
 241
 242 void
 243 _jit_retr_f(jit_state_t *_jit, jit_int32_t u)
 244 {
 245     jit_inc_synth_w(retr_f, u);
 246     if (u != JIT_FRET)
 247         jit_movr_f(JIT_FRET, u);
 248     else
 249         jit_live(JIT_FRET);
 250     jit_ret();
 251     jit_dec_synth();
 252 }
 253
 254 void
 255 _jit_reti_f(jit_state_t *_jit, jit_float32_t u)
 256 {
 257     jit_inc_synth_f(reti_f, u);
 258     jit_movi_f(JIT_FRET, u);
 259     jit_ret();
 260     jit_dec_synth();
 261 }
 262
 263 void
 264 _jit_retr_d(jit_state_t *_jit, jit_int32_t u)
 265 {
 266     jit_inc_synth_w(retr_d, u);
 267     if (u != JIT_FRET)
 268         jit_movr_d(JIT_FRET, u);
 269     else
 270         jit_live(JIT_FRET);
 271     jit_ret();
 272     jit_dec_synth();
 273 }
 274
 275 void
 276 _jit_reti_d(jit_state_t *_jit, jit_float64_t u)
 277 {
 278     jit_inc_synth_d(reti_d, u);
 279     jit_movi_d(JIT_FRET, u);
 280     jit_ret();
 281     jit_dec_synth();
 282 }
 283
 284 void
 285 _jit_epilog(jit_state_t *_jit)
 286 {
 287     assert(_jitc->function);
 288     assert(_jitc->function->epilog->next == NULL);
 289     jit_link(_jitc->function->epilog);
 290     _jitc->function = NULL;
 291 }
 292
 293 jit_bool_t
 294 _jit_arg_register_p(jit_state_t *_jit, jit_node_t *u)
 295 {
 296     if (u->code == jit_code_arg)
 297         return (jit_arg_reg_p(u->u.w));
 298     assert(u->code == jit_code_arg_f || u->code == jit_code_arg_d);
 299     return (jit_arg_f_reg_p(u->u.w));
 300 }
 301
 302 void
 303 _jit_ellipsis(jit_state_t *_jit)
 304 {
 305     jit_inc_synth(ellipsis);
 306     if (_jitc->prepare) {
 307         jit_link_prepare();
 308         assert(!(_jitc->function->call.call & jit_call_varargs));
 309         _jitc->function->call.call |= jit_call_varargs;
 310     }
 311     else {
 312         jit_link_prolog();
 313         assert(!(_jitc->function->self.call & jit_call_varargs));
 314         _jitc->function->self.call |= jit_call_varargs;
 315         _jitc->function->vagp = _jitc->function->self.argi;
 316     }
 317     jit_dec_synth();
 318 }
 319
 320 void
 321 _jit_va_push(jit_state_t *_jit, jit_int32_t u)
 322 {
 323     jit_inc_synth_w(va_push, u);
 324     jit_pushargr(u);
 325     jit_dec_synth();
 326 }
 327
 328 jit_node_t *
 329 _jit_arg(jit_state_t *_jit)
 330 {
 331     jit_node_t          *node;
 332     jit_int32_t          offset;
 333     assert(_jitc->function);
 334     assert(!(_jitc->function->self.call & jit_call_varargs));
 335     if (jit_arg_reg_p(_jitc->function->self.argi))
 336         offset = _jitc->function->self.argi++;
 337     else {
 338         offset = _jitc->function->self.size;
 339         _jitc->function->self.size += sizeof(jit_word_t);
 340     }
 341     node = jit_new_node_ww(jit_code_arg, offset,
 342                            ++_jitc->function->self.argn);
 343     jit_link_prolog();
 344     return (node);
 345 }
 346
 347 jit_node_t *
 348 _jit_arg_f(jit_state_t *_jit)
 349 {
 350     jit_node_t          *node;
 351     jit_int32_t          offset;
 352     assert(_jitc->function);
 353     assert(!(_jitc->function->self.call & jit_call_varargs));
 354     if (jit_arg_f_reg_p(_jitc->function->self.argf))
 355         offset = _jitc->function->self.argf++;
 356     else if (jit_arg_reg_p(_jitc->function->self.argi)) {
 357         offset = _jitc->function->self.argi++;
 358         offset += 8;
 359     }
 360     else {
 361         offset = _jitc->function->self.size;
 362         _jitc->function->self.size += sizeof(jit_word_t);
 363     }
 364     node = jit_new_node_ww(jit_code_arg_f, offset,
 365                            ++_jitc->function->self.argn);
 366     jit_link_prolog();
 367     return (node);
 368 }
 369
 370 jit_node_t *
 371 _jit_arg_d(jit_state_t *_jit)
 372 {
 373     jit_node_t          *node;
 374     jit_int32_t          offset;
 375     assert(_jitc->function);
 376     assert(!(_jitc->function->self.call & jit_call_varargs));
 377     if (jit_arg_f_reg_p(_jitc->function->self.argf))
 378         offset = _jitc->function->self.argf++;
 379     else if (jit_arg_reg_p(_jitc->function->self.argi)) {
 380         offset = _jitc->function->self.argi++;
 381         offset += 8;
 382     }
 383     else {
 384         offset = _jitc->function->self.size;
 385         _jitc->function->self.size += sizeof(jit_word_t);
 386     }
 387     node = jit_new_node_ww(jit_code_arg_d, offset,
 388                            ++_jitc->function->self.argn);
 389     jit_link_prolog();
 390     return (node);
 391 }
 392
 393 void
 394 _jit_getarg_c(jit_state_t *_jit, jit_int32_t u, jit_node_t *v)
 395 {
 396     assert(v->code == jit_code_arg);
 397     jit_inc_synth_wp(getarg_c, u, v);
 398     if (jit_arg_reg_p(v->u.w))
 399         jit_extr_c(u, JIT_RA0 - v->u.w);
 400     else
 401         jit_ldxi_c(u, JIT_FP, v->u.w);
 402     jit_dec_synth();
 403 }
 404
 405 void
 406 _jit_getarg_uc(jit_state_t *_jit, jit_int32_t u, jit_node_t *v)
 407 {
 408     assert(v->code == jit_code_arg);
 409     jit_inc_synth_wp(getarg_uc, u, v);
 410     if (jit_arg_reg_p(v->u.w))
 411         jit_extr_uc(u, JIT_RA0 - v->u.w);
 412     else
 413         jit_ldxi_uc(u, JIT_FP, v->u.w);
 414     jit_dec_synth();
 415 }
 416
 417 void
 418 _jit_getarg_s(jit_state_t *_jit, jit_int32_t u, jit_node_t *v)
 419 {
 420     assert(v->code == jit_code_arg);
 421     jit_inc_synth_wp(getarg_s, u, v);
 422     if (jit_arg_reg_p(v->u.w))
 423         jit_extr_s(u, JIT_RA0 - v->u.w);
 424     else
 425         jit_ldxi_s(u, JIT_FP, v->u.w);
 426     jit_dec_synth();
 427 }
 428
 429 void
 430 _jit_getarg_us(jit_state_t *_jit, jit_int32_t u, jit_node_t *v)
 431 {
 432     assert(v->code == jit_code_arg);
 433     jit_inc_synth_wp(getarg_us, u, v);
 434     if (jit_arg_reg_p(v->u.w))
 435         jit_extr_us(u, JIT_RA0 - v->u.w);
 436     else
 437         jit_ldxi_us(u, JIT_FP, v->u.w);
 438     jit_dec_synth();
 439 }
 440
 441 void
 442 _jit_getarg_i(jit_state_t *_jit, jit_int32_t u, jit_node_t *v)
 443 {
 444     assert(v->code == jit_code_arg);
 445     jit_inc_synth_wp(getarg_i, u, v);
 446     if (jit_arg_reg_p(v->u.w))
 447         jit_extr_i(u, JIT_RA0 - v->u.w);
 448     else
 449         jit_ldxi_i(u, JIT_FP, v->u.w);
 450     jit_dec_synth();
 451 }
 452
 453 void
 454 _jit_getarg_ui(jit_state_t *_jit, jit_int32_t u, jit_node_t *v)
 455 {
 456     assert(v->code == jit_code_arg);
 457     jit_inc_synth_wp(getarg_ui, u, v);
 458     if (jit_arg_reg_p(v->u.w))
 459         jit_extr_ui(u, JIT_RA0 - v->u.w);
 460     else
 461         jit_ldxi_ui(u, JIT_FP, v->u.w);
 462     jit_dec_synth();
 463 }
 464
 465 void
 466 _jit_getarg_l(jit_state_t *_jit, jit_int32_t u, jit_node_t *v)
 467 {
 468     assert(v->code == jit_code_arg);
 469     jit_inc_synth_wp(getarg_l, u, v);
 470     if (jit_arg_reg_p(v->u.w))
 471         jit_movr(u, JIT_RA0 - v->u.w);
 472     else
 473         jit_ldxi_l(u, JIT_FP, v->u.w);
 474     jit_dec_synth();
 475 }
 476
 477 void
 478 _jit_putargr(jit_state_t *_jit, jit_int32_t u, jit_node_t *v)
 479 {
 480     assert(v->code == jit_code_arg);
 481     jit_inc_synth_wp(putargr, u, v);
 482     if (jit_arg_reg_p(v->u.w))
 483         jit_movr(JIT_RA0 - v->u.w, u);
 484     else
 485         jit_stxi(v->u.w, JIT_FP, u);
 486     jit_dec_synth();
 487 }
 488
 489 void
 490 _jit_putargi(jit_state_t *_jit, jit_word_t u, jit_node_t *v)
 491 {
 492     jit_int32_t         regno;
 493     assert(v->code == jit_code_arg);
 494     jit_inc_synth_wp(putargi, u, v);
 495     if (jit_arg_reg_p(v->u.w))
 496         jit_movi(JIT_RA0 - v->u.w, u);
 497     else {
 498         regno = jit_get_reg(jit_class_gpr);
 499         jit_movi(regno, u);
 500         jit_stxi(v->u.w, JIT_FP, regno);
 501         jit_unget_reg(regno);
 502     }
 503     jit_dec_synth();
 504 }
 505
 506 void
 507 _jit_getarg_f(jit_state_t *_jit, jit_int32_t u, jit_node_t *v)
 508 {
 509     assert(v->code == jit_code_arg_f);
 510     jit_inc_synth_wp(getarg_f, u, v);
 511     if (jit_arg_f_reg_p(v->u.w))
 512         jit_movr_f(u, JIT_FA0 - v->u.w);
 513     else if (jit_arg_reg_p(v->u.w - 8))
 514         jit_movr_w_f(u, JIT_RA0 - (v->u.w - 8));
 515     else
 516         jit_ldxi_f(u, JIT_FP, v->u.w);
 517     jit_dec_synth();
 518 }
 519
 520 void
 521 _jit_putargr_f(jit_state_t *_jit, jit_int32_t u, jit_node_t *v)
 522 {
 523     assert(v->code == jit_code_arg_f);
 524     jit_inc_synth_wp(putargr_f, u, v);
 525     if (jit_arg_f_reg_p(v->u.w))
 526         jit_movr_f(JIT_FA0 - v->u.w, u);
 527     else if (jit_arg_reg_p(v->u.w - 8))
 528         jit_movr_f_w(JIT_RA0 - (v->u.w - 8), u);
 529     else
 530         jit_stxi_f(v->u.w, JIT_FP, u);
 531     jit_dec_synth();
 532 }
 533
 534 void
 535 _jit_putargi_f(jit_state_t *_jit, jit_float32_t u, jit_node_t *v)
 536 {
 537     jit_int32_t         regno;
 538     assert(v->code == jit_code_arg_f);
 539     jit_inc_synth_fp(putargi_f, u, v);
 540     if (jit_arg_f_reg_p(v->u.w))
 541         jit_movi_f(JIT_FA0 - v->u.w, u);
 542     else if (jit_arg_reg_p(v->u.w - 8)) {
 543         union {
 544             jit_float32_t       f;
 545             jit_int32_t         i;
 546         } uu;
 547         uu.f = u;
 548         jit_movi(JIT_RA0 - (v->u.w - 8), uu.i);
 549     }
 550     else {
 551         regno = jit_get_reg(jit_class_fpr);
 552         jit_movi_f(regno, u);
 553         jit_stxi_f(v->u.w, JIT_FP, regno);
 554         jit_unget_reg(regno);
 555     }
 556     jit_dec_synth();
 557 }
 558
 559 void
 560 _jit_getarg_d(jit_state_t *_jit, jit_int32_t u, jit_node_t *v)
 561 {
 562     assert(v->code == jit_code_arg_d);
 563     jit_inc_synth_wp(getarg_d, u, v);
 564     if (jit_arg_f_reg_p(v->u.w))
 565         jit_movr_d(u, JIT_FA0 - v->u.w);
 566     else if (jit_arg_reg_p(v->u.w - 8))
 567         jit_movr_w_d(u, JIT_RA0 - (v->u.w - 8));
 568     else
 569         jit_ldxi_d(u, JIT_FP, v->u.w);
 570     jit_dec_synth();
 571 }
 572
 573 void
 574 _jit_putargr_d(jit_state_t *_jit, jit_int32_t u, jit_node_t *v)
 575 {
 576     assert(v->code == jit_code_arg_d);
 577     jit_inc_synth_wp(putargr_d, u, v);
 578     if (jit_arg_reg_p(v->u.w))
 579         jit_movr_d(JIT_FA0 - v->u.w, u);
 580     else if (jit_arg_reg_p(v->u.w - 8))
 581         jit_movr_d_w(JIT_RA0 - (v->u.w - 8), u);
 582     else
 583         jit_stxi_d(v->u.w, JIT_FP, u);
 584     jit_dec_synth();
 585 }
 586
 587 void
 588 _jit_putargi_d(jit_state_t *_jit, jit_float64_t u, jit_node_t *v)
 589 {
 590     jit_int32_t         regno;
 591     assert(v->code == jit_code_arg_d);
 592     jit_inc_synth_dp(putargi_d, u, v);
 593     if (jit_arg_reg_p(v->u.w))
 594         jit_movi_d(JIT_FA0 - v->u.w, u);
 595     else if (jit_arg_reg_p(v->u.w - 8)) {
 596         union {
 597             jit_float64_t       d;
 598             jit_int64_t         w;
 599         } uu;
 600         uu.d = u;
 601         jit_movi(JIT_RA0 - (v->u.w - 8), uu.w);
 602     }
 603     else {
 604         regno = jit_get_reg(jit_class_fpr);
 605         jit_movi_d(regno, u);
 606         jit_stxi_d(v->u.w, JIT_FP, regno);
 607         jit_unget_reg(regno);
 608     }
 609     jit_dec_synth();
 610 }
 611
 612 void
 613 _jit_pushargr(jit_state_t *_jit, jit_int32_t u)
 614 {
 615     assert(_jitc->function);
 616     jit_inc_synth_w(pushargr, u);
 617     jit_link_prepare();
 618     if (jit_arg_reg_p(_jitc->function->call.argi)) {
 619         jit_movr(JIT_RA0 - _jitc->function->call.argi, u);
 620         ++_jitc->function->call.argi;
 621     }
 622     else {
 623         jit_stxi(_jitc->function->call.size, JIT_SP, u);
 624         _jitc->function->call.size += sizeof(jit_word_t);
 625     }
 626     jit_dec_synth();
 627 }
 628
 629 void
 630 _jit_pushargi(jit_state_t *_jit, jit_word_t u)
 631 {
 632     jit_int32_t          regno;
 633     assert(_jitc->function);
 634     jit_inc_synth_w(pushargi, u);
 635     jit_link_prepare();
 636     if (jit_arg_reg_p(_jitc->function->call.argi)) {
 637         jit_movi(JIT_RA0 - _jitc->function->call.argi, u);
 638         ++_jitc->function->call.argi;
 639     }
 640     else {
 641         regno = jit_get_reg(jit_class_gpr);
 642         jit_movi(regno, u);
 643         jit_stxi(_jitc->function->call.size, JIT_SP, regno);
 644         jit_unget_reg(regno);
 645         _jitc->function->call.size += sizeof(jit_word_t);
 646     }
 647     jit_dec_synth();
 648 }
 649
 650 void
 651 _jit_pushargr_f(jit_state_t *_jit, jit_int32_t u)
 652 {
 653     assert(_jitc->function);
 654     jit_inc_synth_w(pushargr_f, u);
 655     jit_link_prepare();
 656     if (jit_arg_f_reg_p(_jitc->function->call.argf) &&
 657         !(_jitc->function->call.call & jit_call_varargs)) {
 658         jit_movr_f(JIT_FA0 - _jitc->function->call.argf, u);
 659         ++_jitc->function->call.argf;
 660     }
 661     else if (jit_arg_reg_p(_jitc->function->call.argi)) {
 662         jit_movr_f_w(JIT_RA0 - _jitc->function->call.argi, u);
 663         ++_jitc->function->call.argi;
 664     }
 665     else {
 666         jit_stxi_f(_jitc->function->call.size, JIT_SP, u);
 667         _jitc->function->call.size += sizeof(jit_word_t);
 668     }
 669     jit_dec_synth();
 670 }
 671
 672 void
 673 _jit_pushargi_f(jit_state_t *_jit, jit_float32_t u)
 674 {
 675     jit_int32_t         regno;
 676     assert(_jitc->function);
 677     jit_inc_synth_f(pushargi_f, u);
 678     jit_link_prepare();
 679     if (jit_arg_f_reg_p(_jitc->function->call.argf) &&
 680         !(_jitc->function->call.call & jit_call_varargs)) {
 681         jit_movi_f(JIT_FA0 - _jitc->function->call.argf, u);
 682         ++_jitc->function->call.argf;
 683     }
 684     else if (jit_arg_reg_p(_jitc->function->call.argi)) {
 685         jit_movi_f_w(JIT_RA0 - _jitc->function->call.argi, u);
 686         ++_jitc->function->call.argi;
 687     }
 688     else {
 689         regno = jit_get_reg(jit_class_fpr);
 690         jit_movi_f(regno, u);
 691         jit_stxi_f(_jitc->function->call.size, JIT_SP, regno);
 692         jit_unget_reg(regno);
 693         _jitc->function->call.size += sizeof(jit_word_t);
 694     }
 695     jit_dec_synth();
 696 }
 697
 698 void
 699 _jit_pushargr_d(jit_state_t *_jit, jit_int32_t u)
 700 {
 701     assert(_jitc->function);
 702     jit_inc_synth_w(pushargr_d, u);
 703     jit_link_prepare();
 704     if (jit_arg_f_reg_p(_jitc->function->call.argf) &&
 705         !(_jitc->function->call.call & jit_call_varargs)) {
 706         jit_movr_d(JIT_FA0 - _jitc->function->call.argf, u);
 707         ++_jitc->function->call.argf;
 708     }
 709     else if (jit_arg_reg_p(_jitc->function->call.argi)) {
 710         jit_movr_d_w(JIT_RA0 - _jitc->function->call.argi, u);
 711         ++_jitc->function->call.argi;
 712     }
 713     else {
 714         jit_stxi_d(_jitc->function->call.size, JIT_SP, u);
 715         _jitc->function->call.size += sizeof(jit_word_t);
 716     }
 717     jit_dec_synth();
 718 }
 719
 720 void
 721 _jit_pushargi_d(jit_state_t *_jit, jit_float64_t u)
 722 {
 723     jit_int32_t         regno;
 724     assert(_jitc->function);
 725     jit_inc_synth_d(pushargi_d, u);
 726     jit_link_prepare();
 727     if (jit_arg_f_reg_p(_jitc->function->call.argf) &&
 728         !(_jitc->function->call.call & jit_call_varargs)) {
 729         jit_movi_d(JIT_FA0 - _jitc->function->call.argf, u);
 730         ++_jitc->function->call.argf;
 731     }
 732     else if (jit_arg_reg_p(_jitc->function->call.argi)) {
 733         jit_movi_d_w(JIT_RA0 - _jitc->function->call.argi, u);
 734         ++_jitc->function->call.argi;
 735     }
 736     else {
 737         regno = jit_get_reg(jit_class_fpr);
 738         jit_movi_d(regno, u);
 739         jit_stxi_d(_jitc->function->call.size, JIT_SP, regno);
 740         jit_unget_reg(regno);
 741         _jitc->function->call.size += sizeof(jit_word_t);
 742     }
 743     jit_dec_synth();
 744 }
 745
 746 jit_bool_t
 747 _jit_regarg_p(jit_state_t *_jit, jit_node_t *node, jit_int32_t regno)
 748 {
 749     jit_int32_t         spec;
 750     spec = jit_class(_rvs[regno].spec);
 751     if (spec & jit_class_arg) {
 752         regno = JIT_RA0 - regno;
 753         if (regno >= 0 && regno < node->v.w)
 754             return (1);
 755         if (spec & jit_class_fpr) {
 756             regno = JIT_FA0 - regno;
 757             if (regno >= 0 && regno < node->w.w)
 758                 return (1);
 759         }
 760     }
 761
 762     return (0);
 763 }
 764
 765 void
 766 _jit_finishr(jit_state_t *_jit, jit_int32_t r0)
 767 {
 768     jit_node_t          *node;
 769     assert(_jitc->function);
 770     jit_inc_synth_w(finishr, r0);
 771     if (_jitc->function->self.alen < _jitc->function->call.size)
 772         _jitc->function->self.alen = _jitc->function->call.size;
 773     node = jit_callr(r0);
 774     node->v.w = _jitc->function->self.argi;
 775     node->w.w = _jitc->function->call.argf;
 776     _jitc->function->call.argi = _jitc->function->call.argf =
 777         _jitc->function->call.size = 0;
 778     _jitc->prepare = 0;
 779     jit_dec_synth();
 780 }
 781
 782 jit_node_t *
 783 _jit_finishi(jit_state_t *_jit, jit_pointer_t i0)
 784 {
 785     jit_node_t          *node;
 786     assert(_jitc->function);
 787     jit_inc_synth_w(finishi, (jit_word_t)i0);
 788     if (_jitc->function->self.alen < _jitc->function->call.size)
 789         _jitc->function->self.alen = _jitc->function->call.size;
 790     node = jit_calli(i0);
 791     node->v.w = _jitc->function->call.argi;
 792     node->w.w = _jitc->function->call.argf;
 793     _jitc->function->call.argi = _jitc->function->call.argf =
 794         _jitc->function->call.size = 0;
 795     _jitc->prepare = 0;
 796     jit_dec_synth();
 797     return (node);
 798 }
 799
 800 void
 801 _jit_retval_c(jit_state_t *_jit, jit_int32_t r0)
 802 {
 803     jit_inc_synth_w(retval_c, r0);
 804     jit_extr_c(r0, JIT_RET);
 805     jit_dec_synth();
 806 }
 807
 808 void
 809 _jit_retval_uc(jit_state_t *_jit, jit_int32_t r0)
 810 {
 811     jit_inc_synth_w(retval_uc, r0);
 812     jit_extr_uc(r0, JIT_RET);
 813     jit_dec_synth();
 814 }
 815
 816 void
 817 _jit_retval_s(jit_state_t *_jit, jit_int32_t r0)
 818 {
 819     jit_inc_synth_w(retval_s, r0);
 820     jit_extr_s(r0, JIT_RET);
 821     jit_dec_synth();
 822 }
 823
 824 void
 825 _jit_retval_us(jit_state_t *_jit, jit_int32_t r0)
 826 {
 827     jit_inc_synth_w(retval_us, r0);
 828     jit_extr_us(r0, JIT_RET);
 829     jit_dec_synth();
 830 }
 831
 832 void
 833 _jit_retval_i(jit_state_t *_jit, jit_int32_t r0)
 834 {
 835     jit_inc_synth_w(retval_i, r0);
 836     jit_extr_i(r0, JIT_RET);
 837     jit_dec_synth();
 838 }
 839
 840 void
 841 _jit_retval_ui(jit_state_t *_jit, jit_int32_t r0)
 842 {
 843     jit_inc_synth_w(retval_ui, r0);
 844     jit_extr_ui(r0, JIT_RET);
 845     jit_dec_synth();
 846 }
 847
 848 void
 849 _jit_retval_l(jit_state_t *_jit, jit_int32_t r0)
 850 {
 851     jit_inc_synth_w(retval_l, r0);
 852     if (r0 != JIT_RET)
 853         jit_movr(r0, JIT_RET);
 854     jit_dec_synth();
 855 }
 856
 857 void
 858 _jit_retval_f(jit_state_t *_jit, jit_int32_t r0)
 859 {
 860     jit_inc_synth_w(retval_f, r0);
 861     if (r0 != JIT_FRET)
 862         jit_movr_f(r0, JIT_FRET);
 863     jit_dec_synth();
 864 }
 865
 866 void
 867 _jit_retval_d(jit_state_t *_jit, jit_int32_t r0)
 868 {
 869     jit_inc_synth_w(retval_d, r0);
 870     if (r0 != JIT_FRET)
 871         jit_movr_d(r0, JIT_FRET);
 872     jit_dec_synth();
 873 }
 874
 875 jit_pointer_t
 876 _emit_code(jit_state_t *_jit)
 877 {
 878     jit_node_t          *node;
 879     jit_node_t          *temp;
 880     jit_word_t           word;
 881     jit_word_t           value;
 882     jit_int32_t          offset;
 883     struct {
 884         jit_node_t      *node;
 885         jit_uint8_t     *data;
 886         jit_word_t       word;
 887 #if DEVEL_DISASSEMBLER
 888         jit_word_t       prevw;
 889 #endif
 890         jit_int32_t      const_offset;
 891         jit_int32_t      patch_offset;
 892     } undo;
 893 #if DEVEL_DISASSEMBLER
 894     jit_word_t           prevw;
 895 #endif
 896
 897     _jitc->function = NULL;
 898
 899     jit_reglive_setup();
 900
 901     undo.word = 0;
 902     undo.node = NULL;
 903     undo.const_offset = undo.patch_offset = 0;
 904 #  define assert_data(node)             /**/
 905 #define case_rr(name, type)                                             \
 906             case jit_code_##name##r##type:                              \
 907                 name##r##type(rn(node->u.w), rn(node->v.w));            \
 908                 break
 909 #define case_rw(name, type)                                             \
 910             case jit_code_##name##i##type:                              \
 911                 name##i##type(rn(node->u.w), node->v.w);                \
 912                 break
 913 #define case_wr(name, type)                                             \
 914             case jit_code_##name##i##type:                              \
 915                 name##i##type(node->u.w, rn(node->v.w));                \
 916                 break
 917 #define case_rrr(name, type)                                            \
 918             case jit_code_##name##r##type:                              \
 919                 name##r##type(rn(node->u.w),                            \
 920                               rn(node->v.w), rn(node->w.w));            \
 921                 break
 922 #define case_rrrr(name, type)                                           \
 923             case jit_code_##name##r##type:                              \
 924                 name##r##type(rn(node->u.q.l), rn(node->u.q.h),         \
 925                               rn(node->v.w), rn(node->w.w));            \
 926                 break
 927 #define case_rrw(name, type)                                            \
 928             case jit_code_##name##i##type:                              \
 929                 name##i##type(rn(node->u.w), rn(node->v.w), node->w.w); \
 930                 break
 931 #define case_rrrw(name, type)                                           \
 932             case jit_code_##name##i##type:                              \
 933                 name##i##type(rn(node->u.q.l), rn(node->u.q.h),         \
 934                               rn(node->v.w), node->w.w);                \
 935                 break
 936 #define case_rrf(name)                                                  \
 937             case jit_code_##name##i_f:                                  \
 938                 assert_data(node);                                      \
 939                 name##i_f(rn(node->u.w), rn(node->v.w), node->w.f);     \
 940                 break
 941 #define case_rrd(name)                                                  \
 942             case jit_code_##name##i_d:                                  \
 943                 assert_data(node);                                      \
 944                 name##i_d(rn(node->u.w), rn(node->v.w), node->w.d);     \
 945                 break
 946 #define case_wrr(name, type)                                            \
 947             case jit_code_##name##i##type:                              \
 948                 name##i##type(node->u.w, rn(node->v.w), rn(node->w.w)); \
 949                 break
 950 #define case_brr(name, type)                                            \
 951             case jit_code_##name##r##type:                              \
 952                 temp = node->u.n;                                       \
 953                 assert(temp->code == jit_code_label ||                  \
 954                        temp->code == jit_code_epilog);                  \
 955                 if (temp->flag & jit_flag_patch)                        \
 956                     name##r##type(temp->u.w, rn(node->v.w),             \
 957                                   rn(node->w.w));                       \
 958                 else {                                                  \
 959                     word = name##r##type(_jit->pc.w,                    \
 960                                          rn(node->v.w), rn(node->w.w)); \
 961                     patch(word, node);                                  \
 962                 }                                                       \
 963                 break
 964 #define case_brw(name, type)                                            \
 965             case jit_code_##name##i##type:                              \
 966                 temp = node->u.n;                                       \
 967                 assert(temp->code == jit_code_label ||                  \
 968                        temp->code == jit_code_epilog);                  \
 969                 if (temp->flag & jit_flag_patch)                        \
 970                     name##i##type(temp->u.w,                            \
 971                                   rn(node->v.w), node->w.w);            \
 972                 else {                                                  \
 973                     word = name##i##type(_jit->pc.w,                    \
 974                                          rn(node->v.w), node->w.w);     \
 975                     patch(word, node);                                  \
 976                 }                                                       \
 977                 break;
 978 #define case_brf(name)                                                  \
 979             case jit_code_##name##i_f:                                  \
 980                 temp = node->u.n;                                       \
 981                 assert(temp->code == jit_code_label ||                  \
 982                        temp->code == jit_code_epilog);                  \
 983                 if (temp->flag & jit_flag_patch)                        \
 984                     name##i_f(temp->u.w, rn(node->v.w), node->w.f);     \
 985                 else {                                                  \
 986                     word = name##i_f(_jit->pc.w, rn(node->v.w),         \
 987                                 node->w.f);                             \
 988                     patch(word, node);                                  \
 989                 }                                                       \
 990                 break
 991 #define case_brd(name)                                                  \
 992             case jit_code_##name##i_d:                                  \
 993                 temp = node->u.n;                                       \
 994                 assert(temp->code == jit_code_label ||                  \
 995                        temp->code == jit_code_epilog);                  \
 996                 if (temp->flag & jit_flag_patch)                        \
 997                     name##i_d(temp->u.w, rn(node->v.w), node->w.d);     \
 998                 else {                                                  \
 999                     word = name##i_d(_jit->pc.w, rn(node->v.w),         \
1000                                 node->w.d);                             \
1001                     patch(word, node);                                  \
1002                 }                                                       \
1003                 break
1004 #if DEVEL_DISASSEMBLER
1005     prevw = _jit->pc.w;
1006 #endif
1007     for (node = _jitc->head; node; node = node->next) {
1008         if (_jit->pc.uc >= _jitc->code.end)
1009             return (NULL);
1010
1011 #if DEVEL_DISASSEMBLER
1012         node->offset = (jit_uword_t)_jit->pc.w - (jit_uword_t)prevw;
1013         prevw = _jit->pc.w;
1014 #endif
1015         value = jit_classify(node->code);
1016         jit_regarg_set(node, value);
1017         switch (node->code) {
1018             case jit_code_align:
1019                 assert(!(node->u.w & (node->u.w - 1)) &&
1020                        node->u.w <= sizeof(jit_word_t));
1021                 if (node->u.w == sizeof(jit_word_t) &&
1022                     (word = _jit->pc.w & (sizeof(jit_word_t) - 1)))
1023                     nop(sizeof(jit_word_t) - word);
1024                 break;
1025             case jit_code_note:         case jit_code_name:
1026                 node->u.w = _jit->pc.w;
1027                 break;
1028             case jit_code_label:
1029                 /* remember label is defined */
1030                 node->flag |= jit_flag_patch;
1031                 node->u.w = _jit->pc.w;
1032                 break;
1033                 case_rrr(add,);
1034                 case_rrw(add,);
1035                 case_rrr(addc,);
1036                 case_rrw(addc,);
1037                 case_rrr(addx,);
1038                 case_rrw(addx,);
1039                 case_rrr(sub,);
1040                 case_rrw(sub,);
1041                 case_rrr(subc,);
1042                 case_rrw(subc,);
1043                 case_rrr(subx,);
1044                 case_rrw(subx,);
1045                 case_rrw(rsb,);
1046                 case_rrr(mul,);
1047                 case_rrw(mul,);
1048                 case_rrrr(qmul,);
1049                 case_rrrw(qmul,);
1050                 case_rrrr(qmul, _u);
1051                 case_rrrw(qmul, _u);
1052                 case_rrr(div,);
1053                 case_rrw(div,);
1054                 case_rrr(div, _u);
1055                 case_rrw(div, _u);
1056                 case_rrrr(qdiv,);
1057                 case_rrrw(qdiv,);
1058                 case_rrrr(qdiv, _u);
1059                 case_rrrw(qdiv, _u);
1060                 case_rrr(rem,);
1061                 case_rrw(rem,);
1062                 case_rrr(rem, _u);
1063                 case_rrw(rem, _u);
1064                 case_rrr(lsh,);
1065                 case_rrw(lsh,);
1066                 case_rrr(rsh,);
1067                 case_rrw(rsh,);
1068                 case_rrr(rsh, _u);
1069                 case_rrw(rsh, _u);
1070                 case_rr(neg,);
1071                 case_rr(com,);
1072                 case_rrr(and,);
1073                 case_rrw(and,);
1074                 case_rrr(or,);
1075                 case_rrw(or,);
1076                 case_rrr(xor,);
1077                 case_rrw(xor,);
1078                 case_rr(trunc, _f_i);
1079                 case_rr(trunc, _d_i);
1080                 case_rr(trunc, _f_l);
1081                 case_rr(trunc, _d_l);
1082                 case_rr(ld, _c);
1083                 case_rw(ld, _c);
1084                 case_rr(ld, _uc);
1085                 case_rw(ld, _uc);
1086                 case_rr(ld, _s);
1087                 case_rw(ld, _s);
1088                 case_rr(ld, _us);
1089                 case_rw(ld, _us);
1090                 case_rr(ld, _i);
1091                 case_rw(ld, _i);
1092                 case_rr(ld, _ui);
1093                 case_rw(ld, _ui);
1094                 case_rr(ld, _l);
1095                 case_rw(ld, _l);
1096                 case_rrr(ldx, _c);
1097                 case_rrw(ldx, _c);
1098                 case_rrr(ldx, _uc);
1099                 case_rrw(ldx, _uc);
1100                 case_rrr(ldx, _s);
1101                 case_rrw(ldx, _s);
1102                 case_rrr(ldx, _us);
1103                 case_rrw(ldx, _us);
1104                 case_rrr(ldx, _i);
1105                 case_rrw(ldx, _i);
1106                 case_rrr(ldx, _ui);
1107                 case_rrw(ldx, _ui);
1108                 case_rrr(ldx, _l);
1109                 case_rrw(ldx, _l);
1110                 case_rr(st, _c);
1111                 case_wr(st, _c);
1112                 case_rr(st, _s);
1113                 case_wr(st, _s);
1114                 case_rr(st, _i);
1115                 case_wr(st, _i);
1116                 case_rr(st, _l);
1117                 case_wr(st, _l);
1118                 case_rrr(stx, _c);
1119                 case_wrr(stx, _c);
1120                 case_rrr(stx, _s);
1121                 case_wrr(stx, _s);
1122                 case_rrr(stx, _i);
1123                 case_wrr(stx, _i);
1124                 case_rrr(stx, _l);
1125                 case_wrr(stx, _l);
1126                 case_rr(hton, _us);
1127                 case_rr(hton, _ui);
1128                 case_rr(hton, _ul);
1129                 case_rr(bswap, _us);
1130                 case_rr(bswap, _ui);
1131                 case_rr(bswap, _ul);
1132                 case_rr(ext, _c);
1133                 case_rr(ext, _uc);
1134                 case_rr(ext, _s);
1135                 case_rr(ext, _us);
1136                 case_rr(ext, _i);
1137                 case_rr(ext, _ui);
1138             case jit_code_casr:
1139                 casr(rn(node->u.w), rn(node->v.w),
1140                      rn(node->w.q.l), rn(node->w.q.h));
1141                 break;
1142             case jit_code_casi:
1143                 casi(rn(node->u.w), node->v.w,
1144                      rn(node->w.q.l), rn(node->w.q.h));
1145                 break;
1146                 case_rrr(movn,);
1147                 case_rrr(movz,);
1148                 case_rr(mov,);
1149             case jit_code_movi:
1150                 if (node->flag & jit_flag_node) {
1151                     temp = node->v.n;
1152                     if (temp->code == jit_code_data ||
1153                         (temp->code == jit_code_label &&
1154                          (temp->flag & jit_flag_patch)))
1155                         movi(rn(node->u.w), temp->u.w);
1156                     else {
1157                         assert(temp->code == jit_code_label ||
1158                                temp->code == jit_code_epilog);
1159                         word = movi_p(rn(node->u.w), temp->u.w);
1160                         patch(word, node);
1161                     }
1162                 }
1163                 else
1164                     movi(rn(node->u.w), node->v.w);
1165                 break;
1166                 case_rrr(lt,);
1167                 case_rrw(lt,);
1168                 case_rrr(lt, _u);
1169                 case_rrw(lt, _u);
1170                 case_rrr(le,);
1171                 case_rrw(le,);
1172                 case_rrr(le, _u);
1173                 case_rrw(le, _u);
1174                 case_rrr(eq,);
1175                 case_rrw(eq,);
1176                 case_rrr(ge,);
1177                 case_rrw(ge,);
1178                 case_rrr(ge, _u);
1179                 case_rrw(ge, _u);
1180                 case_rrr(gt,);
1181                 case_rrw(gt,);
1182                 case_rrr(gt, _u);
1183                 case_rrw(gt, _u);
1184                 case_rrr(ne,);
1185                 case_rrw(ne,);
1186                 case_brr(blt,);
1187                 case_brw(blt,);
1188                 case_brr(blt, _u);
1189                 case_brw(blt, _u);
1190                 case_brr(ble,);
1191                 case_brw(ble,);
1192                 case_brr(ble, _u);
1193                 case_brw(ble, _u);
1194                 case_brr(beq,);
1195                 case_brw(beq,);
1196                 case_brr(bge,);
1197                 case_brw(bge,);
1198                 case_brr(bge, _u);
1199                 case_brw(bge, _u);
1200                 case_brr(bgt,);
1201                 case_brw(bgt,);
1202                 case_brr(bgt, _u);
1203                 case_brw(bgt, _u);
1204                 case_brr(bne,);
1205                 case_brw(bne,);
1206                 case_brr(boadd,);
1207                 case_brw(boadd,);
1208                 case_brr(boadd, _u);
1209                 case_brw(boadd, _u);
1210                 case_brr(bxadd,);
1211                 case_brw(bxadd,);
1212                 case_brr(bxadd, _u);
1213                 case_brw(bxadd, _u);
1214                 case_brr(bosub,);
1215                 case_brw(bosub,);
1216                 case_brr(bosub, _u);
1217                 case_brw(bosub, _u);
1218                 case_brr(bxsub,);
1219                 case_brw(bxsub,);
1220                 case_brr(bxsub, _u);
1221                 case_brw(bxsub, _u);
1222                 case_brr(bms,);
1223                 case_brw(bms,);
1224                 case_brr(bmc,);
1225                 case_brw(bmc,);
1226                 case_rrr(add, _f);
1227                 case_rrf(add);
1228                 case_rrr(sub, _f);
1229                 case_rrf(sub);
1230                 case_rrf(rsb);
1231                 case_rrr(mul, _f);
1232                 case_rrf(mul);
1233                 case_rrr(div, _f);
1234                 case_rrf(div);
1235                 case_rr(abs, _f);
1236                 case_rr(neg, _f);
1237                 case_rr(sqrt, _f);
1238                 case_rr(ext, _f);
1239                 case_rr(ld, _f);
1240                 case_rw(ld, _f);
1241                 case_rrr(ldx, _f);
1242                 case_rrw(ldx, _f);
1243                 case_rr(st, _f);
1244                 case_wr(st, _f);
1245                 case_rrr(stx, _f);
1246                 case_wrr(stx, _f);
1247                 case_rr(mov, _f);
1248             case jit_code_movi_f:
1249                 assert_data(node);
1250                 movi_f(rn(node->u.w), node->v.f);
1251                 break;
1252                 case_rr(ext, _d_f);
1253                 case_rrr(lt, _f);
1254                 case_rrf(lt);
1255                 case_rrr(le, _f);
1256                 case_rrf(le);
1257                 case_rrr(eq, _f);
1258                 case_rrf(eq);
1259                 case_rrr(ge, _f);
1260                 case_rrf(ge);
1261                 case_rrr(gt, _f);
1262                 case_rrf(gt);
1263                 case_rrr(ne, _f);
1264                 case_rrf(ne);
1265                 case_rrr(unlt, _f);
1266                 case_rrf(unlt);
1267                 case_rrr(unle, _f);
1268                 case_rrf(unle);
1269                 case_rrr(uneq, _f);
1270                 case_rrf(uneq);
1271                 case_rrr(unge, _f);
1272                 case_rrf(unge);
1273                 case_rrr(ungt, _f);
1274                 case_rrf(ungt);
1275                 case_rrr(ltgt, _f);
1276                 case_rrf(ltgt);
1277                 case_rrr(ord, _f);
1278                 case_rrf(ord);
1279                 case_rrr(unord, _f);
1280                 case_rrf(unord);
1281                 case_brr(blt, _f);
1282                 case_brf(blt);
1283                 case_brr(ble, _f);
1284                 case_brf(ble);
1285                 case_brr(beq, _f);
1286                 case_brf(beq);
1287                 case_brr(bge, _f);
1288                 case_brf(bge);
1289                 case_brr(bgt, _f);
1290                 case_brf(bgt);
1291                 case_brr(bne, _f);
1292                 case_brf(bne);
1293                 case_brr(bunlt, _f);
1294                 case_brf(bunlt);
1295                 case_brr(bunle, _f);
1296                 case_brf(bunle);
1297                 case_brr(buneq, _f);
1298                 case_brf(buneq);
1299                 case_brr(bunge, _f);
1300                 case_brf(bunge);
1301                 case_brr(bungt, _f);
1302                 case_brf(bungt);
1303                 case_brr(bltgt, _f);
1304                 case_brf(bltgt);
1305                 case_brr(bord, _f);
1306                 case_brf(bord);
1307                 case_brr(bunord, _f);
1308                 case_brf(bunord);
1309                 case_rrr(add, _d);
1310                 case_rrd(add);
1311                 case_rrr(sub, _d);
1312                 case_rrd(sub);
1313                 case_rrd(rsb);
1314                 case_rrr(mul, _d);
1315                 case_rrd(mul);
1316                 case_rrr(div, _d);
1317                 case_rrd(div);
1318                 case_rr(abs, _d);
1319                 case_rr(neg, _d);
1320                 case_rr(sqrt, _d);
1321                 case_rr(ext, _d);
1322                 case_rr(ld, _d);
1323                 case_rw(ld, _d);
1324                 case_rrr(ldx, _d);
1325                 case_rrw(ldx, _d);
1326                 case_rr(st, _d);
1327                 case_wr(st, _d);
1328                 case_rrr(stx, _d);
1329                 case_wrr(stx, _d);
1330                 case_rr(mov, _d);
1331             case jit_code_movi_d:
1332                 assert_data(node);
1333                 movi_d(rn(node->u.w), node->v.d);
1334                 break;
1335                 case_rr(ext, _f_d);
1336                 case_rrr(lt, _d);
1337                 case_rrd(lt);
1338                 case_rrr(le, _d);
1339                 case_rrd(le);
1340                 case_rrr(eq, _d);
1341                 case_rrd(eq);
1342                 case_rrr(ge, _d);
1343                 case_rrd(ge);
1344                 case_rrr(gt, _d);
1345                 case_rrd(gt);
1346                 case_rrr(ne, _d);
1347                 case_rrd(ne);
1348                 case_rrr(unlt, _d);
1349                 case_rrd(unlt);
1350                 case_rrr(unle, _d);
1351                 case_rrd(unle);
1352                 case_rrr(uneq, _d);
1353                 case_rrd(uneq);
1354                 case_rrr(unge, _d);
1355                 case_rrd(unge);
1356                 case_rrr(ungt, _d);
1357                 case_rrd(ungt);
1358                 case_rrr(ltgt, _d);
1359                 case_rrd(ltgt);
1360                 case_rrr(ord, _d);
1361                 case_rrd(ord);
1362                 case_rrr(unord, _d);
1363                 case_rrd(unord);
1364                 case_brr(blt, _d);
1365                 case_brd(blt);
1366                 case_brr(ble, _d);
1367                 case_brd(ble);
1368                 case_brr(beq, _d);
1369                 case_brd(beq);
1370                 case_brr(bge, _d);
1371                 case_brd(bge);
1372                 case_brr(bgt, _d);
1373                 case_brd(bgt);
1374                 case_brr(bne, _d);
1375                 case_brd(bne);
1376                 case_brr(bunlt, _d);
1377                 case_brd(bunlt);
1378                 case_brr(bunle, _d);
1379                 case_brd(bunle);
1380                 case_brr(buneq, _d);
1381                 case_brd(buneq);
1382                 case_brr(bunge, _d);
1383                 case_brd(bunge);
1384                 case_brr(bungt, _d);
1385                 case_brd(bungt);
1386                 case_brr(bltgt, _d);
1387                 case_brd(bltgt);
1388                 case_brr(bord, _d);
1389                 case_brd(bord);
1390                 case_brr(bunord, _d);
1391                 case_brd(bunord);
1392             case jit_code_jmpr:
1393                 jmpr(rn(node->u.w));
1394                 break;
1395             case jit_code_jmpi:
1396                 if (node->flag & jit_flag_node) {
1397                     temp = node->u.n;
1398                     assert(temp->code == jit_code_label ||
1399                            temp->code == jit_code_epilog);
1400                     if (temp->flag & jit_flag_patch)
1401                         jmpi(temp->u.w);
1402                     else {
1403                         word = jmpi_p(_jit->pc.w);
1404                         patch(word, node);
1405                     }
1406                 }
1407                 else
1408                     jmpi(node->u.w);
1409                 break;
1410             case jit_code_callr:
1411                 callr(rn(node->u.w));
1412                 break;
1413             case jit_code_calli:
1414                 if (node->flag & jit_flag_node) {
1415                     temp = node->u.n;
1416                     assert(temp->code == jit_code_label ||
1417                            temp->code == jit_code_epilog);
1418                     if (temp->flag & jit_flag_patch)
1419                         calli(temp->u.w);
1420                     else {
1421                         word = calli_p(_jit->pc.w);
1422                         patch(word, node);
1423                     }
1424                 }
1425                 else
1426                     calli(node->u.w);
1427                 break;
1428             case jit_code_prolog:
1429                 _jitc->function = _jitc->functions.ptr + node->w.w;
1430                 undo.node = node;
1431                 undo.word = _jit->pc.w;
1432 #if DEVEL_DISASSEMBLER
1433                 undo.prevw = prevw;
1434 #endif
1435                 undo.patch_offset = _jitc->patches.offset;
1436             restart_function:
1437                 _jitc->again = 0;
1438                 prolog(node);
1439                 break;
1440             case jit_code_epilog:
1441                 assert(_jitc->function == _jitc->functions.ptr + node->w.w);
1442                 if (_jitc->again) {
1443                     for (temp = undo.node->next;
1444                          temp != node; temp = temp->next) {
1445                         if (temp->code == jit_code_label ||
1446                             temp->code == jit_code_epilog)
1447                             temp->flag &= ~jit_flag_patch;
1448                     }
1449                     temp->flag &= ~jit_flag_patch;
1450                     node = undo.node;
1451                     _jit->pc.w = undo.word;
1452 #if DEVEL_DISASSEMBLER
1453                     prevw = undo.prevw;
1454 #endif
1455                     _jitc->patches.offset = undo.patch_offset;
1456                     goto restart_function;
1457                 }
1458                 /* remember label is defined */
1459                 node->flag |= jit_flag_patch;
1460                 node->u.w = _jit->pc.w;
1461                 epilog(node);
1462                 _jitc->function = NULL;
1463                 break;
1464             case jit_code_movr_w_f:
1465                 movr_w_f(rn(node->u.w), rn(node->v.w));
1466                 break;
1467             case jit_code_movr_f_w:
1468                 movr_f_w(rn(node->u.w), rn(node->v.w));
1469                 break;
1470             case jit_code_movi_f_w:
1471                 assert_data(node);
1472                 movi_f_w(rn(node->u.w), node->v.f);
1473                 break;
1474             case jit_code_movr_w_d:
1475                 movr_w_d(rn(node->u.w), rn(node->v.w));
1476                 break;
1477             case jit_code_movr_d_w:
1478                 movr_d_w(rn(node->u.w), rn(node->v.w));
1479                 break;
1480             case jit_code_movi_d_w:
1481                 assert_data(node);
1482                 movi_d_w(rn(node->u.w), node->v.d);
1483                 break;
1484             case jit_code_va_start:
1485                 vastart(rn(node->u.w));
1486                 break;
1487             case jit_code_va_arg:
1488                 vaarg(rn(node->u.w), rn(node->v.w));
1489                 break;
1490             case jit_code_va_arg_d:
1491                 vaarg_d(rn(node->u.w), rn(node->v.w));
1492                 break;
1493             case jit_code_live:                 case jit_code_ellipsis:
1494             case jit_code_va_push:
1495             case jit_code_allocai:              case jit_code_allocar:
1496             case jit_code_arg:
1497             case jit_code_arg_f:                case jit_code_arg_d:
1498             case jit_code_va_end:
1499             case jit_code_ret:
1500             case jit_code_retr:                 case jit_code_reti:
1501             case jit_code_retr_f:               case jit_code_reti_f:
1502             case jit_code_retr_d:               case jit_code_reti_d:
1503             case jit_code_getarg_c:             case jit_code_getarg_uc:
1504             case jit_code_getarg_s:             case jit_code_getarg_us:
1505             case jit_code_getarg_i:             case jit_code_getarg_ui:
1506             case jit_code_getarg_l:
1507             case jit_code_getarg_f:             case jit_code_getarg_d:
1508             case jit_code_putargr:              case jit_code_putargi:
1509             case jit_code_putargr_f:            case jit_code_putargi_f:
1510             case jit_code_putargr_d:            case jit_code_putargi_d:
1511             case jit_code_pushargr:             case jit_code_pushargi:
1512             case jit_code_pushargr_f:           case jit_code_pushargi_f:
1513             case jit_code_pushargr_d:           case jit_code_pushargi_d:
1514             case jit_code_retval_c:             case jit_code_retval_uc:
1515             case jit_code_retval_s:             case jit_code_retval_us:
1516             case jit_code_retval_i:
1517             case jit_code_retval_ui:            case jit_code_retval_l:
1518             case jit_code_retval_f:             case jit_code_retval_d:
1519             case jit_code_prepare:
1520             case jit_code_finishr:              case jit_code_finishi:
1521                 break;
1522             default:
1523                 abort();
1524         }
1525         if (jit_carry != _NOREG) {
1526             switch (node->code) {
1527                 case jit_code_note:
1528                 case jit_code_addcr:            case jit_code_addci:
1529                 case jit_code_addxr:            case jit_code_addxi:
1530                 case jit_code_subcr:            case jit_code_subci:
1531                 case jit_code_subxr:            case jit_code_subxi:
1532                     break;
1533                 default:
1534                     jit_unget_reg(jit_carry);
1535                     jit_carry = _NOREG;
1536                     break;
1537             }
1538         }
1539         jit_regarg_clr(node, value);
1540         assert(_jitc->regarg == 0 ||
1541                (jit_carry != _NOREG && _jitc->regarg == (1 << jit_carry)));
1542         assert(_jitc->synth == 0);
1543         /* update register live state */
1544         jit_reglive(node);
1545     }
1546 #undef case_brw
1547 #undef case_brr
1548 #undef case_wrr
1549 #undef case_rrw
1550 #undef case_rrr
1551 #undef case_wr
1552 #undef case_rw
1553 #undef case_rr
1554
1555     for (offset = 0; offset < _jitc->patches.offset; offset++) {
1556         node = _jitc->patches.ptr[offset].node;
1557         word = _jitc->patches.ptr[offset].inst;
1558         value = node->code == jit_code_movi ? node->v.n->u.w : node->u.n->u.w;
1559         patch_at(word, value);
1560     }
1561
1562     jit_flush(_jit->code.ptr, _jit->pc.uc);
1563
1564     return (_jit->code.ptr);
1565 }
1566
1567 #define CODE                            1
1568 #  include "jit_riscv-cpu.c"
1569 #  include "jit_riscv-fpu.c"
1570 #  include "jit_fallback.c"
1571 #undef CODE
1572
1573 void
1574 jit_flush(void *fptr, void *tptr)
1575 {
1576 #if defined(__GNUC__)
1577     jit_word_t          f, t, s;
1578
1579     s = sysconf(_SC_PAGE_SIZE);
1580     f = (jit_word_t)fptr & -s;
1581     t = (((jit_word_t)tptr) + s - 1) & -s;
1582     __clear_cache((void *)f, (void *)t);
1583 #endif
1584 }
1585
1586 void
1587 _emit_ldxi(jit_state_t *_jit, jit_int32_t r0, jit_int32_t r1, jit_word_t i0)
1588 {
1589     ldxi(rn(r0), rn(r1), i0);
1590 }
1591
1592 void
1593 _emit_stxi(jit_state_t *_jit, jit_word_t i0, jit_int32_t r0, jit_int32_t r1)
1594 {
1595     stxi(i0, rn(r0), rn(r1));
1596 }
1597
1598 void
1599 _emit_ldxi_d(jit_state_t *_jit, jit_int32_t r0, jit_int32_t r1, jit_word_t i0)
1600 {
1601     ldxi_d(rn(r0), rn(r1), i0);
1602 }
1603
1604 void
1605 _emit_stxi_d(jit_state_t *_jit, jit_word_t i0, jit_int32_t r0, jit_int32_t r1)
1606 {
1607     stxi_d(i0, rn(r0), rn(r1));
1608 }
1609
1610 static void
1611 _patch(jit_state_t *_jit, jit_word_t instr, jit_node_t *node)
1612 {
1613     jit_int32_t          flag;
1614
1615     assert(node->flag & jit_flag_node);
1616     if (node->code == jit_code_movi)
1617         flag = node->v.n->flag;
1618     else
1619         flag = node->u.n->flag;
1620     assert(!(flag & jit_flag_patch));
1621     if (_jitc->patches.offset >= _jitc->patches.length) {
1622         jit_realloc((jit_pointer_t *)&_jitc->patches.ptr,
1623                     _jitc->patches.length * sizeof(jit_patch_t),
1624                     (_jitc->patches.length + 1024) * sizeof(jit_patch_t));
1625         _jitc->patches.length += 1024;
1626     }
1627     _jitc->patches.ptr[_jitc->patches.offset].inst = instr;
1628     _jitc->patches.ptr[_jitc->patches.offset].node = node;
1629     ++_jitc->patches.offset;
1630 }