deps/lightning/lib/jit_riscv.c

   1 /*
   2  * Copyright (C) 2019-2023  Free Software Foundation, Inc.
   3  *
   4  * This file is part of GNU lightning.
   5  *
   6  * GNU lightning is free software; you can redistribute it and/or modify it
   7  * under the terms of the GNU Lesser General Public License as published
   8  * by the Free Software Foundation; either version 3, or (at your option)
   9  * any later version.
  10  *
  11  * GNU lightning is distributed in the hope that it will be useful, but
  12  * WITHOUT ANY WARRANTY; without even the implied warranty of MERCHANTABILITY
  13  * or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU Lesser General Public
  14  * License for more details.
  15  *
  16  * Authors:
  17  *      Paulo Cesar Pereira de Andrade
  18  */
  19
  20 /* callee save                                    + variadic arguments
  21  * align16(ra+fp+s[1-9]+s10+s11+fs[0-9]+fs10+fs11)+align16(a[0-7]) */
  22 #define stack_framesize                 (208 + 64)
  23
  24 #define jit_arg_reg_p(i)                ((i) >= 0 && (i) < 8)
  25 #define jit_arg_f_reg_p(i)              ((i) >= 0 && (i) < 8)
  26
  27 /*
  28  * Types
  29  */
  30 typedef jit_pointer_t jit_va_list_t;
  31
  32 /*
  33  * Prototypes
  34  */
  35 #define compute_framesize()             _compute_framesize(_jit)
  36 static void _compute_framesize(jit_state_t*);
  37 #if __WORDSIZE == 64
  38 #  define load_const(r0, i0)            _load_const(_jit, r0, i0)
  39 static void _load_const(jit_state_t*, jit_int32_t, jit_word_t);
  40 static jit_word_t hash_const(jit_word_t);
  41 #  define put_const(i0)                 _put_const(_jit, i0)
  42 static void _put_const(jit_state_t*, jit_word_t);
  43 #  define get_const(i0)                 _get_const(_jit, i0)
  44 static jit_word_t _get_const(jit_state_t*, jit_word_t);
  45 #endif
  46 #define patch(instr, node)              _patch(_jit, instr, node)
  47 static void _patch(jit_state_t*,jit_word_t,jit_node_t*);
  48
  49 #define PROTO                           1
  50 #  include "jit_riscv-cpu.c"
  51 #  include "jit_riscv-fpu.c"
  52 #  include "jit_fallback.c"
  53 #undef PROTO
  54
  55 /*
  56  * Initialization
  57  */
  58 jit_register_t          _rvs[] = {
  59     { 0x00,                             "zero" },
  60     { 0x01,                             "ra" },
  61     { 0x02,                             "sp" },
  62     { 0x03,                             "gp" },
  63 #if 0           /* Pretend it does not exist, so _NOREG can be used in
  64                  * a 64 bit bitmask */
  65     { 0x04,                             "tp" },
  66 #endif
  67     { rc(gpr) | 0x05,                   "t0" },
  68     { rc(gpr) | 0x06,                   "t1" },
  69     { rc(gpr) | 0x07,                   "t2" },
  70     { rc(gpr) | 0x1c,                   "t3" },
  71     { rc(gpr) | 0x1d,                   "t4" },
  72     { rc(gpr) | 0x1e,                   "t5" },
  73     { rc(gpr) | 0x1f,                   "t6" },
  74     { 0x08,                             "fp" },
  75     { rc(sav) | rc(gpr) | 0x09,         "s1" },
  76     { rc(sav) | rc(gpr) | 0x12,         "s2" },
  77     { rc(sav) | rc(gpr) | 0x13,         "s3" },
  78     { rc(sav) | rc(gpr) | 0x14,         "s4" },
  79     { rc(sav) | rc(gpr) | 0x15,         "s5" },
  80     { rc(sav) | rc(gpr) | 0x16,         "s6" },
  81     { rc(sav) | rc(gpr) | 0x17,         "s7" },
  82     { rc(sav) | rc(gpr) | 0x18,         "s8" },
  83     { rc(sav) | rc(gpr) | 0x19,         "s9" },
  84     { rc(sav) | rc(gpr) | 0x1a,         "s10" },
  85     { rc(sav) | rc(gpr) | 0x1b,         "s11" },
  86     { rc(arg) | rc(gpr) | 0x11,         "a7" },
  87     { rc(arg) | rc(gpr) | 0x10,         "a6" },
  88     { rc(arg) | rc(gpr) | 0x0f,         "a5" },
  89     { rc(arg) | rc(gpr) | 0x0e,         "a4" },
  90     { rc(arg) | rc(gpr) | 0x0d,         "a3" },
  91     { rc(arg) | rc(gpr) | 0x0c,         "a2" },
  92     { rc(arg) | rc(gpr) | 0x0b,         "a1" },
  93     { rc(arg) | rc(gpr) | 0x0a,         "a0" },
  94     { rc(fpr) | 0x00,                   "ft0" },
  95     { rc(fpr) | 0x01,                   "ft1" },
  96     { rc(fpr) | 0x02,                   "ft2" },
  97     { rc(fpr) | 0x03,                   "ft3" },
  98     { rc(fpr) | 0x04,                   "ft4" },
  99     { rc(fpr) | 0x05,                   "ft5" },
 100     { rc(fpr) | 0x06,                   "ft6" },
 101     { rc(fpr) | 0x07,                   "ft7" },
 102     { rc(fpr) | 0x1c,                   "ft8" },
 103     { rc(fpr) | 0x1d,                   "ft9" },
 104     { rc(fpr) | 0x1e,                   "ft10" },
 105     { rc(fpr) | 0x1f,                   "ft11" },
 106     { rc(sav) | rc(fpr) | 0x08,         "fs0" },
 107     { rc(sav) | rc(fpr) | 0x09,         "fs1" },
 108     { rc(sav) | rc(fpr) | 0x12,         "fs2" },
 109     { rc(sav) | rc(fpr) | 0x13,         "fs3" },
 110     { rc(sav) | rc(fpr) | 0x14,         "fs4" },
 111     { rc(sav) | rc(fpr) | 0x15,         "fs5" },
 112     { rc(sav) | rc(fpr) | 0x16,         "fs6" },
 113     { rc(sav) | rc(fpr) | 0x17,         "fs7" },
 114     { rc(sav) | rc(fpr) | 0x18,         "fs8" },
 115     { rc(sav) | rc(fpr) | 0x19,         "fs9" },
 116     { rc(sav) | rc(fpr) | 0x1a,         "fs10" },
 117     { rc(sav) | rc(fpr) | 0x1b,         "fs11" },
 118     { rc(arg) | rc(fpr) | 0x11,         "fa7" },
 119     { rc(arg) | rc(fpr) | 0x10,         "fa6" },
 120     { rc(arg) | rc(fpr) | 0x0f,         "fa5" },
 121     { rc(arg) | rc(fpr) | 0x0e,         "fa4" },
 122     { rc(arg) | rc(fpr) | 0x0d,         "fa3" },
 123     { rc(arg) | rc(fpr) | 0x0c,         "fa2" },
 124     { rc(arg) | rc(fpr) | 0x0b,         "fa1" },
 125     { rc(arg) | rc(fpr) | 0x0a,         "fa0" },
 126     { _NOREG,                           "<none>" },
 127 };
 128
 129 static jit_int32_t iregs[] = {
 130     _S1, _S2, _S3, _S4, _S5, _S6, _S7, _S8, _S9, _S10, _S11
 131 };
 132
 133 static jit_int32_t fregs[] = {
 134     _FS0, _FS1, _FS2, _FS3, _FS4, _FS5, _FS6, _FS7, _FS8, _FS9, _FS10, _FS11
 135 };
 136
 137 /*
 138  * Implementation
 139  */
 140 void
 141 jit_get_cpu(void)
 142 {
 143 }
 144
 145 void
 146 _jit_init(jit_state_t *_jit)
 147 {
 148     _jitc->reglen = jit_size(_rvs) - 1;
 149     jit_carry = _NOREG;
 150 }
 151
 152 void
 153 _jit_prolog(jit_state_t *_jit)
 154 {
 155     jit_int32_t          offset;
 156
 157     if (_jitc->function)
 158         jit_epilog();
 159     assert(jit_regset_cmp_ui(&_jitc->regarg, 0) == 0);
 160     jit_regset_set_ui(&_jitc->regsav, 0);
 161     offset = _jitc->functions.offset;
 162     if (offset >= _jitc->functions.length) {
 163         jit_realloc((jit_pointer_t *)&_jitc->functions.ptr,
 164                     _jitc->functions.length * sizeof(jit_function_t),
 165                     (_jitc->functions.length + 16) * sizeof(jit_function_t));
 166         _jitc->functions.length += 16;
 167     }
 168     _jitc->function = _jitc->functions.ptr + _jitc->functions.offset++;
 169     _jitc->function->self.size = stack_framesize;
 170     _jitc->function->self.argi = _jitc->function->self.argf =
 171         _jitc->function->self.alen = 0;
 172     _jitc->function->self.aoff = 0;
 173     _jitc->function->self.call = jit_call_default;
 174     jit_alloc((jit_pointer_t *)&_jitc->function->regoff,
 175               _jitc->reglen * sizeof(jit_int32_t));
 176
 177     /* _no_link here does not mean the jit_link() call can be removed
 178      * by rewriting as:
 179      * _jitc->function->prolog = jit_new_node(jit_code_prolog);
 180      */
 181     _jitc->function->prolog = jit_new_node_no_link(jit_code_prolog);
 182     jit_link(_jitc->function->prolog);
 183     _jitc->function->prolog->w.w = offset;
 184     _jitc->function->epilog = jit_new_node_no_link(jit_code_epilog);
 185     /*  u:      label value
 186      *  v:      offset in blocks vector
 187      *  w:      offset in functions vector
 188      */
 189     _jitc->function->epilog->w.w = offset;
 190
 191     jit_regset_new(&_jitc->function->regset);
 192 }
 193
 194 jit_int32_t
 195 _jit_allocai(jit_state_t *_jit, jit_int32_t length)
 196 {
 197     assert(_jitc->function);
 198     jit_check_frame();
 199     switch (length) {
 200         case 0: case 1:                                         break;
 201         case 2:         _jitc->function->self.aoff &= -2;       break;
 202         case 3: case 4: _jitc->function->self.aoff &= -4;       break;
 203         default:        _jitc->function->self.aoff &= -8;       break;
 204     }
 205     _jitc->function->self.aoff -= length;
 206     if (!_jitc->realize) {
 207         jit_inc_synth_ww(allocai, _jitc->function->self.aoff, length);
 208         jit_dec_synth();
 209     }
 210     return (_jitc->function->self.aoff);
 211 }
 212
 213 void
 214 _jit_allocar(jit_state_t *_jit, jit_int32_t u, jit_int32_t v)
 215 {
 216     jit_int32_t          r0;
 217     assert(_jitc->function);
 218     jit_inc_synth_ww(allocar, u, v);
 219     if (!_jitc->function->allocar) {
 220         _jitc->function->aoffoff = jit_allocai(sizeof(jit_int32_t));
 221         _jitc->function->allocar = 1;
 222     }
 223     r0 = jit_get_reg(jit_class_gpr);
 224     jit_negr(r0, v);
 225     jit_andi(r0, r0, -16);
 226     jit_ldxi_i(u, JIT_FP, _jitc->function->aoffoff);
 227     jit_addr(u, u, r0);
 228     jit_addr(JIT_SP, JIT_SP, r0);
 229     jit_stxi_i(_jitc->function->aoffoff, JIT_FP, u);
 230     jit_unget_reg(r0);
 231     jit_dec_synth();
 232 }
 233
 234 void
 235 _jit_ret(jit_state_t *_jit)
 236 {
 237     jit_node_t          *instr;
 238     assert(_jitc->function);
 239     jit_inc_synth(ret);
 240     /* jump to epilog */
 241     instr = jit_jmpi();
 242     jit_patch_at(instr, _jitc->function->epilog);
 243     jit_dec_synth();
 244 }
 245
 246 void
 247 _jit_retr(jit_state_t *_jit, jit_int32_t u, jit_code_t code)
 248 {
 249     jit_code_inc_synth_w(code, u);
 250     jit_movr(JIT_RET, u);
 251     jit_ret();
 252     jit_dec_synth();
 253 }
 254
 255 void
 256 _jit_reti(jit_state_t *_jit, jit_word_t u, jit_code_t code)
 257 {
 258     jit_code_inc_synth_w(code, u);
 259     jit_movi(JIT_RET, u);
 260     jit_ret();
 261     jit_dec_synth();
 262 }
 263
 264 void
 265 _jit_retr_f(jit_state_t *_jit, jit_int32_t u)
 266 {
 267     jit_inc_synth_w(retr_f, u);
 268     if (u != JIT_FRET)
 269         jit_movr_f(JIT_FRET, u);
 270     else
 271         jit_live(JIT_FRET);
 272     jit_ret();
 273     jit_dec_synth();
 274 }
 275
 276 void
 277 _jit_reti_f(jit_state_t *_jit, jit_float32_t u)
 278 {
 279     jit_inc_synth_f(reti_f, u);
 280     jit_movi_f(JIT_FRET, u);
 281     jit_ret();
 282     jit_dec_synth();
 283 }
 284
 285 void
 286 _jit_retr_d(jit_state_t *_jit, jit_int32_t u)
 287 {
 288     jit_inc_synth_w(retr_d, u);
 289     if (u != JIT_FRET)
 290         jit_movr_d(JIT_FRET, u);
 291     else
 292         jit_live(JIT_FRET);
 293     jit_ret();
 294     jit_dec_synth();
 295 }
 296
 297 void
 298 _jit_reti_d(jit_state_t *_jit, jit_float64_t u)
 299 {
 300     jit_inc_synth_d(reti_d, u);
 301     jit_movi_d(JIT_FRET, u);
 302     jit_ret();
 303     jit_dec_synth();
 304 }
 305
 306 void
 307 _jit_epilog(jit_state_t *_jit)
 308 {
 309     assert(_jitc->function);
 310     assert(_jitc->function->epilog->next == NULL);
 311     jit_link(_jitc->function->epilog);
 312     _jitc->function = NULL;
 313 }
 314
 315 jit_bool_t
 316 _jit_arg_register_p(jit_state_t *_jit, jit_node_t *u)
 317 {
 318     if (u->code >= jit_code_arg_c && u->code <= jit_code_arg)
 319         return (jit_arg_reg_p(u->u.w));
 320     assert(u->code == jit_code_arg_f || u->code == jit_code_arg_d);
 321     return (jit_arg_f_reg_p(u->u.w) || jit_arg_reg_p(u->u.w - 8));
 322 }
 323
 324 void
 325 _jit_ellipsis(jit_state_t *_jit)
 326 {
 327     jit_inc_synth(ellipsis);
 328     jit_check_frame();
 329     if (_jitc->prepare) {
 330         jit_link_prepare();
 331         assert(!(_jitc->function->call.call & jit_call_varargs));
 332         _jitc->function->call.call |= jit_call_varargs;
 333     }
 334     else {
 335         jit_link_prolog();
 336         assert(!(_jitc->function->self.call & jit_call_varargs));
 337         _jitc->function->self.call |= jit_call_varargs;
 338         _jitc->function->vagp = _jitc->function->self.argi;
 339     }
 340     jit_dec_synth();
 341 }
 342
 343 void
 344 _jit_va_push(jit_state_t *_jit, jit_int32_t u)
 345 {
 346     jit_inc_synth_w(va_push, u);
 347     jit_pushargr(u);
 348     jit_dec_synth();
 349 }
 350
 351 jit_node_t *
 352 _jit_arg(jit_state_t *_jit, jit_code_t code)
 353 {
 354     jit_node_t          *node;
 355     jit_int32_t          offset;
 356     assert(_jitc->function);
 357     assert(!(_jitc->function->self.call & jit_call_varargs));
 358 #if STRONG_TYPE_CHECKING
 359     assert(code >= jit_code_arg_c && code <= jit_code_arg);
 360 #endif
 361     if (jit_arg_reg_p(_jitc->function->self.argi))
 362         offset = _jitc->function->self.argi++;
 363     else {
 364         offset = _jitc->function->self.size;
 365         _jitc->function->self.size += sizeof(jit_word_t);
 366         jit_check_frame();
 367     }
 368     node = jit_new_node_ww(code, offset,
 369                            ++_jitc->function->self.argn);
 370     jit_link_prolog();
 371     return (node);
 372 }
 373
 374 jit_node_t *
 375 _jit_arg_f(jit_state_t *_jit)
 376 {
 377     jit_node_t          *node;
 378     jit_int32_t          offset;
 379     assert(_jitc->function);
 380     assert(!(_jitc->function->self.call & jit_call_varargs));
 381     if (jit_arg_f_reg_p(_jitc->function->self.argf))
 382         offset = _jitc->function->self.argf++;
 383     else if (jit_arg_reg_p(_jitc->function->self.argi)) {
 384         offset = _jitc->function->self.argi++;
 385         offset += 8;
 386     }
 387     else {
 388         offset = _jitc->function->self.size;
 389         _jitc->function->self.size += sizeof(jit_word_t);
 390         jit_check_frame();
 391     }
 392     node = jit_new_node_ww(jit_code_arg_f, offset,
 393                            ++_jitc->function->self.argn);
 394     jit_link_prolog();
 395     return (node);
 396 }
 397
 398 jit_node_t *
 399 _jit_arg_d(jit_state_t *_jit)
 400 {
 401     jit_node_t          *node;
 402     jit_int32_t          offset;
 403     assert(_jitc->function);
 404     assert(!(_jitc->function->self.call & jit_call_varargs));
 405     if (jit_arg_f_reg_p(_jitc->function->self.argf))
 406         offset = _jitc->function->self.argf++;
 407     else if (jit_arg_reg_p(_jitc->function->self.argi)) {
 408         offset = _jitc->function->self.argi++;
 409         offset += 8;
 410     }
 411     else {
 412         offset = _jitc->function->self.size;
 413         _jitc->function->self.size += sizeof(jit_word_t);
 414         jit_check_frame();
 415     }
 416     node = jit_new_node_ww(jit_code_arg_d, offset,
 417                            ++_jitc->function->self.argn);
 418     jit_link_prolog();
 419     return (node);
 420 }
 421
 422 void
 423 _jit_getarg_c(jit_state_t *_jit, jit_int32_t u, jit_node_t *v)
 424 {
 425     assert_arg_type(v->code, jit_code_arg_c);
 426     jit_inc_synth_wp(getarg_c, u, v);
 427     if (jit_arg_reg_p(v->u.w))
 428         jit_extr_c(u, JIT_RA0 - v->u.w);
 429     else {
 430         jit_node_t      *node = jit_ldxi_c(u, JIT_FP, v->u.w);
 431         jit_link_alist(node);
 432     }
 433     jit_dec_synth();
 434 }
 435
 436 void
 437 _jit_getarg_uc(jit_state_t *_jit, jit_int32_t u, jit_node_t *v)
 438 {
 439     assert_arg_type(v->code, jit_code_arg_c);
 440     jit_inc_synth_wp(getarg_uc, u, v);
 441     if (jit_arg_reg_p(v->u.w))
 442         jit_extr_uc(u, JIT_RA0 - v->u.w);
 443     else {
 444         jit_node_t      *node = jit_ldxi_uc(u, JIT_FP, v->u.w);
 445         jit_link_alist(node);
 446     }
 447     jit_dec_synth();
 448 }
 449
 450 void
 451 _jit_getarg_s(jit_state_t *_jit, jit_int32_t u, jit_node_t *v)
 452 {
 453     assert_arg_type(v->code, jit_code_arg_s);
 454     jit_inc_synth_wp(getarg_s, u, v);
 455     if (jit_arg_reg_p(v->u.w))
 456         jit_extr_s(u, JIT_RA0 - v->u.w);
 457     else {
 458         jit_node_t      *node = jit_ldxi_s(u, JIT_FP, v->u.w);
 459         jit_link_alist(node);
 460     }
 461     jit_dec_synth();
 462 }
 463
 464 void
 465 _jit_getarg_us(jit_state_t *_jit, jit_int32_t u, jit_node_t *v)
 466 {
 467     assert_arg_type(v->code, jit_code_arg_s);
 468     jit_inc_synth_wp(getarg_us, u, v);
 469     if (jit_arg_reg_p(v->u.w))
 470         jit_extr_us(u, JIT_RA0 - v->u.w);
 471     else {
 472         jit_node_t      *node = jit_ldxi_us(u, JIT_FP, v->u.w);
 473         jit_link_alist(node);
 474     }
 475     jit_dec_synth();
 476 }
 477
 478 void
 479 _jit_getarg_i(jit_state_t *_jit, jit_int32_t u, jit_node_t *v)
 480 {
 481     assert_arg_type(v->code, jit_code_arg_i);
 482     jit_inc_synth_wp(getarg_i, u, v);
 483     if (jit_arg_reg_p(v->u.w))
 484         jit_extr_i(u, JIT_RA0 - v->u.w);
 485     else {
 486         jit_node_t      *node = jit_ldxi_i(u, JIT_FP, v->u.w);
 487         jit_link_alist(node);
 488     }
 489     jit_dec_synth();
 490 }
 491
 492 void
 493 _jit_getarg_ui(jit_state_t *_jit, jit_int32_t u, jit_node_t *v)
 494 {
 495     assert_arg_type(v->code, jit_code_arg_i);
 496     jit_inc_synth_wp(getarg_ui, u, v);
 497     if (jit_arg_reg_p(v->u.w))
 498         jit_extr_ui(u, JIT_RA0 - v->u.w);
 499     else {
 500         jit_node_t      *node = jit_ldxi_ui(u, JIT_FP, v->u.w);
 501         jit_link_alist(node);
 502     }
 503     jit_dec_synth();
 504 }
 505
 506 void
 507 _jit_getarg_l(jit_state_t *_jit, jit_int32_t u, jit_node_t *v)
 508 {
 509     assert_arg_type(v->code, jit_code_arg_l);
 510     jit_inc_synth_wp(getarg_l, u, v);
 511     if (jit_arg_reg_p(v->u.w))
 512         jit_movr(u, JIT_RA0 - v->u.w);
 513     else {
 514         jit_node_t      *node = jit_ldxi_l(u, JIT_FP, v->u.w);
 515         jit_link_alist(node);
 516     }
 517     jit_dec_synth();
 518 }
 519
 520 void
 521 _jit_putargr(jit_state_t *_jit, jit_int32_t u, jit_node_t *v, jit_code_t code)
 522 {
 523     assert_putarg_type(code, v->code);
 524     jit_code_inc_synth_wp(code, u, v);
 525     if (jit_arg_reg_p(v->u.w))
 526         jit_movr(JIT_RA0 - v->u.w, u);
 527     else {
 528         jit_node_t      *node = jit_stxi(v->u.w, JIT_FP, u);
 529         jit_link_alist(node);
 530     }
 531     jit_dec_synth();
 532 }
 533
 534 void
 535 _jit_putargi(jit_state_t *_jit, jit_word_t u, jit_node_t *v, jit_code_t code)
 536 {
 537     jit_int32_t         regno;
 538     assert_putarg_type(code, v->code);
 539     jit_code_inc_synth_wp(code, u, v);
 540     if (jit_arg_reg_p(v->u.w))
 541         jit_movi(JIT_RA0 - v->u.w, u);
 542     else {
 543         jit_node_t      *node;
 544         regno = jit_get_reg(jit_class_gpr);
 545         jit_movi(regno, u);
 546         node = jit_stxi(v->u.w, JIT_FP, regno);
 547         jit_link_alist(node);
 548         jit_unget_reg(regno);
 549     }
 550     jit_dec_synth();
 551 }
 552
 553 void
 554 _jit_getarg_f(jit_state_t *_jit, jit_int32_t u, jit_node_t *v)
 555 {
 556     assert(v->code == jit_code_arg_f);
 557     jit_inc_synth_wp(getarg_f, u, v);
 558     if (jit_arg_f_reg_p(v->u.w))
 559         jit_movr_f(u, JIT_FA0 - v->u.w);
 560     else if (jit_arg_reg_p(v->u.w - 8))
 561         jit_movr_w_f(u, JIT_RA0 - (v->u.w - 8));
 562     else {
 563         jit_node_t      *node = jit_ldxi_f(u, JIT_FP, v->u.w);
 564         jit_link_alist(node);
 565     }
 566     jit_dec_synth();
 567 }
 568
 569 void
 570 _jit_putargr_f(jit_state_t *_jit, jit_int32_t u, jit_node_t *v)
 571 {
 572     assert(v->code == jit_code_arg_f);
 573     jit_inc_synth_wp(putargr_f, u, v);
 574     if (jit_arg_f_reg_p(v->u.w))
 575         jit_movr_f(JIT_FA0 - v->u.w, u);
 576     else if (jit_arg_reg_p(v->u.w - 8))
 577         jit_movr_f_w(JIT_RA0 - (v->u.w - 8), u);
 578     else {
 579         jit_node_t      *node = jit_stxi_f(v->u.w, JIT_FP, u);
 580         jit_link_alist(node);
 581     }
 582     jit_dec_synth();
 583 }
 584
 585 void
 586 _jit_putargi_f(jit_state_t *_jit, jit_float32_t u, jit_node_t *v)
 587 {
 588     jit_int32_t         regno;
 589     assert(v->code == jit_code_arg_f);
 590     jit_inc_synth_fp(putargi_f, u, v);
 591     if (jit_arg_f_reg_p(v->u.w))
 592         jit_movi_f(JIT_FA0 - v->u.w, u);
 593     else if (jit_arg_reg_p(v->u.w - 8))
 594         jit_movi_f_w(JIT_RA0 - (v->u.w - 8), u);
 595     else {
 596         jit_node_t      *node;
 597         regno = jit_get_reg(jit_class_fpr);
 598         jit_movi_f(regno, u);
 599         node = jit_stxi_f(v->u.w, JIT_FP, regno);
 600         jit_link_alist(node);
 601         jit_unget_reg(regno);
 602     }
 603     jit_dec_synth();
 604 }
 605
 606 void
 607 _jit_getarg_d(jit_state_t *_jit, jit_int32_t u, jit_node_t *v)
 608 {
 609     assert(v->code == jit_code_arg_d);
 610     jit_inc_synth_wp(getarg_d, u, v);
 611     if (jit_arg_f_reg_p(v->u.w))
 612         jit_movr_d(u, JIT_FA0 - v->u.w);
 613     else if (jit_arg_reg_p(v->u.w - 8))
 614         jit_movr_w_d(u, JIT_RA0 - (v->u.w - 8));
 615     else {
 616         jit_node_t      *node = jit_ldxi_d(u, JIT_FP, v->u.w);
 617         jit_link_alist(node);
 618     }
 619     jit_dec_synth();
 620 }
 621
 622 void
 623 _jit_putargr_d(jit_state_t *_jit, jit_int32_t u, jit_node_t *v)
 624 {
 625     assert(v->code == jit_code_arg_d);
 626     jit_inc_synth_wp(putargr_d, u, v);
 627     if (jit_arg_reg_p(v->u.w))
 628         jit_movr_d(JIT_FA0 - v->u.w, u);
 629     else if (jit_arg_reg_p(v->u.w - 8))
 630         jit_movr_d_w(JIT_RA0 - (v->u.w - 8), u);
 631     else {
 632         jit_node_t      *node = jit_stxi_d(v->u.w, JIT_FP, u);
 633         jit_link_alist(node);
 634     }
 635     jit_dec_synth();
 636 }
 637
 638 void
 639 _jit_putargi_d(jit_state_t *_jit, jit_float64_t u, jit_node_t *v)
 640 {
 641     jit_int32_t         regno;
 642     assert(v->code == jit_code_arg_d);
 643     jit_inc_synth_dp(putargi_d, u, v);
 644     if (jit_arg_reg_p(v->u.w))
 645         jit_movi_d(JIT_FA0 - v->u.w, u);
 646     else if (jit_arg_reg_p(v->u.w - 8))
 647         jit_movi_d_w(JIT_RA0 - (v->u.w - 8), u);
 648     else {
 649         jit_node_t      *node;
 650         regno = jit_get_reg(jit_class_fpr);
 651         jit_movi_d(regno, u);
 652         node = jit_stxi_d(v->u.w, JIT_FP, regno);
 653         jit_link_alist(node);
 654         jit_unget_reg(regno);
 655     }
 656     jit_dec_synth();
 657 }
 658
 659 void
 660 _jit_pushargr(jit_state_t *_jit, jit_int32_t u, jit_code_t code)
 661 {
 662     assert(_jitc->function);
 663     jit_code_inc_synth_w(code, u);
 664     jit_link_prepare();
 665     if (jit_arg_reg_p(_jitc->function->call.argi)) {
 666         jit_movr(JIT_RA0 - _jitc->function->call.argi, u);
 667         ++_jitc->function->call.argi;
 668     }
 669     else {
 670         jit_stxi(_jitc->function->call.size, JIT_SP, u);
 671         _jitc->function->call.size += sizeof(jit_word_t);
 672         jit_check_frame();
 673     }
 674     jit_dec_synth();
 675 }
 676
 677 void
 678 _jit_pushargi(jit_state_t *_jit, jit_word_t u, jit_code_t code)
 679 {
 680     jit_int32_t          regno;
 681     assert(_jitc->function);
 682     jit_code_inc_synth_w(code, u);
 683     jit_link_prepare();
 684     if (jit_arg_reg_p(_jitc->function->call.argi)) {
 685         jit_movi(JIT_RA0 - _jitc->function->call.argi, u);
 686         ++_jitc->function->call.argi;
 687     }
 688     else {
 689         regno = jit_get_reg(jit_class_gpr);
 690         jit_movi(regno, u);
 691         jit_stxi(_jitc->function->call.size, JIT_SP, regno);
 692         jit_unget_reg(regno);
 693         _jitc->function->call.size += sizeof(jit_word_t);
 694         jit_check_frame();
 695     }
 696     jit_dec_synth();
 697 }
 698
 699 void
 700 _jit_pushargr_f(jit_state_t *_jit, jit_int32_t u)
 701 {
 702     assert(_jitc->function);
 703     jit_inc_synth_w(pushargr_f, u);
 704     jit_link_prepare();
 705     if (jit_arg_f_reg_p(_jitc->function->call.argf) &&
 706         !(_jitc->function->call.call & jit_call_varargs)) {
 707         jit_movr_f(JIT_FA0 - _jitc->function->call.argf, u);
 708         ++_jitc->function->call.argf;
 709     }
 710     else if (jit_arg_reg_p(_jitc->function->call.argi)) {
 711         jit_movr_f_w(JIT_RA0 - _jitc->function->call.argi, u);
 712         ++_jitc->function->call.argi;
 713     }
 714     else {
 715         jit_stxi_f(_jitc->function->call.size, JIT_SP, u);
 716         _jitc->function->call.size += sizeof(jit_word_t);
 717         jit_check_frame();
 718     }
 719     jit_dec_synth();
 720 }
 721
 722 void
 723 _jit_pushargi_f(jit_state_t *_jit, jit_float32_t u)
 724 {
 725     jit_int32_t         regno;
 726     assert(_jitc->function);
 727     jit_inc_synth_f(pushargi_f, u);
 728     jit_link_prepare();
 729     if (jit_arg_f_reg_p(_jitc->function->call.argf) &&
 730         !(_jitc->function->call.call & jit_call_varargs)) {
 731         jit_movi_f(JIT_FA0 - _jitc->function->call.argf, u);
 732         ++_jitc->function->call.argf;
 733     }
 734     else if (jit_arg_reg_p(_jitc->function->call.argi)) {
 735         jit_movi_f_w(JIT_RA0 - _jitc->function->call.argi, u);
 736         ++_jitc->function->call.argi;
 737     }
 738     else {
 739         regno = jit_get_reg(jit_class_fpr);
 740         jit_movi_f(regno, u);
 741         jit_stxi_f(_jitc->function->call.size, JIT_SP, regno);
 742         jit_unget_reg(regno);
 743         _jitc->function->call.size += sizeof(jit_word_t);
 744         jit_check_frame();
 745     }
 746     jit_dec_synth();
 747 }
 748
 749 void
 750 _jit_pushargr_d(jit_state_t *_jit, jit_int32_t u)
 751 {
 752     assert(_jitc->function);
 753     jit_inc_synth_w(pushargr_d, u);
 754     jit_link_prepare();
 755     if (jit_arg_f_reg_p(_jitc->function->call.argf) &&
 756         !(_jitc->function->call.call & jit_call_varargs)) {
 757         jit_movr_d(JIT_FA0 - _jitc->function->call.argf, u);
 758         ++_jitc->function->call.argf;
 759     }
 760     else if (jit_arg_reg_p(_jitc->function->call.argi)) {
 761         jit_movr_d_w(JIT_RA0 - _jitc->function->call.argi, u);
 762         ++_jitc->function->call.argi;
 763     }
 764     else {
 765         jit_stxi_d(_jitc->function->call.size, JIT_SP, u);
 766         _jitc->function->call.size += sizeof(jit_word_t);
 767         jit_check_frame();
 768     }
 769     jit_dec_synth();
 770 }
 771
 772 void
 773 _jit_pushargi_d(jit_state_t *_jit, jit_float64_t u)
 774 {
 775     jit_int32_t         regno;
 776     assert(_jitc->function);
 777     jit_inc_synth_d(pushargi_d, u);
 778     jit_link_prepare();
 779     if (jit_arg_f_reg_p(_jitc->function->call.argf) &&
 780         !(_jitc->function->call.call & jit_call_varargs)) {
 781         jit_movi_d(JIT_FA0 - _jitc->function->call.argf, u);
 782         ++_jitc->function->call.argf;
 783     }
 784     else if (jit_arg_reg_p(_jitc->function->call.argi)) {
 785         jit_movi_d_w(JIT_RA0 - _jitc->function->call.argi, u);
 786         ++_jitc->function->call.argi;
 787     }
 788     else {
 789         regno = jit_get_reg(jit_class_fpr);
 790         jit_movi_d(regno, u);
 791         jit_stxi_d(_jitc->function->call.size, JIT_SP, regno);
 792         jit_unget_reg(regno);
 793         _jitc->function->call.size += sizeof(jit_word_t);
 794         jit_check_frame();
 795     }
 796     jit_dec_synth();
 797 }
 798
 799 jit_bool_t
 800 _jit_regarg_p(jit_state_t *_jit, jit_node_t *node, jit_int32_t regno)
 801 {
 802     jit_int32_t         spec;
 803     spec = jit_class(_rvs[regno].spec);
 804     if (spec & jit_class_arg) {
 805         regno = JIT_RA0 - regno;
 806         if (regno >= 0 && regno < node->v.w)
 807             return (1);
 808         if (spec & jit_class_fpr) {
 809             regno = JIT_FA0 - regno;
 810             if (regno >= 0 && regno < node->w.w)
 811                 return (1);
 812         }
 813     }
 814
 815     return (0);
 816 }
 817
 818 void
 819 _jit_finishr(jit_state_t *_jit, jit_int32_t r0)
 820 {
 821     jit_node_t          *node;
 822     assert(_jitc->function);
 823     jit_check_frame();
 824     jit_inc_synth_w(finishr, r0);
 825     if (_jitc->function->self.alen < _jitc->function->call.size)
 826         _jitc->function->self.alen = _jitc->function->call.size;
 827     node = jit_callr(r0);
 828     node->v.w = _jitc->function->self.argi;
 829     node->w.w = _jitc->function->call.argf;
 830     _jitc->function->call.argi = _jitc->function->call.argf =
 831         _jitc->function->call.size = 0;
 832     _jitc->prepare = 0;
 833     jit_dec_synth();
 834 }
 835
 836 jit_node_t *
 837 _jit_finishi(jit_state_t *_jit, jit_pointer_t i0)
 838 {
 839     jit_node_t          *node;
 840     assert(_jitc->function);
 841     jit_check_frame();
 842     jit_inc_synth_w(finishi, (jit_word_t)i0);
 843     if (_jitc->function->self.alen < _jitc->function->call.size)
 844         _jitc->function->self.alen = _jitc->function->call.size;
 845     node = jit_calli(i0);
 846     node->v.w = _jitc->function->call.argi;
 847     node->w.w = _jitc->function->call.argf;
 848     _jitc->function->call.argi = _jitc->function->call.argf =
 849         _jitc->function->call.size = 0;
 850     _jitc->prepare = 0;
 851     jit_dec_synth();
 852     return (node);
 853 }
 854
 855 void
 856 _jit_retval_c(jit_state_t *_jit, jit_int32_t r0)
 857 {
 858     jit_inc_synth_w(retval_c, r0);
 859     jit_extr_c(r0, JIT_RET);
 860     jit_dec_synth();
 861 }
 862
 863 void
 864 _jit_retval_uc(jit_state_t *_jit, jit_int32_t r0)
 865 {
 866     jit_inc_synth_w(retval_uc, r0);
 867     jit_extr_uc(r0, JIT_RET);
 868     jit_dec_synth();
 869 }
 870
 871 void
 872 _jit_retval_s(jit_state_t *_jit, jit_int32_t r0)
 873 {
 874     jit_inc_synth_w(retval_s, r0);
 875     jit_extr_s(r0, JIT_RET);
 876     jit_dec_synth();
 877 }
 878
 879 void
 880 _jit_retval_us(jit_state_t *_jit, jit_int32_t r0)
 881 {
 882     jit_inc_synth_w(retval_us, r0);
 883     jit_extr_us(r0, JIT_RET);
 884     jit_dec_synth();
 885 }
 886
 887 void
 888 _jit_retval_i(jit_state_t *_jit, jit_int32_t r0)
 889 {
 890     jit_inc_synth_w(retval_i, r0);
 891     jit_extr_i(r0, JIT_RET);
 892     jit_dec_synth();
 893 }
 894
 895 void
 896 _jit_retval_ui(jit_state_t *_jit, jit_int32_t r0)
 897 {
 898     jit_inc_synth_w(retval_ui, r0);
 899     jit_extr_ui(r0, JIT_RET);
 900     jit_dec_synth();
 901 }
 902
 903 void
 904 _jit_retval_l(jit_state_t *_jit, jit_int32_t r0)
 905 {
 906     jit_inc_synth_w(retval_l, r0);
 907     if (r0 != JIT_RET)
 908         jit_movr(r0, JIT_RET);
 909     jit_dec_synth();
 910 }
 911
 912 void
 913 _jit_retval_f(jit_state_t *_jit, jit_int32_t r0)
 914 {
 915     jit_inc_synth_w(retval_f, r0);
 916     if (r0 != JIT_FRET)
 917         jit_movr_f(r0, JIT_FRET);
 918     jit_dec_synth();
 919 }
 920
 921 void
 922 _jit_retval_d(jit_state_t *_jit, jit_int32_t r0)
 923 {
 924     jit_inc_synth_w(retval_d, r0);
 925     if (r0 != JIT_FRET)
 926         jit_movr_d(r0, JIT_FRET);
 927     jit_dec_synth();
 928 }
 929
 930 jit_pointer_t
 931 _emit_code(jit_state_t *_jit)
 932 {
 933     jit_node_t          *node;
 934     jit_node_t          *temp;
 935     jit_word_t           word;
 936     jit_word_t           value;
 937     jit_int32_t          offset;
 938     struct {
 939         jit_node_t      *node;
 940         jit_uint8_t     *data;
 941         jit_word_t       word;
 942         jit_function_t   func;
 943 #if DEVEL_DISASSEMBLER
 944         jit_word_t       prevw;
 945 #endif
 946         jit_int32_t      const_offset;
 947         jit_int32_t      patch_offset;
 948     } undo;
 949 #if DEVEL_DISASSEMBLER
 950     jit_word_t           prevw;
 951 #endif
 952
 953 #if __WORDSIZE == 64
 954     if (!_jitc->consts.hash.table) {
 955         jit_alloc((jit_pointer_t *)&_jitc->consts.hash.table,
 956                   16 * sizeof(jit_const_t *));
 957         _jitc->consts.hash.size = 16;
 958         jit_alloc((jit_pointer_t *)&_jitc->consts.pool.ptr,
 959                   sizeof(jit_const_t *));
 960         jit_alloc((jit_pointer_t *)_jitc->consts.pool.ptr,
 961                   1024 * sizeof(jit_const_t));
 962         _jitc->consts.pool.length = 1;
 963     }
 964     /* Reset table if starting over jit generation */
 965     else
 966         memset(_jitc->consts.hash.table, 0,
 967                _jitc->consts.hash.size * sizeof(jit_word_t));
 968     for (offset = 0; offset < _jitc->consts.pool.length; offset++) {
 969         jit_int32_t      i;
 970         jit_const_t     *list = _jitc->consts.pool.ptr[offset];
 971         for (i = 0; i < 1023; ++i, ++list)
 972             list->next = list + 1;
 973         if (offset + 1 < _jitc->consts.pool.length)
 974             list->next = _jitc->consts.pool.ptr[offset + 1];
 975         else
 976             list->next = NULL;
 977     }
 978     _jitc->consts.pool.list = _jitc->consts.pool.ptr[0];
 979     _jitc->consts.hash.count = 0;
 980     if (!_jitc->consts.vector.instrs) {
 981         jit_alloc((jit_pointer_t *)&_jitc->consts.vector.instrs,
 982                   16 * sizeof(jit_word_t));
 983         jit_alloc((jit_pointer_t *)&_jitc->consts.vector.values,
 984                   16 * sizeof(jit_word_t));
 985         _jitc->consts.vector.length = 16;
 986     }
 987     _jitc->consts.vector.offset = 0;
 988 #endif
 989
 990     _jitc->function = NULL;
 991
 992     jit_reglive_setup();
 993
 994     undo.word = 0;
 995     undo.node = NULL;
 996     undo.const_offset = undo.patch_offset = 0;
 997 #  define assert_data(node)             /**/
 998 #define case_rr(name, type)                                             \
 999             case jit_code_##name##r##type:                              \
1000                 name##r##type(rn(node->u.w), rn(node->v.w));            \
1001                 break
1002 #define case_rw(name, type)                                             \
1003             case jit_code_##name##i##type:                              \
1004                 name##i##type(rn(node->u.w), node->v.w);                \
1005                 break
1006 #define case_wr(name, type)                                             \
1007             case jit_code_##name##i##type:                              \
1008                 name##i##type(node->u.w, rn(node->v.w));                \
1009                 break
1010 #define case_rrr(name, type)                                            \
1011             case jit_code_##name##r##type:                              \
1012                 name##r##type(rn(node->u.w),                            \
1013                               rn(node->v.w), rn(node->w.w));            \
1014                 break
1015 #define case_rrrr(name, type)                                           \
1016             case jit_code_##name##r##type:                              \
1017                 name##r##type(rn(node->u.q.l), rn(node->u.q.h),         \
1018                               rn(node->v.w), rn(node->w.w));            \
1019                 break
1020 #define case_rrw(name, type)                                            \
1021             case jit_code_##name##i##type:                              \
1022                 name##i##type(rn(node->u.w), rn(node->v.w), node->w.w); \
1023                 break
1024 #define case_rrrw(name, type)                                           \
1025             case jit_code_##name##i##type:                              \
1026                 name##i##type(rn(node->u.q.l), rn(node->u.q.h),         \
1027                               rn(node->v.w), node->w.w);                \
1028                 break
1029 #define case_rrf(name)                                                  \
1030             case jit_code_##name##i_f:                                  \
1031                 assert_data(node);                                      \
1032                 name##i_f(rn(node->u.w), rn(node->v.w), node->w.f);     \
1033                 break
1034 #define case_rrd(name)                                                  \
1035             case jit_code_##name##i_d:                                  \
1036                 assert_data(node);                                      \
1037                 name##i_d(rn(node->u.w), rn(node->v.w), node->w.d);     \
1038                 break
1039 #define case_wrr(name, type)                                            \
1040             case jit_code_##name##i##type:                              \
1041                 name##i##type(node->u.w, rn(node->v.w), rn(node->w.w)); \
1042                 break
1043 #define case_brr(name, type)                                            \
1044             case jit_code_##name##r##type:                              \
1045                 temp = node->u.n;                                       \
1046                 assert(temp->code == jit_code_label ||                  \
1047                        temp->code == jit_code_epilog);                  \
1048                 if (temp->flag & jit_flag_patch)                        \
1049                     name##r##type(temp->u.w, rn(node->v.w),             \
1050                                   rn(node->w.w));                       \
1051                 else {                                                  \
1052                     word = name##r##type(_jit->pc.w,                    \
1053                                          rn(node->v.w), rn(node->w.w)); \
1054                     patch(word, node);                                  \
1055                 }                                                       \
1056                 break
1057 #define case_brw(name, type)                                            \
1058             case jit_code_##name##i##type:                              \
1059                 temp = node->u.n;                                       \
1060                 assert(temp->code == jit_code_label ||                  \
1061                        temp->code == jit_code_epilog);                  \
1062                 if (temp->flag & jit_flag_patch)                        \
1063                     name##i##type(temp->u.w,                            \
1064                                   rn(node->v.w), node->w.w);            \
1065                 else {                                                  \
1066                     word = name##i##type(_jit->pc.w,                    \
1067                                          rn(node->v.w), node->w.w);     \
1068                     patch(word, node);                                  \
1069                 }                                                       \
1070                 break;
1071 #define case_brf(name)                                                  \
1072             case jit_code_##name##i_f:                                  \
1073                 temp = node->u.n;                                       \
1074                 assert(temp->code == jit_code_label ||                  \
1075                        temp->code == jit_code_epilog);                  \
1076                 if (temp->flag & jit_flag_patch)                        \
1077                     name##i_f(temp->u.w, rn(node->v.w), node->w.f);     \
1078                 else {                                                  \
1079                     word = name##i_f(_jit->pc.w, rn(node->v.w),         \
1080                                 node->w.f);                             \
1081                     patch(word, node);                                  \
1082                 }                                                       \
1083                 break
1084 #define case_brd(name)                                                  \
1085             case jit_code_##name##i_d:                                  \
1086                 temp = node->u.n;                                       \
1087                 assert(temp->code == jit_code_label ||                  \
1088                        temp->code == jit_code_epilog);                  \
1089                 if (temp->flag & jit_flag_patch)                        \
1090                     name##i_d(temp->u.w, rn(node->v.w), node->w.d);     \
1091                 else {                                                  \
1092                     word = name##i_d(_jit->pc.w, rn(node->v.w),         \
1093                                 node->w.d);                             \
1094                     patch(word, node);                                  \
1095                 }                                                       \
1096                 break
1097 #if DEVEL_DISASSEMBLER
1098     prevw = _jit->pc.w;
1099 #endif
1100     for (node = _jitc->head; node; node = node->next) {
1101         if (_jit->pc.uc >= _jitc->code.end)
1102             return (NULL);
1103
1104 #if DEVEL_DISASSEMBLER
1105         node->offset = (jit_uword_t)_jit->pc.w - (jit_uword_t)prevw;
1106         prevw = _jit->pc.w;
1107 #endif
1108         value = jit_classify(node->code);
1109         jit_regarg_set(node, value);
1110         switch (node->code) {
1111             case jit_code_align:
1112                 /* Must align to a power of two */
1113                 assert(!(node->u.w & (node->u.w - 1)));
1114                 if ((word = _jit->pc.w & (node->u.w - 1)))
1115                     nop(node->u.w - word);
1116                 break;
1117             case jit_code_skip:
1118                 nop((node->u.w + 3) & ~3);
1119                 break;
1120             case jit_code_note:         case jit_code_name:
1121                 node->u.w = _jit->pc.w;
1122                 break;
1123             case jit_code_label:
1124                 /* remember label is defined */
1125                 node->flag |= jit_flag_patch;
1126                 node->u.w = _jit->pc.w;
1127                 break;
1128                 case_rrr(add,);
1129                 case_rrw(add,);
1130                 case_rrr(addc,);
1131                 case_rrw(addc,);
1132                 case_rrr(addx,);
1133                 case_rrw(addx,);
1134                 case_rrr(sub,);
1135                 case_rrw(sub,);
1136                 case_rrr(subc,);
1137                 case_rrw(subc,);
1138                 case_rrr(subx,);
1139                 case_rrw(subx,);
1140                 case_rrw(rsb,);
1141                 case_rrr(mul,);
1142                 case_rrw(mul,);
1143                 case_rrrr(qmul,);
1144                 case_rrrw(qmul,);
1145                 case_rrrr(qmul, _u);
1146                 case_rrrw(qmul, _u);
1147                 case_rrr(div,);
1148                 case_rrw(div,);
1149                 case_rrr(div, _u);
1150                 case_rrw(div, _u);
1151                 case_rrrr(qdiv,);
1152                 case_rrrw(qdiv,);
1153                 case_rrrr(qdiv, _u);
1154                 case_rrrw(qdiv, _u);
1155                 case_rrr(rem,);
1156                 case_rrw(rem,);
1157                 case_rrr(rem, _u);
1158                 case_rrw(rem, _u);
1159                 case_rrr(lsh,);
1160                 case_rrw(lsh,);
1161                 case_rrr(rsh,);
1162                 case_rrw(rsh,);
1163                 case_rrr(rsh, _u);
1164                 case_rrw(rsh, _u);
1165                 case_rr(neg,);
1166                 case_rr(com,);
1167 #define clor(r0, r1)    fallback_clo(r0, r1)
1168 #define clzr(r0, r1)    fallback_clz(r0, r1)
1169 #define ctor(r0, r1)    fallback_cto(r0, r1)
1170 #define ctzr(r0, r1)    fallback_ctz(r0, r1)
1171                 case_rr(clo,);
1172                 case_rr(clz,);
1173                 case_rr(cto,);
1174                 case_rr(ctz,);
1175                 case_rrr(and,);
1176                 case_rrw(and,);
1177                 case_rrr(or,);
1178                 case_rrw(or,);
1179                 case_rrr(xor,);
1180                 case_rrw(xor,);
1181                 case_rr(trunc, _f_i);
1182                 case_rr(trunc, _d_i);
1183                 case_rr(trunc, _f_l);
1184                 case_rr(trunc, _d_l);
1185                 case_rr(ld, _c);
1186                 case_rw(ld, _c);
1187                 case_rr(ld, _uc);
1188                 case_rw(ld, _uc);
1189                 case_rr(ld, _s);
1190                 case_rw(ld, _s);
1191                 case_rr(ld, _us);
1192                 case_rw(ld, _us);
1193                 case_rr(ld, _i);
1194                 case_rw(ld, _i);
1195                 case_rr(ld, _ui);
1196                 case_rw(ld, _ui);
1197                 case_rr(ld, _l);
1198                 case_rw(ld, _l);
1199                 case_rrr(ldx, _c);
1200                 case_rrw(ldx, _c);
1201                 case_rrr(ldx, _uc);
1202                 case_rrw(ldx, _uc);
1203                 case_rrr(ldx, _s);
1204                 case_rrw(ldx, _s);
1205                 case_rrr(ldx, _us);
1206                 case_rrw(ldx, _us);
1207                 case_rrr(ldx, _i);
1208                 case_rrw(ldx, _i);
1209                 case_rrr(ldx, _ui);
1210                 case_rrw(ldx, _ui);
1211                 case_rrr(ldx, _l);
1212                 case_rrw(ldx, _l);
1213                 case_rr(st, _c);
1214                 case_wr(st, _c);
1215                 case_rr(st, _s);
1216                 case_wr(st, _s);
1217                 case_rr(st, _i);
1218                 case_wr(st, _i);
1219                 case_rr(st, _l);
1220                 case_wr(st, _l);
1221                 case_rrr(stx, _c);
1222                 case_wrr(stx, _c);
1223                 case_rrr(stx, _s);
1224                 case_wrr(stx, _s);
1225                 case_rrr(stx, _i);
1226                 case_wrr(stx, _i);
1227                 case_rrr(stx, _l);
1228                 case_wrr(stx, _l);
1229                 case_rr(hton, _us);
1230                 case_rr(hton, _ui);
1231                 case_rr(hton, _ul);
1232                 case_rr(bswap, _us);
1233                 case_rr(bswap, _ui);
1234                 case_rr(bswap, _ul);
1235                 case_rr(ext, _c);
1236                 case_rr(ext, _uc);
1237                 case_rr(ext, _s);
1238                 case_rr(ext, _us);
1239                 case_rr(ext, _i);
1240                 case_rr(ext, _ui);
1241             case jit_code_casr:
1242                 casr(rn(node->u.w), rn(node->v.w),
1243                      rn(node->w.q.l), rn(node->w.q.h));
1244                 break;
1245             case jit_code_casi:
1246                 casi(rn(node->u.w), node->v.w,
1247                      rn(node->w.q.l), rn(node->w.q.h));
1248                 break;
1249                 case_rrr(movn,);
1250                 case_rrr(movz,);
1251                 case_rr(mov,);
1252             case jit_code_movi:
1253                 if (node->flag & jit_flag_node) {
1254                     temp = node->v.n;
1255                     if (temp->code == jit_code_data ||
1256                         (temp->code == jit_code_label &&
1257                          (temp->flag & jit_flag_patch)))
1258                         movi(rn(node->u.w), temp->u.w);
1259                     else {
1260                         assert(temp->code == jit_code_label ||
1261                                temp->code == jit_code_epilog);
1262                         word = movi_p(rn(node->u.w), temp->u.w);
1263                         patch(word, node);
1264                     }
1265                 }
1266                 else
1267                     movi(rn(node->u.w), node->v.w);
1268                 break;
1269                 case_rrr(lt,);
1270                 case_rrw(lt,);
1271                 case_rrr(lt, _u);
1272                 case_rrw(lt, _u);
1273                 case_rrr(le,);
1274                 case_rrw(le,);
1275                 case_rrr(le, _u);
1276                 case_rrw(le, _u);
1277                 case_rrr(eq,);
1278                 case_rrw(eq,);
1279                 case_rrr(ge,);
1280                 case_rrw(ge,);
1281                 case_rrr(ge, _u);
1282                 case_rrw(ge, _u);
1283                 case_rrr(gt,);
1284                 case_rrw(gt,);
1285                 case_rrr(gt, _u);
1286                 case_rrw(gt, _u);
1287                 case_rrr(ne,);
1288                 case_rrw(ne,);
1289                 case_brr(blt,);
1290                 case_brw(blt,);
1291                 case_brr(blt, _u);
1292                 case_brw(blt, _u);
1293                 case_brr(ble,);
1294                 case_brw(ble,);
1295                 case_brr(ble, _u);
1296                 case_brw(ble, _u);
1297                 case_brr(beq,);
1298                 case_brw(beq,);
1299                 case_brr(bge,);
1300                 case_brw(bge,);
1301                 case_brr(bge, _u);
1302                 case_brw(bge, _u);
1303                 case_brr(bgt,);
1304                 case_brw(bgt,);
1305                 case_brr(bgt, _u);
1306                 case_brw(bgt, _u);
1307                 case_brr(bne,);
1308                 case_brw(bne,);
1309                 case_brr(boadd,);
1310                 case_brw(boadd,);
1311                 case_brr(boadd, _u);
1312                 case_brw(boadd, _u);
1313                 case_brr(bxadd,);
1314                 case_brw(bxadd,);
1315                 case_brr(bxadd, _u);
1316                 case_brw(bxadd, _u);
1317                 case_brr(bosub,);
1318                 case_brw(bosub,);
1319                 case_brr(bosub, _u);
1320                 case_brw(bosub, _u);
1321                 case_brr(bxsub,);
1322                 case_brw(bxsub,);
1323                 case_brr(bxsub, _u);
1324                 case_brw(bxsub, _u);
1325                 case_brr(bms,);
1326                 case_brw(bms,);
1327                 case_brr(bmc,);
1328                 case_brw(bmc,);
1329                 case_rrr(add, _f);
1330                 case_rrf(add);
1331                 case_rrr(sub, _f);
1332                 case_rrf(sub);
1333                 case_rrf(rsb);
1334                 case_rrr(mul, _f);
1335                 case_rrf(mul);
1336                 case_rrr(div, _f);
1337                 case_rrf(div);
1338                 case_rr(abs, _f);
1339                 case_rr(neg, _f);
1340                 case_rr(sqrt, _f);
1341                 case_rr(ext, _f);
1342                 case_rr(ld, _f);
1343                 case_rw(ld, _f);
1344                 case_rrr(ldx, _f);
1345                 case_rrw(ldx, _f);
1346                 case_rr(st, _f);
1347                 case_wr(st, _f);
1348                 case_rrr(stx, _f);
1349                 case_wrr(stx, _f);
1350                 case_rr(mov, _f);
1351             case jit_code_movi_f:
1352                 assert_data(node);
1353                 movi_f(rn(node->u.w), node->v.f);
1354                 break;
1355                 case_rr(ext, _d_f);
1356                 case_rrr(lt, _f);
1357                 case_rrf(lt);
1358                 case_rrr(le, _f);
1359                 case_rrf(le);
1360                 case_rrr(eq, _f);
1361                 case_rrf(eq);
1362                 case_rrr(ge, _f);
1363                 case_rrf(ge);
1364                 case_rrr(gt, _f);
1365                 case_rrf(gt);
1366                 case_rrr(ne, _f);
1367                 case_rrf(ne);
1368                 case_rrr(unlt, _f);
1369                 case_rrf(unlt);
1370                 case_rrr(unle, _f);
1371                 case_rrf(unle);
1372                 case_rrr(uneq, _f);
1373                 case_rrf(uneq);
1374                 case_rrr(unge, _f);
1375                 case_rrf(unge);
1376                 case_rrr(ungt, _f);
1377                 case_rrf(ungt);
1378                 case_rrr(ltgt, _f);
1379                 case_rrf(ltgt);
1380                 case_rrr(ord, _f);
1381                 case_rrf(ord);
1382                 case_rrr(unord, _f);
1383                 case_rrf(unord);
1384                 case_brr(blt, _f);
1385                 case_brf(blt);
1386                 case_brr(ble, _f);
1387                 case_brf(ble);
1388                 case_brr(beq, _f);
1389                 case_brf(beq);
1390                 case_brr(bge, _f);
1391                 case_brf(bge);
1392                 case_brr(bgt, _f);
1393                 case_brf(bgt);
1394                 case_brr(bne, _f);
1395                 case_brf(bne);
1396                 case_brr(bunlt, _f);
1397                 case_brf(bunlt);
1398                 case_brr(bunle, _f);
1399                 case_brf(bunle);
1400                 case_brr(buneq, _f);
1401                 case_brf(buneq);
1402                 case_brr(bunge, _f);
1403                 case_brf(bunge);
1404                 case_brr(bungt, _f);
1405                 case_brf(bungt);
1406                 case_brr(bltgt, _f);
1407                 case_brf(bltgt);
1408                 case_brr(bord, _f);
1409                 case_brf(bord);
1410                 case_brr(bunord, _f);
1411                 case_brf(bunord);
1412                 case_rrr(add, _d);
1413                 case_rrd(add);
1414                 case_rrr(sub, _d);
1415                 case_rrd(sub);
1416                 case_rrd(rsb);
1417                 case_rrr(mul, _d);
1418                 case_rrd(mul);
1419                 case_rrr(div, _d);
1420                 case_rrd(div);
1421                 case_rr(abs, _d);
1422                 case_rr(neg, _d);
1423                 case_rr(sqrt, _d);
1424                 case_rr(ext, _d);
1425                 case_rr(ld, _d);
1426                 case_rw(ld, _d);
1427                 case_rrr(ldx, _d);
1428                 case_rrw(ldx, _d);
1429                 case_rr(st, _d);
1430                 case_wr(st, _d);
1431                 case_rrr(stx, _d);
1432                 case_wrr(stx, _d);
1433                 case_rr(mov, _d);
1434             case jit_code_movi_d:
1435                 assert_data(node);
1436                 movi_d(rn(node->u.w), node->v.d);
1437                 break;
1438                 case_rr(ext, _f_d);
1439                 case_rrr(lt, _d);
1440                 case_rrd(lt);
1441                 case_rrr(le, _d);
1442                 case_rrd(le);
1443                 case_rrr(eq, _d);
1444                 case_rrd(eq);
1445                 case_rrr(ge, _d);
1446                 case_rrd(ge);
1447                 case_rrr(gt, _d);
1448                 case_rrd(gt);
1449                 case_rrr(ne, _d);
1450                 case_rrd(ne);
1451                 case_rrr(unlt, _d);
1452                 case_rrd(unlt);
1453                 case_rrr(unle, _d);
1454                 case_rrd(unle);
1455                 case_rrr(uneq, _d);
1456                 case_rrd(uneq);
1457                 case_rrr(unge, _d);
1458                 case_rrd(unge);
1459                 case_rrr(ungt, _d);
1460                 case_rrd(ungt);
1461                 case_rrr(ltgt, _d);
1462                 case_rrd(ltgt);
1463                 case_rrr(ord, _d);
1464                 case_rrd(ord);
1465                 case_rrr(unord, _d);
1466                 case_rrd(unord);
1467                 case_brr(blt, _d);
1468                 case_brd(blt);
1469                 case_brr(ble, _d);
1470                 case_brd(ble);
1471                 case_brr(beq, _d);
1472                 case_brd(beq);
1473                 case_brr(bge, _d);
1474                 case_brd(bge);
1475                 case_brr(bgt, _d);
1476                 case_brd(bgt);
1477                 case_brr(bne, _d);
1478                 case_brd(bne);
1479                 case_brr(bunlt, _d);
1480                 case_brd(bunlt);
1481                 case_brr(bunle, _d);
1482                 case_brd(bunle);
1483                 case_brr(buneq, _d);
1484                 case_brd(buneq);
1485                 case_brr(bunge, _d);
1486                 case_brd(bunge);
1487                 case_brr(bungt, _d);
1488                 case_brd(bungt);
1489                 case_brr(bltgt, _d);
1490                 case_brd(bltgt);
1491                 case_brr(bord, _d);
1492                 case_brd(bord);
1493                 case_brr(bunord, _d);
1494                 case_brd(bunord);
1495             case jit_code_jmpr:
1496                 jit_check_frame();
1497                 jmpr(rn(node->u.w));
1498                 break;
1499             case jit_code_jmpi:
1500                 if (node->flag & jit_flag_node) {
1501                     temp = node->u.n;
1502                     assert(temp->code == jit_code_label ||
1503                            temp->code == jit_code_epilog);
1504                     if (temp->flag & jit_flag_patch)
1505                         jmpi(temp->u.w);
1506                     else {
1507                         word = _jit->code.length -
1508                             (_jit->pc.uc - _jit->code.ptr);
1509                         if (simm20_p(word))
1510                             word = jmpi(_jit->pc.w);
1511                         else
1512                         word = jmpi_p(_jit->pc.w);
1513                         patch(word, node);
1514                     }
1515                 }
1516                 else {
1517                     jit_check_frame();
1518                     jmpi(node->u.w);
1519                 }
1520                 break;
1521             case jit_code_callr:
1522                 jit_check_frame();
1523                 callr(rn(node->u.w));
1524                 break;
1525             case jit_code_calli:
1526                 if (node->flag & jit_flag_node) {
1527                     temp = node->u.n;
1528                     assert(temp->code == jit_code_label ||
1529                            temp->code == jit_code_epilog);
1530                     if (temp->flag & jit_flag_patch)
1531                         calli(temp->u.w);
1532                     else {
1533                         word = _jit->code.length -
1534                             (_jit->pc.uc - _jit->code.ptr);
1535                         if (simm20_p(word))
1536                             word = calli(_jit->pc.w);
1537                         else
1538                             word = calli_p(_jit->pc.w);
1539                         patch(word, node);
1540                     }
1541                 }
1542                 else {
1543                     jit_check_frame();
1544                     calli(node->u.w);
1545                 }
1546                 break;
1547             case jit_code_prolog:
1548                 _jitc->function = _jitc->functions.ptr + node->w.w;
1549                 undo.node = node;
1550                 undo.word = _jit->pc.w;
1551                 memcpy(&undo.func, _jitc->function, sizeof(undo.func));
1552 #if DEVEL_DISASSEMBLER
1553                 undo.prevw = prevw;
1554 #endif
1555                 undo.patch_offset = _jitc->patches.offset;
1556             restart_function:
1557                 compute_framesize();
1558                 patch_alist(0);
1559                 _jitc->again = 0;
1560                 prolog(node);
1561                 break;
1562             case jit_code_epilog:
1563                 assert(_jitc->function == _jitc->functions.ptr + node->w.w);
1564                 if (_jitc->again) {
1565                     for (temp = undo.node->next;
1566                          temp != node; temp = temp->next) {
1567                         if (temp->code == jit_code_label ||
1568                             temp->code == jit_code_epilog)
1569                             temp->flag &= ~jit_flag_patch;
1570                     }
1571                     temp->flag &= ~jit_flag_patch;
1572                     node = undo.node;
1573                     _jit->pc.w = undo.word;
1574                     /* undo.func.self.aoff and undo.func.regset should not
1575                      * be undone, as they will be further updated, and are
1576                      * the reason of the undo. */
1577                     undo.func.self.aoff = _jitc->function->frame +
1578                         _jitc->function->self.aoff;
1579                     undo.func.need_frame = _jitc->function->need_frame;
1580                     jit_regset_set(&undo.func.regset, &_jitc->function->regset);
1581                     /* allocar information also does not need to be undone */
1582                     undo.func.aoffoff = _jitc->function->aoffoff;
1583                     undo.func.allocar = _jitc->function->allocar;
1584                     /* this will be recomputed but undo anyway to have it
1585                      * better self documented.*/
1586                     undo.func.need_stack = _jitc->function->need_stack;
1587                     memcpy(_jitc->function, &undo.func, sizeof(undo.func));
1588 #if DEVEL_DISASSEMBLER
1589                     prevw = undo.prevw;
1590 #endif
1591                     _jitc->patches.offset = undo.patch_offset;
1592                     patch_alist(1);
1593                     goto restart_function;
1594                 }
1595                 /* remember label is defined */
1596                 node->flag |= jit_flag_patch;
1597                 node->u.w = _jit->pc.w;
1598                 epilog(node);
1599                 _jitc->function = NULL;
1600                 break;
1601             case jit_code_movr_w_f:
1602                 movr_w_f(rn(node->u.w), rn(node->v.w));
1603                 break;
1604             case jit_code_movr_f_w:
1605                 movr_f_w(rn(node->u.w), rn(node->v.w));
1606                 break;
1607             case jit_code_movi_f_w:
1608                 assert_data(node);
1609                 movi_f_w(rn(node->u.w), node->v.f);
1610                 break;
1611             case jit_code_movr_w_d:
1612                 movr_w_d(rn(node->u.w), rn(node->v.w));
1613                 break;
1614             case jit_code_movr_d_w:
1615                 movr_d_w(rn(node->u.w), rn(node->v.w));
1616                 break;
1617             case jit_code_movi_d_w:
1618                 assert_data(node);
1619                 movi_d_w(rn(node->u.w), node->v.d);
1620                 break;
1621             case jit_code_va_start:
1622                 vastart(rn(node->u.w));
1623                 break;
1624             case jit_code_va_arg:
1625                 vaarg(rn(node->u.w), rn(node->v.w));
1626                 break;
1627             case jit_code_va_arg_d:
1628                 vaarg_d(rn(node->u.w), rn(node->v.w));
1629                 break;
1630             case jit_code_live:                 case jit_code_ellipsis:
1631             case jit_code_va_push:
1632             case jit_code_allocai:              case jit_code_allocar:
1633             case jit_code_arg_c:                case jit_code_arg_s:
1634             case jit_code_arg_i:
1635             case jit_code_arg_l:
1636             case jit_code_arg_f:                case jit_code_arg_d:
1637             case jit_code_va_end:
1638             case jit_code_ret:
1639             case jit_code_retr_c:               case jit_code_reti_c:
1640             case jit_code_retr_uc:              case jit_code_reti_uc:
1641             case jit_code_retr_s:               case jit_code_reti_s:
1642             case jit_code_retr_us:              case jit_code_reti_us:
1643             case jit_code_retr_i:               case jit_code_reti_i:
1644             case jit_code_retr_ui:              case jit_code_reti_ui:
1645             case jit_code_retr_l:               case jit_code_reti_l:
1646             case jit_code_retr_f:               case jit_code_reti_f:
1647             case jit_code_retr_d:               case jit_code_reti_d:
1648             case jit_code_getarg_c:             case jit_code_getarg_uc:
1649             case jit_code_getarg_s:             case jit_code_getarg_us:
1650             case jit_code_getarg_i:             case jit_code_getarg_ui:
1651             case jit_code_getarg_l:
1652             case jit_code_getarg_f:             case jit_code_getarg_d:
1653             case jit_code_putargr_c:            case jit_code_putargi_c:
1654             case jit_code_putargr_uc:           case jit_code_putargi_uc:
1655             case jit_code_putargr_s:            case jit_code_putargi_s:
1656             case jit_code_putargr_us:           case jit_code_putargi_us:
1657             case jit_code_putargr_i:            case jit_code_putargi_i:
1658             case jit_code_putargr_ui:           case jit_code_putargi_ui:
1659             case jit_code_putargr_l:            case jit_code_putargi_l:
1660             case jit_code_putargr_f:            case jit_code_putargi_f:
1661             case jit_code_putargr_d:            case jit_code_putargi_d:
1662             case jit_code_pushargr_c:           case jit_code_pushargi_c:
1663             case jit_code_pushargr_uc:          case jit_code_pushargi_uc:
1664             case jit_code_pushargr_s:           case jit_code_pushargi_s:
1665             case jit_code_pushargr_us:          case jit_code_pushargi_us:
1666             case jit_code_pushargr_i:           case jit_code_pushargi_i:
1667             case jit_code_pushargr_ui:          case jit_code_pushargi_ui:
1668             case jit_code_pushargr_l:           case jit_code_pushargi_l:
1669             case jit_code_pushargr_f:           case jit_code_pushargi_f:
1670             case jit_code_pushargr_d:           case jit_code_pushargi_d:
1671             case jit_code_retval_c:             case jit_code_retval_uc:
1672             case jit_code_retval_s:             case jit_code_retval_us:
1673             case jit_code_retval_i:
1674             case jit_code_retval_ui:            case jit_code_retval_l:
1675             case jit_code_retval_f:             case jit_code_retval_d:
1676             case jit_code_prepare:
1677             case jit_code_finishr:              case jit_code_finishi:
1678                 break;
1679             default:
1680                 abort();
1681         }
1682         if (jit_carry != _NOREG) {
1683             switch (node->code) {
1684                 case jit_code_note:
1685                 case jit_code_addcr:            case jit_code_addci:
1686                 case jit_code_addxr:            case jit_code_addxi:
1687                 case jit_code_subcr:            case jit_code_subci:
1688                 case jit_code_subxr:            case jit_code_subxi:
1689                     break;
1690                 default:
1691                     jit_unget_reg(jit_carry);
1692                     jit_carry = _NOREG;
1693                     break;
1694             }
1695         }
1696         jit_regarg_clr(node, value);
1697         assert(_jitc->regarg == 0 ||
1698                (jit_carry != _NOREG && _jitc->regarg == (1 << jit_carry)));
1699         assert(_jitc->synth == 0);
1700         /* update register live state */
1701         jit_reglive(node);
1702     }
1703 #undef case_brw
1704 #undef case_brr
1705 #undef case_wrr
1706 #undef case_rrw
1707 #undef case_rrr
1708 #undef case_wr
1709 #undef case_rw
1710 #undef case_rr
1711
1712 #if __WORDSIZE == 64
1713     /* Record all constants to be patched */
1714     for (offset = 0; offset < _jitc->patches.offset; offset++) {
1715         node = _jitc->patches.ptr[offset].node;
1716         value = node->code == jit_code_movi ? node->v.n->u.w : node->u.n->u.w;
1717         put_const(value);
1718     }
1719     /* Record all direct constants */
1720     for (offset = 0; offset < _jitc->consts.vector.offset; offset++)
1721         put_const(_jitc->consts.vector.values[offset]);
1722     /* Now actually inject constants at the end of code buffer */
1723     if (_jitc->consts.hash.count) {
1724         jit_const_t     *entry;
1725         /* Insert nop if aligned at 4 bytes */
1726         if (_jit->pc.w % sizeof(jit_word_t))
1727             nop(_jit->pc.w % sizeof(jit_word_t));
1728         for (offset = 0; offset < _jitc->consts.hash.size; offset++) {
1729             entry = _jitc->consts.hash.table[offset];
1730             for (; entry; entry = entry->next) {
1731                 /* Make sure to not write out of bounds */
1732                 if (_jit->pc.uc >= _jitc->code.end)
1733                     return (NULL);
1734                 entry->address = _jit->pc.w;
1735                 *_jit->pc.ul++ = entry->value;
1736             }
1737         }
1738     }
1739 #endif
1740
1741     for (offset = 0; offset < _jitc->patches.offset; offset++) {
1742         node = _jitc->patches.ptr[offset].node;
1743         word = _jitc->patches.ptr[offset].inst;
1744         value = node->code == jit_code_movi ? node->v.n->u.w : node->u.n->u.w;
1745         patch_at(word, value);
1746     }
1747
1748 #if __WORDSIZE == 64
1749     /* Patch direct complex constants */
1750     if (_jitc->consts.vector.instrs) {
1751         for (offset = 0; offset < _jitc->consts.vector.offset; offset++)
1752             patch_at(_jitc->consts.vector.instrs[offset],
1753                      _jitc->consts.vector.values[offset]);
1754         jit_free((jit_pointer_t *)&_jitc->consts.vector.instrs);
1755         jit_free((jit_pointer_t *)&_jitc->consts.vector.values);
1756     }
1757
1758     /* Hash table no longer need */
1759     if (_jitc->consts.hash.table) {
1760         jit_free((jit_pointer_t *)&_jitc->consts.hash.table);
1761         for (offset = 0; offset < _jitc->consts.pool.length; offset++)
1762             jit_free((jit_pointer_t *)_jitc->consts.pool.ptr + offset);
1763         jit_free((jit_pointer_t *)&_jitc->consts.pool.ptr);
1764     }
1765 #endif
1766
1767     jit_flush(_jit->code.ptr, _jit->pc.uc);
1768
1769     return (_jit->code.ptr);
1770 }
1771
1772 #define CODE                            1
1773 #  include "jit_riscv-cpu.c"
1774 #  include "jit_riscv-fpu.c"
1775 #  include "jit_fallback.c"
1776 #undef CODE
1777
1778 static void
1779 _load_const(jit_state_t *_jit, jit_int32_t reg, jit_word_t value)
1780 {
1781     if (_jitc->consts.vector.offset >= _jitc->consts.vector.length) {
1782         jit_word_t      new_size = _jitc->consts.vector.length *
1783                                    2 * sizeof(jit_word_t);
1784         jit_realloc((jit_pointer_t *)&_jitc->consts.vector.instrs,
1785                     _jitc->consts.vector.length * sizeof(jit_word_t), new_size);
1786         jit_realloc((jit_pointer_t *)&_jitc->consts.vector.values,
1787                     _jitc->consts.vector.length * sizeof(jit_word_t), new_size);
1788         _jitc->consts.vector.length *= 2;
1789     }
1790     _jitc->consts.vector.instrs[_jitc->consts.vector.offset] = _jit->pc.w;
1791     _jitc->consts.vector.values[_jitc->consts.vector.offset] = value;
1792     ++_jitc->consts.vector.offset;
1793     /* Resolve later the pc relative address */
1794     put_const(value);
1795     AUIPC(reg, 0);
1796     ADDI(reg, reg, 0);
1797     LD(reg, reg, 0);
1798 }
1799
1800 static jit_word_t
1801 hash_const(jit_word_t value)
1802 {
1803     const jit_uint8_t   *ptr;
1804     jit_word_t           i, key;
1805     for (i = key = 0, ptr = (jit_uint8_t *)&value; i < 4; ++i)
1806         key = (key << (key & 1)) ^ ptr[i];
1807     return (key);
1808
1809 }
1810
1811 static void
1812 _put_const(jit_state_t *_jit, jit_word_t value)
1813 {
1814     jit_word_t           key;
1815     jit_const_t         *entry;
1816
1817     /* Check if already inserted in table */
1818     key = hash_const(value) % _jitc->consts.hash.size;
1819     for (entry = _jitc->consts.hash.table[key]; entry; entry = entry->next) {
1820         if (entry->value == value)
1821             return;
1822     }
1823
1824     /* Check if need to increase pool size */
1825     if (_jitc->consts.pool.list->next == NULL) {
1826         jit_const_t     *list;
1827         jit_word_t       offset;
1828         jit_word_t       new_size = (_jitc->consts.pool.length + 1) *
1829                                     sizeof(jit_const_t*);
1830         jit_realloc((jit_pointer_t *)&_jitc->consts.pool.ptr,
1831                     _jitc->consts.pool.length * sizeof(jit_const_t*), new_size);
1832         jit_alloc((jit_pointer_t *)
1833                   _jitc->consts.pool.ptr + _jitc->consts.pool.length,
1834                   1024 * sizeof(jit_const_t));
1835         list = _jitc->consts.pool.ptr[_jitc->consts.pool.length];
1836         _jitc->consts.pool.list->next = list;
1837         for (offset = 0; offset < 1023; ++offset, ++list)
1838             list->next = list + 1;
1839         list->next = NULL;
1840         ++_jitc->consts.pool.length;
1841     }
1842
1843     /* Rehash if more than 75% used table */
1844     if (_jitc->consts.hash.count > (_jitc->consts.hash.size / 4) * 3) {
1845         jit_word_t        i, k;
1846         jit_const_t      *next;
1847         jit_const_t     **table;
1848         jit_alloc((jit_pointer_t *)&table,
1849                   _jitc->consts.hash.size * 2 * sizeof(jit_const_t *));
1850         for (i = 0; i < _jitc->consts.hash.size; ++i) {
1851             for (entry = _jitc->consts.hash.table[i]; entry; entry = next) {
1852                 next = entry->next;
1853                 k = hash_const(entry->value) % (_jitc->consts.hash.size * 2);
1854                 entry->next = table[k];
1855                 table[k] = entry;
1856             }
1857         }
1858         jit_free((jit_pointer_t *)&_jitc->consts.hash.table);
1859         _jitc->consts.hash.size *= 2;
1860         _jitc->consts.hash.table = table;
1861     }
1862
1863     /* Insert in hash */
1864     entry = _jitc->consts.pool.list;
1865     _jitc->consts.pool.list =  entry->next;
1866     ++_jitc->consts.hash.count;
1867     entry->value = value;
1868     entry->next = _jitc->consts.hash.table[key];
1869     _jitc->consts.hash.table[key] = entry;
1870 }
1871
1872 static jit_word_t
1873 _get_const(jit_state_t *_jit, jit_word_t value)
1874 {
1875     jit_word_t           key;
1876     jit_const_t         *entry;
1877     key = hash_const(value) % _jitc->consts.hash.size;
1878     for (entry = _jitc->consts.hash.table[key]; entry; entry = entry->next) {
1879         if (entry->value == value)
1880             return (entry->address);
1881     }
1882     /* Only the final patch should call get_const() */
1883     abort();
1884 }
1885
1886 void
1887 jit_flush(void *fptr, void *tptr)
1888 {
1889 #if defined(__GNUC__)
1890     jit_word_t          f, t, s;
1891
1892     s = sysconf(_SC_PAGE_SIZE);
1893     f = (jit_word_t)fptr & -s;
1894     t = (((jit_word_t)tptr) + s - 1) & -s;
1895     __clear_cache((void *)f, (void *)t);
1896 #endif
1897 }
1898
1899 void
1900 _emit_ldxi(jit_state_t *_jit, jit_int32_t r0, jit_int32_t r1, jit_word_t i0)
1901 {
1902     ldxi(rn(r0), rn(r1), i0);
1903 }
1904
1905 void
1906 _emit_stxi(jit_state_t *_jit, jit_word_t i0, jit_int32_t r0, jit_int32_t r1)
1907 {
1908     stxi(i0, rn(r0), rn(r1));
1909 }
1910
1911 void
1912 _emit_ldxi_d(jit_state_t *_jit, jit_int32_t r0, jit_int32_t r1, jit_word_t i0)
1913 {
1914     ldxi_d(rn(r0), rn(r1), i0);
1915 }
1916
1917 void
1918 _emit_stxi_d(jit_state_t *_jit, jit_word_t i0, jit_int32_t r0, jit_int32_t r1)
1919 {
1920     stxi_d(i0, rn(r0), rn(r1));
1921 }
1922
1923 #if __WORDSIZE != 64
1924 # error "only 64 bit ports tested"
1925 #endif
1926 static void
1927 _compute_framesize(jit_state_t *_jit)
1928 {
1929     jit_int32_t         reg;
1930     _jitc->framesize = 16;      /* ra+fp */
1931     for (reg = 0; reg < jit_size(iregs); reg++)
1932         if (jit_regset_tstbit(&_jitc->function->regset, iregs[reg]))
1933             _jitc->framesize += sizeof(jit_word_t);
1934
1935     for (reg = 0; reg < jit_size(fregs); reg++)
1936         if (jit_regset_tstbit(&_jitc->function->regset, fregs[reg]))
1937             _jitc->framesize += sizeof(jit_float64_t);
1938
1939     /* Space to store variadic arguments */
1940     if (_jitc->function->self.call & jit_call_varargs)
1941         _jitc->framesize += (8 - _jitc->function->vagp) * 8;
1942
1943     /* Make sure functions called have a 16 byte aligned stack */
1944     _jitc->framesize = (_jitc->framesize + 15) & -16;
1945 }
1946
1947 static void
1948 _patch(jit_state_t *_jit, jit_word_t instr, jit_node_t *node)
1949 {
1950     jit_int32_t          flag;
1951
1952     assert(node->flag & jit_flag_node);
1953     if (node->code == jit_code_movi)
1954         flag = node->v.n->flag;
1955     else
1956         flag = node->u.n->flag;
1957     assert(!(flag & jit_flag_patch));
1958     if (_jitc->patches.offset >= _jitc->patches.length) {
1959         jit_realloc((jit_pointer_t *)&_jitc->patches.ptr,
1960                     _jitc->patches.length * sizeof(jit_patch_t),
1961                     (_jitc->patches.length + 1024) * sizeof(jit_patch_t));
1962         _jitc->patches.length += 1024;
1963     }
1964     _jitc->patches.ptr[_jitc->patches.offset].inst = instr;
1965     _jitc->patches.ptr[_jitc->patches.offset].node = node;
1966     ++_jitc->patches.offset;
1967 }