deps/lightning/lib/jit_loongarch.c

   1 /*
   2  * Copyright (C) 2022-2023  Free Software Foundation, Inc.
   3  *
   4  * This file is part of GNU lightning.
   5  *
   6  * GNU lightning is free software; you can redistribute it and/or modify it
   7  * under the terms of the GNU Lesser General Public License as published
   8  * by the Free Software Foundation; either version 3, or (at your option)
   9  * any later version.
  10  *
  11  * GNU lightning is distributed in the hope that it will be useful, but
  12  * WITHOUT ANY WARRANTY; without even the implied warranty of MERCHANTABILITY
  13  * or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU Lesser General Public
  14  * License for more details.
  15  *
  16  * Authors:
  17  *      Paulo Cesar Pereira de Andrade
  18  */
  19
  20 /* callee save                          + variadic arguments
  21  * align16(ra+fp+s[0-8]+fs[0-7])        +       align16(a[0-7]) */
  22 #define stack_framesize                 (144 + 64)
  23
  24 #define jit_arg_reg_p(i)                ((i) >= 0 && (i) < 8)
  25 #define jit_arg_f_reg_p(i)              ((i) >= 0 && (i) < 8)
  26
  27 /*
  28  * Types
  29  */
  30 typedef struct jit_pointer_t jit_va_list_t;
  31
  32 /*
  33  * Prototypes
  34  */
  35 #define compute_framesize()             _compute_framesize(_jit)
  36 static void _compute_framesize(jit_state_t*);
  37 #define patch(instr, node)              _patch(_jit, instr, node)
  38 static void _patch(jit_state_t*,jit_word_t,jit_node_t*);
  39
  40 #define PROTO                           1
  41 # include "jit_loongarch-cpu.c"
  42 # include "jit_loongarch-fpu.c"
  43 # include "jit_fallback.c"
  44 #undef PROTO
  45
  46 /*
  47  * Initialization
  48  */
  49 jit_cpu_t               jit_cpu;
  50 jit_register_t          _rvs[] = {
  51     { rc(gpr) | 0x14,                   "$t8" },
  52     { rc(gpr) | 0x13,                   "$t7" },
  53     { rc(gpr) | 0x12,                   "$t6" },
  54     { rc(gpr) | 0x11,                   "$t5" },
  55     { rc(gpr) | 0x10,                   "$t4" },
  56     { rc(gpr) | 0x0f,                   "$t3" },
  57     { rc(gpr) | 0x0e,                   "$t2" },
  58     { rc(gpr) | 0x0d,                   "$t1" },
  59     { rc(gpr) | 0x0c,                   "$t0" },
  60     { rc(sav) | rc(gpr) | 0x1f,         "$s8" },
  61     { rc(sav) | rc(gpr) | 0x1e,         "$s7" },
  62     { rc(sav) | rc(gpr) | 0x1d,         "$s6" },
  63     { rc(sav) | rc(gpr) | 0x1c,         "$s5" },
  64     { rc(sav) | rc(gpr) | 0x1b,         "$s4" },
  65     { rc(sav) | rc(gpr) | 0x1a,         "$s3" },
  66     { rc(sav) | rc(gpr) | 0x19,         "$s2" },
  67     { rc(sav) | rc(gpr) | 0x18,         "$s1" },
  68     { rc(sav) | rc(gpr) | 0x17,         "$s0" },
  69     { rc(arg) | rc(gpr) | 0x0b,         "$a7" },
  70     { rc(arg) | rc(gpr) | 0x0a,         "$a6" },
  71     { rc(arg) | rc(gpr) | 0x09,         "$a5" },
  72     { rc(arg) | rc(gpr) | 0x08,         "$a4" },
  73     { rc(arg) | rc(gpr) | 0x07,         "$a3" },
  74     { rc(arg) | rc(gpr) | 0x06,         "$a2" },
  75     { rc(arg) | rc(gpr) | 0x05,         "$a1" },
  76     { rc(arg) | rc(gpr) | 0x04,         "$a0" },
  77     { 0x16,                             "$fp" },
  78     { 0x15,                             "<reserved>" },
  79     { 0x00,                             "$zero" },
  80     { 0x01,                             "$ra" },
  81     { 0x02,                             "$tp" },
  82     { 0x03,                             "$sp" },
  83     { rc(fpr) | 0x08,                   "$ft0" },
  84     { rc(fpr) | 0x09,                   "$ft1" },
  85     { rc(fpr) | 0x0a,                   "$ft2" },
  86     { rc(fpr) | 0x0b,                   "$ft3" },
  87     { rc(fpr) | 0x0c,                   "$ft4" },
  88     { rc(fpr) | 0x0d,                   "$ft5" },
  89     { rc(fpr) | 0x0e,                   "$ft6" },
  90     { rc(fpr) | 0x0f,                   "$ft7" },
  91     { rc(fpr) | 0x10,                   "$ft8" },
  92     { rc(fpr) | 0x11,                   "$ft9" },
  93     { rc(fpr) | 0x12,                   "$ft10" },
  94     { rc(fpr) | 0x13,                   "$ft11" },
  95     { rc(fpr) | 0x14,                   "$ft12" },
  96     { rc(fpr) | 0x15,                   "$ft13" },
  97     { rc(fpr) | 0x16,                   "$ft14" },
  98     { rc(fpr) | 0x17,                   "$ft15" },
  99     { rc(arg) | rc(fpr) | 0x07,         "$fa7" },
 100     { rc(arg) | rc(fpr) | 0x06,         "$fa6" },
 101     { rc(arg) | rc(fpr) | 0x05,         "$fa5" },
 102     { rc(arg) | rc(fpr) | 0x04,         "$fa4" },
 103     { rc(arg) | rc(fpr) | 0x03,         "$fa3" },
 104     { rc(arg) | rc(fpr) | 0x02,         "$fa2" },
 105     { rc(arg) | rc(fpr) | 0x01,         "$fa1" },
 106     { rc(arg) | rc(fpr) | 0x00,         "$fa0" },
 107     { rc(sav) | rc(fpr) | 0x1f,         "$fs7" },
 108     { rc(sav) | rc(fpr) | 0x1e,         "$fs6" },
 109     { rc(sav) | rc(fpr) | 0x1d,         "$fs5" },
 110     { rc(sav) | rc(fpr) | 0x1c,         "$fs4" },
 111     { rc(sav) | rc(fpr) | 0x1b,         "$fs3" },
 112     { rc(sav) | rc(fpr) | 0x1a,         "$fs2" },
 113     { rc(sav) | rc(fpr) | 0x19,         "$fs1" },
 114     { rc(sav) | rc(fpr) | 0x18,         "$fs0" },
 115     { _NOREG,                           "<none>" },
 116 };
 117
 118 static jit_int32_t iregs[] = {
 119     _S0, _S1, _S2, _S3, _S4, _S5, _S6, _S7, _S8
 120 };
 121
 122 static jit_int32_t fregs[] = {
 123     _FS0, _FS1, _FS2, _FS3, _FS4, _FS5, _FS6, _FS7
 124 };
 125
 126 /*
 127  * Implementation
 128  */
 129 void
 130 jit_get_cpu(void)
 131 {
 132 }
 133
 134 void
 135 _jit_init(jit_state_t *_jit)
 136 {
 137     _jitc->reglen = jit_size(_rvs) - 1;
 138     jit_carry = _NOREG;
 139     /* By default assume kernel or cpu will handle unaligned load/store */
 140     jit_cpu.unaligned = 0;
 141 }
 142
 143 void
 144 _jit_prolog(jit_state_t *_jit)
 145 {
 146     jit_int32_t          offset;
 147
 148     if (_jitc->function)
 149         jit_epilog();
 150     assert(jit_regset_cmp_ui(&_jitc->regarg, 0) == 0);
 151     jit_regset_set_ui(&_jitc->regsav, 0);
 152     offset = _jitc->functions.offset;
 153     if (offset >= _jitc->functions.length) {
 154         jit_realloc((jit_pointer_t *)&_jitc->functions.ptr,
 155                     _jitc->functions.length * sizeof(jit_function_t),
 156                     (_jitc->functions.length + 16) * sizeof(jit_function_t));
 157         _jitc->functions.length += 16;
 158     }
 159     _jitc->function = _jitc->functions.ptr + _jitc->functions.offset++;
 160     _jitc->function->self.size = stack_framesize;
 161     _jitc->function->self.argi = _jitc->function->self.argf =
 162         _jitc->function->self.aoff = _jitc->function->self.alen = 0;
 163     _jitc->function->self.call = jit_call_default;
 164     jit_alloc((jit_pointer_t *)&_jitc->function->regoff,
 165               _jitc->reglen * sizeof(jit_int32_t));
 166
 167     /* _no_link here does not mean the jit_link() call can be removed
 168      * by rewriting as:
 169      * _jitc->function->prolog = jit_new_node(jit_code_prolog);
 170      */
 171     _jitc->function->prolog = jit_new_node_no_link(jit_code_prolog);
 172     jit_link(_jitc->function->prolog);
 173     _jitc->function->prolog->w.w = offset;
 174     _jitc->function->epilog = jit_new_node_no_link(jit_code_epilog);
 175     /*  u:      label value
 176      *  v:      offset in blocks vector
 177      *  w:      offset in functions vector
 178      */
 179     _jitc->function->epilog->w.w = offset;
 180
 181     jit_regset_new(&_jitc->function->regset);
 182 }
 183
 184 jit_int32_t
 185 _jit_allocai(jit_state_t *_jit, jit_int32_t length)
 186 {
 187     assert(_jitc->function);
 188     jit_check_frame();
 189     switch (length) {
 190         case 0: case 1:                                         break;
 191         case 2:         _jitc->function->self.aoff &= -2;       break;
 192         case 3: case 4: _jitc->function->self.aoff &= -4;       break;
 193         default:        _jitc->function->self.aoff &= -8;       break;
 194     }
 195     _jitc->function->self.aoff -= length;
 196     if (!_jitc->realize) {
 197         jit_inc_synth_ww(allocai, _jitc->function->self.aoff, length);
 198         jit_dec_synth();
 199     }
 200     return (_jitc->function->self.aoff);
 201 }
 202
 203 void
 204 _jit_allocar(jit_state_t *_jit, jit_int32_t u, jit_int32_t v)
 205 {
 206     jit_int32_t          reg;
 207     assert(_jitc->function);
 208     jit_inc_synth_ww(allocar, u, v);
 209     if (!_jitc->function->allocar) {
 210         _jitc->function->aoffoff = jit_allocai(sizeof(jit_int32_t));
 211         _jitc->function->allocar = 1;
 212     }
 213     reg = jit_get_reg(jit_class_gpr);
 214     jit_negr(reg, v);
 215     jit_andi(reg, reg, -16);
 216     jit_ldxi_i(u, JIT_FP, _jitc->function->aoffoff);
 217     jit_addr(u, u, reg);
 218     jit_addr(JIT_SP, JIT_SP, reg);
 219     jit_stxi_i(_jitc->function->aoffoff, JIT_FP, u);
 220     jit_unget_reg(reg);
 221     jit_dec_synth();
 222 }
 223
 224 void
 225 _jit_ret(jit_state_t *_jit)
 226 {
 227     jit_node_t          *instr;
 228     assert(_jitc->function);
 229     jit_inc_synth(ret);
 230     /* jump to epilog */
 231     instr = jit_jmpi();
 232     jit_patch_at(instr, _jitc->function->epilog);
 233     jit_dec_synth();
 234 }
 235
 236 void
 237 _jit_retr(jit_state_t *_jit, jit_int32_t u, jit_code_t code)
 238 {
 239     jit_code_inc_synth_w(code, u);
 240     jit_movr(JIT_RET, u);
 241     jit_ret();
 242     jit_dec_synth();
 243 }
 244
 245 void
 246 _jit_reti(jit_state_t *_jit, jit_word_t u, jit_code_t code)
 247 {
 248     jit_code_inc_synth_w(code, u);
 249     jit_movi(JIT_RET, u);
 250     jit_ret();
 251     jit_dec_synth();
 252 }
 253
 254 void
 255 _jit_retr_f(jit_state_t *_jit, jit_int32_t u)
 256 {
 257     jit_inc_synth_w(retr_f, u);
 258     if (JIT_FRET != u)
 259         jit_movr_f(JIT_FRET, u);
 260     else
 261         jit_live(JIT_FRET);
 262     jit_ret();
 263     jit_dec_synth();
 264 }
 265
 266 void
 267 _jit_reti_f(jit_state_t *_jit, jit_float32_t u)
 268 {
 269     jit_inc_synth_f(reti_f, u);
 270     jit_movi_f(JIT_FRET, u);
 271     jit_ret();
 272     jit_dec_synth();
 273 }
 274
 275 void
 276 _jit_retr_d(jit_state_t *_jit, jit_int32_t u)
 277 {
 278     jit_inc_synth_w(retr_d, u);
 279     if (JIT_FRET != u)
 280         jit_movr_d(JIT_FRET, u);
 281     else
 282         jit_live(JIT_FRET);
 283     jit_ret();
 284     jit_dec_synth();
 285 }
 286
 287 void
 288 _jit_reti_d(jit_state_t *_jit, jit_float64_t u)
 289 {
 290     jit_inc_synth_d(reti_d, u);
 291     jit_movi_d(JIT_FRET, u);
 292     jit_ret();
 293     jit_dec_synth();
 294 }
 295
 296 void
 297 _jit_epilog(jit_state_t *_jit)
 298 {
 299     assert(_jitc->function);
 300     assert(_jitc->function->epilog->next == NULL);
 301     jit_link(_jitc->function->epilog);
 302     _jitc->function = NULL;
 303 }
 304
 305 jit_bool_t
 306 _jit_arg_register_p(jit_state_t *_jit, jit_node_t *u)
 307 {
 308     if (u->code >= jit_code_arg_c && u->code <= jit_code_arg)
 309         return (jit_arg_reg_p(u->u.w));
 310     assert(u->code == jit_code_arg_f || u->code == jit_code_arg_d);
 311     return (jit_arg_f_reg_p(u->u.w) || jit_arg_reg_p(u->u.w - 8));
 312 }
 313
 314 void
 315 _jit_ellipsis(jit_state_t *_jit)
 316 {
 317     jit_inc_synth(ellipsis);
 318     jit_check_frame();
 319     if (_jitc->prepare) {
 320         jit_link_prepare();
 321         assert(!(_jitc->function->call.call & jit_call_varargs));
 322         _jitc->function->call.call |= jit_call_varargs;
 323     }
 324     else {
 325         jit_link_prolog();
 326         assert(!(_jitc->function->self.call & jit_call_varargs));
 327         _jitc->function->self.call |= jit_call_varargs;
 328         _jitc->function->vagp = _jitc->function->self.argi;
 329     }
 330     jit_dec_synth();
 331 }
 332
 333 void
 334 _jit_va_push(jit_state_t *_jit, jit_int32_t u)
 335 {
 336     jit_inc_synth_w(va_push, u);
 337     jit_pushargr(u);
 338     jit_dec_synth();
 339 }
 340
 341 jit_node_t *
 342 _jit_arg(jit_state_t *_jit, jit_code_t code)
 343 {
 344     jit_node_t          *node;
 345     jit_int32_t          offset;
 346     assert(_jitc->function);
 347     assert(!(_jitc->function->self.call & jit_call_varargs));
 348 #if STRONG_TYPE_CHECKING
 349     assert(code >= jit_code_arg_c && code <= jit_code_arg);
 350 #endif
 351     if (jit_arg_reg_p(_jitc->function->self.argi))
 352         offset = _jitc->function->self.argi++;
 353     else {
 354         offset = _jitc->function->self.size;
 355         _jitc->function->self.size += sizeof(jit_word_t);
 356         jit_check_frame();
 357     }
 358     node = jit_new_node_ww(code, offset,
 359                            ++_jitc->function->self.argn);
 360     jit_link_prolog();
 361     return (node);
 362 }
 363
 364 jit_node_t *
 365 _jit_arg_f(jit_state_t *_jit)
 366 {
 367     jit_node_t          *node;
 368     jit_int32_t          offset;
 369     assert(_jitc->function);
 370     assert(!(_jitc->function->self.call & jit_call_varargs));
 371     if (jit_arg_f_reg_p(_jitc->function->self.argf))
 372         offset = _jitc->function->self.argf++;
 373     else if (jit_arg_reg_p(_jitc->function->self.argi)) {
 374         offset = _jitc->function->self.argi++;
 375         offset += 8;
 376     }
 377     else {
 378         offset = _jitc->function->self.size;
 379         _jitc->function->self.size += sizeof(jit_word_t);
 380         jit_check_frame();
 381     }
 382     node = jit_new_node_ww(jit_code_arg_f, offset,
 383                            ++_jitc->function->self.argn);
 384     jit_link_prolog();
 385     return (node);
 386 }
 387
 388 jit_node_t *
 389 _jit_arg_d(jit_state_t *_jit)
 390 {
 391     jit_node_t          *node;
 392     jit_int32_t          offset;
 393     assert(_jitc->function);
 394     assert(!(_jitc->function->self.call & jit_call_varargs));
 395     if (jit_arg_f_reg_p(_jitc->function->self.argf))
 396         offset = _jitc->function->self.argf++;
 397     else if (jit_arg_reg_p(_jitc->function->self.argi)) {
 398         offset = _jitc->function->self.argi++;
 399         offset += 8;
 400     }
 401     else {
 402         offset = _jitc->function->self.size;
 403         _jitc->function->self.size += sizeof(jit_word_t);
 404         jit_check_frame();
 405     }
 406     node = jit_new_node_ww(jit_code_arg_d, offset,
 407                            ++_jitc->function->self.argn);
 408     jit_link_prolog();
 409     return (node);
 410 }
 411
 412 void
 413 _jit_getarg_c(jit_state_t *_jit, jit_int32_t u, jit_node_t *v)
 414 {
 415     assert_arg_type(v->code, jit_code_arg_c);
 416     jit_inc_synth_wp(getarg_c, u, v);
 417     if (jit_arg_reg_p(v->u.w))
 418         jit_extr_c(u, _A0 - v->u.w);
 419     else {
 420         jit_node_t      *node = jit_ldxi_c(u, JIT_FP, v->u.w);
 421         jit_link_alist(node);
 422     }
 423     jit_dec_synth();
 424 }
 425
 426 void
 427 _jit_getarg_uc(jit_state_t *_jit, jit_int32_t u, jit_node_t *v)
 428 {
 429     assert_arg_type(v->code, jit_code_arg_c);
 430     jit_inc_synth_wp(getarg_uc, u, v);
 431     if (jit_arg_reg_p(v->u.w))
 432         jit_extr_uc(u, _A0 - v->u.w);
 433     else {
 434         jit_node_t      *node = jit_ldxi_uc(u, JIT_FP, v->u.w);
 435         jit_link_alist(node);
 436     }
 437     jit_dec_synth();
 438 }
 439
 440 void
 441 _jit_getarg_s(jit_state_t *_jit, jit_int32_t u, jit_node_t *v)
 442 {
 443     assert_arg_type(v->code, jit_code_arg_s);
 444     jit_inc_synth_wp(getarg_s, u, v);
 445     if (jit_arg_reg_p(v->u.w))
 446         jit_extr_s(u, _A0 - v->u.w);
 447     else {
 448         jit_node_t      *node = jit_ldxi_s(u, JIT_FP, v->u.w);
 449         jit_link_alist(node);
 450     }
 451     jit_dec_synth();
 452 }
 453
 454 void
 455 _jit_getarg_us(jit_state_t *_jit, jit_int32_t u, jit_node_t *v)
 456 {
 457     assert_arg_type(v->code, jit_code_arg_s);
 458     jit_inc_synth_wp(getarg_us, u, v);
 459     if (jit_arg_reg_p(v->u.w))
 460         jit_extr_us(u, _A0 - v->u.w);
 461     else {
 462         jit_node_t      *node = jit_ldxi_us(u, JIT_FP, v->u.w);
 463         jit_link_alist(node);
 464     }
 465     jit_dec_synth();
 466 }
 467
 468 void
 469 _jit_getarg_i(jit_state_t *_jit, jit_int32_t u, jit_node_t *v)
 470 {
 471     assert_arg_type(v->code, jit_code_arg_i);
 472     jit_inc_synth_wp(getarg_i, u, v);
 473     if (jit_arg_reg_p(v->u.w))
 474         jit_extr_i(u, _A0 - v->u.w);
 475     else {
 476         jit_node_t      *node = jit_ldxi_i(u, JIT_FP, v->u.w);
 477         jit_link_alist(node);
 478     }
 479     jit_dec_synth();
 480 }
 481
 482 void
 483 _jit_getarg_ui(jit_state_t *_jit, jit_int32_t u, jit_node_t *v)
 484 {
 485     assert_arg_type(v->code, jit_code_arg_i);
 486     jit_inc_synth_wp(getarg_ui, u, v);
 487     if (jit_arg_reg_p(v->u.w))
 488         jit_extr_ui(u, _A0 - v->u.w);
 489     else {
 490         jit_node_t      *node = jit_ldxi_ui(u, JIT_FP, v->u.w);
 491         jit_link_alist(node);
 492     }
 493     jit_dec_synth();
 494 }
 495
 496 void
 497 _jit_getarg_l(jit_state_t *_jit, jit_int32_t u, jit_node_t *v)
 498 {
 499     assert_arg_type(v->code, jit_code_arg_l);
 500     jit_inc_synth_wp(getarg_l, u, v);
 501     if (jit_arg_reg_p(v->u.w))
 502         jit_movr(u, _A0 - v->u.w);
 503     else {
 504         jit_node_t      *node = jit_ldxi_l(u, JIT_FP, v->u.w);
 505         jit_link_alist(node);
 506     }
 507     jit_dec_synth();
 508 }
 509
 510 void
 511 _jit_putargr(jit_state_t *_jit, jit_int32_t u, jit_node_t *v, jit_code_t code)
 512 {
 513     assert_putarg_type(code, v->code);
 514     jit_code_inc_synth_wp(code, u, v);
 515     if (jit_arg_reg_p(v->u.w))
 516         jit_movr(_A0 - v->u.w, u);
 517     else {
 518         jit_node_t      *node = jit_stxi(v->u.w, JIT_FP, u);
 519         jit_link_alist(node);
 520     }
 521     jit_dec_synth();
 522 }
 523
 524 void
 525 _jit_putargi(jit_state_t *_jit, jit_word_t u, jit_node_t *v, jit_code_t code)
 526 {
 527     jit_int32_t         regno;
 528     assert_putarg_type(code, v->code);
 529     jit_code_inc_synth_wp(code, u, v);
 530     if (jit_arg_reg_p(v->u.w))
 531         jit_movi(_A0 - v->u.w, u);
 532     else {
 533         jit_node_t      *node;
 534         regno = jit_get_reg(jit_class_gpr);
 535         jit_movi(regno, u);
 536         node = jit_stxi(v->u.w, JIT_FP, regno);
 537         jit_link_alist(node);
 538         jit_unget_reg(regno);
 539     }
 540     jit_dec_synth();
 541 }
 542
 543 void
 544 _jit_getarg_f(jit_state_t *_jit, jit_int32_t u, jit_node_t *v)
 545 {
 546     assert(v->code == jit_code_arg_f);
 547     jit_inc_synth_wp(getarg_f, u, v);
 548     if (jit_arg_f_reg_p(v->u.w))
 549         jit_movr_f(u, _FA0 - v->u.w);
 550     else if (jit_arg_reg_p(v->u.w - 8))
 551         jit_movr_w_f(u, JIT_RA0 - (v->u.w - 8));
 552     else {
 553         jit_node_t      *node = jit_ldxi_f(u, JIT_FP, v->u.w);
 554         jit_link_alist(node);
 555     }
 556     jit_dec_synth();
 557 }
 558
 559 void
 560 _jit_putargr_f(jit_state_t *_jit, jit_int32_t u, jit_node_t *v)
 561 {
 562     assert(v->code == jit_code_arg_f);
 563     jit_inc_synth_wp(putargr_f, u, v);
 564     if (jit_arg_f_reg_p(v->u.w))
 565         jit_movr_f(_FA0 - v->u.w, u);
 566     else if (jit_arg_reg_p(v->u.w - 8))
 567         jit_movr_f_w(JIT_RA0 - (v->u.w - 8), u);
 568     else {
 569         jit_node_t      *node = jit_stxi_f(v->u.w, JIT_FP, u);
 570         jit_link_alist(node);
 571     }
 572     jit_dec_synth();
 573 }
 574
 575 void
 576 _jit_putargi_f(jit_state_t *_jit, jit_float32_t u, jit_node_t *v)
 577 {
 578     jit_int32_t         regno;
 579     assert(v->code == jit_code_arg_f);
 580     jit_inc_synth_fp(putargi_f, u, v);
 581     if (jit_arg_f_reg_p(v->u.w))
 582         jit_movi_f(_FA0 - v->u.w, u);
 583     else if (jit_arg_reg_p(v->u.w - 8))
 584         jit_movi_f_w(JIT_RA0 - (v->u.w - 8), u);
 585     else {
 586         jit_node_t      *node;
 587         regno = jit_get_reg(jit_class_fpr);
 588         jit_movi_f(regno, u);
 589         node = jit_stxi_f(v->u.w, JIT_FP, regno);
 590         jit_link_alist(node);
 591         jit_unget_reg(regno);
 592     }
 593     jit_dec_synth();
 594 }
 595
 596 void
 597 _jit_getarg_d(jit_state_t *_jit, jit_int32_t u, jit_node_t *v)
 598 {
 599     assert(v->code == jit_code_arg_d);
 600     jit_inc_synth_wp(getarg_d, u, v);
 601     if (jit_arg_f_reg_p(v->u.w))
 602         jit_movr_d(u, _FA0 - v->u.w);
 603     else if (jit_arg_reg_p(v->u.w - 8))
 604         jit_movr_w_d(u, JIT_RA0 - (v->u.w - 8));
 605     else {
 606         jit_node_t      *node = jit_ldxi_d(u, JIT_FP, v->u.w);
 607         jit_link_alist(node);
 608     }
 609     jit_dec_synth();
 610 }
 611
 612 void
 613 _jit_putargr_d(jit_state_t *_jit, jit_int32_t u, jit_node_t *v)
 614 {
 615     assert(v->code == jit_code_arg_d);
 616     jit_inc_synth_wp(putargr_d, u, v);
 617     if (jit_arg_f_reg_p(v->u.w))
 618         jit_movr_d(_FA0 - v->u.w, u);
 619     else if (jit_arg_reg_p(v->u.w - 8))
 620         jit_movr_d_w(JIT_RA0 - (v->u.w - 8), u);
 621     else {
 622         jit_node_t      *node = jit_stxi_d(v->u.w, JIT_FP, u);
 623         jit_link_alist(node);
 624     }
 625     jit_dec_synth();
 626 }
 627
 628 void
 629 _jit_putargi_d(jit_state_t *_jit, jit_float64_t u, jit_node_t *v)
 630 {
 631     jit_int32_t         regno;
 632     assert(v->code == jit_code_arg_d);
 633     jit_inc_synth_dp(putargi_d, u, v);
 634     if (jit_arg_f_reg_p(v->u.w))
 635         jit_movi_d(_FA0 - v->u.w, u);
 636     else if (jit_arg_reg_p(v->u.w - 8))
 637         jit_movi_d_w(JIT_RA0 - (v->u.w - 8), u);
 638     else {
 639         jit_node_t      *node;
 640         regno = jit_get_reg(jit_class_fpr);
 641         jit_movi_d(regno, u);
 642         node = jit_stxi_d(v->u.w, JIT_FP, regno);
 643         jit_link_alist(node);
 644         jit_unget_reg(regno);
 645     }
 646     jit_dec_synth();
 647 }
 648
 649 void
 650 _jit_pushargr(jit_state_t *_jit, jit_int32_t u, jit_code_t code)
 651 {
 652     assert(_jitc->function);
 653     jit_code_inc_synth_w(code, u);
 654     jit_link_prepare();
 655     if (jit_arg_reg_p(_jitc->function->call.argi)) {
 656         jit_movr(_A0 - _jitc->function->call.argi, u);
 657         ++_jitc->function->call.argi;
 658     }
 659     else {
 660         jit_stxi(_jitc->function->call.size, JIT_SP, u);
 661         _jitc->function->call.size += sizeof(jit_word_t);
 662         jit_check_frame();
 663     }
 664     jit_dec_synth();
 665 }
 666
 667 void
 668 _jit_pushargi(jit_state_t *_jit, jit_word_t u, jit_code_t code)
 669 {
 670     jit_int32_t          regno;
 671     assert(_jitc->function);
 672     jit_code_inc_synth_w(code, u);
 673     jit_link_prepare();
 674     if (jit_arg_reg_p(_jitc->function->call.argi)) {
 675         jit_movi(_A0 - _jitc->function->call.argi, u);
 676         ++_jitc->function->call.argi;
 677     }
 678     else {
 679         regno = jit_get_reg(jit_class_gpr);
 680         jit_movi(regno, u);
 681         jit_stxi(_jitc->function->call.size, JIT_SP, regno);
 682         jit_unget_reg(regno);
 683         _jitc->function->call.size += sizeof(jit_word_t);
 684         jit_check_frame();
 685     }
 686     jit_dec_synth();
 687 }
 688
 689 void
 690 _jit_pushargr_f(jit_state_t *_jit, jit_int32_t u)
 691 {
 692     assert(_jitc->function);
 693     jit_inc_synth_w(pushargr_f, u);
 694     jit_link_prepare();
 695     if (jit_arg_f_reg_p(_jitc->function->call.argf) &&
 696         !(_jitc->function->call.call & jit_call_varargs)) {
 697         jit_movr_f(_FA0 - _jitc->function->call.argf, u);
 698         ++_jitc->function->call.argf;
 699     }
 700     else if (jit_arg_reg_p(_jitc->function->call.argi)) {
 701         jit_movr_f_w(_A0 - _jitc->function->call.argi, u);
 702         ++_jitc->function->call.argi;
 703     }
 704     else {
 705         jit_stxi_f(_jitc->function->call.size, JIT_SP, u);
 706         _jitc->function->call.size += sizeof(jit_word_t);
 707         jit_check_frame();
 708     }
 709     jit_dec_synth();
 710 }
 711
 712 void
 713 _jit_pushargi_f(jit_state_t *_jit, jit_float32_t u)
 714 {
 715     jit_int32_t         regno;
 716     assert(_jitc->function);
 717     jit_inc_synth_f(pushargi_f, u);
 718     jit_link_prepare();
 719     if (jit_arg_f_reg_p(_jitc->function->call.argf) &&
 720         !(_jitc->function->call.call & jit_call_varargs)) {
 721         jit_movi_f(_FA0 - _jitc->function->call.argf, u);
 722         ++_jitc->function->call.argf;
 723     }
 724     else if (jit_arg_reg_p(_jitc->function->call.argi)) {
 725         jit_movi_f_w(_A0 - _jitc->function->call.argi, u);
 726         ++_jitc->function->call.argi;
 727     }
 728     else {
 729         regno = jit_get_reg(jit_class_fpr);
 730         jit_movi_f(regno, u);
 731         jit_stxi_f(_jitc->function->call.size, JIT_SP, regno);
 732         jit_unget_reg(regno);
 733         _jitc->function->call.size += sizeof(jit_word_t);
 734         jit_check_frame();
 735     }
 736     jit_dec_synth();
 737 }
 738
 739 void
 740 _jit_pushargr_d(jit_state_t *_jit, jit_int32_t u)
 741 {
 742     assert(_jitc->function);
 743     jit_inc_synth_w(pushargr_d, u);
 744     jit_link_prepare();
 745     if (jit_arg_f_reg_p(_jitc->function->call.argf) &&
 746         !(_jitc->function->call.call & jit_call_varargs)) {
 747         jit_movr_d(_FA0 - _jitc->function->call.argf, u);
 748         ++_jitc->function->call.argf;
 749     }
 750     else if (jit_arg_reg_p(_jitc->function->call.argi)) {
 751         jit_movr_d_w(_A0 - _jitc->function->call.argi, u);
 752         ++_jitc->function->call.argi;
 753     }
 754     else {
 755         jit_stxi_d(_jitc->function->call.size, JIT_SP, u);
 756         _jitc->function->call.size += sizeof(jit_word_t);
 757         jit_check_frame();
 758     }
 759     jit_dec_synth();
 760 }
 761
 762 void
 763 _jit_pushargi_d(jit_state_t *_jit, jit_float64_t u)
 764 {
 765     jit_int32_t         regno;
 766     assert(_jitc->function);
 767     jit_inc_synth_d(pushargi_d, u);
 768     jit_link_prepare();
 769     if (jit_arg_f_reg_p(_jitc->function->call.argf) &&
 770         !(_jitc->function->call.call & jit_call_varargs)) {
 771         jit_movi_d(_FA0 - _jitc->function->call.argf, u);
 772         ++_jitc->function->call.argf;
 773     }
 774     else if (jit_arg_reg_p(_jitc->function->call.argi)) {
 775         jit_movi_d_w(_A0 - _jitc->function->call.argi, u);
 776         ++_jitc->function->call.argi;
 777     }
 778     else {
 779         regno = jit_get_reg(jit_class_fpr);
 780         jit_movi_d(regno, u);
 781         jit_stxi_d(_jitc->function->call.size, JIT_SP, regno);
 782         jit_unget_reg(regno);
 783         _jitc->function->call.size += sizeof(jit_word_t);
 784         jit_check_frame();
 785     }
 786     jit_dec_synth();
 787 }
 788
 789 jit_bool_t
 790 _jit_regarg_p(jit_state_t *_jit, jit_node_t *node, jit_int32_t regno)
 791 {
 792     jit_int32_t         spec;
 793     spec = jit_class(_rvs[regno].spec);
 794     if (spec & jit_class_arg) {
 795         regno = _A0 - regno;
 796         if (regno >= 0 && regno < node->v.w)
 797             return (1);
 798         if (spec & jit_class_fpr) {
 799             regno = _FA0 - regno;
 800             if (regno >= 0 && regno < node->w.w)
 801                 return (1);
 802         }
 803     }
 804     return (0);
 805 }
 806
 807 void
 808 _jit_finishr(jit_state_t *_jit, jit_int32_t r0)
 809 {
 810     jit_node_t          *node;
 811     assert(_jitc->function);
 812     jit_check_frame();
 813     jit_inc_synth_w(finishr, r0);
 814     if (_jitc->function->self.alen < _jitc->function->call.size)
 815         _jitc->function->self.alen = _jitc->function->call.size;
 816     node = jit_callr(r0);
 817     node->v.w = _jitc->function->call.argi;
 818     node->w.w = _jitc->function->call.argf;
 819     _jitc->function->call.argi = _jitc->function->call.argf =
 820         _jitc->function->call.size = 0;
 821     _jitc->prepare = 0;
 822     jit_dec_synth();
 823 }
 824
 825 jit_node_t *
 826 _jit_finishi(jit_state_t *_jit, jit_pointer_t i0)
 827 {
 828     jit_node_t          *node;
 829     assert(_jitc->function);
 830     jit_check_frame();
 831     jit_inc_synth_w(finishi, (jit_word_t)i0);
 832     if (_jitc->function->self.alen < _jitc->function->call.size)
 833         _jitc->function->self.alen = _jitc->function->call.size;
 834     node = jit_calli(i0);
 835     node->v.w = _jitc->function->call.argi;
 836     node->w.w = _jitc->function->call.argf;
 837     _jitc->function->call.argi = _jitc->function->call.argf =
 838         _jitc->function->call.size = 0;
 839     _jitc->prepare = 0;
 840     jit_dec_synth();
 841     return (node);
 842 }
 843
 844 void
 845 _jit_retval_c(jit_state_t *_jit, jit_int32_t r0)
 846 {
 847     jit_inc_synth_w(retval_c, r0);
 848     jit_extr_c(r0, JIT_RET);
 849     jit_dec_synth();
 850 }
 851
 852 void
 853 _jit_retval_uc(jit_state_t *_jit, jit_int32_t r0)
 854 {
 855     jit_inc_synth_w(retval_uc, r0);
 856     jit_extr_uc(r0, JIT_RET);
 857     jit_dec_synth();
 858 }
 859
 860 void
 861 _jit_retval_s(jit_state_t *_jit, jit_int32_t r0)
 862 {
 863     jit_inc_synth_w(retval_s, r0);
 864     jit_extr_s(r0, JIT_RET);
 865     jit_dec_synth();
 866 }
 867
 868 void
 869 _jit_retval_us(jit_state_t *_jit, jit_int32_t r0)
 870 {
 871     jit_inc_synth_w(retval_us, r0);
 872     jit_extr_us(r0, JIT_RET);
 873     jit_dec_synth();
 874 }
 875
 876 void
 877 _jit_retval_i(jit_state_t *_jit, jit_int32_t r0)
 878 {
 879     jit_inc_synth_w(retval_i, r0);
 880     jit_extr_i(r0, JIT_RET);
 881     jit_dec_synth();
 882 }
 883
 884 void
 885 _jit_retval_ui(jit_state_t *_jit, jit_int32_t r0)
 886 {
 887     jit_inc_synth_w(retval_ui, r0);
 888     jit_extr_ui(r0, JIT_RET);
 889     jit_dec_synth();
 890 }
 891
 892 void
 893 _jit_retval_l(jit_state_t *_jit, jit_int32_t r0)
 894 {
 895     jit_inc_synth_w(retval_l, r0);
 896     if (r0 != JIT_RET)
 897     jit_movr(r0, JIT_RET);
 898     jit_dec_synth();
 899 }
 900
 901 void
 902 _jit_retval_f(jit_state_t *_jit, jit_int32_t r0)
 903 {
 904     jit_inc_synth_w(retval_f, r0);
 905     if (r0 != JIT_FRET)
 906         jit_movr_f(r0, JIT_FRET);
 907     jit_dec_synth();
 908 }
 909
 910 void
 911 _jit_retval_d(jit_state_t *_jit, jit_int32_t r0)
 912 {
 913     jit_inc_synth_w(retval_d, r0);
 914     if (r0 != JIT_FRET)
 915         jit_movr_d(r0, JIT_FRET);
 916     jit_dec_synth();
 917 }
 918
 919 jit_pointer_t
 920 _emit_code(jit_state_t *_jit)
 921 {
 922     jit_node_t          *node;
 923     jit_node_t          *temp;
 924     jit_word_t           word;
 925     jit_int32_t          value;
 926     jit_int32_t          offset;
 927     struct {
 928         jit_node_t      *node;
 929         jit_word_t       word;
 930         jit_function_t   func;
 931 #if DEVEL_DISASSEMBLER
 932         jit_word_t       prevw;
 933 #endif
 934         jit_int32_t      patch_offset;
 935     } undo;
 936 #if DEVEL_DISASSEMBLER
 937     jit_word_t           prevw;
 938 #endif
 939
 940     _jitc->function = NULL;
 941
 942     jit_reglive_setup();
 943
 944     undo.word = 0;
 945     undo.node = NULL;
 946     undo.patch_offset = 0;
 947
 948 #define assert_data(node)               /**/
 949 #define case_rr(name, type)                                             \
 950             case jit_code_##name##r##type:                              \
 951                 name##r##type(rn(node->u.w), rn(node->v.w));            \
 952                 break
 953 #define case_rw(name, type)                                             \
 954             case jit_code_##name##i##type:                              \
 955                 name##i##type(rn(node->u.w), node->v.w);                \
 956                 break
 957 #define case_wr(name, type)                                             \
 958             case jit_code_##name##i##type:                              \
 959                 name##i##type(node->u.w, rn(node->v.w));                \
 960                 break
 961 #define case_rrr(name, type)                                            \
 962             case jit_code_##name##r##type:                              \
 963                 name##r##type(rn(node->u.w),                            \
 964                               rn(node->v.w), rn(node->w.w));            \
 965                 break
 966 #define case_rrrr(name, type)                                           \
 967             case jit_code_##name##r##type:                              \
 968                 name##r##type(rn(node->u.q.l), rn(node->u.q.h),         \
 969                               rn(node->v.w), rn(node->w.w));            \
 970                 break
 971 #define case_rqr(name, type)                                            \
 972             case jit_code_##name##r##type:                              \
 973                 name##r##type(rn(node->u.w), rn(node->v.q.l),           \
 974                               rn(node->v.q.h), rn(node->w.w));          \
 975             case jit_code_##name##i##type:                              \
 976                 break;
 977 #define case_rrw(name, type)                                            \
 978             case jit_code_##name##i##type:                              \
 979                 name##i##type(rn(node->u.w), rn(node->v.w), node->w.w); \
 980                 break
 981 #define case_rrrw(name, type)                                           \
 982             case jit_code_##name##i##type:                              \
 983                 name##i##type(rn(node->u.q.l), rn(node->u.q.h),         \
 984                               rn(node->v.w), node->w.w);                \
 985                 break
 986 #define case_rrf(name)                                                  \
 987             case jit_code_##name##i_f:                                  \
 988                 assert_data(node);                                      \
 989                 name##i_f(rn(node->u.w), rn(node->v.w), node->w.f);     \
 990                 break
 991 #define case_rrd(name)                                                  \
 992             case jit_code_##name##i_d:                                  \
 993                 assert_data(node);                                      \
 994                 name##i_d(rn(node->u.w), rn(node->v.w),node->w.d);      \
 995                 break
 996 #define case_wrr(name, type)                                            \
 997             case jit_code_##name##i##type:                              \
 998                 name##i##type(node->u.w, rn(node->v.w), rn(node->w.w)); \
 999                 break
1000 #define case_brr(name, type)                                            \
1001             case jit_code_##name##r##type:                              \
1002                 temp = node->u.n;                                       \
1003                 assert(temp->code == jit_code_label ||                  \
1004                        temp->code == jit_code_epilog);                  \
1005                 if (temp->flag & jit_flag_patch)                        \
1006                     name##r##type(temp->u.w, rn(node->v.w),             \
1007                                   rn(node->w.w));                       \
1008                 else {                                                  \
1009                     word = name##r##type(_jit->pc.w,                    \
1010                                          rn(node->v.w),                 \
1011                                          rn(node->w.w));                \
1012                     patch(word, node);                                  \
1013                 }                                                       \
1014                 break
1015 #define case_brw(name, type)                                            \
1016             case jit_code_##name##i##type:                              \
1017                 temp = node->u.n;                                       \
1018                 assert(temp->code == jit_code_label ||                  \
1019                        temp->code == jit_code_epilog);                  \
1020                 if (temp->flag & jit_flag_patch)                        \
1021                     name##i##type(temp->u.w,                            \
1022                                   rn(node->v.w), node->w.w);            \
1023                 else {                                                  \
1024                     word = name##i##type(_jit->pc.w,                    \
1025                                          rn(node->v.w), node->w.w);     \
1026                     patch(word, node);                                  \
1027                 }                                                       \
1028                 break;
1029 #define case_brf(name)                                                  \
1030             case jit_code_##name##i_f:                                  \
1031                 temp = node->u.n;                                       \
1032                 assert(temp->code == jit_code_label ||                  \
1033                        temp->code == jit_code_epilog);                  \
1034                 if (temp->flag & jit_flag_patch)                        \
1035                     name##i_f(temp->u.w, rn(node->v.w), node->w.f);     \
1036                 else {                                                  \
1037                     word = name##i_f(_jit->pc.w, rn(node->v.w),         \
1038                                      node->w.f);                        \
1039                     patch(word, node);                                  \
1040                 }                                                       \
1041                 break
1042 #define case_brd(name)                                                  \
1043             case jit_code_##name##i_d:                                  \
1044                 temp = node->u.n;                                       \
1045                 assert(temp->code == jit_code_label ||                  \
1046                        temp->code == jit_code_epilog);                  \
1047                 if (temp->flag & jit_flag_patch)                        \
1048                     name##i_d(temp->u.w, rn(node->v.w), node->w.d);     \
1049                 else {                                                  \
1050                     word = name##i_d(_jit->pc.w, rn(node->v.w),         \
1051                                      node->w.d);                        \
1052                     patch(word, node);                                  \
1053                 }                                                       \
1054                 break
1055 #if DEVEL_DISASSEMBLER
1056     prevw = _jit->pc.w;
1057 #endif
1058     for (node = _jitc->head; node; node = node->next) {
1059         if (_jit->pc.uc >= _jitc->code.end)
1060             return (NULL);
1061
1062 #if DEVEL_DISASSEMBLER
1063         node->offset = (jit_uword_t)_jit->pc.w - (jit_uword_t)prevw;
1064         prevw = _jit->pc.w;
1065 #endif
1066         value = jit_classify(node->code);
1067         jit_regarg_set(node, value);
1068         switch (node->code) {
1069             case jit_code_align:
1070                 /* Must align to a power of two */
1071                 assert(!(node->u.w & (node->u.w - 1)));
1072                 if ((word = _jit->pc.w & (node->u.w - 1)))
1073                     nop(node->u.w - word);
1074                 break;
1075             case jit_code_skip:
1076                 nop((node->u.w + 3) & ~3);
1077                 break;
1078             case jit_code_note:         case jit_code_name:
1079                 node->u.w = _jit->pc.w;
1080                 break;
1081             case jit_code_label:
1082                 /* remember label is defined */
1083                 node->flag |= jit_flag_patch;
1084                 node->u.w = _jit->pc.w;
1085                 break;
1086                 case_rrr(add,);
1087                 case_rrw(add,);
1088                 case_rrr(addc,);
1089                 case_rrw(addc,);
1090                 case_rrr(addx,);
1091                 case_rrw(addx,);
1092                 case_rrr(sub,);
1093                 case_rrw(sub,);
1094                 case_rrr(subc,);
1095                 case_rrw(subc,);
1096                 case_rrr(subx,);
1097                 case_rrw(subx,);
1098                 case_rrw(rsb,);
1099                 case_rrr(mul,);
1100                 case_rrw(mul,);
1101                 case_rrr(hmul,);
1102                 case_rrw(hmul,);
1103                 case_rrr(hmul, _u);
1104                 case_rrw(hmul, _u);
1105                 case_rrrr(qmul,);
1106                 case_rrrw(qmul,);
1107                 case_rrrr(qmul, _u);
1108                 case_rrrw(qmul, _u);
1109                 case_rrr(div,);
1110                 case_rrw(div,);
1111                 case_rrr(div, _u);
1112                 case_rrw(div, _u);
1113                 case_rrr(rem,);
1114                 case_rrw(rem,);
1115                 case_rrr(rem, _u);
1116                 case_rrw(rem, _u);
1117                 case_rrrr(qdiv,);
1118                 case_rrrw(qdiv,);
1119                 case_rrrr(qdiv, _u);
1120                 case_rrrw(qdiv, _u);
1121                 case_rrr(lsh,);
1122                 case_rrw(lsh,);
1123 #define qlshr(r0, r1, r2, r3)   fallback_qlshr(r0, r1, r2, r3)
1124 #define qlshi(r0, r1, r2, i0)   fallback_qlshi(r0, r1, r2, i0)
1125 #define qlshr_u(r0, r1, r2, r3) fallback_qlshr_u(r0, r1, r2, r3)
1126 #define qlshi_u(r0, r1, r2, i0) fallback_qlshi_u(r0, r1, r2, i0)
1127                 case_rrrr(qlsh,);
1128                 case_rrrw(qlsh,);
1129                 case_rrrr(qlsh, _u);
1130                 case_rrrw(qlsh, _u);
1131                 case_rrr(rsh,);
1132                 case_rrw(rsh,);
1133                 case_rrr(rsh, _u);
1134                 case_rrw(rsh, _u);
1135 #define qrshr(r0, r1, r2, r3)   fallback_qrshr(r0, r1, r2, r3)
1136 #define qrshi(r0, r1, r2, i0)   fallback_qrshi(r0, r1, r2, i0)
1137 #define qrshr_u(r0, r1, r2, r3) fallback_qrshr_u(r0, r1, r2, r3)
1138 #define qrshi_u(r0, r1, r2, i0) fallback_qrshi_u(r0, r1, r2, i0)
1139                 case_rrrr(qrsh,);
1140                 case_rrrw(qrsh,);
1141                 case_rrrr(qrsh, _u);
1142                 case_rrrw(qrsh, _u);
1143                 case_rrr(lrot,);
1144                 case_rrw(lrot,);
1145                 case_rrr(rrot,);
1146                 case_rrw(rrot,);
1147                 case_rr(neg,);
1148                 case_rr(com,);
1149                 case_rr(clo,);
1150                 case_rr(clz,);
1151                 case_rr(cto,);
1152                 case_rr(ctz,);
1153                 case_rr(rbit,);
1154 #define popcntr(r0, r1) fallback_popcnt(r0, r1)
1155                 case_rr(popcnt,);
1156                 case_rrr(and,);
1157                 case_rrw(and,);
1158                 case_rrr(or,);
1159                 case_rrw(or,);
1160                 case_rrr(xor,);
1161                 case_rrw(xor,);
1162                 case_rr(trunc, _f_i);
1163                 case_rr(trunc, _d_i);
1164                 case_rr(trunc, _f_l);
1165                 case_rr(trunc, _d_l);
1166                 case_rr(ld, _c);
1167                 case_rw(ld, _c);
1168                 case_rr(ld, _uc);
1169                 case_rw(ld, _uc);
1170                 case_rr(ld, _s);
1171                 case_rw(ld, _s);
1172                 case_rr(ld, _us);
1173                 case_rw(ld, _us);
1174                 case_rr(ld, _i);
1175                 case_rw(ld, _i);
1176                 case_rr(ld, _ui);
1177                 case_rw(ld, _ui);
1178                 case_rr(ld, _l);
1179                 case_rw(ld, _l);
1180                 case_rrr(ldx, _c);
1181                 case_rrw(ldx, _c);
1182                 case_rrr(ldx, _uc);
1183                 case_rrw(ldx, _uc);
1184                 case_rrr(ldx, _s);
1185                 case_rrw(ldx, _s);
1186                 case_rrr(ldx, _us);
1187                 case_rrw(ldx, _us);
1188                 case_rrr(ldx, _i);
1189                 case_rrw(ldx, _i);
1190                 case_rrr(ldx, _ui);
1191                 case_rrw(ldx, _ui);
1192                 case_rrr(ldx, _l);
1193                 case_rrw(ldx, _l);
1194             case jit_code_unldr:
1195                 unldr(rn(node->u.w), rn(node->v.w), node->w.w);
1196                 break;
1197             case jit_code_unldi:
1198                 unldi(rn(node->u.w), node->v.w, node->w.w);
1199                 break;
1200             case jit_code_unldr_u:
1201                 unldr_u(rn(node->u.w), rn(node->v.w), node->w.w);
1202                 break;
1203             case jit_code_unldi_u:
1204                 unldi_u(rn(node->u.w), node->v.w, node->w.w);
1205                 break;
1206                 case_rr(st, _c);
1207                 case_wr(st, _c);
1208                 case_rr(st, _s);
1209                 case_wr(st, _s);
1210                 case_rr(st, _i);
1211                 case_wr(st, _i);
1212                 case_rr(st, _l);
1213                 case_wr(st, _l);
1214                 case_rrr(stx, _c);
1215                 case_wrr(stx, _c);
1216                 case_rrr(stx, _s);
1217                 case_wrr(stx, _s);
1218                 case_rrr(stx, _i);
1219                 case_wrr(stx, _i);
1220                 case_rrr(stx, _l);
1221                 case_wrr(stx, _l);
1222             case jit_code_unstr:
1223                 unstr(rn(node->u.w), rn(node->v.w), node->w.w);
1224                 break;
1225             case jit_code_unsti:
1226                 unsti(node->u.w, rn(node->v.w), node->w.w);
1227                 break;
1228                 case_rr(hton, _us);
1229                 case_rr(hton, _ui);
1230                 case_rr(hton, _ul);
1231                 case_rr(bswap, _us);
1232                 case_rr(bswap, _ui);
1233                 case_rr(bswap, _ul);
1234             case jit_code_extr:
1235                 extr(rn(node->u.w), rn(node->v.w), node->w.q.l, node->w.q.h);
1236                 break;
1237             case jit_code_extr_u:
1238                 extr_u(rn(node->u.w), rn(node->v.w), node->w.q.l, node->w.q.h);
1239                 break;
1240             case jit_code_depr:
1241                 depr(rn(node->u.w), rn(node->v.w), node->w.q.l, node->w.q.h);
1242                 break;
1243             case jit_code_depi:
1244                 depi(rn(node->u.w), node->v.w, node->w.q.l, node->w.q.h);
1245                 break;
1246                 case_rr(ext, _c);
1247                 case_rr(ext, _uc);
1248                 case_rr(ext, _s);
1249                 case_rr(ext, _us);
1250                 case_rr(ext, _i);
1251                 case_rr(ext, _ui);
1252             case jit_code_casr:
1253                 casr(rn(node->u.w), rn(node->v.w),
1254                      rn(node->w.q.l), rn(node->w.q.h));
1255                 break;
1256             case jit_code_casi:
1257                 casi(rn(node->u.w), node->v.w,
1258                      rn(node->w.q.l), rn(node->w.q.h));
1259                 break;
1260                 case_rrr(movn,);
1261                 case_rrr(movz,);
1262                 case_rr(mov,);
1263             case jit_code_movi:
1264                 if (node->flag & jit_flag_node) {
1265                     temp = node->v.n;
1266                     if (temp->code == jit_code_data ||
1267                         (temp->code == jit_code_label &&
1268                          (temp->flag & jit_flag_patch)))
1269                         movi(rn(node->u.w), temp->u.w);
1270                     else {
1271                         assert(temp->code == jit_code_label ||
1272                                temp->code == jit_code_epilog);
1273                         word = movi_p(rn(node->u.w), temp->u.w);
1274                         patch(word, node);
1275                     }
1276                 }
1277                 else
1278                     movi(rn(node->u.w), node->v.w);
1279                 break;
1280                 case_rrr(lt,);
1281                 case_rrw(lt,);
1282                 case_rrr(lt, _u);
1283                 case_rrw(lt, _u);
1284                 case_rrr(le,);
1285                 case_rrw(le,);
1286                 case_rrr(le, _u);
1287                 case_rrw(le, _u);
1288                 case_rrr(eq,);
1289                 case_rrw(eq,);
1290                 case_rrr(ge,);
1291                 case_rrw(ge,);
1292                 case_rrr(ge, _u);
1293                 case_rrw(ge, _u);
1294                 case_rrr(gt,);
1295                 case_rrw(gt,);
1296                 case_rrr(gt, _u);
1297                 case_rrw(gt, _u);
1298                 case_rrr(ne,);
1299                 case_rrw(ne,);
1300                 case_brr(blt,);
1301                 case_brw(blt,);
1302                 case_brr(blt, _u);
1303                 case_brw(blt, _u);
1304                 case_brr(ble,);
1305                 case_brw(ble,);
1306                 case_brr(ble, _u);
1307                 case_brw(ble, _u);
1308                 case_brr(beq,);
1309                 case_brw(beq,);
1310                 case_brr(bge,);
1311                 case_brw(bge,);
1312                 case_brr(bge, _u);
1313                 case_brw(bge, _u);
1314                 case_brr(bgt,);
1315                 case_brw(bgt,);
1316                 case_brr(bgt, _u);
1317                 case_brw(bgt, _u);
1318                 case_brr(bne,);
1319                 case_brw(bne,);
1320                 case_brr(boadd,);
1321                 case_brw(boadd,);
1322                 case_brr(boadd, _u);
1323                 case_brw(boadd, _u);
1324                 case_brr(bxadd,);
1325                 case_brw(bxadd,);
1326                 case_brr(bxadd, _u);
1327                 case_brw(bxadd, _u);
1328                 case_brr(bosub,);
1329                 case_brw(bosub,);
1330                 case_brr(bosub, _u);
1331                 case_brw(bosub, _u);
1332                 case_brr(bxsub,);
1333                 case_brw(bxsub,);
1334                 case_brr(bxsub, _u);
1335                 case_brw(bxsub, _u);
1336                 case_brr(bms,);
1337                 case_brw(bms,);
1338                 case_brr(bmc,);
1339                 case_brw(bmc,);
1340                 case_rrr(add, _f);
1341                 case_rrf(add);
1342                 case_rrr(sub, _f);
1343                 case_rrf(sub);
1344                 case_rrf(rsb);
1345                 case_rrr(mul, _f);
1346                 case_rrf(mul);
1347                 case_rrr(div, _f);
1348                 case_rrf(div);
1349                 case_rr(abs, _f);
1350                 case_rr(neg, _f);
1351                 case_rr(sqrt, _f);
1352                 case_rqr(fma, _f);
1353                 case_rqr(fms, _f);
1354                 case_rqr(fnma, _f);
1355                 case_rqr(fnms, _f);
1356                 case_rr(ext, _f);
1357                 case_rr(ld, _f);
1358                 case_rw(ld, _f);
1359                 case_rrr(ldx, _f);
1360                 case_rrw(ldx, _f);
1361             case jit_code_unldr_x:
1362                 unldr_x(rn(node->u.w), rn(node->v.w), node->w.w);
1363                 break;
1364             case jit_code_unldi_x:
1365                 unldi_x(rn(node->u.w), node->v.w, node->w.w);
1366                 break;
1367                 case_rr(st, _f);
1368                 case_wr(st, _f);
1369                 case_rrr(stx, _f);
1370                 case_wrr(stx, _f);
1371             case jit_code_unstr_x:
1372                 unstr_x(rn(node->u.w), rn(node->v.w), node->w.w);
1373                 break;
1374             case jit_code_unsti_x:
1375                 unsti_x(node->u.w, rn(node->v.w), node->w.w);
1376                 break;
1377                 case_rr(mov, _f);
1378             case jit_code_movi_f:
1379                 assert_data(node);
1380                 movi_f(rn(node->u.w), node->v.f);
1381                 break;
1382                 case_rr(ext, _d_f);
1383                 case_rrr(lt, _f);
1384                 case_rrf(lt);
1385                 case_rrr(le, _f);
1386                 case_rrf(le);
1387                 case_rrr(eq, _f);
1388                 case_rrf(eq);
1389                 case_rrr(ge, _f);
1390                 case_rrf(ge);
1391                 case_rrr(gt, _f);
1392                 case_rrf(gt);
1393                 case_rrr(ne, _f);
1394                 case_rrf(ne);
1395                 case_rrr(unlt, _f);
1396                 case_rrf(unlt);
1397                 case_rrr(unle, _f);
1398                 case_rrf(unle);
1399                 case_rrr(uneq, _f);
1400                 case_rrf(uneq);
1401                 case_rrr(unge, _f);
1402                 case_rrf(unge);
1403                 case_rrr(ungt, _f);
1404                 case_rrf(ungt);
1405                 case_rrr(ltgt, _f);
1406                 case_rrf(ltgt);
1407                 case_rrr(ord, _f);
1408                 case_rrf(ord);
1409                 case_rrr(unord, _f);
1410                 case_rrf(unord);
1411                 case_brr(blt, _f);
1412                 case_brf(blt);
1413                 case_brr(ble, _f);
1414                 case_brf(ble);
1415                 case_brr(beq, _f);
1416                 case_brf(beq);
1417                 case_brr(bge, _f);
1418                 case_brf(bge);
1419                 case_brr(bgt, _f);
1420                 case_brf(bgt);
1421                 case_brr(bne, _f);
1422                 case_brf(bne);
1423                 case_brr(bunlt, _f);
1424                 case_brf(bunlt);
1425                 case_brr(bunle, _f);
1426                 case_brf(bunle);
1427                 case_brr(buneq, _f);
1428                 case_brf(buneq);
1429                 case_brr(bunge, _f);
1430                 case_brf(bunge);
1431                 case_brr(bungt, _f);
1432                 case_brf(bungt);
1433                 case_brr(bltgt, _f);
1434                 case_brf(bltgt);
1435                 case_brr(bord, _f);
1436                 case_brf(bord);
1437                 case_brr(bunord, _f);
1438                 case_brf(bunord);
1439                 case_rrr(add, _d);
1440                 case_rrd(add);
1441                 case_rrr(sub, _d);
1442                 case_rrd(sub);
1443                 case_rrd(rsb);
1444                 case_rrr(mul, _d);
1445                 case_rrd(mul);
1446                 case_rrr(div, _d);
1447                 case_rrd(div);
1448                 case_rr(abs, _d);
1449                 case_rr(neg, _d);
1450                 case_rr(sqrt, _d);
1451                 case_rqr(fma, _d);
1452                 case_rqr(fms, _d);
1453                 case_rqr(fnma, _d);
1454                 case_rqr(fnms, _d);
1455                 case_rr(ext, _d);
1456                 case_rr(ld, _d);
1457                 case_rw(ld, _d);
1458                 case_rrr(ldx, _d);
1459                 case_rrw(ldx, _d);
1460                 case_rr(st, _d);
1461                 case_wr(st, _d);
1462                 case_rrr(stx, _d);
1463                 case_wrr(stx, _d);
1464                 case_rr(mov, _d);
1465             case jit_code_movi_d:
1466                 assert_data(node);
1467                 movi_d(rn(node->u.w), node->v.d);
1468                 break;
1469                 case_rr(ext, _f_d);
1470                 case_rrr(lt, _d);
1471                 case_rrd(lt);
1472                 case_rrr(le, _d);
1473                 case_rrd(le);
1474                 case_rrr(eq, _d);
1475                 case_rrd(eq);
1476                 case_rrr(ge, _d);
1477                 case_rrd(ge);
1478                 case_rrr(gt, _d);
1479                 case_rrd(gt);
1480                 case_rrr(ne, _d);
1481                 case_rrd(ne);
1482                 case_rrr(unlt, _d);
1483                 case_rrd(unlt);
1484                 case_rrr(unle, _d);
1485                 case_rrd(unle);
1486                 case_rrr(uneq, _d);
1487                 case_rrd(uneq);
1488                 case_rrr(unge, _d);
1489                 case_rrd(unge);
1490                 case_rrr(ungt, _d);
1491                 case_rrd(ungt);
1492                 case_rrr(ltgt, _d);
1493                 case_rrd(ltgt);
1494                 case_rrr(ord, _d);
1495                 case_rrd(ord);
1496                 case_rrr(unord, _d);
1497                 case_rrd(unord);
1498                 case_brr(blt, _d);
1499                 case_brd(blt);
1500                 case_brr(ble, _d);
1501                 case_brd(ble);
1502                 case_brr(beq, _d);
1503                 case_brd(beq);
1504                 case_brr(bge, _d);
1505                 case_brd(bge);
1506                 case_brr(bgt, _d);
1507                 case_brd(bgt);
1508                 case_brr(bne, _d);
1509                 case_brd(bne);
1510                 case_brr(bunlt, _d);
1511                 case_brd(bunlt);
1512                 case_brr(bunle, _d);
1513                 case_brd(bunle);
1514                 case_brr(buneq, _d);
1515                 case_brd(buneq);
1516                 case_brr(bunge, _d);
1517                 case_brd(bunge);
1518                 case_brr(bungt, _d);
1519                 case_brd(bungt);
1520                 case_brr(bltgt, _d);
1521                 case_brd(bltgt);
1522                 case_brr(bord, _d);
1523                 case_brd(bord);
1524                 case_brr(bunord, _d);
1525                 case_brd(bunord);
1526             case jit_code_jmpr:
1527                 jit_check_frame();
1528                 jmpr(rn(node->u.w));
1529                 break;
1530             case jit_code_jmpi:
1531                 if (node->flag & jit_flag_node) {
1532                     temp = node->u.n;
1533                     assert(temp->code == jit_code_label ||
1534                            temp->code == jit_code_epilog);
1535                     if (temp->flag & jit_flag_patch)
1536                         jmpi(temp->u.w);
1537                     else {
1538                         word = _jit->code.length -
1539                             (_jit->pc.uc - _jit->code.ptr);
1540                         if (can_sign_extend_si26_p(word))
1541                             word = jmpi(_jit->pc.w);
1542                         else
1543                             word = jmpi_p(_jit->pc.w);
1544                         patch(word, node);
1545                     }
1546                 }
1547                 else {
1548                     jit_check_frame();
1549                     jmpi(node->u.w);
1550                 }
1551                 break;
1552             case jit_code_callr:
1553                 jit_check_frame();
1554                 callr(rn(node->u.w));
1555                 break;
1556             case jit_code_calli:
1557                 if (node->flag & jit_flag_node) {
1558                     temp = node->u.n;
1559                     assert(temp->code == jit_code_label ||
1560                            temp->code == jit_code_epilog);
1561                     if (temp->flag & jit_flag_patch)
1562                         calli(temp->u.w);
1563                     else {
1564                         word = _jit->code.length -
1565                             (_jit->pc.uc - _jit->code.ptr);
1566                         if (can_sign_extend_si26_p(word))
1567                             word = calli(_jit->pc.w);
1568                         else
1569                             word = calli_p(_jit->pc.w);
1570                         patch(word, node);
1571                     }
1572                 }
1573                 else {
1574                     jit_check_frame();
1575                     calli(node->u.w);
1576                 }
1577                 break;
1578             case jit_code_prolog:
1579                 _jitc->function = _jitc->functions.ptr + node->w.w;
1580                 undo.node = node;
1581                 undo.word = _jit->pc.w;
1582                 memcpy(&undo.func, _jitc->function, sizeof(undo.func));
1583 #if DEVEL_DISASSEMBLER
1584                 undo.prevw = prevw;
1585 #endif
1586                 undo.patch_offset = _jitc->patches.offset;
1587             restart_function:
1588                 compute_framesize();
1589                 patch_alist(0);
1590                 _jitc->again = 0;
1591                 prolog(node);
1592                 break;
1593             case jit_code_epilog:
1594                 assert(_jitc->function == _jitc->functions.ptr + node->w.w);
1595                 if (_jitc->again) {
1596                     for (temp = undo.node->next;
1597                          temp != node; temp = temp->next) {
1598                         if (temp->code == jit_code_label ||
1599                             temp->code == jit_code_epilog)
1600                             temp->flag &= ~jit_flag_patch;
1601                     }
1602                     temp->flag &= ~jit_flag_patch;
1603                     node = undo.node;
1604                     _jit->pc.w = undo.word;
1605                     /* undo.func.self.aoff and undo.func.regset should not
1606                      * be undone, as they will be further updated, and are
1607                      * the reason of the undo. */
1608                     undo.func.self.aoff = _jitc->function->frame +
1609                         _jitc->function->self.aoff;
1610                     undo.func.need_frame = _jitc->function->need_frame;
1611                     jit_regset_set(&undo.func.regset, &_jitc->function->regset);
1612                     /* allocar information also does not need to be undone */
1613                     undo.func.aoffoff = _jitc->function->aoffoff;
1614                     undo.func.allocar = _jitc->function->allocar;
1615                     /* this will be recomputed but undo anyway to have it
1616                      * better self documented.*/
1617                     undo.func.need_stack = _jitc->function->need_stack;
1618                     memcpy(_jitc->function, &undo.func, sizeof(undo.func));
1619 #if DEVEL_DISASSEMBLER
1620                     prevw = undo.prevw;
1621 #endif
1622                     _jitc->patches.offset = undo.patch_offset;
1623                     patch_alist(1);
1624                     goto restart_function;
1625                 }
1626                 if (node->link && (word = _jit->pc.w & 3))
1627                     nop(4 - word);
1628                 /* remember label is defined */
1629                 node->flag |= jit_flag_patch;
1630                 node->u.w = _jit->pc.w;
1631                 epilog(node);
1632                 _jitc->function = NULL;
1633                 break;
1634             case jit_code_movr_w_f:
1635                 movr_w_f(rn(node->u.w), rn(node->v.w));
1636                 break;
1637             case jit_code_movr_f_w:
1638                 movr_f_w(rn(node->u.w), rn(node->v.w));
1639                 break;
1640             case jit_code_movi_f_w:
1641                 assert_data(node);
1642                 movi_f_w(rn(node->u.w), node->v.f);
1643                 break;
1644             case jit_code_movi_w_f:
1645                 movi_w_f(rn(node->u.w), node->v.w);
1646                 break;
1647             case jit_code_movr_w_d:
1648                 movr_w_d(rn(node->u.w), rn(node->v.w));
1649                 break;
1650             case jit_code_movr_d_w:
1651                 movr_d_w(rn(node->u.w), rn(node->v.w));
1652                 break;
1653             case jit_code_movi_d_w:
1654                 assert_data(node);
1655                 movi_d_w(rn(node->u.w), node->v.d);
1656                 break;
1657             case jit_code_movi_w_d:
1658                 movi_w_d(rn(node->u.w), node->v.w);
1659                 break;
1660             case jit_code_va_start:
1661                 vastart(rn(node->u.w));
1662                 break;
1663             case jit_code_va_arg:
1664                 vaarg(rn(node->u.w), rn(node->v.w));
1665                 break;
1666             case jit_code_va_arg_d:
1667                 vaarg_d(rn(node->u.w), rn(node->v.w));
1668                 break;
1669             case jit_code_live:                 case jit_code_ellipsis:
1670             case jit_code_va_push:
1671             case jit_code_allocai:              case jit_code_allocar:
1672             case jit_code_arg_c:                case jit_code_arg_s:
1673             case jit_code_arg_i:                case jit_code_arg_l:
1674             case jit_code_arg_f:                case jit_code_arg_d:
1675             case jit_code_va_end:
1676             case jit_code_ret:
1677             case jit_code_retr_c:               case jit_code_reti_c:
1678             case jit_code_retr_uc:              case jit_code_reti_uc:
1679             case jit_code_retr_s:               case jit_code_reti_s:
1680             case jit_code_retr_us:              case jit_code_reti_us:
1681             case jit_code_retr_i:               case jit_code_reti_i:
1682             case jit_code_retr_ui:              case jit_code_reti_ui:
1683             case jit_code_retr_l:               case jit_code_reti_l:
1684             case jit_code_retr_f:               case jit_code_reti_f:
1685             case jit_code_retr_d:               case jit_code_reti_d:
1686             case jit_code_getarg_c:             case jit_code_getarg_uc:
1687             case jit_code_getarg_s:             case jit_code_getarg_us:
1688             case jit_code_getarg_i:
1689             case jit_code_getarg_ui:            case jit_code_getarg_l:
1690             case jit_code_getarg_f:             case jit_code_getarg_d:
1691             case jit_code_putargr_c:            case jit_code_putargi_c:
1692             case jit_code_putargr_uc:           case jit_code_putargi_uc:
1693             case jit_code_putargr_s:            case jit_code_putargi_s:
1694             case jit_code_putargr_us:           case jit_code_putargi_us:
1695             case jit_code_putargr_i:            case jit_code_putargi_i:
1696             case jit_code_putargr_ui:           case jit_code_putargi_ui:
1697             case jit_code_putargr_l:            case jit_code_putargi_l:
1698             case jit_code_putargr_f:            case jit_code_putargi_f:
1699             case jit_code_putargr_d:            case jit_code_putargi_d:
1700             case jit_code_pushargr_c:           case jit_code_pushargi_c:
1701             case jit_code_pushargr_uc:          case jit_code_pushargi_uc:
1702             case jit_code_pushargr_s:           case jit_code_pushargi_s:
1703             case jit_code_pushargr_us:          case jit_code_pushargi_us:
1704             case jit_code_pushargr_i:           case jit_code_pushargi_i:
1705             case jit_code_pushargr_ui:          case jit_code_pushargi_ui:
1706             case jit_code_pushargr_l:           case jit_code_pushargi_l:
1707             case jit_code_pushargr_f:           case jit_code_pushargi_f:
1708             case jit_code_pushargr_d:           case jit_code_pushargi_d:
1709             case jit_code_retval_c:             case jit_code_retval_uc:
1710             case jit_code_retval_s:             case jit_code_retval_us:
1711             case jit_code_retval_i:
1712             case jit_code_retval_ui:            case jit_code_retval_l:
1713             case jit_code_retval_f:             case jit_code_retval_d:
1714             case jit_code_prepare:
1715             case jit_code_finishr:              case jit_code_finishi:
1716             case jit_code_negi_f:               case jit_code_absi_f:
1717             case jit_code_sqrti_f:              case jit_code_negi_d:
1718             case jit_code_absi_d:               case jit_code_sqrti_d:
1719                 break;
1720             case jit_code_negi:
1721                 negi(rn(node->u.w), node->v.w);
1722                 break;
1723             case jit_code_comi:
1724                 comi(rn(node->u.w), node->v.w);
1725                 break;
1726             case jit_code_exti_c:
1727                 exti_c(rn(node->u.w), node->v.w);
1728                 break;
1729             case jit_code_exti_uc:
1730                 exti_uc(rn(node->u.w), node->v.w);
1731                 break;
1732             case jit_code_exti_s:
1733                 exti_s(rn(node->u.w), node->v.w);
1734                 break;
1735             case jit_code_exti_us:
1736                 exti_us(rn(node->u.w), node->v.w);
1737                 break;
1738             case jit_code_bswapi_us:
1739                 bswapi_us(rn(node->u.w), node->v.w);
1740                 break;
1741             case jit_code_bswapi_ui:
1742                 bswapi_ui(rn(node->u.w), node->v.w);
1743                 break;
1744             case jit_code_htoni_us:
1745                 htoni_us(rn(node->u.w), node->v.w);
1746                 break;
1747             case jit_code_htoni_ui:
1748                 htoni_ui(rn(node->u.w), node->v.w);
1749                 break;
1750 #if __WORDSIZE == 64
1751             case jit_code_exti_i:
1752                 exti_i(rn(node->u.w), node->v.w);
1753                 break;
1754             case jit_code_exti_ui:
1755                 exti_ui(rn(node->u.w), node->v.w);
1756                 break;
1757             case jit_code_bswapi_ul:
1758                 bswapi_ul(rn(node->u.w), node->v.w);
1759                 break;
1760             case jit_code_htoni_ul:
1761                 htoni_ul(rn(node->u.w), node->v.w);
1762                 break;
1763 #endif
1764             case jit_code_cloi:
1765                 cloi(rn(node->u.w), node->v.w);
1766                 break;
1767             case jit_code_clzi:
1768                 clzi(rn(node->u.w), node->v.w);
1769                 break;
1770             case jit_code_ctoi:
1771                 ctoi(rn(node->u.w), node->v.w);
1772                 break;
1773             case jit_code_ctzi:
1774                 ctzi(rn(node->u.w), node->v.w);
1775                 break;
1776             case jit_code_rbiti:
1777                 rbiti(rn(node->u.w), node->v.w);
1778                 break;
1779             case jit_code_popcnti:
1780                 popcnti(rn(node->u.w), node->v.w);
1781                 break;
1782             case jit_code_exti:
1783                 exti(rn(node->u.w), node->v.w, node->w.q.l, node->w.q.h);
1784                 break;
1785             case jit_code_exti_u:
1786                 exti_u(rn(node->u.w), node->v.w, node->w.q.l, node->w.q.h);
1787                 break;
1788             default:
1789                 abort();
1790         }
1791         if (jit_carry != _NOREG) {
1792             switch (node->code) {
1793                 case jit_code_note:
1794                 case jit_code_addcr:            case jit_code_addci:
1795                 case jit_code_addxr:            case jit_code_addxi:
1796                 case jit_code_subcr:            case jit_code_subci:
1797                 case jit_code_subxr:            case jit_code_subxi:
1798                     break;
1799                 default:
1800                     jit_unget_reg(jit_carry);
1801                     jit_carry = _NOREG;
1802                     break;
1803             }
1804         }
1805         jit_regarg_clr(node, value);
1806         assert(_jitc->regarg == 0 ||
1807                (jit_carry != _NOREG && _jitc->regarg == (1 << jit_carry)));
1808         assert(_jitc->synth == 0);
1809         /* update register live state */
1810         jit_reglive(node);
1811     }
1812 #undef case_brw
1813 #undef case_brr
1814 #undef case_wrr
1815 #undef case_rrw
1816 #undef case_rrr
1817 #undef case_wr
1818 #undef case_rw
1819 #undef case_rr
1820
1821     for (offset = 0; offset < _jitc->patches.offset; offset++) {
1822         node = _jitc->patches.ptr[offset].node;
1823         word = node->code == jit_code_movi ? node->v.n->u.w : node->u.n->u.w;
1824         patch_at(_jitc->patches.ptr[offset].inst, word);
1825     }
1826
1827     jit_flush(_jit->code.ptr, _jit->pc.uc);
1828
1829     return (_jit->code.ptr);
1830 }
1831
1832 #define CODE                            1
1833 #  include "jit_loongarch-cpu.c"
1834 #  include "jit_loongarch-fpu.c"
1835 #  include "jit_fallback.c"
1836 #undef CODE
1837
1838 void
1839 jit_flush(void *fptr, void *tptr)
1840 {
1841 #if defined(__GNUC__)
1842     jit_word_t          f, t, s;
1843
1844     s = sysconf(_SC_PAGE_SIZE);
1845     f = (jit_word_t)fptr & -s;
1846     t = (((jit_word_t)tptr) + s - 1) & -s;
1847     __clear_cache((void *)f, (void *)t);
1848 #endif
1849 }
1850
1851 void
1852 _emit_ldxi(jit_state_t *_jit, jit_int32_t r0, jit_int32_t r1, jit_word_t i0)
1853 {
1854     ldxi(rn(r0), rn(r1), i0);
1855 }
1856
1857 void
1858 _emit_stxi(jit_state_t *_jit, jit_word_t i0, jit_int32_t r0, jit_int32_t r1)
1859 {
1860     stxi(i0, rn(r0), rn(r1));
1861 }
1862
1863 void
1864 _emit_ldxi_d(jit_state_t *_jit, jit_int32_t r0, jit_int32_t r1, jit_word_t i0)
1865 {
1866     ldxi_d(rn(r0), rn(r1), i0);
1867 }
1868
1869 void
1870 _emit_stxi_d(jit_state_t *_jit, jit_word_t i0, jit_int32_t r0, jit_int32_t r1)
1871 {
1872     stxi_d(i0, rn(r0), rn(r1));
1873 }
1874
1875 static void
1876 _compute_framesize(jit_state_t *_jit)
1877 {
1878     jit_int32_t         reg;
1879     _jitc->framesize = 16;      /* ra+fp */
1880     for (reg = 0; reg < jit_size(iregs); reg++)
1881         if (jit_regset_tstbit(&_jitc->function->regset, iregs[reg]))
1882             _jitc->framesize += sizeof(jit_word_t);
1883
1884     for (reg = 0; reg < jit_size(fregs); reg++)
1885         if (jit_regset_tstbit(&_jitc->function->regset, fregs[reg]))
1886             _jitc->framesize += sizeof(jit_float64_t);
1887
1888     /* Space to store variadic arguments */
1889     if (_jitc->function->self.call & jit_call_varargs)
1890         _jitc->framesize += (8 - _jitc->function->vagp) * 8;
1891
1892     /* Make sure functions called have a 16 byte aligned stack */
1893     _jitc->framesize = (_jitc->framesize + 15) & -16;
1894 }
1895
1896 static void
1897 _patch(jit_state_t *_jit, jit_word_t instr, jit_node_t *node)
1898 {
1899     jit_int32_t          flag;
1900
1901     assert(node->flag & jit_flag_node);
1902     if (node->code == jit_code_movi)
1903         flag = node->v.n->flag;
1904     else
1905         flag = node->u.n->flag;
1906     assert(!(flag & jit_flag_patch));
1907     if (_jitc->patches.offset >= _jitc->patches.length) {
1908         jit_realloc((jit_pointer_t *)&_jitc->patches.ptr,
1909                     _jitc->patches.length * sizeof(jit_patch_t),
1910                     (_jitc->patches.length + 1024) * sizeof(jit_patch_t));
1911         _jitc->patches.length += 1024;
1912     }
1913     _jitc->patches.ptr[_jitc->patches.offset].inst = instr;
1914     _jitc->patches.ptr[_jitc->patches.offset].node = node;
1915     ++_jitc->patches.offset;
1916 }