deps/lightning/lib/jit_loongarch.c

   1 /*
   2  * Copyright (C) 2022-2023  Free Software Foundation, Inc.
   3  *
   4  * This file is part of GNU lightning.
   5  *
   6  * GNU lightning is free software; you can redistribute it and/or modify it
   7  * under the terms of the GNU Lesser General Public License as published
   8  * by the Free Software Foundation; either version 3, or (at your option)
   9  * any later version.
  10  *
  11  * GNU lightning is distributed in the hope that it will be useful, but
  12  * WITHOUT ANY WARRANTY; without even the implied warranty of MERCHANTABILITY
  13  * or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU Lesser General Public
  14  * License for more details.
  15  *
  16  * Authors:
  17  *      Paulo Cesar Pereira de Andrade
  18  */
  19
  20 /* callee save                          + variadic arguments
  21  * align16(ra+fp+s[0-8]+fs[0-7])        +       align16(a[0-7]) */
  22 #define stack_framesize                 (144 + 64)
  23
  24 #define jit_arg_reg_p(i)                ((i) >= 0 && (i) < 8)
  25 #define jit_arg_f_reg_p(i)              ((i) >= 0 && (i) < 8)
  26
  27 /*
  28  * Types
  29  */
  30 typedef struct jit_pointer_t jit_va_list_t;
  31
  32 /*
  33  * Prototypes
  34  */
  35 #define compute_framesize()             _compute_framesize(_jit)
  36 static void _compute_framesize(jit_state_t*);
  37 #define patch(instr, node)              _patch(_jit, instr, node)
  38 static void _patch(jit_state_t*,jit_word_t,jit_node_t*);
  39
  40 #define PROTO                           1
  41 # include "jit_loongarch-cpu.c"
  42 # include "jit_loongarch-fpu.c"
  43 #undef PROTO
  44
  45 /*
  46  * Initialization
  47  */
  48 jit_register_t          _rvs[] = {
  49     { rc(gpr) | 0x14,                   "$t8" },
  50     { rc(gpr) | 0x13,                   "$t7" },
  51     { rc(gpr) | 0x12,                   "$t6" },
  52     { rc(gpr) | 0x11,                   "$t5" },
  53     { rc(gpr) | 0x10,                   "$t4" },
  54     { rc(gpr) | 0x0f,                   "$t3" },
  55     { rc(gpr) | 0x0e,                   "$t2" },
  56     { rc(gpr) | 0x0d,                   "$t1" },
  57     { rc(gpr) | 0x0c,                   "$t0" },
  58     { rc(sav) | rc(gpr) | 0x1f,         "$s8" },
  59     { rc(sav) | rc(gpr) | 0x1e,         "$s7" },
  60     { rc(sav) | rc(gpr) | 0x1d,         "$s6" },
  61     { rc(sav) | rc(gpr) | 0x1c,         "$s5" },
  62     { rc(sav) | rc(gpr) | 0x1b,         "$s4" },
  63     { rc(sav) | rc(gpr) | 0x1a,         "$s3" },
  64     { rc(sav) | rc(gpr) | 0x19,         "$s2" },
  65     { rc(sav) | rc(gpr) | 0x18,         "$s1" },
  66     { rc(sav) | rc(gpr) | 0x17,         "$s0" },
  67     { rc(arg) | rc(gpr) | 0x0b,         "$a7" },
  68     { rc(arg) | rc(gpr) | 0x0a,         "$a6" },
  69     { rc(arg) | rc(gpr) | 0x09,         "$a5" },
  70     { rc(arg) | rc(gpr) | 0x08,         "$a4" },
  71     { rc(arg) | rc(gpr) | 0x07,         "$a3" },
  72     { rc(arg) | rc(gpr) | 0x06,         "$a2" },
  73     { rc(arg) | rc(gpr) | 0x05,         "$a1" },
  74     { rc(arg) | rc(gpr) | 0x04,         "$a0" },
  75     { 0x16,                             "$fp" },
  76     { 0x15,                             "<reserved>" },
  77     { 0x00,                             "$zero" },
  78     { 0x01,                             "$ra" },
  79     { 0x02,                             "$tp" },
  80     { 0x03,                             "$sp" },
  81     { rc(fpr) | 0x08,                   "$ft0" },
  82     { rc(fpr) | 0x09,                   "$ft1" },
  83     { rc(fpr) | 0x0a,                   "$ft2" },
  84     { rc(fpr) | 0x0b,                   "$ft3" },
  85     { rc(fpr) | 0x0c,                   "$ft4" },
  86     { rc(fpr) | 0x0d,                   "$ft5" },
  87     { rc(fpr) | 0x0e,                   "$ft6" },
  88     { rc(fpr) | 0x0f,                   "$ft7" },
  89     { rc(fpr) | 0x10,                   "$ft8" },
  90     { rc(fpr) | 0x11,                   "$ft9" },
  91     { rc(fpr) | 0x12,                   "$ft10" },
  92     { rc(fpr) | 0x13,                   "$ft11" },
  93     { rc(fpr) | 0x14,                   "$ft12" },
  94     { rc(fpr) | 0x15,                   "$ft13" },
  95     { rc(fpr) | 0x16,                   "$ft14" },
  96     { rc(fpr) | 0x17,                   "$ft15" },
  97     { rc(arg) | rc(fpr) | 0x07,         "$fa7" },
  98     { rc(arg) | rc(fpr) | 0x06,         "$fa6" },
  99     { rc(arg) | rc(fpr) | 0x05,         "$fa5" },
 100     { rc(arg) | rc(fpr) | 0x04,         "$fa4" },
 101     { rc(arg) | rc(fpr) | 0x03,         "$fa3" },
 102     { rc(arg) | rc(fpr) | 0x02,         "$fa2" },
 103     { rc(arg) | rc(fpr) | 0x01,         "$fa1" },
 104     { rc(arg) | rc(fpr) | 0x00,         "$fa0" },
 105     { rc(sav) | rc(fpr) | 0x1f,         "$fs7" },
 106     { rc(sav) | rc(fpr) | 0x1e,         "$fs6" },
 107     { rc(sav) | rc(fpr) | 0x1d,         "$fs5" },
 108     { rc(sav) | rc(fpr) | 0x1c,         "$fs4" },
 109     { rc(sav) | rc(fpr) | 0x1b,         "$fs3" },
 110     { rc(sav) | rc(fpr) | 0x1a,         "$fs2" },
 111     { rc(sav) | rc(fpr) | 0x19,         "$fs1" },
 112     { rc(sav) | rc(fpr) | 0x18,         "$fs0" },
 113     { _NOREG,                           "<none>" },
 114 };
 115
 116 static jit_int32_t iregs[] = {
 117     _S0, _S1, _S2, _S3, _S4, _S5, _S6, _S7, _S8
 118 };
 119
 120 static jit_int32_t fregs[] = {
 121     _FS0, _FS1, _FS2, _FS3, _FS4, _FS5, _FS6, _FS7
 122 };
 123
 124 /*
 125  * Implementation
 126  */
 127 void
 128 jit_get_cpu(void)
 129 {
 130 }
 131
 132 void
 133 _jit_init(jit_state_t *_jit)
 134 {
 135     _jitc->reglen = jit_size(_rvs) - 1;
 136     jit_carry = _NOREG;
 137 }
 138
 139 void
 140 _jit_prolog(jit_state_t *_jit)
 141 {
 142     jit_int32_t          offset;
 143
 144     if (_jitc->function)
 145         jit_epilog();
 146     assert(jit_regset_cmp_ui(&_jitc->regarg, 0) == 0);
 147     jit_regset_set_ui(&_jitc->regsav, 0);
 148     offset = _jitc->functions.offset;
 149     if (offset >= _jitc->functions.length) {
 150         jit_realloc((jit_pointer_t *)&_jitc->functions.ptr,
 151                     _jitc->functions.length * sizeof(jit_function_t),
 152                     (_jitc->functions.length + 16) * sizeof(jit_function_t));
 153         _jitc->functions.length += 16;
 154     }
 155     _jitc->function = _jitc->functions.ptr + _jitc->functions.offset++;
 156     _jitc->function->self.size = stack_framesize;
 157     _jitc->function->self.argi = _jitc->function->self.argf =
 158         _jitc->function->self.aoff = _jitc->function->self.alen = 0;
 159     _jitc->function->self.call = jit_call_default;
 160     jit_alloc((jit_pointer_t *)&_jitc->function->regoff,
 161               _jitc->reglen * sizeof(jit_int32_t));
 162
 163     /* _no_link here does not mean the jit_link() call can be removed
 164      * by rewriting as:
 165      * _jitc->function->prolog = jit_new_node(jit_code_prolog);
 166      */
 167     _jitc->function->prolog = jit_new_node_no_link(jit_code_prolog);
 168     jit_link(_jitc->function->prolog);
 169     _jitc->function->prolog->w.w = offset;
 170     _jitc->function->epilog = jit_new_node_no_link(jit_code_epilog);
 171     /*  u:      label value
 172      *  v:      offset in blocks vector
 173      *  w:      offset in functions vector
 174      */
 175     _jitc->function->epilog->w.w = offset;
 176
 177     jit_regset_new(&_jitc->function->regset);
 178 }
 179
 180 jit_int32_t
 181 _jit_allocai(jit_state_t *_jit, jit_int32_t length)
 182 {
 183     assert(_jitc->function);
 184     jit_check_frame();
 185     switch (length) {
 186         case 0: case 1:                                         break;
 187         case 2:         _jitc->function->self.aoff &= -2;       break;
 188         case 3: case 4: _jitc->function->self.aoff &= -4;       break;
 189         default:        _jitc->function->self.aoff &= -8;       break;
 190     }
 191     _jitc->function->self.aoff -= length;
 192     if (!_jitc->realize) {
 193         jit_inc_synth_ww(allocai, _jitc->function->self.aoff, length);
 194         jit_dec_synth();
 195     }
 196     return (_jitc->function->self.aoff);
 197 }
 198
 199 void
 200 _jit_allocar(jit_state_t *_jit, jit_int32_t u, jit_int32_t v)
 201 {
 202     jit_int32_t          reg;
 203     assert(_jitc->function);
 204     jit_inc_synth_ww(allocar, u, v);
 205     if (!_jitc->function->allocar) {
 206         _jitc->function->aoffoff = jit_allocai(sizeof(jit_int32_t));
 207         _jitc->function->allocar = 1;
 208     }
 209     reg = jit_get_reg(jit_class_gpr);
 210     jit_negr(reg, v);
 211     jit_andi(reg, reg, -16);
 212     jit_ldxi_i(u, JIT_FP, _jitc->function->aoffoff);
 213     jit_addr(u, u, reg);
 214     jit_addr(JIT_SP, JIT_SP, reg);
 215     jit_stxi_i(_jitc->function->aoffoff, JIT_FP, u);
 216     jit_unget_reg(reg);
 217     jit_dec_synth();
 218 }
 219
 220 void
 221 _jit_ret(jit_state_t *_jit)
 222 {
 223     jit_node_t          *instr;
 224     assert(_jitc->function);
 225     jit_inc_synth(ret);
 226     /* jump to epilog */
 227     instr = jit_jmpi();
 228     jit_patch_at(instr, _jitc->function->epilog);
 229     jit_dec_synth();
 230 }
 231
 232 void
 233 _jit_retr(jit_state_t *_jit, jit_int32_t u, jit_code_t code)
 234 {
 235     jit_code_inc_synth_w(code, u);
 236     jit_movr(JIT_RET, u);
 237     jit_ret();
 238     jit_dec_synth();
 239 }
 240
 241 void
 242 _jit_reti(jit_state_t *_jit, jit_word_t u, jit_code_t code)
 243 {
 244     jit_code_inc_synth_w(code, u);
 245     jit_movi(JIT_RET, u);
 246     jit_ret();
 247     jit_dec_synth();
 248 }
 249
 250 void
 251 _jit_retr_f(jit_state_t *_jit, jit_int32_t u)
 252 {
 253     jit_inc_synth_w(retr_f, u);
 254     if (JIT_FRET != u)
 255         jit_movr_f(JIT_FRET, u);
 256     else
 257         jit_live(JIT_FRET);
 258     jit_ret();
 259     jit_dec_synth();
 260 }
 261
 262 void
 263 _jit_reti_f(jit_state_t *_jit, jit_float32_t u)
 264 {
 265     jit_inc_synth_f(reti_f, u);
 266     jit_movi_f(JIT_FRET, u);
 267     jit_ret();
 268     jit_dec_synth();
 269 }
 270
 271 void
 272 _jit_retr_d(jit_state_t *_jit, jit_int32_t u)
 273 {
 274     jit_inc_synth_w(retr_d, u);
 275     if (JIT_FRET != u)
 276         jit_movr_d(JIT_FRET, u);
 277     else
 278         jit_live(JIT_FRET);
 279     jit_ret();
 280     jit_dec_synth();
 281 }
 282
 283 void
 284 _jit_reti_d(jit_state_t *_jit, jit_float64_t u)
 285 {
 286     jit_inc_synth_d(reti_d, u);
 287     jit_movi_d(JIT_FRET, u);
 288     jit_ret();
 289     jit_dec_synth();
 290 }
 291
 292 void
 293 _jit_epilog(jit_state_t *_jit)
 294 {
 295     assert(_jitc->function);
 296     assert(_jitc->function->epilog->next == NULL);
 297     jit_link(_jitc->function->epilog);
 298     _jitc->function = NULL;
 299 }
 300
 301 jit_bool_t
 302 _jit_arg_register_p(jit_state_t *_jit, jit_node_t *u)
 303 {
 304     if (u->code >= jit_code_arg_c && u->code <= jit_code_arg)
 305         return (jit_arg_reg_p(u->u.w));
 306     assert(u->code == jit_code_arg_f || u->code == jit_code_arg_d);
 307     return (jit_arg_f_reg_p(u->u.w) || jit_arg_reg_p(u->u.w - 8));
 308 }
 309
 310 void
 311 _jit_ellipsis(jit_state_t *_jit)
 312 {
 313     jit_inc_synth(ellipsis);
 314     jit_check_frame();
 315     if (_jitc->prepare) {
 316         jit_link_prepare();
 317         assert(!(_jitc->function->call.call & jit_call_varargs));
 318         _jitc->function->call.call |= jit_call_varargs;
 319     }
 320     else {
 321         jit_link_prolog();
 322         assert(!(_jitc->function->self.call & jit_call_varargs));
 323         _jitc->function->self.call |= jit_call_varargs;
 324         _jitc->function->vagp = _jitc->function->self.argi;
 325     }
 326     jit_dec_synth();
 327 }
 328
 329 void
 330 _jit_va_push(jit_state_t *_jit, jit_int32_t u)
 331 {
 332     jit_inc_synth_w(va_push, u);
 333     jit_pushargr(u);
 334     jit_dec_synth();
 335 }
 336
 337 jit_node_t *
 338 _jit_arg(jit_state_t *_jit, jit_code_t code)
 339 {
 340     jit_node_t          *node;
 341     jit_int32_t          offset;
 342     assert(_jitc->function);
 343     assert(!(_jitc->function->self.call & jit_call_varargs));
 344 #if STRONG_TYPE_CHECKING
 345     assert(code >= jit_code_arg_c && code <= jit_code_arg);
 346 #endif
 347     if (jit_arg_reg_p(_jitc->function->self.argi))
 348         offset = _jitc->function->self.argi++;
 349     else {
 350         offset = _jitc->function->self.size;
 351         _jitc->function->self.size += sizeof(jit_word_t);
 352         jit_check_frame();
 353     }
 354     node = jit_new_node_ww(code, offset,
 355                            ++_jitc->function->self.argn);
 356     jit_link_prolog();
 357     return (node);
 358 }
 359
 360 jit_node_t *
 361 _jit_arg_f(jit_state_t *_jit)
 362 {
 363     jit_node_t          *node;
 364     jit_int32_t          offset;
 365     assert(_jitc->function);
 366     assert(!(_jitc->function->self.call & jit_call_varargs));
 367     if (jit_arg_f_reg_p(_jitc->function->self.argf))
 368         offset = _jitc->function->self.argf++;
 369     else if (jit_arg_reg_p(_jitc->function->self.argi)) {
 370         offset = _jitc->function->self.argi++;
 371         offset += 8;
 372     }
 373     else {
 374         offset = _jitc->function->self.size;
 375         _jitc->function->self.size += sizeof(jit_word_t);
 376         jit_check_frame();
 377     }
 378     node = jit_new_node_ww(jit_code_arg_f, offset,
 379                            ++_jitc->function->self.argn);
 380     jit_link_prolog();
 381     return (node);
 382 }
 383
 384 jit_node_t *
 385 _jit_arg_d(jit_state_t *_jit)
 386 {
 387     jit_node_t          *node;
 388     jit_int32_t          offset;
 389     assert(_jitc->function);
 390     assert(!(_jitc->function->self.call & jit_call_varargs));
 391     if (jit_arg_f_reg_p(_jitc->function->self.argf))
 392         offset = _jitc->function->self.argf++;
 393     else if (jit_arg_reg_p(_jitc->function->self.argi)) {
 394         offset = _jitc->function->self.argi++;
 395         offset += 8;
 396     }
 397     else {
 398         offset = _jitc->function->self.size;
 399         _jitc->function->self.size += sizeof(jit_word_t);
 400         jit_check_frame();
 401     }
 402     node = jit_new_node_ww(jit_code_arg_d, offset,
 403                            ++_jitc->function->self.argn);
 404     jit_link_prolog();
 405     return (node);
 406 }
 407
 408 void
 409 _jit_getarg_c(jit_state_t *_jit, jit_int32_t u, jit_node_t *v)
 410 {
 411     assert_arg_type(v->code, jit_code_arg_c);
 412     jit_inc_synth_wp(getarg_c, u, v);
 413     if (jit_arg_reg_p(v->u.w))
 414         jit_extr_c(u, _A0 - v->u.w);
 415     else {
 416         jit_node_t      *node = jit_ldxi_c(u, JIT_FP, v->u.w);
 417         jit_link_alist(node);
 418     }
 419     jit_dec_synth();
 420 }
 421
 422 void
 423 _jit_getarg_uc(jit_state_t *_jit, jit_int32_t u, jit_node_t *v)
 424 {
 425     assert_arg_type(v->code, jit_code_arg_c);
 426     jit_inc_synth_wp(getarg_uc, u, v);
 427     if (jit_arg_reg_p(v->u.w))
 428         jit_extr_uc(u, _A0 - v->u.w);
 429     else {
 430         jit_node_t      *node = jit_ldxi_uc(u, JIT_FP, v->u.w);
 431         jit_link_alist(node);
 432     }
 433     jit_dec_synth();
 434 }
 435
 436 void
 437 _jit_getarg_s(jit_state_t *_jit, jit_int32_t u, jit_node_t *v)
 438 {
 439     assert_arg_type(v->code, jit_code_arg_s);
 440     jit_inc_synth_wp(getarg_s, u, v);
 441     if (jit_arg_reg_p(v->u.w))
 442         jit_extr_s(u, _A0 - v->u.w);
 443     else {
 444         jit_node_t      *node = jit_ldxi_s(u, JIT_FP, v->u.w);
 445         jit_link_alist(node);
 446     }
 447     jit_dec_synth();
 448 }
 449
 450 void
 451 _jit_getarg_us(jit_state_t *_jit, jit_int32_t u, jit_node_t *v)
 452 {
 453     assert_arg_type(v->code, jit_code_arg_s);
 454     jit_inc_synth_wp(getarg_us, u, v);
 455     if (jit_arg_reg_p(v->u.w))
 456         jit_extr_us(u, _A0 - v->u.w);
 457     else {
 458         jit_node_t      *node = jit_ldxi_us(u, JIT_FP, v->u.w);
 459         jit_link_alist(node);
 460     }
 461     jit_dec_synth();
 462 }
 463
 464 void
 465 _jit_getarg_i(jit_state_t *_jit, jit_int32_t u, jit_node_t *v)
 466 {
 467     assert_arg_type(v->code, jit_code_arg_i);
 468     jit_inc_synth_wp(getarg_i, u, v);
 469     if (jit_arg_reg_p(v->u.w))
 470         jit_extr_i(u, _A0 - v->u.w);
 471     else {
 472         jit_node_t      *node = jit_ldxi_i(u, JIT_FP, v->u.w);
 473         jit_link_alist(node);
 474     }
 475     jit_dec_synth();
 476 }
 477
 478 void
 479 _jit_getarg_ui(jit_state_t *_jit, jit_int32_t u, jit_node_t *v)
 480 {
 481     assert_arg_type(v->code, jit_code_arg_i);
 482     jit_inc_synth_wp(getarg_ui, u, v);
 483     if (jit_arg_reg_p(v->u.w))
 484         jit_extr_ui(u, _A0 - v->u.w);
 485     else {
 486         jit_node_t      *node = jit_ldxi_ui(u, JIT_FP, v->u.w);
 487         jit_link_alist(node);
 488     }
 489     jit_dec_synth();
 490 }
 491
 492 void
 493 _jit_getarg_l(jit_state_t *_jit, jit_int32_t u, jit_node_t *v)
 494 {
 495     assert_arg_type(v->code, jit_code_arg_l);
 496     jit_inc_synth_wp(getarg_l, u, v);
 497     if (jit_arg_reg_p(v->u.w))
 498         jit_movr(u, _A0 - v->u.w);
 499     else {
 500         jit_node_t      *node = jit_ldxi_l(u, JIT_FP, v->u.w);
 501         jit_link_alist(node);
 502     }
 503     jit_dec_synth();
 504 }
 505
 506 void
 507 _jit_putargr(jit_state_t *_jit, jit_int32_t u, jit_node_t *v, jit_code_t code)
 508 {
 509     assert_putarg_type(code, v->code);
 510     jit_code_inc_synth_wp(code, u, v);
 511     if (jit_arg_reg_p(v->u.w))
 512         jit_movr(_A0 - v->u.w, u);
 513     else {
 514         jit_node_t      *node = jit_stxi(v->u.w, JIT_FP, u);
 515         jit_link_alist(node);
 516     }
 517     jit_dec_synth();
 518 }
 519
 520 void
 521 _jit_putargi(jit_state_t *_jit, jit_word_t u, jit_node_t *v, jit_code_t code)
 522 {
 523     jit_int32_t         regno;
 524     assert_putarg_type(code, v->code);
 525     jit_code_inc_synth_wp(code, u, v);
 526     if (jit_arg_reg_p(v->u.w))
 527         jit_movi(_A0 - v->u.w, u);
 528     else {
 529         jit_node_t      *node;
 530         regno = jit_get_reg(jit_class_gpr);
 531         jit_movi(regno, u);
 532         node = jit_stxi(v->u.w, JIT_FP, regno);
 533         jit_link_alist(node);
 534         jit_unget_reg(regno);
 535     }
 536     jit_dec_synth();
 537 }
 538
 539 void
 540 _jit_getarg_f(jit_state_t *_jit, jit_int32_t u, jit_node_t *v)
 541 {
 542     assert(v->code == jit_code_arg_f);
 543     jit_inc_synth_wp(getarg_f, u, v);
 544     if (jit_arg_f_reg_p(v->u.w))
 545         jit_movr_f(u, _FA0 - v->u.w);
 546     else if (jit_arg_reg_p(v->u.w - 8))
 547         jit_movr_w_f(u, JIT_RA0 - (v->u.w - 8));
 548     else {
 549         jit_node_t      *node = jit_ldxi_f(u, JIT_FP, v->u.w);
 550         jit_link_alist(node);
 551     }
 552     jit_dec_synth();
 553 }
 554
 555 void
 556 _jit_putargr_f(jit_state_t *_jit, jit_int32_t u, jit_node_t *v)
 557 {
 558     assert(v->code == jit_code_arg_f);
 559     jit_inc_synth_wp(putargr_f, u, v);
 560     if (jit_arg_f_reg_p(v->u.w))
 561         jit_movr_f(_FA0 - v->u.w, u);
 562     else if (jit_arg_reg_p(v->u.w - 8))
 563         jit_movr_f_w(JIT_RA0 - (v->u.w - 8), u);
 564     else {
 565         jit_node_t      *node = jit_stxi_f(v->u.w, JIT_FP, u);
 566         jit_link_alist(node);
 567     }
 568     jit_dec_synth();
 569 }
 570
 571 void
 572 _jit_putargi_f(jit_state_t *_jit, jit_float32_t u, jit_node_t *v)
 573 {
 574     jit_int32_t         regno;
 575     assert(v->code == jit_code_arg_f);
 576     jit_inc_synth_fp(putargi_f, u, v);
 577     if (jit_arg_f_reg_p(v->u.w))
 578         jit_movi_f(_FA0 - v->u.w, u);
 579     else if (jit_arg_reg_p(v->u.w - 8))
 580         jit_movi_f_w(JIT_RA0 - (v->u.w - 8), u);
 581     else {
 582         jit_node_t      *node;
 583         regno = jit_get_reg(jit_class_fpr);
 584         jit_movi_f(regno, u);
 585         node = jit_stxi_f(v->u.w, JIT_FP, regno);
 586         jit_link_alist(node);
 587         jit_unget_reg(regno);
 588     }
 589     jit_dec_synth();
 590 }
 591
 592 void
 593 _jit_getarg_d(jit_state_t *_jit, jit_int32_t u, jit_node_t *v)
 594 {
 595     assert(v->code == jit_code_arg_d);
 596     jit_inc_synth_wp(getarg_d, u, v);
 597     if (jit_arg_f_reg_p(v->u.w))
 598         jit_movr_d(u, _FA0 - v->u.w);
 599     else if (jit_arg_reg_p(v->u.w - 8))
 600         jit_movr_w_d(u, JIT_RA0 - (v->u.w - 8));
 601     else {
 602         jit_node_t      *node = jit_ldxi_d(u, JIT_FP, v->u.w);
 603         jit_link_alist(node);
 604     }
 605     jit_dec_synth();
 606 }
 607
 608 void
 609 _jit_putargr_d(jit_state_t *_jit, jit_int32_t u, jit_node_t *v)
 610 {
 611     assert(v->code == jit_code_arg_d);
 612     jit_inc_synth_wp(putargr_d, u, v);
 613     if (jit_arg_f_reg_p(v->u.w))
 614         jit_movr_d(_FA0 - v->u.w, u);
 615     else if (jit_arg_reg_p(v->u.w - 8))
 616         jit_movr_d_w(JIT_RA0 - (v->u.w - 8), u);
 617     else {
 618         jit_node_t      *node = jit_stxi_d(v->u.w, JIT_FP, u);
 619         jit_link_alist(node);
 620     }
 621     jit_dec_synth();
 622 }
 623
 624 void
 625 _jit_putargi_d(jit_state_t *_jit, jit_float64_t u, jit_node_t *v)
 626 {
 627     jit_int32_t         regno;
 628     assert(v->code == jit_code_arg_d);
 629     jit_inc_synth_dp(putargi_d, u, v);
 630     if (jit_arg_f_reg_p(v->u.w))
 631         jit_movi_d(_FA0 - v->u.w, u);
 632     else if (jit_arg_reg_p(v->u.w - 8))
 633         jit_movi_d_w(JIT_RA0 - (v->u.w - 8), u);
 634     else {
 635         jit_node_t      *node;
 636         regno = jit_get_reg(jit_class_fpr);
 637         jit_movi_d(regno, u);
 638         node = jit_stxi_d(v->u.w, JIT_FP, regno);
 639         jit_link_alist(node);
 640         jit_unget_reg(regno);
 641     }
 642     jit_dec_synth();
 643 }
 644
 645 void
 646 _jit_pushargr(jit_state_t *_jit, jit_int32_t u, jit_code_t code)
 647 {
 648     assert(_jitc->function);
 649     jit_code_inc_synth_w(code, u);
 650     jit_link_prepare();
 651     if (jit_arg_reg_p(_jitc->function->call.argi)) {
 652         jit_movr(_A0 - _jitc->function->call.argi, u);
 653         ++_jitc->function->call.argi;
 654     }
 655     else {
 656         jit_stxi(_jitc->function->call.size, JIT_SP, u);
 657         _jitc->function->call.size += sizeof(jit_word_t);
 658         jit_check_frame();
 659     }
 660     jit_dec_synth();
 661 }
 662
 663 void
 664 _jit_pushargi(jit_state_t *_jit, jit_word_t u, jit_code_t code)
 665 {
 666     jit_int32_t          regno;
 667     assert(_jitc->function);
 668     jit_code_inc_synth_w(code, u);
 669     jit_link_prepare();
 670     if (jit_arg_reg_p(_jitc->function->call.argi)) {
 671         jit_movi(_A0 - _jitc->function->call.argi, u);
 672         ++_jitc->function->call.argi;
 673     }
 674     else {
 675         regno = jit_get_reg(jit_class_gpr);
 676         jit_movi(regno, u);
 677         jit_stxi(_jitc->function->call.size, JIT_SP, regno);
 678         jit_unget_reg(regno);
 679         _jitc->function->call.size += sizeof(jit_word_t);
 680         jit_check_frame();
 681     }
 682     jit_dec_synth();
 683 }
 684
 685 void
 686 _jit_pushargr_f(jit_state_t *_jit, jit_int32_t u)
 687 {
 688     assert(_jitc->function);
 689     jit_inc_synth_w(pushargr_f, u);
 690     jit_link_prepare();
 691     if (jit_arg_f_reg_p(_jitc->function->call.argf) &&
 692         !(_jitc->function->call.call & jit_call_varargs)) {
 693         jit_movr_f(_FA0 - _jitc->function->call.argf, u);
 694         ++_jitc->function->call.argf;
 695     }
 696     else if (jit_arg_reg_p(_jitc->function->call.argi)) {
 697         jit_movr_f_w(_A0 - _jitc->function->call.argi, u);
 698         ++_jitc->function->call.argi;
 699     }
 700     else {
 701         jit_stxi_f(_jitc->function->call.size, JIT_SP, u);
 702         _jitc->function->call.size += sizeof(jit_word_t);
 703         jit_check_frame();
 704     }
 705     jit_dec_synth();
 706 }
 707
 708 void
 709 _jit_pushargi_f(jit_state_t *_jit, jit_float32_t u)
 710 {
 711     jit_int32_t         regno;
 712     assert(_jitc->function);
 713     jit_inc_synth_f(pushargi_f, u);
 714     jit_link_prepare();
 715     if (jit_arg_f_reg_p(_jitc->function->call.argf) &&
 716         !(_jitc->function->call.call & jit_call_varargs)) {
 717         jit_movi_f(_FA0 - _jitc->function->call.argf, u);
 718         ++_jitc->function->call.argf;
 719     }
 720     else if (jit_arg_reg_p(_jitc->function->call.argi)) {
 721         jit_movi_f_w(_A0 - _jitc->function->call.argi, u);
 722         ++_jitc->function->call.argi;
 723     }
 724     else {
 725         regno = jit_get_reg(jit_class_fpr);
 726         jit_movi_f(regno, u);
 727         jit_stxi_f(_jitc->function->call.size, JIT_SP, regno);
 728         jit_unget_reg(regno);
 729         _jitc->function->call.size += sizeof(jit_word_t);
 730         jit_check_frame();
 731     }
 732     jit_dec_synth();
 733 }
 734
 735 void
 736 _jit_pushargr_d(jit_state_t *_jit, jit_int32_t u)
 737 {
 738     assert(_jitc->function);
 739     jit_inc_synth_w(pushargr_d, u);
 740     jit_link_prepare();
 741     if (jit_arg_f_reg_p(_jitc->function->call.argf) &&
 742         !(_jitc->function->call.call & jit_call_varargs)) {
 743         jit_movr_d(_FA0 - _jitc->function->call.argf, u);
 744         ++_jitc->function->call.argf;
 745     }
 746     else if (jit_arg_reg_p(_jitc->function->call.argi)) {
 747         jit_movr_d_w(_A0 - _jitc->function->call.argi, u);
 748         ++_jitc->function->call.argi;
 749     }
 750     else {
 751         jit_stxi_d(_jitc->function->call.size, JIT_SP, u);
 752         _jitc->function->call.size += sizeof(jit_word_t);
 753         jit_check_frame();
 754     }
 755     jit_dec_synth();
 756 }
 757
 758 void
 759 _jit_pushargi_d(jit_state_t *_jit, jit_float64_t u)
 760 {
 761     jit_int32_t         regno;
 762     assert(_jitc->function);
 763     jit_inc_synth_d(pushargi_d, u);
 764     jit_link_prepare();
 765     if (jit_arg_f_reg_p(_jitc->function->call.argf) &&
 766         !(_jitc->function->call.call & jit_call_varargs)) {
 767         jit_movi_d(_FA0 - _jitc->function->call.argf, u);
 768         ++_jitc->function->call.argf;
 769     }
 770     else if (jit_arg_reg_p(_jitc->function->call.argi)) {
 771         jit_movi_d_w(_A0 - _jitc->function->call.argi, u);
 772         ++_jitc->function->call.argi;
 773     }
 774     else {
 775         regno = jit_get_reg(jit_class_fpr);
 776         jit_movi_d(regno, u);
 777         jit_stxi_d(_jitc->function->call.size, JIT_SP, regno);
 778         jit_unget_reg(regno);
 779         _jitc->function->call.size += sizeof(jit_word_t);
 780         jit_check_frame();
 781     }
 782     jit_dec_synth();
 783 }
 784
 785 jit_bool_t
 786 _jit_regarg_p(jit_state_t *_jit, jit_node_t *node, jit_int32_t regno)
 787 {
 788     jit_int32_t         spec;
 789     spec = jit_class(_rvs[regno].spec);
 790     if (spec & jit_class_arg) {
 791         regno = _A0 - regno;
 792         if (regno >= 0 && regno < node->v.w)
 793             return (1);
 794         if (spec & jit_class_fpr) {
 795             regno = _FA0 - regno;
 796             if (regno >= 0 && regno < node->w.w)
 797                 return (1);
 798         }
 799     }
 800     return (0);
 801 }
 802
 803 void
 804 _jit_finishr(jit_state_t *_jit, jit_int32_t r0)
 805 {
 806     jit_node_t          *node;
 807     assert(_jitc->function);
 808     jit_check_frame();
 809     jit_inc_synth_w(finishr, r0);
 810     if (_jitc->function->self.alen < _jitc->function->call.size)
 811         _jitc->function->self.alen = _jitc->function->call.size;
 812     node = jit_callr(r0);
 813     node->v.w = _jitc->function->call.argi;
 814     node->w.w = _jitc->function->call.argf;
 815     _jitc->function->call.argi = _jitc->function->call.argf =
 816         _jitc->function->call.size = 0;
 817     _jitc->prepare = 0;
 818     jit_dec_synth();
 819 }
 820
 821 jit_node_t *
 822 _jit_finishi(jit_state_t *_jit, jit_pointer_t i0)
 823 {
 824     jit_node_t          *node;
 825     assert(_jitc->function);
 826     jit_check_frame();
 827     jit_inc_synth_w(finishi, (jit_word_t)i0);
 828     if (_jitc->function->self.alen < _jitc->function->call.size)
 829         _jitc->function->self.alen = _jitc->function->call.size;
 830     node = jit_calli(i0);
 831     node->v.w = _jitc->function->call.argi;
 832     node->w.w = _jitc->function->call.argf;
 833     _jitc->function->call.argi = _jitc->function->call.argf =
 834         _jitc->function->call.size = 0;
 835     _jitc->prepare = 0;
 836     jit_dec_synth();
 837     return (node);
 838 }
 839
 840 void
 841 _jit_retval_c(jit_state_t *_jit, jit_int32_t r0)
 842 {
 843     jit_inc_synth_w(retval_c, r0);
 844     jit_extr_c(r0, JIT_RET);
 845     jit_dec_synth();
 846 }
 847
 848 void
 849 _jit_retval_uc(jit_state_t *_jit, jit_int32_t r0)
 850 {
 851     jit_inc_synth_w(retval_uc, r0);
 852     jit_extr_uc(r0, JIT_RET);
 853     jit_dec_synth();
 854 }
 855
 856 void
 857 _jit_retval_s(jit_state_t *_jit, jit_int32_t r0)
 858 {
 859     jit_inc_synth_w(retval_s, r0);
 860     jit_extr_s(r0, JIT_RET);
 861     jit_dec_synth();
 862 }
 863
 864 void
 865 _jit_retval_us(jit_state_t *_jit, jit_int32_t r0)
 866 {
 867     jit_inc_synth_w(retval_us, r0);
 868     jit_extr_us(r0, JIT_RET);
 869     jit_dec_synth();
 870 }
 871
 872 void
 873 _jit_retval_i(jit_state_t *_jit, jit_int32_t r0)
 874 {
 875     jit_inc_synth_w(retval_i, r0);
 876     jit_extr_i(r0, JIT_RET);
 877     jit_dec_synth();
 878 }
 879
 880 void
 881 _jit_retval_ui(jit_state_t *_jit, jit_int32_t r0)
 882 {
 883     jit_inc_synth_w(retval_ui, r0);
 884     jit_extr_ui(r0, JIT_RET);
 885     jit_dec_synth();
 886 }
 887
 888 void
 889 _jit_retval_l(jit_state_t *_jit, jit_int32_t r0)
 890 {
 891     jit_inc_synth_w(retval_l, r0);
 892     if (r0 != JIT_RET)
 893     jit_movr(r0, JIT_RET);
 894     jit_dec_synth();
 895 }
 896
 897 void
 898 _jit_retval_f(jit_state_t *_jit, jit_int32_t r0)
 899 {
 900     jit_inc_synth_w(retval_f, r0);
 901     if (r0 != JIT_FRET)
 902         jit_movr_f(r0, JIT_FRET);
 903     jit_dec_synth();
 904 }
 905
 906 void
 907 _jit_retval_d(jit_state_t *_jit, jit_int32_t r0)
 908 {
 909     jit_inc_synth_w(retval_d, r0);
 910     if (r0 != JIT_FRET)
 911         jit_movr_d(r0, JIT_FRET);
 912     jit_dec_synth();
 913 }
 914
 915 jit_pointer_t
 916 _emit_code(jit_state_t *_jit)
 917 {
 918     jit_node_t          *node;
 919     jit_node_t          *temp;
 920     jit_word_t           word;
 921     jit_int32_t          value;
 922     jit_int32_t          offset;
 923     struct {
 924         jit_node_t      *node;
 925         jit_word_t       word;
 926         jit_function_t   func;
 927 #if DEVEL_DISASSEMBLER
 928         jit_word_t       prevw;
 929 #endif
 930         jit_int32_t      patch_offset;
 931     } undo;
 932 #if DEVEL_DISASSEMBLER
 933     jit_word_t           prevw;
 934 #endif
 935
 936     _jitc->function = NULL;
 937
 938     jit_reglive_setup();
 939
 940     undo.word = 0;
 941     undo.node = NULL;
 942     undo.patch_offset = 0;
 943
 944 #define assert_data(node)               /**/
 945 #define case_rr(name, type)                                             \
 946             case jit_code_##name##r##type:                              \
 947                 name##r##type(rn(node->u.w), rn(node->v.w));            \
 948                 break
 949 #define case_rw(name, type)                                             \
 950             case jit_code_##name##i##type:                              \
 951                 name##i##type(rn(node->u.w), node->v.w);                \
 952                 break
 953 #define case_wr(name, type)                                             \
 954             case jit_code_##name##i##type:                              \
 955                 name##i##type(node->u.w, rn(node->v.w));                \
 956                 break
 957 #define case_rrr(name, type)                                            \
 958             case jit_code_##name##r##type:                              \
 959                 name##r##type(rn(node->u.w),                            \
 960                               rn(node->v.w), rn(node->w.w));            \
 961                 break
 962 #define case_rrrr(name, type)                                           \
 963             case jit_code_##name##r##type:                              \
 964                 name##r##type(rn(node->u.q.l), rn(node->u.q.h),         \
 965                               rn(node->v.w), rn(node->w.w));            \
 966                 break
 967 #define case_rrw(name, type)                                            \
 968             case jit_code_##name##i##type:                              \
 969                 name##i##type(rn(node->u.w), rn(node->v.w), node->w.w); \
 970                 break
 971 #define case_rrrw(name, type)                                           \
 972             case jit_code_##name##i##type:                              \
 973                 name##i##type(rn(node->u.q.l), rn(node->u.q.h),         \
 974                               rn(node->v.w), node->w.w);                \
 975                 break
 976 #define case_rrf(name)                                                  \
 977             case jit_code_##name##i_f:                                  \
 978                 assert_data(node);                                      \
 979                 name##i_f(rn(node->u.w), rn(node->v.w), node->w.f);     \
 980                 break
 981 #define case_rrd(name)                                                  \
 982             case jit_code_##name##i_d:                                  \
 983                 assert_data(node);                                      \
 984                 name##i_d(rn(node->u.w), rn(node->v.w),node->w.d);      \
 985                 break
 986 #define case_wrr(name, type)                                            \
 987             case jit_code_##name##i##type:                              \
 988                 name##i##type(node->u.w, rn(node->v.w), rn(node->w.w)); \
 989                 break
 990 #define case_brr(name, type)                                            \
 991             case jit_code_##name##r##type:                              \
 992                 temp = node->u.n;                                       \
 993                 assert(temp->code == jit_code_label ||                  \
 994                        temp->code == jit_code_epilog);                  \
 995                 if (temp->flag & jit_flag_patch)                        \
 996                     name##r##type(temp->u.w, rn(node->v.w),             \
 997                                   rn(node->w.w));                       \
 998                 else {                                                  \
 999                     word = name##r##type(_jit->pc.w,                    \
1000                                          rn(node->v.w),                 \
1001                                          rn(node->w.w));                \
1002                     patch(word, node);                                  \
1003                 }                                                       \
1004                 break
1005 #define case_brw(name, type)                                            \
1006             case jit_code_##name##i##type:                              \
1007                 temp = node->u.n;                                       \
1008                 assert(temp->code == jit_code_label ||                  \
1009                        temp->code == jit_code_epilog);                  \
1010                 if (temp->flag & jit_flag_patch)                        \
1011                     name##i##type(temp->u.w,                            \
1012                                   rn(node->v.w), node->w.w);            \
1013                 else {                                                  \
1014                     word = name##i##type(_jit->pc.w,                    \
1015                                          rn(node->v.w), node->w.w);     \
1016                     patch(word, node);                                  \
1017                 }                                                       \
1018                 break;
1019 #define case_brf(name)                                                  \
1020             case jit_code_##name##i_f:                                  \
1021                 temp = node->u.n;                                       \
1022                 assert(temp->code == jit_code_label ||                  \
1023                        temp->code == jit_code_epilog);                  \
1024                 if (temp->flag & jit_flag_patch)                        \
1025                     name##i_f(temp->u.w, rn(node->v.w), node->w.f);     \
1026                 else {                                                  \
1027                     word = name##i_f(_jit->pc.w, rn(node->v.w),         \
1028                                      node->w.f);                        \
1029                     patch(word, node);                                  \
1030                 }                                                       \
1031                 break
1032 #define case_brd(name)                                                  \
1033             case jit_code_##name##i_d:                                  \
1034                 temp = node->u.n;                                       \
1035                 assert(temp->code == jit_code_label ||                  \
1036                        temp->code == jit_code_epilog);                  \
1037                 if (temp->flag & jit_flag_patch)                        \
1038                     name##i_d(temp->u.w, rn(node->v.w), node->w.d);     \
1039                 else {                                                  \
1040                     word = name##i_d(_jit->pc.w, rn(node->v.w),         \
1041                                      node->w.d);                        \
1042                     patch(word, node);                                  \
1043                 }                                                       \
1044                 break
1045 #if DEVEL_DISASSEMBLER
1046     prevw = _jit->pc.w;
1047 #endif
1048     for (node = _jitc->head; node; node = node->next) {
1049         if (_jit->pc.uc >= _jitc->code.end)
1050             return (NULL);
1051
1052 #if DEVEL_DISASSEMBLER
1053         node->offset = (jit_uword_t)_jit->pc.w - (jit_uword_t)prevw;
1054         prevw = _jit->pc.w;
1055 #endif
1056         value = jit_classify(node->code);
1057         jit_regarg_set(node, value);
1058         switch (node->code) {
1059             case jit_code_align:
1060                 /* Must align to a power of two */
1061                 assert(!(node->u.w & (node->u.w - 1)));
1062                 if ((word = _jit->pc.w & (node->u.w - 1)))
1063                     nop(node->u.w - word);
1064                 break;
1065             case jit_code_skip:
1066                 nop((node->u.w + 3) & ~3);
1067                 break;
1068             case jit_code_note:         case jit_code_name:
1069                 node->u.w = _jit->pc.w;
1070                 break;
1071             case jit_code_label:
1072                 /* remember label is defined */
1073                 node->flag |= jit_flag_patch;
1074                 node->u.w = _jit->pc.w;
1075                 break;
1076                 case_rrr(add,);
1077                 case_rrw(add,);
1078                 case_rrr(addc,);
1079                 case_rrw(addc,);
1080                 case_rrr(addx,);
1081                 case_rrw(addx,);
1082                 case_rrr(sub,);
1083                 case_rrw(sub,);
1084                 case_rrr(subc,);
1085                 case_rrw(subc,);
1086                 case_rrr(subx,);
1087                 case_rrw(subx,);
1088                 case_rrw(rsb,);
1089                 case_rrr(mul,);
1090                 case_rrw(mul,);
1091                 case_rrrr(qmul,);
1092                 case_rrrw(qmul,);
1093                 case_rrrr(qmul, _u);
1094                 case_rrrw(qmul, _u);
1095                 case_rrr(div,);
1096                 case_rrw(div,);
1097                 case_rrr(div, _u);
1098                 case_rrw(div, _u);
1099                 case_rrr(rem,);
1100                 case_rrw(rem,);
1101                 case_rrr(rem, _u);
1102                 case_rrw(rem, _u);
1103                 case_rrrr(qdiv,);
1104                 case_rrrw(qdiv,);
1105                 case_rrrr(qdiv, _u);
1106                 case_rrrw(qdiv, _u);
1107                 case_rrr(lsh,);
1108                 case_rrw(lsh,);
1109                 case_rrr(rsh,);
1110                 case_rrw(rsh,);
1111                 case_rrr(rsh, _u);
1112                 case_rrw(rsh, _u);
1113                 case_rr(neg,);
1114                 case_rr(com,);
1115                 case_rr(clo,);
1116                 case_rr(clz,);
1117                 case_rr(cto,);
1118                 case_rr(ctz,);
1119                 case_rrr(and,);
1120                 case_rrw(and,);
1121                 case_rrr(or,);
1122                 case_rrw(or,);
1123                 case_rrr(xor,);
1124                 case_rrw(xor,);
1125                 case_rr(trunc, _f_i);
1126                 case_rr(trunc, _d_i);
1127                 case_rr(trunc, _f_l);
1128                 case_rr(trunc, _d_l);
1129                 case_rr(ld, _c);
1130                 case_rw(ld, _c);
1131                 case_rr(ld, _uc);
1132                 case_rw(ld, _uc);
1133                 case_rr(ld, _s);
1134                 case_rw(ld, _s);
1135                 case_rr(ld, _us);
1136                 case_rw(ld, _us);
1137                 case_rr(ld, _i);
1138                 case_rw(ld, _i);
1139                 case_rr(ld, _ui);
1140                 case_rw(ld, _ui);
1141                 case_rr(ld, _l);
1142                 case_rw(ld, _l);
1143                 case_rrr(ldx, _c);
1144                 case_rrw(ldx, _c);
1145                 case_rrr(ldx, _uc);
1146                 case_rrw(ldx, _uc);
1147                 case_rrr(ldx, _s);
1148                 case_rrw(ldx, _s);
1149                 case_rrr(ldx, _us);
1150                 case_rrw(ldx, _us);
1151                 case_rrr(ldx, _i);
1152                 case_rrw(ldx, _i);
1153                 case_rrr(ldx, _ui);
1154                 case_rrw(ldx, _ui);
1155                 case_rrr(ldx, _l);
1156                 case_rrw(ldx, _l);
1157                 case_rr(st, _c);
1158                 case_wr(st, _c);
1159                 case_rr(st, _s);
1160                 case_wr(st, _s);
1161                 case_rr(st, _i);
1162                 case_wr(st, _i);
1163                 case_rr(st, _l);
1164                 case_wr(st, _l);
1165                 case_rrr(stx, _c);
1166                 case_wrr(stx, _c);
1167                 case_rrr(stx, _s);
1168                 case_wrr(stx, _s);
1169                 case_rrr(stx, _i);
1170                 case_wrr(stx, _i);
1171                 case_rrr(stx, _l);
1172                 case_wrr(stx, _l);
1173                 case_rr(hton, _us);
1174                 case_rr(hton, _ui);
1175                 case_rr(hton, _ul);
1176                 case_rr(bswap, _us);
1177                 case_rr(bswap, _ui);
1178                 case_rr(bswap, _ul);
1179                 case_rr(ext, _c);
1180                 case_rr(ext, _uc);
1181                 case_rr(ext, _s);
1182                 case_rr(ext, _us);
1183                 case_rr(ext, _i);
1184                 case_rr(ext, _ui);
1185             case jit_code_casr:
1186                 casr(rn(node->u.w), rn(node->v.w),
1187                      rn(node->w.q.l), rn(node->w.q.h));
1188                 break;
1189             case jit_code_casi:
1190                 casi(rn(node->u.w), node->v.w,
1191                      rn(node->w.q.l), rn(node->w.q.h));
1192                 break;
1193                 case_rrr(movn,);
1194                 case_rrr(movz,);
1195                 case_rr(mov,);
1196             case jit_code_movi:
1197                 if (node->flag & jit_flag_node) {
1198                     temp = node->v.n;
1199                     if (temp->code == jit_code_data ||
1200                         (temp->code == jit_code_label &&
1201                          (temp->flag & jit_flag_patch)))
1202                         movi(rn(node->u.w), temp->u.w);
1203                     else {
1204                         assert(temp->code == jit_code_label ||
1205                                temp->code == jit_code_epilog);
1206                         word = movi_p(rn(node->u.w), temp->u.w);
1207                         patch(word, node);
1208                     }
1209                 }
1210                 else
1211                     movi(rn(node->u.w), node->v.w);
1212                 break;
1213                 case_rrr(lt,);
1214                 case_rrw(lt,);
1215                 case_rrr(lt, _u);
1216                 case_rrw(lt, _u);
1217                 case_rrr(le,);
1218                 case_rrw(le,);
1219                 case_rrr(le, _u);
1220                 case_rrw(le, _u);
1221                 case_rrr(eq,);
1222                 case_rrw(eq,);
1223                 case_rrr(ge,);
1224                 case_rrw(ge,);
1225                 case_rrr(ge, _u);
1226                 case_rrw(ge, _u);
1227                 case_rrr(gt,);
1228                 case_rrw(gt,);
1229                 case_rrr(gt, _u);
1230                 case_rrw(gt, _u);
1231                 case_rrr(ne,);
1232                 case_rrw(ne,);
1233                 case_brr(blt,);
1234                 case_brw(blt,);
1235                 case_brr(blt, _u);
1236                 case_brw(blt, _u);
1237                 case_brr(ble,);
1238                 case_brw(ble,);
1239                 case_brr(ble, _u);
1240                 case_brw(ble, _u);
1241                 case_brr(beq,);
1242                 case_brw(beq,);
1243                 case_brr(bge,);
1244                 case_brw(bge,);
1245                 case_brr(bge, _u);
1246                 case_brw(bge, _u);
1247                 case_brr(bgt,);
1248                 case_brw(bgt,);
1249                 case_brr(bgt, _u);
1250                 case_brw(bgt, _u);
1251                 case_brr(bne,);
1252                 case_brw(bne,);
1253                 case_brr(boadd,);
1254                 case_brw(boadd,);
1255                 case_brr(boadd, _u);
1256                 case_brw(boadd, _u);
1257                 case_brr(bxadd,);
1258                 case_brw(bxadd,);
1259                 case_brr(bxadd, _u);
1260                 case_brw(bxadd, _u);
1261                 case_brr(bosub,);
1262                 case_brw(bosub,);
1263                 case_brr(bosub, _u);
1264                 case_brw(bosub, _u);
1265                 case_brr(bxsub,);
1266                 case_brw(bxsub,);
1267                 case_brr(bxsub, _u);
1268                 case_brw(bxsub, _u);
1269                 case_brr(bms,);
1270                 case_brw(bms,);
1271                 case_brr(bmc,);
1272                 case_brw(bmc,);
1273                 case_rrr(add, _f);
1274                 case_rrf(add);
1275                 case_rrr(sub, _f);
1276                 case_rrf(sub);
1277                 case_rrf(rsb);
1278                 case_rrr(mul, _f);
1279                 case_rrf(mul);
1280                 case_rrr(div, _f);
1281                 case_rrf(div);
1282                 case_rr(abs, _f);
1283                 case_rr(neg, _f);
1284                 case_rr(sqrt, _f);
1285                 case_rr(ext, _f);
1286                 case_rr(ld, _f);
1287                 case_rw(ld, _f);
1288                 case_rrr(ldx, _f);
1289                 case_rrw(ldx, _f);
1290                 case_rr(st, _f);
1291                 case_wr(st, _f);
1292                 case_rrr(stx, _f);
1293                 case_wrr(stx, _f);
1294                 case_rr(mov, _f);
1295             case jit_code_movi_f:
1296                 assert_data(node);
1297                 movi_f(rn(node->u.w), node->v.f);
1298                 break;
1299                 case_rr(ext, _d_f);
1300                 case_rrr(lt, _f);
1301                 case_rrf(lt);
1302                 case_rrr(le, _f);
1303                 case_rrf(le);
1304                 case_rrr(eq, _f);
1305                 case_rrf(eq);
1306                 case_rrr(ge, _f);
1307                 case_rrf(ge);
1308                 case_rrr(gt, _f);
1309                 case_rrf(gt);
1310                 case_rrr(ne, _f);
1311                 case_rrf(ne);
1312                 case_rrr(unlt, _f);
1313                 case_rrf(unlt);
1314                 case_rrr(unle, _f);
1315                 case_rrf(unle);
1316                 case_rrr(uneq, _f);
1317                 case_rrf(uneq);
1318                 case_rrr(unge, _f);
1319                 case_rrf(unge);
1320                 case_rrr(ungt, _f);
1321                 case_rrf(ungt);
1322                 case_rrr(ltgt, _f);
1323                 case_rrf(ltgt);
1324                 case_rrr(ord, _f);
1325                 case_rrf(ord);
1326                 case_rrr(unord, _f);
1327                 case_rrf(unord);
1328                 case_brr(blt, _f);
1329                 case_brf(blt);
1330                 case_brr(ble, _f);
1331                 case_brf(ble);
1332                 case_brr(beq, _f);
1333                 case_brf(beq);
1334                 case_brr(bge, _f);
1335                 case_brf(bge);
1336                 case_brr(bgt, _f);
1337                 case_brf(bgt);
1338                 case_brr(bne, _f);
1339                 case_brf(bne);
1340                 case_brr(bunlt, _f);
1341                 case_brf(bunlt);
1342                 case_brr(bunle, _f);
1343                 case_brf(bunle);
1344                 case_brr(buneq, _f);
1345                 case_brf(buneq);
1346                 case_brr(bunge, _f);
1347                 case_brf(bunge);
1348                 case_brr(bungt, _f);
1349                 case_brf(bungt);
1350                 case_brr(bltgt, _f);
1351                 case_brf(bltgt);
1352                 case_brr(bord, _f);
1353                 case_brf(bord);
1354                 case_brr(bunord, _f);
1355                 case_brf(bunord);
1356                 case_rrr(add, _d);
1357                 case_rrd(add);
1358                 case_rrr(sub, _d);
1359                 case_rrd(sub);
1360                 case_rrd(rsb);
1361                 case_rrr(mul, _d);
1362                 case_rrd(mul);
1363                 case_rrr(div, _d);
1364                 case_rrd(div);
1365                 case_rr(abs, _d);
1366                 case_rr(neg, _d);
1367                 case_rr(sqrt, _d);
1368                 case_rr(ext, _d);
1369                 case_rr(ld, _d);
1370                 case_rw(ld, _d);
1371                 case_rrr(ldx, _d);
1372                 case_rrw(ldx, _d);
1373                 case_rr(st, _d);
1374                 case_wr(st, _d);
1375                 case_rrr(stx, _d);
1376                 case_wrr(stx, _d);
1377                 case_rr(mov, _d);
1378             case jit_code_movi_d:
1379                 assert_data(node);
1380                 movi_d(rn(node->u.w), node->v.d);
1381                 break;
1382                 case_rr(ext, _f_d);
1383                 case_rrr(lt, _d);
1384                 case_rrd(lt);
1385                 case_rrr(le, _d);
1386                 case_rrd(le);
1387                 case_rrr(eq, _d);
1388                 case_rrd(eq);
1389                 case_rrr(ge, _d);
1390                 case_rrd(ge);
1391                 case_rrr(gt, _d);
1392                 case_rrd(gt);
1393                 case_rrr(ne, _d);
1394                 case_rrd(ne);
1395                 case_rrr(unlt, _d);
1396                 case_rrd(unlt);
1397                 case_rrr(unle, _d);
1398                 case_rrd(unle);
1399                 case_rrr(uneq, _d);
1400                 case_rrd(uneq);
1401                 case_rrr(unge, _d);
1402                 case_rrd(unge);
1403                 case_rrr(ungt, _d);
1404                 case_rrd(ungt);
1405                 case_rrr(ltgt, _d);
1406                 case_rrd(ltgt);
1407                 case_rrr(ord, _d);
1408                 case_rrd(ord);
1409                 case_rrr(unord, _d);
1410                 case_rrd(unord);
1411                 case_brr(blt, _d);
1412                 case_brd(blt);
1413                 case_brr(ble, _d);
1414                 case_brd(ble);
1415                 case_brr(beq, _d);
1416                 case_brd(beq);
1417                 case_brr(bge, _d);
1418                 case_brd(bge);
1419                 case_brr(bgt, _d);
1420                 case_brd(bgt);
1421                 case_brr(bne, _d);
1422                 case_brd(bne);
1423                 case_brr(bunlt, _d);
1424                 case_brd(bunlt);
1425                 case_brr(bunle, _d);
1426                 case_brd(bunle);
1427                 case_brr(buneq, _d);
1428                 case_brd(buneq);
1429                 case_brr(bunge, _d);
1430                 case_brd(bunge);
1431                 case_brr(bungt, _d);
1432                 case_brd(bungt);
1433                 case_brr(bltgt, _d);
1434                 case_brd(bltgt);
1435                 case_brr(bord, _d);
1436                 case_brd(bord);
1437                 case_brr(bunord, _d);
1438                 case_brd(bunord);
1439             case jit_code_jmpr:
1440                 jit_check_frame();
1441                 jmpr(rn(node->u.w));
1442                 break;
1443             case jit_code_jmpi:
1444                 if (node->flag & jit_flag_node) {
1445                     temp = node->u.n;
1446                     assert(temp->code == jit_code_label ||
1447                            temp->code == jit_code_epilog);
1448                     if (temp->flag & jit_flag_patch)
1449                         jmpi(temp->u.w);
1450                     else {
1451                         word = _jit->code.length -
1452                             (_jit->pc.uc - _jit->code.ptr);
1453                         if (can_sign_extend_si26_p(word))
1454                             word = jmpi(_jit->pc.w);
1455                         else
1456                             word = jmpi_p(_jit->pc.w);
1457                         patch(word, node);
1458                     }
1459                 }
1460                 else {
1461                     jit_check_frame();
1462                     jmpi(node->u.w);
1463                 }
1464                 break;
1465             case jit_code_callr:
1466                 jit_check_frame();
1467                 callr(rn(node->u.w));
1468                 break;
1469             case jit_code_calli:
1470                 if (node->flag & jit_flag_node) {
1471                     temp = node->u.n;
1472                     assert(temp->code == jit_code_label ||
1473                            temp->code == jit_code_epilog);
1474                     if (temp->flag & jit_flag_patch)
1475                         calli(temp->u.w);
1476                     else {
1477                         word = _jit->code.length -
1478                             (_jit->pc.uc - _jit->code.ptr);
1479                         if (can_sign_extend_si26_p(word))
1480                             word = calli(_jit->pc.w);
1481                         else
1482                             word = calli_p(_jit->pc.w);
1483                         patch(word, node);
1484                     }
1485                 }
1486                 else {
1487                     jit_check_frame();
1488                     calli(node->u.w);
1489                 }
1490                 break;
1491             case jit_code_prolog:
1492                 _jitc->function = _jitc->functions.ptr + node->w.w;
1493                 undo.node = node;
1494                 undo.word = _jit->pc.w;
1495                 memcpy(&undo.func, _jitc->function, sizeof(undo.func));
1496 #if DEVEL_DISASSEMBLER
1497                 undo.prevw = prevw;
1498 #endif
1499                 undo.patch_offset = _jitc->patches.offset;
1500             restart_function:
1501                 compute_framesize();
1502                 patch_alist(0);
1503                 _jitc->again = 0;
1504                 prolog(node);
1505                 break;
1506             case jit_code_epilog:
1507                 assert(_jitc->function == _jitc->functions.ptr + node->w.w);
1508                 if (_jitc->again) {
1509                     for (temp = undo.node->next;
1510                          temp != node; temp = temp->next) {
1511                         if (temp->code == jit_code_label ||
1512                             temp->code == jit_code_epilog)
1513                             temp->flag &= ~jit_flag_patch;
1514                     }
1515                     temp->flag &= ~jit_flag_patch;
1516                     node = undo.node;
1517                     _jit->pc.w = undo.word;
1518                     /* undo.func.self.aoff and undo.func.regset should not
1519                      * be undone, as they will be further updated, and are
1520                      * the reason of the undo. */
1521                     undo.func.self.aoff = _jitc->function->frame +
1522                         _jitc->function->self.aoff;
1523                     undo.func.need_frame = _jitc->function->need_frame;
1524                     jit_regset_set(&undo.func.regset, &_jitc->function->regset);
1525                     /* allocar information also does not need to be undone */
1526                     undo.func.aoffoff = _jitc->function->aoffoff;
1527                     undo.func.allocar = _jitc->function->allocar;
1528                     /* this will be recomputed but undo anyway to have it
1529                      * better self documented.*/
1530                     undo.func.need_stack = _jitc->function->need_stack;
1531                     memcpy(_jitc->function, &undo.func, sizeof(undo.func));
1532 #if DEVEL_DISASSEMBLER
1533                     prevw = undo.prevw;
1534 #endif
1535                     _jitc->patches.offset = undo.patch_offset;
1536                     patch_alist(1);
1537                     goto restart_function;
1538                 }
1539                 if (node->link && (word = _jit->pc.w & 3))
1540                     nop(4 - word);
1541                 /* remember label is defined */
1542                 node->flag |= jit_flag_patch;
1543                 node->u.w = _jit->pc.w;
1544                 epilog(node);
1545                 _jitc->function = NULL;
1546                 break;
1547             case jit_code_movr_w_f:
1548                 movr_w_f(rn(node->u.w), rn(node->v.w));
1549                 break;
1550             case jit_code_movr_f_w:
1551                 movr_f_w(rn(node->u.w), rn(node->v.w));
1552                 break;
1553             case jit_code_movi_f_w:
1554                 assert_data(node);
1555                 movi_f_w(rn(node->u.w), node->v.f);
1556                 break;
1557             case jit_code_movr_w_d:
1558                 movr_w_d(rn(node->u.w), rn(node->v.w));
1559                 break;
1560             case jit_code_movr_d_w:
1561                 movr_d_w(rn(node->u.w), rn(node->v.w));
1562                 break;
1563             case jit_code_movi_d_w:
1564                 assert_data(node);
1565                 movi_d_w(rn(node->u.w), node->v.d);
1566                 break;
1567             case jit_code_va_start:
1568                 vastart(rn(node->u.w));
1569                 break;
1570             case jit_code_va_arg:
1571                 vaarg(rn(node->u.w), rn(node->v.w));
1572                 break;
1573             case jit_code_va_arg_d:
1574                 vaarg_d(rn(node->u.w), rn(node->v.w));
1575                 break;
1576             case jit_code_live:                 case jit_code_ellipsis:
1577             case jit_code_va_push:
1578             case jit_code_allocai:              case jit_code_allocar:
1579             case jit_code_arg_c:                case jit_code_arg_s:
1580             case jit_code_arg_i:                case jit_code_arg_l:
1581             case jit_code_arg_f:                case jit_code_arg_d:
1582             case jit_code_va_end:
1583             case jit_code_ret:
1584             case jit_code_retr_c:               case jit_code_reti_c:
1585             case jit_code_retr_uc:              case jit_code_reti_uc:
1586             case jit_code_retr_s:               case jit_code_reti_s:
1587             case jit_code_retr_us:              case jit_code_reti_us:
1588             case jit_code_retr_i:               case jit_code_reti_i:
1589             case jit_code_retr_ui:              case jit_code_reti_ui:
1590             case jit_code_retr_l:               case jit_code_reti_l:
1591             case jit_code_retr_f:               case jit_code_reti_f:
1592             case jit_code_retr_d:               case jit_code_reti_d:
1593             case jit_code_getarg_c:             case jit_code_getarg_uc:
1594             case jit_code_getarg_s:             case jit_code_getarg_us:
1595             case jit_code_getarg_i:
1596             case jit_code_getarg_ui:            case jit_code_getarg_l:
1597             case jit_code_getarg_f:             case jit_code_getarg_d:
1598             case jit_code_putargr_c:            case jit_code_putargi_c:
1599             case jit_code_putargr_uc:           case jit_code_putargi_uc:
1600             case jit_code_putargr_s:            case jit_code_putargi_s:
1601             case jit_code_putargr_us:           case jit_code_putargi_us:
1602             case jit_code_putargr_i:            case jit_code_putargi_i:
1603             case jit_code_putargr_ui:           case jit_code_putargi_ui:
1604             case jit_code_putargr_l:            case jit_code_putargi_l:
1605             case jit_code_putargr_f:            case jit_code_putargi_f:
1606             case jit_code_putargr_d:            case jit_code_putargi_d:
1607             case jit_code_pushargr_c:           case jit_code_pushargi_c:
1608             case jit_code_pushargr_uc:          case jit_code_pushargi_uc:
1609             case jit_code_pushargr_s:           case jit_code_pushargi_s:
1610             case jit_code_pushargr_us:          case jit_code_pushargi_us:
1611             case jit_code_pushargr_i:           case jit_code_pushargi_i:
1612             case jit_code_pushargr_ui:          case jit_code_pushargi_ui:
1613             case jit_code_pushargr_l:           case jit_code_pushargi_l:
1614             case jit_code_pushargr_f:           case jit_code_pushargi_f:
1615             case jit_code_pushargr_d:           case jit_code_pushargi_d:
1616             case jit_code_retval_c:             case jit_code_retval_uc:
1617             case jit_code_retval_s:             case jit_code_retval_us:
1618             case jit_code_retval_i:
1619             case jit_code_retval_ui:            case jit_code_retval_l:
1620             case jit_code_retval_f:             case jit_code_retval_d:
1621             case jit_code_prepare:
1622             case jit_code_finishr:              case jit_code_finishi:
1623                 break;
1624             default:
1625                 abort();
1626         }
1627         if (jit_carry != _NOREG) {
1628             switch (node->code) {
1629                 case jit_code_note:
1630                 case jit_code_addcr:            case jit_code_addci:
1631                 case jit_code_addxr:            case jit_code_addxi:
1632                 case jit_code_subcr:            case jit_code_subci:
1633                 case jit_code_subxr:            case jit_code_subxi:
1634                     break;
1635                 default:
1636                     jit_unget_reg(jit_carry);
1637                     jit_carry = _NOREG;
1638                     break;
1639             }
1640         }
1641         jit_regarg_clr(node, value);
1642         assert(_jitc->regarg == 0 ||
1643                (jit_carry != _NOREG && _jitc->regarg == (1 << jit_carry)));
1644         assert(_jitc->synth == 0);
1645         /* update register live state */
1646         jit_reglive(node);
1647     }
1648 #undef case_brw
1649 #undef case_brr
1650 #undef case_wrr
1651 #undef case_rrw
1652 #undef case_rrr
1653 #undef case_wr
1654 #undef case_rw
1655 #undef case_rr
1656
1657     for (offset = 0; offset < _jitc->patches.offset; offset++) {
1658         node = _jitc->patches.ptr[offset].node;
1659         word = node->code == jit_code_movi ? node->v.n->u.w : node->u.n->u.w;
1660         patch_at(_jitc->patches.ptr[offset].inst, word);
1661     }
1662
1663     jit_flush(_jit->code.ptr, _jit->pc.uc);
1664
1665     return (_jit->code.ptr);
1666 }
1667
1668 #define CODE                            1
1669 #  include "jit_loongarch-cpu.c"
1670 #  include "jit_loongarch-fpu.c"
1671 #undef CODE
1672
1673 void
1674 jit_flush(void *fptr, void *tptr)
1675 {
1676 #if defined(__GNUC__)
1677     jit_word_t          f, t, s;
1678
1679     s = sysconf(_SC_PAGE_SIZE);
1680     f = (jit_word_t)fptr & -s;
1681     t = (((jit_word_t)tptr) + s - 1) & -s;
1682     __clear_cache((void *)f, (void *)t);
1683 #endif
1684 }
1685
1686 void
1687 _emit_ldxi(jit_state_t *_jit, jit_int32_t r0, jit_int32_t r1, jit_word_t i0)
1688 {
1689     ldxi(rn(r0), rn(r1), i0);
1690 }
1691
1692 void
1693 _emit_stxi(jit_state_t *_jit, jit_word_t i0, jit_int32_t r0, jit_int32_t r1)
1694 {
1695     stxi(i0, rn(r0), rn(r1));
1696 }
1697
1698 void
1699 _emit_ldxi_d(jit_state_t *_jit, jit_int32_t r0, jit_int32_t r1, jit_word_t i0)
1700 {
1701     ldxi_d(rn(r0), rn(r1), i0);
1702 }
1703
1704 void
1705 _emit_stxi_d(jit_state_t *_jit, jit_word_t i0, jit_int32_t r0, jit_int32_t r1)
1706 {
1707     stxi_d(i0, rn(r0), rn(r1));
1708 }
1709
1710 static void
1711 _compute_framesize(jit_state_t *_jit)
1712 {
1713     jit_int32_t         reg;
1714     _jitc->framesize = 16;      /* ra+fp */
1715     for (reg = 0; reg < jit_size(iregs); reg++)
1716         if (jit_regset_tstbit(&_jitc->function->regset, iregs[reg]))
1717             _jitc->framesize += sizeof(jit_word_t);
1718
1719     for (reg = 0; reg < jit_size(fregs); reg++)
1720         if (jit_regset_tstbit(&_jitc->function->regset, fregs[reg]))
1721             _jitc->framesize += sizeof(jit_float64_t);
1722
1723     /* Space to store variadic arguments */
1724     if (_jitc->function->self.call & jit_call_varargs)
1725         _jitc->framesize += (8 - _jitc->function->vagp) * 8;
1726
1727     /* Make sure functions called have a 16 byte aligned stack */
1728     _jitc->framesize = (_jitc->framesize + 15) & -16;
1729 }
1730
1731 static void
1732 _patch(jit_state_t *_jit, jit_word_t instr, jit_node_t *node)
1733 {
1734     jit_int32_t          flag;
1735
1736     assert(node->flag & jit_flag_node);
1737     if (node->code == jit_code_movi)
1738         flag = node->v.n->flag;
1739     else
1740         flag = node->u.n->flag;
1741     assert(!(flag & jit_flag_patch));
1742     if (_jitc->patches.offset >= _jitc->patches.length) {
1743         jit_realloc((jit_pointer_t *)&_jitc->patches.ptr,
1744                     _jitc->patches.length * sizeof(jit_patch_t),
1745                     (_jitc->patches.length + 1024) * sizeof(jit_patch_t));
1746         _jitc->patches.length += 1024;
1747     }
1748     _jitc->patches.ptr[_jitc->patches.offset].inst = instr;
1749     _jitc->patches.ptr[_jitc->patches.offset].node = node;
1750     ++_jitc->patches.offset;
1751 }