deps/lightning/lib/jit_loongarch.c

   1 /*
   2  * Copyright (C) 2022  Free Software Foundation, Inc.
   3  *
   4  * This file is part of GNU lightning.
   5  *
   6  * GNU lightning is free software; you can redistribute it and/or modify it
   7  * under the terms of the GNU Lesser General Public License as published
   8  * by the Free Software Foundation; either version 3, or (at your option)
   9  * any later version.
  10  *
  11  * GNU lightning is distributed in the hope that it will be useful, but
  12  * WITHOUT ANY WARRANTY; without even the implied warranty of MERCHANTABILITY
  13  * or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU Lesser General Public
  14  * License for more details.
  15  *
  16  * Authors:
  17  *      Paulo Cesar Pereira de Andrade
  18  */
  19
  20 #define jit_arg_reg_p(i)                ((i) >= 0 && (i) < 8)
  21 #define jit_arg_f_reg_p(i)              ((i) >= 0 && (i) < 8)
  22
  23 /*
  24  * Types
  25  */
  26 typedef struct jit_pointer_t jit_va_list_t;
  27
  28 /*
  29  * Prototypes
  30  */
  31 #define patch(instr, node)              _patch(_jit, instr, node)
  32 static void _patch(jit_state_t*,jit_word_t,jit_node_t*);
  33
  34 #define PROTO                           1
  35 # include "jit_loongarch-cpu.c"
  36 # include "jit_loongarch-fpu.c"
  37 #undef PROTO
  38
  39 /*
  40  * Initialization
  41  */
  42 jit_register_t          _rvs[] = {
  43     { rc(gpr) | 0x14,                   "$t8" },
  44     { rc(gpr) | 0x13,                   "$t7" },
  45     { rc(gpr) | 0x12,                   "$t6" },
  46     { rc(gpr) | 0x11,                   "$t5" },
  47     { rc(gpr) | 0x10,                   "$t4" },
  48     { rc(gpr) | 0x0f,                   "$t3" },
  49     { rc(gpr) | 0x0e,                   "$t2" },
  50     { rc(gpr) | 0x0d,                   "$t1" },
  51     { rc(gpr) | 0x0c,                   "$t0" },
  52     { rc(sav) | rc(gpr) | 0x1f,         "$s8" },
  53     { rc(sav) | rc(gpr) | 0x1e,         "$s7" },
  54     { rc(sav) | rc(gpr) | 0x1d,         "$s6" },
  55     { rc(sav) | rc(gpr) | 0x1c,         "$s5" },
  56     { rc(sav) | rc(gpr) | 0x1b,         "$s4" },
  57     { rc(sav) | rc(gpr) | 0x1a,         "$s3" },
  58     { rc(sav) | rc(gpr) | 0x19,         "$s2" },
  59     { rc(sav) | rc(gpr) | 0x18,         "$s1" },
  60     { rc(sav) | rc(gpr) | 0x17,         "$s0" },
  61     { rc(arg) | rc(gpr) | 0x0b,         "$a7" },
  62     { rc(arg) | rc(gpr) | 0x0a,         "$a6" },
  63     { rc(arg) | rc(gpr) | 0x09,         "$a5" },
  64     { rc(arg) | rc(gpr) | 0x08,         "$a4" },
  65     { rc(arg) | rc(gpr) | 0x07,         "$a3" },
  66     { rc(arg) | rc(gpr) | 0x06,         "$a2" },
  67     { rc(arg) | rc(gpr) | 0x05,         "$a1" },
  68     { rc(arg) | rc(gpr) | 0x04,         "$a0" },
  69     { 0x16,                             "$fp" },
  70     { 0x15,                             "<reserved>" },
  71     { 0x00,                             "$zero" },
  72     { 0x01,                             "$ra" },
  73     { 0x02,                             "$tp" },
  74     { 0x03,                             "$sp" },
  75     { rc(fpr) | 0x08,                   "$ft0" },
  76     { rc(fpr) | 0x09,                   "$ft1" },
  77     { rc(fpr) | 0x0a,                   "$ft2" },
  78     { rc(fpr) | 0x0b,                   "$ft3" },
  79     { rc(fpr) | 0x0c,                   "$ft4" },
  80     { rc(fpr) | 0x0d,                   "$ft5" },
  81     { rc(fpr) | 0x0e,                   "$ft6" },
  82     { rc(fpr) | 0x0f,                   "$ft7" },
  83     { rc(fpr) | 0x10,                   "$ft8" },
  84     { rc(fpr) | 0x11,                   "$ft9" },
  85     { rc(fpr) | 0x12,                   "$ft10" },
  86     { rc(fpr) | 0x13,                   "$ft11" },
  87     { rc(fpr) | 0x14,                   "$ft12" },
  88     { rc(fpr) | 0x15,                   "$ft13" },
  89     { rc(fpr) | 0x16,                   "$ft14" },
  90     { rc(fpr) | 0x17,                   "$ft15" },
  91     { rc(arg) | rc(fpr) | 0x07,         "$fa7" },
  92     { rc(arg) | rc(fpr) | 0x06,         "$fa6" },
  93     { rc(arg) | rc(fpr) | 0x05,         "$fa5" },
  94     { rc(arg) | rc(fpr) | 0x04,         "$fa4" },
  95     { rc(arg) | rc(fpr) | 0x03,         "$fa3" },
  96     { rc(arg) | rc(fpr) | 0x02,         "$fa2" },
  97     { rc(arg) | rc(fpr) | 0x01,         "$fa1" },
  98     { rc(arg) | rc(fpr) | 0x00,         "$fa0" },
  99     { rc(sav) | rc(fpr) | 0x1f,         "$fs7" },
 100     { rc(sav) | rc(fpr) | 0x1e,         "$fs6" },
 101     { rc(sav) | rc(fpr) | 0x1d,         "$fs5" },
 102     { rc(sav) | rc(fpr) | 0x1c,         "$fs4" },
 103     { rc(sav) | rc(fpr) | 0x1b,         "$fs3" },
 104     { rc(sav) | rc(fpr) | 0x1a,         "$fs2" },
 105     { rc(sav) | rc(fpr) | 0x19,         "$fs1" },
 106     { rc(sav) | rc(fpr) | 0x18,         "$fs0" },
 107     { _NOREG,                           "<none>" },
 108 };
 109
 110 /*
 111  * Implementation
 112  */
 113 void
 114 jit_get_cpu(void)
 115 {
 116 }
 117
 118 void
 119 _jit_init(jit_state_t *_jit)
 120 {
 121     _jitc->reglen = jit_size(_rvs) - 1;
 122     jit_carry = _NOREG;
 123 }
 124
 125 void
 126 _jit_prolog(jit_state_t *_jit)
 127 {
 128     jit_int32_t          offset;
 129
 130     if (_jitc->function)
 131         jit_epilog();
 132     assert(jit_regset_cmp_ui(&_jitc->regarg, 0) == 0);
 133     jit_regset_set_ui(&_jitc->regsav, 0);
 134     offset = _jitc->functions.offset;
 135     if (offset >= _jitc->functions.length) {
 136         jit_realloc((jit_pointer_t *)&_jitc->functions.ptr,
 137                     _jitc->functions.length * sizeof(jit_function_t),
 138                     (_jitc->functions.length + 16) * sizeof(jit_function_t));
 139         _jitc->functions.length += 16;
 140     }
 141     _jitc->function = _jitc->functions.ptr + _jitc->functions.offset++;
 142     _jitc->function->self.size = stack_framesize;
 143     _jitc->function->self.argi = _jitc->function->self.argf =
 144         _jitc->function->self.aoff = _jitc->function->self.alen = 0;
 145     _jitc->function->self.call = jit_call_default;
 146     jit_alloc((jit_pointer_t *)&_jitc->function->regoff,
 147               _jitc->reglen * sizeof(jit_int32_t));
 148
 149     /* _no_link here does not mean the jit_link() call can be removed
 150      * by rewriting as:
 151      * _jitc->function->prolog = jit_new_node(jit_code_prolog);
 152      */
 153     _jitc->function->prolog = jit_new_node_no_link(jit_code_prolog);
 154     jit_link(_jitc->function->prolog);
 155     _jitc->function->prolog->w.w = offset;
 156     _jitc->function->epilog = jit_new_node_no_link(jit_code_epilog);
 157     /*  u:      label value
 158      *  v:      offset in blocks vector
 159      *  w:      offset in functions vector
 160      */
 161     _jitc->function->epilog->w.w = offset;
 162
 163     jit_regset_new(&_jitc->function->regset);
 164 }
 165
 166 jit_int32_t
 167 _jit_allocai(jit_state_t *_jit, jit_int32_t length)
 168 {
 169     assert(_jitc->function);
 170     switch (length) {
 171         case 0: case 1:                                         break;
 172         case 2:         _jitc->function->self.aoff &= -2;       break;
 173         case 3: case 4: _jitc->function->self.aoff &= -4;       break;
 174         default:        _jitc->function->self.aoff &= -8;       break;
 175     }
 176     _jitc->function->self.aoff -= length;
 177     if (!_jitc->realize) {
 178         jit_inc_synth_ww(allocai, _jitc->function->self.aoff, length);
 179         jit_dec_synth();
 180     }
 181     return (_jitc->function->self.aoff);
 182 }
 183
 184 void
 185 _jit_allocar(jit_state_t *_jit, jit_int32_t u, jit_int32_t v)
 186 {
 187     jit_int32_t          reg;
 188     assert(_jitc->function);
 189     jit_inc_synth_ww(allocar, u, v);
 190     if (!_jitc->function->allocar) {
 191         _jitc->function->aoffoff = jit_allocai(sizeof(jit_int32_t));
 192         _jitc->function->allocar = 1;
 193     }
 194     reg = jit_get_reg(jit_class_gpr);
 195     jit_negr(reg, v);
 196     jit_andi(reg, reg, -16);
 197     jit_ldxi_i(u, JIT_FP, _jitc->function->aoffoff);
 198     jit_addr(u, u, reg);
 199     jit_addr(JIT_SP, JIT_SP, reg);
 200     jit_stxi_i(_jitc->function->aoffoff, JIT_FP, u);
 201     jit_unget_reg(reg);
 202     jit_dec_synth();
 203 }
 204
 205 void
 206 _jit_ret(jit_state_t *_jit)
 207 {
 208     jit_node_t          *instr;
 209     assert(_jitc->function);
 210     jit_inc_synth(ret);
 211     /* jump to epilog */
 212     instr = jit_jmpi();
 213     jit_patch_at(instr, _jitc->function->epilog);
 214     jit_dec_synth();
 215 }
 216
 217 void
 218 _jit_retr(jit_state_t *_jit, jit_int32_t u)
 219 {
 220     jit_inc_synth_w(retr, u);
 221     if (JIT_RET != u)
 222         jit_movr(JIT_RET, u);
 223     jit_live(JIT_RET);
 224     jit_ret();
 225     jit_dec_synth();
 226 }
 227
 228 void
 229 _jit_reti(jit_state_t *_jit, jit_word_t u)
 230 {
 231     jit_inc_synth_w(reti, u);
 232     jit_movi(JIT_RET, u);
 233     jit_ret();
 234     jit_dec_synth();
 235 }
 236
 237 void
 238 _jit_retr_f(jit_state_t *_jit, jit_int32_t u)
 239 {
 240     jit_inc_synth_w(retr_f, u);
 241     if (JIT_FRET != u)
 242         jit_movr_f(JIT_FRET, u);
 243     else
 244         jit_live(JIT_FRET);
 245     jit_ret();
 246     jit_dec_synth();
 247 }
 248
 249 void
 250 _jit_reti_f(jit_state_t *_jit, jit_float32_t u)
 251 {
 252     jit_inc_synth_f(reti_f, u);
 253     jit_movi_f(JIT_FRET, u);
 254     jit_ret();
 255     jit_dec_synth();
 256 }
 257
 258 void
 259 _jit_retr_d(jit_state_t *_jit, jit_int32_t u)
 260 {
 261     jit_inc_synth_w(retr_d, u);
 262     if (JIT_FRET != u)
 263         jit_movr_d(JIT_FRET, u);
 264     else
 265         jit_live(JIT_FRET);
 266     jit_ret();
 267     jit_dec_synth();
 268 }
 269
 270 void
 271 _jit_reti_d(jit_state_t *_jit, jit_float64_t u)
 272 {
 273     jit_inc_synth_d(reti_d, u);
 274     jit_movi_d(JIT_FRET, u);
 275     jit_ret();
 276     jit_dec_synth();
 277 }
 278
 279 void
 280 _jit_epilog(jit_state_t *_jit)
 281 {
 282     assert(_jitc->function);
 283     assert(_jitc->function->epilog->next == NULL);
 284     jit_link(_jitc->function->epilog);
 285     _jitc->function = NULL;
 286 }
 287
 288 jit_bool_t
 289 _jit_arg_register_p(jit_state_t *_jit, jit_node_t *u)
 290 {
 291     if (u->code == jit_code_arg)
 292         return (jit_arg_reg_p(u->u.w));
 293     assert(u->code == jit_code_arg_f || u->code == jit_code_arg_d);
 294     return (jit_arg_f_reg_p(u->u.w));
 295 }
 296
 297 void
 298 _jit_ellipsis(jit_state_t *_jit)
 299 {
 300     jit_inc_synth(ellipsis);
 301     if (_jitc->prepare) {
 302         jit_link_prepare();
 303         assert(!(_jitc->function->call.call & jit_call_varargs));
 304         _jitc->function->call.call |= jit_call_varargs;
 305     }
 306     else {
 307         jit_link_prolog();
 308         assert(!(_jitc->function->self.call & jit_call_varargs));
 309         _jitc->function->self.call |= jit_call_varargs;
 310         _jitc->function->vagp = _jitc->function->self.argi;
 311     }
 312     jit_dec_synth();
 313 }
 314
 315 void
 316 _jit_va_push(jit_state_t *_jit, jit_int32_t u)
 317 {
 318     jit_inc_synth_w(va_push, u);
 319     jit_pushargr(u);
 320     jit_dec_synth();
 321 }
 322
 323 jit_node_t *
 324 _jit_arg(jit_state_t *_jit)
 325 {
 326     jit_node_t          *node;
 327     jit_int32_t          offset;
 328     assert(_jitc->function);
 329     assert(!(_jitc->function->self.call & jit_call_varargs));
 330     if (jit_arg_reg_p(_jitc->function->self.argi))
 331         offset = _jitc->function->self.argi++;
 332     else {
 333         offset = _jitc->function->self.size;
 334         _jitc->function->self.size += sizeof(jit_word_t);
 335     }
 336     node = jit_new_node_ww(jit_code_arg, offset,
 337                            ++_jitc->function->self.argn);
 338     jit_link_prolog();
 339     return (node);
 340 }
 341
 342 jit_node_t *
 343 _jit_arg_f(jit_state_t *_jit)
 344 {
 345     jit_node_t          *node;
 346     jit_int32_t          offset;
 347     assert(_jitc->function);
 348     assert(!(_jitc->function->self.call & jit_call_varargs));
 349     if (jit_arg_f_reg_p(_jitc->function->self.argf))
 350         offset = _jitc->function->self.argf++;
 351     else if (jit_arg_reg_p(_jitc->function->self.argi)) {
 352         offset = _jitc->function->self.argi++;
 353         offset += 8;
 354     }
 355     else {
 356         offset = _jitc->function->self.size;
 357         _jitc->function->self.size += sizeof(jit_word_t);
 358     }
 359     node = jit_new_node_ww(jit_code_arg_f, offset,
 360                            ++_jitc->function->self.argn);
 361     jit_link_prolog();
 362     return (node);
 363 }
 364
 365 jit_node_t *
 366 _jit_arg_d(jit_state_t *_jit)
 367 {
 368     jit_node_t          *node;
 369     jit_int32_t          offset;
 370     assert(_jitc->function);
 371     assert(!(_jitc->function->self.call & jit_call_varargs));
 372     if (jit_arg_f_reg_p(_jitc->function->self.argf))
 373         offset = _jitc->function->self.argf++;
 374     else if (jit_arg_reg_p(_jitc->function->self.argi)) {
 375         offset = _jitc->function->self.argi++;
 376         offset += 8;
 377     }
 378     else {
 379         offset = _jitc->function->self.size;
 380         _jitc->function->self.size += sizeof(jit_word_t);
 381     }
 382     node = jit_new_node_ww(jit_code_arg_d, offset,
 383                            ++_jitc->function->self.argn);
 384     jit_link_prolog();
 385     return (node);
 386 }
 387
 388 void
 389 _jit_getarg_c(jit_state_t *_jit, jit_int32_t u, jit_node_t *v)
 390 {
 391     assert(v->code == jit_code_arg);
 392     jit_inc_synth_wp(getarg_c, u, v);
 393     if (jit_arg_reg_p(v->u.w))
 394         jit_extr_c(u, _A0 - v->u.w);
 395     else
 396         jit_ldxi_c(u, JIT_FP, v->u.w);
 397     jit_dec_synth();
 398 }
 399
 400 void
 401 _jit_getarg_uc(jit_state_t *_jit, jit_int32_t u, jit_node_t *v)
 402 {
 403     assert(v->code == jit_code_arg);
 404     jit_inc_synth_wp(getarg_uc, u, v);
 405     if (jit_arg_reg_p(v->u.w))
 406         jit_extr_uc(u, _A0 - v->u.w);
 407     else
 408         jit_ldxi_uc(u, JIT_FP, v->u.w);
 409     jit_dec_synth();
 410 }
 411
 412 void
 413 _jit_getarg_s(jit_state_t *_jit, jit_int32_t u, jit_node_t *v)
 414 {
 415     assert(v->code == jit_code_arg);
 416     jit_inc_synth_wp(getarg_s, u, v);
 417     if (jit_arg_reg_p(v->u.w))
 418         jit_extr_s(u, _A0 - v->u.w);
 419     else
 420         jit_ldxi_s(u, JIT_FP, v->u.w);
 421     jit_dec_synth();
 422 }
 423
 424 void
 425 _jit_getarg_us(jit_state_t *_jit, jit_int32_t u, jit_node_t *v)
 426 {
 427     assert(v->code == jit_code_arg);
 428     jit_inc_synth_wp(getarg_us, u, v);
 429     if (jit_arg_reg_p(v->u.w))
 430         jit_extr_us(u, _A0 - v->u.w);
 431     else
 432         jit_ldxi_us(u, JIT_FP, v->u.w);
 433     jit_dec_synth();
 434 }
 435
 436 void
 437 _jit_getarg_i(jit_state_t *_jit, jit_int32_t u, jit_node_t *v)
 438 {
 439     assert(v->code == jit_code_arg);
 440     jit_inc_synth_wp(getarg_i, u, v);
 441     if (jit_arg_reg_p(v->u.w))
 442         jit_extr_i(u, _A0 - v->u.w);
 443     else
 444         jit_ldxi_i(u, JIT_FP, v->u.w);
 445     jit_dec_synth();
 446 }
 447
 448 void
 449 _jit_getarg_ui(jit_state_t *_jit, jit_int32_t u, jit_node_t *v)
 450 {
 451     assert(v->code == jit_code_arg);
 452     jit_inc_synth_wp(getarg_ui, u, v);
 453     if (jit_arg_reg_p(v->u.w))
 454         jit_extr_ui(u, _A0 - v->u.w);
 455     else
 456         jit_ldxi_ui(u, JIT_FP, v->u.w);
 457     jit_dec_synth();
 458 }
 459
 460 void
 461 _jit_getarg_l(jit_state_t *_jit, jit_int32_t u, jit_node_t *v)
 462 {
 463     assert(v->code == jit_code_arg);
 464     jit_inc_synth_wp(getarg_l, u, v);
 465     if (jit_arg_reg_p(v->u.w))
 466         jit_movr(u, _A0 - v->u.w);
 467     else
 468         jit_ldxi_l(u, JIT_FP, v->u.w);
 469     jit_dec_synth();
 470 }
 471
 472 void
 473 _jit_putargr(jit_state_t *_jit, jit_int32_t u, jit_node_t *v)
 474 {
 475     assert(v->code == jit_code_arg);
 476     jit_inc_synth_wp(putargr, u, v);
 477     if (jit_arg_reg_p(v->u.w))
 478         jit_movr(_A0 - v->u.w, u);
 479     else
 480         jit_stxi(v->u.w, JIT_FP, u);
 481     jit_dec_synth();
 482 }
 483
 484 void
 485 _jit_putargi(jit_state_t *_jit, jit_word_t u, jit_node_t *v)
 486 {
 487     jit_int32_t         regno;
 488     assert(v->code == jit_code_arg);
 489     jit_inc_synth_wp(putargi, u, v);
 490     if (jit_arg_reg_p(v->u.w))
 491         jit_movi(_A0 - v->u.w, u);
 492     else {
 493         regno = jit_get_reg(jit_class_gpr);
 494         jit_movi(regno, u);
 495         jit_stxi(v->u.w, JIT_FP, regno);
 496         jit_unget_reg(regno);
 497     }
 498     jit_dec_synth();
 499 }
 500
 501 void
 502 _jit_getarg_f(jit_state_t *_jit, jit_int32_t u, jit_node_t *v)
 503 {
 504     assert(v->code == jit_code_arg_f);
 505     jit_inc_synth_wp(getarg_f, u, v);
 506     if (jit_arg_f_reg_p(v->u.w))
 507         jit_movr_f(u, _FA0 - v->u.w);
 508     else if (jit_arg_reg_p(v->u.w - 8))
 509         jit_movr_w_f(u, JIT_RA0 - (v->u.w - 8));
 510     else
 511         jit_ldxi_f(u, JIT_FP, v->u.w);
 512     jit_dec_synth();
 513 }
 514
 515 void
 516 _jit_putargr_f(jit_state_t *_jit, jit_int32_t u, jit_node_t *v)
 517 {
 518     assert(v->code == jit_code_arg_f);
 519     jit_inc_synth_wp(putargr_f, u, v);
 520     if (jit_arg_f_reg_p(v->u.w))
 521         jit_movr_f(_FA0 - v->u.w, u);
 522     else if (jit_arg_reg_p(v->u.w - 8))
 523         jit_movr_f_w(JIT_RA0 - (v->u.w - 8), u);
 524     else
 525         jit_stxi_f(v->u.w, JIT_FP, u);
 526     jit_dec_synth();
 527 }
 528
 529 void
 530 _jit_putargi_f(jit_state_t *_jit, jit_float32_t u, jit_node_t *v)
 531 {
 532     jit_int32_t         regno;
 533     assert(v->code == jit_code_arg_f);
 534     jit_inc_synth_fp(putargi_f, u, v);
 535     if (jit_arg_f_reg_p(v->u.w))
 536         jit_movi_f(_FA0 - v->u.w, u);
 537     else if (jit_arg_reg_p(v->u.w - 8)) {
 538         union {
 539             jit_float32_t       f;
 540             jit_int32_t         i;
 541         } uu;
 542         uu.f = u;
 543         jit_movi(JIT_RA0 - (v->u.w - 8), uu.i);
 544     }
 545     else {
 546         regno = jit_get_reg(jit_class_fpr);
 547         jit_movi_f(regno, u);
 548         jit_stxi_f(v->u.w, JIT_FP, regno);
 549         jit_unget_reg(regno);
 550     }
 551     jit_dec_synth();
 552 }
 553
 554 void
 555 _jit_getarg_d(jit_state_t *_jit, jit_int32_t u, jit_node_t *v)
 556 {
 557     assert(v->code == jit_code_arg_d);
 558     jit_inc_synth_wp(getarg_d, u, v);
 559     if (jit_arg_f_reg_p(v->u.w))
 560         jit_movr_d(u, _FA0 - v->u.w);
 561     else if (jit_arg_reg_p(v->u.w - 8))
 562         jit_movr_w_d(u, JIT_RA0 - (v->u.w - 8));
 563     else
 564         jit_ldxi_d(u, JIT_FP, v->u.w);
 565     jit_dec_synth();
 566 }
 567
 568 void
 569 _jit_putargr_d(jit_state_t *_jit, jit_int32_t u, jit_node_t *v)
 570 {
 571     assert(v->code == jit_code_arg_d);
 572     jit_inc_synth_wp(putargr_d, u, v);
 573     if (jit_arg_f_reg_p(v->u.w))
 574         jit_movr_d(_FA0 - v->u.w, u);
 575     else if (jit_arg_reg_p(v->u.w - 8))
 576         jit_movr_d_w(JIT_RA0 - (v->u.w - 8), u);
 577     else
 578         jit_stxi_d(v->u.w, JIT_FP, u);
 579     jit_dec_synth();
 580 }
 581
 582 void
 583 _jit_putargi_d(jit_state_t *_jit, jit_float64_t u, jit_node_t *v)
 584 {
 585     jit_int32_t         regno;
 586     assert(v->code == jit_code_arg_d);
 587     jit_inc_synth_dp(putargi_d, u, v);
 588     if (jit_arg_f_reg_p(v->u.w))
 589         jit_movi_d(_FA0 - v->u.w, u);
 590     else if (jit_arg_reg_p(v->u.w - 8)) {
 591         union {
 592             jit_float64_t       d;
 593             jit_int64_t         w;
 594         } uu;
 595         uu.d = u;
 596         jit_movi(JIT_RA0 - (v->u.w - 8), uu.w);
 597     }
 598     else {
 599         regno = jit_get_reg(jit_class_fpr);
 600         jit_movi_d(regno, u);
 601         jit_stxi_d(v->u.w, JIT_FP, regno);
 602         jit_unget_reg(regno);
 603     }
 604     jit_dec_synth();
 605 }
 606
 607 void
 608 _jit_pushargr(jit_state_t *_jit, jit_int32_t u)
 609 {
 610     assert(_jitc->function);
 611     jit_inc_synth_w(pushargr, u);
 612     jit_link_prepare();
 613     if (jit_arg_reg_p(_jitc->function->call.argi)) {
 614         jit_movr(_A0 - _jitc->function->call.argi, u);
 615         ++_jitc->function->call.argi;
 616     }
 617     else {
 618         jit_stxi(_jitc->function->call.size, JIT_SP, u);
 619         _jitc->function->call.size += sizeof(jit_word_t);
 620     }
 621     jit_dec_synth();
 622 }
 623
 624 void
 625 _jit_pushargi(jit_state_t *_jit, jit_word_t u)
 626 {
 627     jit_int32_t          regno;
 628     assert(_jitc->function);
 629     jit_inc_synth_w(pushargi, u);
 630     jit_link_prepare();
 631     if (jit_arg_reg_p(_jitc->function->call.argi)) {
 632         jit_movi(_A0 - _jitc->function->call.argi, u);
 633         ++_jitc->function->call.argi;
 634     }
 635     else {
 636         regno = jit_get_reg(jit_class_gpr);
 637         jit_movi(regno, u);
 638         jit_stxi(_jitc->function->call.size, JIT_SP, regno);
 639         jit_unget_reg(regno);
 640         _jitc->function->call.size += sizeof(jit_word_t);
 641     }
 642     jit_dec_synth();
 643 }
 644
 645 void
 646 _jit_pushargr_f(jit_state_t *_jit, jit_int32_t u)
 647 {
 648     assert(_jitc->function);
 649     jit_inc_synth_w(pushargr_f, u);
 650     jit_link_prepare();
 651     if (jit_arg_f_reg_p(_jitc->function->call.argf) &&
 652         !(_jitc->function->call.call & jit_call_varargs)) {
 653         jit_movr_f(_FA0 - _jitc->function->call.argf, u);
 654         ++_jitc->function->call.argf;
 655     }
 656     else if (jit_arg_reg_p(_jitc->function->call.argi)) {
 657         jit_movr_f_w(_A0 - _jitc->function->call.argi, u);
 658         ++_jitc->function->call.argi;
 659     }
 660     else {
 661         jit_stxi_f(_jitc->function->call.size, JIT_SP, u);
 662         _jitc->function->call.size += sizeof(jit_word_t);
 663     }
 664     jit_dec_synth();
 665 }
 666
 667 void
 668 _jit_pushargi_f(jit_state_t *_jit, jit_float32_t u)
 669 {
 670     jit_int32_t         regno;
 671     assert(_jitc->function);
 672     jit_inc_synth_f(pushargi_f, u);
 673     jit_link_prepare();
 674     if (jit_arg_f_reg_p(_jitc->function->call.argf) &&
 675         !(_jitc->function->call.call & jit_call_varargs)) {
 676         jit_movi_f(_FA0 - _jitc->function->call.argf, u);
 677         ++_jitc->function->call.argf;
 678     }
 679     else if (jit_arg_reg_p(_jitc->function->call.argi)) {
 680         jit_movi_f_w(_A0 - _jitc->function->call.argi, u);
 681         ++_jitc->function->call.argi;
 682     }
 683     else {
 684         regno = jit_get_reg(jit_class_fpr);
 685         jit_movi_f(regno, u);
 686         jit_stxi_f(_jitc->function->call.size, JIT_SP, regno);
 687         jit_unget_reg(regno);
 688         _jitc->function->call.size += sizeof(jit_word_t);
 689     }
 690     jit_dec_synth();
 691 }
 692
 693 void
 694 _jit_pushargr_d(jit_state_t *_jit, jit_int32_t u)
 695 {
 696     assert(_jitc->function);
 697     jit_inc_synth_w(pushargr_d, u);
 698     jit_link_prepare();
 699     if (jit_arg_f_reg_p(_jitc->function->call.argf) &&
 700         !(_jitc->function->call.call & jit_call_varargs)) {
 701         jit_movr_d(_FA0 - _jitc->function->call.argf, u);
 702         ++_jitc->function->call.argf;
 703     }
 704     else if (jit_arg_reg_p(_jitc->function->call.argi)) {
 705         jit_movr_d_w(_A0 - _jitc->function->call.argi, u);
 706         ++_jitc->function->call.argi;
 707     }
 708     else {
 709         jit_stxi_d(_jitc->function->call.size, JIT_SP, u);
 710         _jitc->function->call.size += sizeof(jit_word_t);
 711     }
 712     jit_dec_synth();
 713 }
 714
 715 void
 716 _jit_pushargi_d(jit_state_t *_jit, jit_float64_t u)
 717 {
 718     jit_int32_t         regno;
 719     assert(_jitc->function);
 720     jit_inc_synth_d(pushargi_d, u);
 721     jit_link_prepare();
 722     if (jit_arg_f_reg_p(_jitc->function->call.argf) &&
 723         !(_jitc->function->call.call & jit_call_varargs)) {
 724         jit_movi_d(_FA0 - _jitc->function->call.argf, u);
 725         ++_jitc->function->call.argf;
 726     }
 727     else if (jit_arg_reg_p(_jitc->function->call.argi)) {
 728         jit_movi_d_w(_A0 - _jitc->function->call.argi, u);
 729         ++_jitc->function->call.argi;
 730     }
 731     else {
 732         regno = jit_get_reg(jit_class_fpr);
 733         jit_movi_d(regno, u);
 734         jit_stxi_d(_jitc->function->call.size, JIT_SP, regno);
 735         jit_unget_reg(regno);
 736         _jitc->function->call.size += sizeof(jit_word_t);
 737     }
 738     jit_dec_synth();
 739 }
 740
 741 jit_bool_t
 742 _jit_regarg_p(jit_state_t *_jit, jit_node_t *node, jit_int32_t regno)
 743 {
 744     jit_int32_t         spec;
 745     spec = jit_class(_rvs[regno].spec);
 746     if (spec & jit_class_arg) {
 747         regno = _A0 - regno;
 748         if (regno >= 0 && regno < node->v.w)
 749             return (1);
 750         if (spec & jit_class_fpr) {
 751             regno = _FA0 - regno;
 752             if (regno >= 0 && regno < node->w.w)
 753                 return (1);
 754         }
 755     }
 756     return (0);
 757 }
 758
 759 void
 760 _jit_finishr(jit_state_t *_jit, jit_int32_t r0)
 761 {
 762     jit_node_t          *node;
 763     assert(_jitc->function);
 764     jit_inc_synth_w(finishr, r0);
 765     if (_jitc->function->self.alen < _jitc->function->call.size)
 766         _jitc->function->self.alen = _jitc->function->call.size;
 767     node = jit_callr(r0);
 768     node->v.w = _jitc->function->call.argi;
 769     node->w.w = _jitc->function->call.argf;
 770     _jitc->function->call.argi = _jitc->function->call.argf =
 771         _jitc->function->call.size = 0;
 772     _jitc->prepare = 0;
 773     jit_dec_synth();
 774 }
 775
 776 jit_node_t *
 777 _jit_finishi(jit_state_t *_jit, jit_pointer_t i0)
 778 {
 779     jit_node_t          *node;
 780     assert(_jitc->function);
 781     jit_inc_synth_w(finishi, (jit_word_t)i0);
 782     if (_jitc->function->self.alen < _jitc->function->call.size)
 783         _jitc->function->self.alen = _jitc->function->call.size;
 784     node = jit_calli(i0);
 785     node->v.w = _jitc->function->call.argi;
 786     node->w.w = _jitc->function->call.argf;
 787     _jitc->function->call.argi = _jitc->function->call.argf =
 788         _jitc->function->call.size = 0;
 789     _jitc->prepare = 0;
 790     jit_dec_synth();
 791     return (node);
 792 }
 793
 794 void
 795 _jit_retval_c(jit_state_t *_jit, jit_int32_t r0)
 796 {
 797     jit_inc_synth_w(retval_c, r0);
 798     jit_extr_c(r0, JIT_RET);
 799     jit_dec_synth();
 800 }
 801
 802 void
 803 _jit_retval_uc(jit_state_t *_jit, jit_int32_t r0)
 804 {
 805     jit_inc_synth_w(retval_uc, r0);
 806     jit_extr_uc(r0, JIT_RET);
 807     jit_dec_synth();
 808 }
 809
 810 void
 811 _jit_retval_s(jit_state_t *_jit, jit_int32_t r0)
 812 {
 813     jit_inc_synth_w(retval_s, r0);
 814     jit_extr_s(r0, JIT_RET);
 815     jit_dec_synth();
 816 }
 817
 818 void
 819 _jit_retval_us(jit_state_t *_jit, jit_int32_t r0)
 820 {
 821     jit_inc_synth_w(retval_us, r0);
 822     jit_extr_us(r0, JIT_RET);
 823     jit_dec_synth();
 824 }
 825
 826 void
 827 _jit_retval_i(jit_state_t *_jit, jit_int32_t r0)
 828 {
 829     jit_inc_synth_w(retval_i, r0);
 830     jit_extr_i(r0, JIT_RET);
 831     jit_dec_synth();
 832 }
 833
 834 void
 835 _jit_retval_ui(jit_state_t *_jit, jit_int32_t r0)
 836 {
 837     jit_inc_synth_w(retval_ui, r0);
 838     jit_extr_ui(r0, JIT_RET);
 839     jit_dec_synth();
 840 }
 841
 842 void
 843 _jit_retval_l(jit_state_t *_jit, jit_int32_t r0)
 844 {
 845     jit_inc_synth_w(retval_l, r0);
 846     if (r0 != JIT_RET)
 847     jit_movr(r0, JIT_RET);
 848     jit_dec_synth();
 849 }
 850
 851 void
 852 _jit_retval_f(jit_state_t *_jit, jit_int32_t r0)
 853 {
 854     jit_inc_synth_w(retval_f, r0);
 855     if (r0 != JIT_FRET)
 856         jit_movr_f(r0, JIT_FRET);
 857     jit_dec_synth();
 858 }
 859
 860 void
 861 _jit_retval_d(jit_state_t *_jit, jit_int32_t r0)
 862 {
 863     jit_inc_synth_w(retval_d, r0);
 864     if (r0 != JIT_FRET)
 865         jit_movr_d(r0, JIT_FRET);
 866     jit_dec_synth();
 867 }
 868
 869 jit_pointer_t
 870 _emit_code(jit_state_t *_jit)
 871 {
 872     jit_node_t          *node;
 873     jit_node_t          *temp;
 874     jit_word_t           word;
 875     jit_int32_t          value;
 876     jit_int32_t          offset;
 877     struct {
 878         jit_node_t      *node;
 879         jit_word_t       word;
 880 #if DEVEL_DISASSEMBLER
 881         jit_word_t       prevw;
 882 #endif
 883         jit_int32_t      patch_offset;
 884     } undo;
 885 #if DEVEL_DISASSEMBLER
 886     jit_word_t           prevw;
 887 #endif
 888
 889     _jitc->function = NULL;
 890
 891     jit_reglive_setup();
 892
 893     undo.word = 0;
 894     undo.node = NULL;
 895     undo.patch_offset = 0;
 896
 897 #define assert_data(node)               /**/
 898 #define case_rr(name, type)                                             \
 899             case jit_code_##name##r##type:                              \
 900                 name##r##type(rn(node->u.w), rn(node->v.w));            \
 901                 break
 902 #define case_rw(name, type)                                             \
 903             case jit_code_##name##i##type:                              \
 904                 name##i##type(rn(node->u.w), node->v.w);                \
 905                 break
 906 #define case_wr(name, type)                                             \
 907             case jit_code_##name##i##type:                              \
 908                 name##i##type(node->u.w, rn(node->v.w));                \
 909                 break
 910 #define case_rrr(name, type)                                            \
 911             case jit_code_##name##r##type:                              \
 912                 name##r##type(rn(node->u.w),                            \
 913                               rn(node->v.w), rn(node->w.w));            \
 914                 break
 915 #define case_rrrr(name, type)                                           \
 916             case jit_code_##name##r##type:                              \
 917                 name##r##type(rn(node->u.q.l), rn(node->u.q.h),         \
 918                               rn(node->v.w), rn(node->w.w));            \
 919                 break
 920 #define case_rrw(name, type)                                            \
 921             case jit_code_##name##i##type:                              \
 922                 name##i##type(rn(node->u.w), rn(node->v.w), node->w.w); \
 923                 break
 924 #define case_rrrw(name, type)                                           \
 925             case jit_code_##name##i##type:                              \
 926                 name##i##type(rn(node->u.q.l), rn(node->u.q.h),         \
 927                               rn(node->v.w), node->w.w);                \
 928                 break
 929 #define case_rrf(name)                                                  \
 930             case jit_code_##name##i_f:                                  \
 931                 assert_data(node);                                      \
 932                 name##i_f(rn(node->u.w), rn(node->v.w), node->w.f);     \
 933                 break
 934 #define case_rrd(name)                                                  \
 935             case jit_code_##name##i_d:                                  \
 936                 assert_data(node);                                      \
 937                 name##i_d(rn(node->u.w), rn(node->v.w),node->w.d);      \
 938                 break
 939 #define case_wrr(name, type)                                            \
 940             case jit_code_##name##i##type:                              \
 941                 name##i##type(node->u.w, rn(node->v.w), rn(node->w.w)); \
 942                 break
 943 #define case_brr(name, type)                                            \
 944             case jit_code_##name##r##type:                              \
 945                 temp = node->u.n;                                       \
 946                 assert(temp->code == jit_code_label ||                  \
 947                        temp->code == jit_code_epilog);                  \
 948                 if (temp->flag & jit_flag_patch)                        \
 949                     name##r##type(temp->u.w, rn(node->v.w),             \
 950                                   rn(node->w.w));                       \
 951                 else {                                                  \
 952                     word = name##r##type(_jit->pc.w,                    \
 953                                          rn(node->v.w),                 \
 954                                          rn(node->w.w));                \
 955                     patch(word, node);                                  \
 956                 }                                                       \
 957                 break
 958 #define case_brw(name, type)                                            \
 959             case jit_code_##name##i##type:                              \
 960                 temp = node->u.n;                                       \
 961                 assert(temp->code == jit_code_label ||                  \
 962                        temp->code == jit_code_epilog);                  \
 963                 if (temp->flag & jit_flag_patch)                        \
 964                     name##i##type(temp->u.w,                            \
 965                                   rn(node->v.w), node->w.w);            \
 966                 else {                                                  \
 967                     word = name##i##type(_jit->pc.w,                    \
 968                                          rn(node->v.w), node->w.w);     \
 969                     patch(word, node);                                  \
 970                 }                                                       \
 971                 break;
 972 #define case_brf(name)                                                  \
 973             case jit_code_##name##i_f:                                  \
 974                 temp = node->u.n;                                       \
 975                 assert(temp->code == jit_code_label ||                  \
 976                        temp->code == jit_code_epilog);                  \
 977                 if (temp->flag & jit_flag_patch)                        \
 978                     name##i_f(temp->u.w, rn(node->v.w), node->w.f);     \
 979                 else {                                                  \
 980                     word = name##i_f(_jit->pc.w, rn(node->v.w),         \
 981                                      node->w.f);                        \
 982                     patch(word, node);                                  \
 983                 }                                                       \
 984                 break
 985 #define case_brd(name)                                                  \
 986             case jit_code_##name##i_d:                                  \
 987                 temp = node->u.n;                                       \
 988                 assert(temp->code == jit_code_label ||                  \
 989                        temp->code == jit_code_epilog);                  \
 990                 if (temp->flag & jit_flag_patch)                        \
 991                     name##i_d(temp->u.w, rn(node->v.w), node->w.d);     \
 992                 else {                                                  \
 993                     word = name##i_d(_jit->pc.w, rn(node->v.w),         \
 994                                      node->w.d);                        \
 995                     patch(word, node);                                  \
 996                 }                                                       \
 997                 break
 998 #if DEVEL_DISASSEMBLER
 999     prevw = _jit->pc.w;
1000 #endif
1001     for (node = _jitc->head; node; node = node->next) {
1002         if (_jit->pc.uc >= _jitc->code.end)
1003             return (NULL);
1004
1005 #if DEVEL_DISASSEMBLER
1006         node->offset = (jit_uword_t)_jit->pc.w - (jit_uword_t)prevw;
1007         prevw = _jit->pc.w;
1008 #endif
1009         value = jit_classify(node->code);
1010         jit_regarg_set(node, value);
1011         switch (node->code) {
1012             case jit_code_align:
1013                 /* Must align to a power of two */
1014                 assert(!(node->u.w & (node->u.w - 1)));
1015                 if ((word = _jit->pc.w & (node->u.w - 1)))
1016                     nop(node->u.w - word);
1017                 break;
1018             case jit_code_note:         case jit_code_name:
1019                 node->u.w = _jit->pc.w;
1020                 break;
1021             case jit_code_label:
1022                 /* remember label is defined */
1023                 node->flag |= jit_flag_patch;
1024                 node->u.w = _jit->pc.w;
1025                 break;
1026                 case_rrr(add,);
1027                 case_rrw(add,);
1028                 case_rrr(addc,);
1029                 case_rrw(addc,);
1030                 case_rrr(addx,);
1031                 case_rrw(addx,);
1032                 case_rrr(sub,);
1033                 case_rrw(sub,);
1034                 case_rrr(subc,);
1035                 case_rrw(subc,);
1036                 case_rrr(subx,);
1037                 case_rrw(subx,);
1038                 case_rrw(rsb,);
1039                 case_rrr(mul,);
1040                 case_rrw(mul,);
1041                 case_rrrr(qmul,);
1042                 case_rrrw(qmul,);
1043                 case_rrrr(qmul, _u);
1044                 case_rrrw(qmul, _u);
1045                 case_rrr(div,);
1046                 case_rrw(div,);
1047                 case_rrr(div, _u);
1048                 case_rrw(div, _u);
1049                 case_rrr(rem,);
1050                 case_rrw(rem,);
1051                 case_rrr(rem, _u);
1052                 case_rrw(rem, _u);
1053                 case_rrrr(qdiv,);
1054                 case_rrrw(qdiv,);
1055                 case_rrrr(qdiv, _u);
1056                 case_rrrw(qdiv, _u);
1057                 case_rrr(lsh,);
1058                 case_rrw(lsh,);
1059                 case_rrr(rsh,);
1060                 case_rrw(rsh,);
1061                 case_rrr(rsh, _u);
1062                 case_rrw(rsh, _u);
1063                 case_rr(neg,);
1064                 case_rr(com,);
1065                 case_rrr(and,);
1066                 case_rrw(and,);
1067                 case_rrr(or,);
1068                 case_rrw(or,);
1069                 case_rrr(xor,);
1070                 case_rrw(xor,);
1071                 case_rr(trunc, _f_i);
1072                 case_rr(trunc, _d_i);
1073                 case_rr(trunc, _f_l);
1074                 case_rr(trunc, _d_l);
1075                 case_rr(ld, _c);
1076                 case_rw(ld, _c);
1077                 case_rr(ld, _uc);
1078                 case_rw(ld, _uc);
1079                 case_rr(ld, _s);
1080                 case_rw(ld, _s);
1081                 case_rr(ld, _us);
1082                 case_rw(ld, _us);
1083                 case_rr(ld, _i);
1084                 case_rw(ld, _i);
1085                 case_rr(ld, _ui);
1086                 case_rw(ld, _ui);
1087                 case_rr(ld, _l);
1088                 case_rw(ld, _l);
1089                 case_rrr(ldx, _c);
1090                 case_rrw(ldx, _c);
1091                 case_rrr(ldx, _uc);
1092                 case_rrw(ldx, _uc);
1093                 case_rrr(ldx, _s);
1094                 case_rrw(ldx, _s);
1095                 case_rrr(ldx, _us);
1096                 case_rrw(ldx, _us);
1097                 case_rrr(ldx, _i);
1098                 case_rrw(ldx, _i);
1099                 case_rrr(ldx, _ui);
1100                 case_rrw(ldx, _ui);
1101                 case_rrr(ldx, _l);
1102                 case_rrw(ldx, _l);
1103                 case_rr(st, _c);
1104                 case_wr(st, _c);
1105                 case_rr(st, _s);
1106                 case_wr(st, _s);
1107                 case_rr(st, _i);
1108                 case_wr(st, _i);
1109                 case_rr(st, _l);
1110                 case_wr(st, _l);
1111                 case_rrr(stx, _c);
1112                 case_wrr(stx, _c);
1113                 case_rrr(stx, _s);
1114                 case_wrr(stx, _s);
1115                 case_rrr(stx, _i);
1116                 case_wrr(stx, _i);
1117                 case_rrr(stx, _l);
1118                 case_wrr(stx, _l);
1119                 case_rr(hton, _us);
1120                 case_rr(hton, _ui);
1121                 case_rr(hton, _ul);
1122                 case_rr(bswap, _us);
1123                 case_rr(bswap, _ui);
1124                 case_rr(bswap, _ul);
1125                 case_rr(ext, _c);
1126                 case_rr(ext, _uc);
1127                 case_rr(ext, _s);
1128                 case_rr(ext, _us);
1129                 case_rr(ext, _i);
1130                 case_rr(ext, _ui);
1131             case jit_code_casr:
1132                 casr(rn(node->u.w), rn(node->v.w),
1133                      rn(node->w.q.l), rn(node->w.q.h));
1134                 break;
1135             case jit_code_casi:
1136                 casi(rn(node->u.w), node->v.w,
1137                      rn(node->w.q.l), rn(node->w.q.h));
1138                 break;
1139                 case_rrr(movn,);
1140                 case_rrr(movz,);
1141                 case_rr(mov,);
1142             case jit_code_movi:
1143                 if (node->flag & jit_flag_node) {
1144                     temp = node->v.n;
1145                     if (temp->code == jit_code_data ||
1146                         (temp->code == jit_code_label &&
1147                          (temp->flag & jit_flag_patch)))
1148                         movi(rn(node->u.w), temp->u.w);
1149                     else {
1150                         assert(temp->code == jit_code_label ||
1151                                temp->code == jit_code_epilog);
1152                         word = movi_p(rn(node->u.w), temp->u.w);
1153                         patch(word, node);
1154                     }
1155                 }
1156                 else
1157                     movi(rn(node->u.w), node->v.w);
1158                 break;
1159                 case_rrr(lt,);
1160                 case_rrw(lt,);
1161                 case_rrr(lt, _u);
1162                 case_rrw(lt, _u);
1163                 case_rrr(le,);
1164                 case_rrw(le,);
1165                 case_rrr(le, _u);
1166                 case_rrw(le, _u);
1167                 case_rrr(eq,);
1168                 case_rrw(eq,);
1169                 case_rrr(ge,);
1170                 case_rrw(ge,);
1171                 case_rrr(ge, _u);
1172                 case_rrw(ge, _u);
1173                 case_rrr(gt,);
1174                 case_rrw(gt,);
1175                 case_rrr(gt, _u);
1176                 case_rrw(gt, _u);
1177                 case_rrr(ne,);
1178                 case_rrw(ne,);
1179                 case_brr(blt,);
1180                 case_brw(blt,);
1181                 case_brr(blt, _u);
1182                 case_brw(blt, _u);
1183                 case_brr(ble,);
1184                 case_brw(ble,);
1185                 case_brr(ble, _u);
1186                 case_brw(ble, _u);
1187                 case_brr(beq,);
1188                 case_brw(beq,);
1189                 case_brr(bge,);
1190                 case_brw(bge,);
1191                 case_brr(bge, _u);
1192                 case_brw(bge, _u);
1193                 case_brr(bgt,);
1194                 case_brw(bgt,);
1195                 case_brr(bgt, _u);
1196                 case_brw(bgt, _u);
1197                 case_brr(bne,);
1198                 case_brw(bne,);
1199                 case_brr(boadd,);
1200                 case_brw(boadd,);
1201                 case_brr(boadd, _u);
1202                 case_brw(boadd, _u);
1203                 case_brr(bxadd,);
1204                 case_brw(bxadd,);
1205                 case_brr(bxadd, _u);
1206                 case_brw(bxadd, _u);
1207                 case_brr(bosub,);
1208                 case_brw(bosub,);
1209                 case_brr(bosub, _u);
1210                 case_brw(bosub, _u);
1211                 case_brr(bxsub,);
1212                 case_brw(bxsub,);
1213                 case_brr(bxsub, _u);
1214                 case_brw(bxsub, _u);
1215                 case_brr(bms,);
1216                 case_brw(bms,);
1217                 case_brr(bmc,);
1218                 case_brw(bmc,);
1219                 case_rrr(add, _f);
1220                 case_rrf(add);
1221                 case_rrr(sub, _f);
1222                 case_rrf(sub);
1223                 case_rrf(rsb);
1224                 case_rrr(mul, _f);
1225                 case_rrf(mul);
1226                 case_rrr(div, _f);
1227                 case_rrf(div);
1228                 case_rr(abs, _f);
1229                 case_rr(neg, _f);
1230                 case_rr(sqrt, _f);
1231                 case_rr(ext, _f);
1232                 case_rr(ld, _f);
1233                 case_rw(ld, _f);
1234                 case_rrr(ldx, _f);
1235                 case_rrw(ldx, _f);
1236                 case_rr(st, _f);
1237                 case_wr(st, _f);
1238                 case_rrr(stx, _f);
1239                 case_wrr(stx, _f);
1240                 case_rr(mov, _f);
1241             case jit_code_movi_f:
1242                 assert_data(node);
1243                 movi_f(rn(node->u.w), node->v.f);
1244                 break;
1245                 case_rr(ext, _d_f);
1246                 case_rrr(lt, _f);
1247                 case_rrf(lt);
1248                 case_rrr(le, _f);
1249                 case_rrf(le);
1250                 case_rrr(eq, _f);
1251                 case_rrf(eq);
1252                 case_rrr(ge, _f);
1253                 case_rrf(ge);
1254                 case_rrr(gt, _f);
1255                 case_rrf(gt);
1256                 case_rrr(ne, _f);
1257                 case_rrf(ne);
1258                 case_rrr(unlt, _f);
1259                 case_rrf(unlt);
1260                 case_rrr(unle, _f);
1261                 case_rrf(unle);
1262                 case_rrr(uneq, _f);
1263                 case_rrf(uneq);
1264                 case_rrr(unge, _f);
1265                 case_rrf(unge);
1266                 case_rrr(ungt, _f);
1267                 case_rrf(ungt);
1268                 case_rrr(ltgt, _f);
1269                 case_rrf(ltgt);
1270                 case_rrr(ord, _f);
1271                 case_rrf(ord);
1272                 case_rrr(unord, _f);
1273                 case_rrf(unord);
1274                 case_brr(blt, _f);
1275                 case_brf(blt);
1276                 case_brr(ble, _f);
1277                 case_brf(ble);
1278                 case_brr(beq, _f);
1279                 case_brf(beq);
1280                 case_brr(bge, _f);
1281                 case_brf(bge);
1282                 case_brr(bgt, _f);
1283                 case_brf(bgt);
1284                 case_brr(bne, _f);
1285                 case_brf(bne);
1286                 case_brr(bunlt, _f);
1287                 case_brf(bunlt);
1288                 case_brr(bunle, _f);
1289                 case_brf(bunle);
1290                 case_brr(buneq, _f);
1291                 case_brf(buneq);
1292                 case_brr(bunge, _f);
1293                 case_brf(bunge);
1294                 case_brr(bungt, _f);
1295                 case_brf(bungt);
1296                 case_brr(bltgt, _f);
1297                 case_brf(bltgt);
1298                 case_brr(bord, _f);
1299                 case_brf(bord);
1300                 case_brr(bunord, _f);
1301                 case_brf(bunord);
1302                 case_rrr(add, _d);
1303                 case_rrd(add);
1304                 case_rrr(sub, _d);
1305                 case_rrd(sub);
1306                 case_rrd(rsb);
1307                 case_rrr(mul, _d);
1308                 case_rrd(mul);
1309                 case_rrr(div, _d);
1310                 case_rrd(div);
1311                 case_rr(abs, _d);
1312                 case_rr(neg, _d);
1313                 case_rr(sqrt, _d);
1314                 case_rr(ext, _d);
1315                 case_rr(ld, _d);
1316                 case_rw(ld, _d);
1317                 case_rrr(ldx, _d);
1318                 case_rrw(ldx, _d);
1319                 case_rr(st, _d);
1320                 case_wr(st, _d);
1321                 case_rrr(stx, _d);
1322                 case_wrr(stx, _d);
1323                 case_rr(mov, _d);
1324             case jit_code_movi_d:
1325                 assert_data(node);
1326                 movi_d(rn(node->u.w), node->v.d);
1327                 break;
1328                 case_rr(ext, _f_d);
1329                 case_rrr(lt, _d);
1330                 case_rrd(lt);
1331                 case_rrr(le, _d);
1332                 case_rrd(le);
1333                 case_rrr(eq, _d);
1334                 case_rrd(eq);
1335                 case_rrr(ge, _d);
1336                 case_rrd(ge);
1337                 case_rrr(gt, _d);
1338                 case_rrd(gt);
1339                 case_rrr(ne, _d);
1340                 case_rrd(ne);
1341                 case_rrr(unlt, _d);
1342                 case_rrd(unlt);
1343                 case_rrr(unle, _d);
1344                 case_rrd(unle);
1345                 case_rrr(uneq, _d);
1346                 case_rrd(uneq);
1347                 case_rrr(unge, _d);
1348                 case_rrd(unge);
1349                 case_rrr(ungt, _d);
1350                 case_rrd(ungt);
1351                 case_rrr(ltgt, _d);
1352                 case_rrd(ltgt);
1353                 case_rrr(ord, _d);
1354                 case_rrd(ord);
1355                 case_rrr(unord, _d);
1356                 case_rrd(unord);
1357                 case_brr(blt, _d);
1358                 case_brd(blt);
1359                 case_brr(ble, _d);
1360                 case_brd(ble);
1361                 case_brr(beq, _d);
1362                 case_brd(beq);
1363                 case_brr(bge, _d);
1364                 case_brd(bge);
1365                 case_brr(bgt, _d);
1366                 case_brd(bgt);
1367                 case_brr(bne, _d);
1368                 case_brd(bne);
1369                 case_brr(bunlt, _d);
1370                 case_brd(bunlt);
1371                 case_brr(bunle, _d);
1372                 case_brd(bunle);
1373                 case_brr(buneq, _d);
1374                 case_brd(buneq);
1375                 case_brr(bunge, _d);
1376                 case_brd(bunge);
1377                 case_brr(bungt, _d);
1378                 case_brd(bungt);
1379                 case_brr(bltgt, _d);
1380                 case_brd(bltgt);
1381                 case_brr(bord, _d);
1382                 case_brd(bord);
1383                 case_brr(bunord, _d);
1384                 case_brd(bunord);
1385             case jit_code_jmpr:
1386                 jmpr(rn(node->u.w));
1387                 break;
1388             case jit_code_jmpi:
1389                 if (node->flag & jit_flag_node) {
1390                     temp = node->u.n;
1391                     assert(temp->code == jit_code_label ||
1392                            temp->code == jit_code_epilog);
1393                     if (temp->flag & jit_flag_patch)
1394                         jmpi(temp->u.w);
1395                     else {
1396                         word = jmpi_p(_jit->pc.w);
1397                         patch(word, node);
1398                     }
1399                 }
1400                 else
1401                     jmpi(node->u.w);
1402                 break;
1403             case jit_code_callr:
1404                 callr(rn(node->u.w));
1405                 break;
1406             case jit_code_calli:
1407                 if (node->flag & jit_flag_node) {
1408                     temp = node->u.n;
1409                     assert(temp->code == jit_code_label ||
1410                            temp->code == jit_code_epilog);
1411                     if (temp->flag & jit_flag_patch)
1412                         calli(temp->u.w);
1413                     else {
1414                         word = calli_p(_jit->pc.w);
1415                         patch(word, node);
1416                     }
1417                 }
1418                 else
1419                     calli(node->u.w);
1420                 break;
1421             case jit_code_prolog:
1422                 _jitc->function = _jitc->functions.ptr + node->w.w;
1423                 undo.node = node;
1424                 undo.word = _jit->pc.w;
1425 #if DEVEL_DISASSEMBLER
1426                 undo.prevw = prevw;
1427 #endif
1428                 undo.patch_offset = _jitc->patches.offset;
1429             restart_function:
1430                 _jitc->again = 0;
1431                 prolog(node);
1432                 break;
1433             case jit_code_epilog:
1434                 assert(_jitc->function == _jitc->functions.ptr + node->w.w);
1435                 if (_jitc->again) {
1436                     for (temp = undo.node->next;
1437                          temp != node; temp = temp->next) {
1438                         if (temp->code == jit_code_label ||
1439                             temp->code == jit_code_epilog)
1440                             temp->flag &= ~jit_flag_patch;
1441                     }
1442                     temp->flag &= ~jit_flag_patch;
1443                     node = undo.node;
1444                     _jit->pc.w = undo.word;
1445 #if DEVEL_DISASSEMBLER
1446                     prevw = undo.prevw;
1447 #endif
1448                     _jitc->patches.offset = undo.patch_offset;
1449                     goto restart_function;
1450                 }
1451                 if (node->link && (word = _jit->pc.w & 3))
1452                     nop(4 - word);
1453                 /* remember label is defined */
1454                 node->flag |= jit_flag_patch;
1455                 node->u.w = _jit->pc.w;
1456                 epilog(node);
1457                 _jitc->function = NULL;
1458                 break;
1459             case jit_code_movr_w_f:
1460                 movr_w_f(rn(node->u.w), rn(node->v.w));
1461                 break;
1462             case jit_code_movr_f_w:
1463                 movr_f_w(rn(node->u.w), rn(node->v.w));
1464                 break;
1465             case jit_code_movi_f_w:
1466                 assert_data(node);
1467                 movi_f_w(rn(node->u.w), node->v.f);
1468                 break;
1469             case jit_code_movr_w_d:
1470                 movr_w_d(rn(node->u.w), rn(node->v.w));
1471                 break;
1472             case jit_code_movr_d_w:
1473                 movr_d_w(rn(node->u.w), rn(node->v.w));
1474                 break;
1475             case jit_code_movi_d_w:
1476                 assert_data(node);
1477                 movi_d_w(rn(node->u.w), node->v.d);
1478                 break;
1479             case jit_code_va_start:
1480                 vastart(rn(node->u.w));
1481                 break;
1482             case jit_code_va_arg:
1483                 vaarg(rn(node->u.w), rn(node->v.w));
1484                 break;
1485             case jit_code_va_arg_d:
1486                 vaarg_d(rn(node->u.w), rn(node->v.w));
1487                 break;
1488             case jit_code_live:                 case jit_code_ellipsis:
1489             case jit_code_va_push:
1490             case jit_code_allocai:              case jit_code_allocar:
1491             case jit_code_arg:
1492             case jit_code_arg_f:                case jit_code_arg_d:
1493             case jit_code_va_end:
1494             case jit_code_ret:
1495             case jit_code_retr:                 case jit_code_reti:
1496             case jit_code_retr_f:               case jit_code_reti_f:
1497             case jit_code_retr_d:               case jit_code_reti_d:
1498             case jit_code_getarg_c:             case jit_code_getarg_uc:
1499             case jit_code_getarg_s:             case jit_code_getarg_us:
1500             case jit_code_getarg_i:
1501             case jit_code_getarg_ui:            case jit_code_getarg_l:
1502             case jit_code_getarg_f:             case jit_code_getarg_d:
1503             case jit_code_putargr:              case jit_code_putargi:
1504             case jit_code_putargr_f:            case jit_code_putargi_f:
1505             case jit_code_putargr_d:            case jit_code_putargi_d:
1506             case jit_code_pushargr:             case jit_code_pushargi:
1507             case jit_code_pushargr_f:           case jit_code_pushargi_f:
1508             case jit_code_pushargr_d:           case jit_code_pushargi_d:
1509             case jit_code_retval_c:             case jit_code_retval_uc:
1510             case jit_code_retval_s:             case jit_code_retval_us:
1511             case jit_code_retval_i:
1512             case jit_code_retval_ui:            case jit_code_retval_l:
1513             case jit_code_retval_f:             case jit_code_retval_d:
1514             case jit_code_prepare:
1515             case jit_code_finishr:              case jit_code_finishi:
1516                 break;
1517             default:
1518                 abort();
1519         }
1520         if (jit_carry != _NOREG) {
1521             switch (node->code) {
1522                 case jit_code_note:
1523                 case jit_code_addcr:            case jit_code_addci:
1524                 case jit_code_addxr:            case jit_code_addxi:
1525                 case jit_code_subcr:            case jit_code_subci:
1526                 case jit_code_subxr:            case jit_code_subxi:
1527                     break;
1528                 default:
1529                     jit_unget_reg(jit_carry);
1530                     jit_carry = _NOREG;
1531                     break;
1532             }
1533         }
1534         jit_regarg_clr(node, value);
1535         assert(_jitc->regarg == 0 ||
1536                (jit_carry != _NOREG && _jitc->regarg == (1 << jit_carry)));
1537         assert(_jitc->synth == 0);
1538         /* update register live state */
1539         jit_reglive(node);
1540     }
1541 #undef case_brw
1542 #undef case_brr
1543 #undef case_wrr
1544 #undef case_rrw
1545 #undef case_rrr
1546 #undef case_wr
1547 #undef case_rw
1548 #undef case_rr
1549
1550     for (offset = 0; offset < _jitc->patches.offset; offset++) {
1551         node = _jitc->patches.ptr[offset].node;
1552         word = node->code == jit_code_movi ? node->v.n->u.w : node->u.n->u.w;
1553         patch_at(_jitc->patches.ptr[offset].inst, word);
1554     }
1555
1556     jit_flush(_jit->code.ptr, _jit->pc.uc);
1557
1558     return (_jit->code.ptr);
1559 }
1560
1561 #define CODE                            1
1562 #  include "jit_loongarch-cpu.c"
1563 #  include "jit_loongarch-fpu.c"
1564 #undef CODE
1565
1566 void
1567 jit_flush(void *fptr, void *tptr)
1568 {
1569 #if defined(__GNUC__)
1570     jit_word_t          f, t, s;
1571
1572     s = sysconf(_SC_PAGE_SIZE);
1573     f = (jit_word_t)fptr & -s;
1574     t = (((jit_word_t)tptr) + s - 1) & -s;
1575     __clear_cache((void *)f, (void *)t);
1576 #endif
1577 }
1578
1579 void
1580 _emit_ldxi(jit_state_t *_jit, jit_int32_t r0, jit_int32_t r1, jit_word_t i0)
1581 {
1582     ldxi(rn(r0), rn(r1), i0);
1583 }
1584
1585 void
1586 _emit_stxi(jit_state_t *_jit, jit_word_t i0, jit_int32_t r0, jit_int32_t r1)
1587 {
1588     stxi(i0, rn(r0), rn(r1));
1589 }
1590
1591 void
1592 _emit_ldxi_d(jit_state_t *_jit, jit_int32_t r0, jit_int32_t r1, jit_word_t i0)
1593 {
1594     ldxi_d(rn(r0), rn(r1), i0);
1595 }
1596
1597 void
1598 _emit_stxi_d(jit_state_t *_jit, jit_word_t i0, jit_int32_t r0, jit_int32_t r1)
1599 {
1600     stxi_d(i0, rn(r0), rn(r1));
1601 }
1602
1603 static void
1604 _patch(jit_state_t *_jit, jit_word_t instr, jit_node_t *node)
1605 {
1606     jit_int32_t          flag;
1607
1608     assert(node->flag & jit_flag_node);
1609     if (node->code == jit_code_movi)
1610         flag = node->v.n->flag;
1611     else
1612         flag = node->u.n->flag;
1613     assert(!(flag & jit_flag_patch));
1614     if (_jitc->patches.offset >= _jitc->patches.length) {
1615         jit_realloc((jit_pointer_t *)&_jitc->patches.ptr,
1616                     _jitc->patches.length * sizeof(jit_patch_t),
1617                     (_jitc->patches.length + 1024) * sizeof(jit_patch_t));
1618         _jitc->patches.length += 1024;
1619     }
1620     _jitc->patches.ptr[_jitc->patches.offset].inst = instr;
1621     _jitc->patches.ptr[_jitc->patches.offset].node = node;
1622     ++_jitc->patches.offset;
1623 }