deps/lightning/lib/jit_ppc.c

   1 /*
   2  * Copyright (C) 2012-2023  Free Software Foundation, Inc.
   3  *
   4  * This file is part of GNU lightning.
   5  *
   6  * GNU lightning is free software; you can redistribute it and/or modify it
   7  * under the terms of the GNU Lesser General Public License as published
   8  * by the Free Software Foundation; either version 3, or (at your option)
   9  * any later version.
  10  *
  11  * GNU lightning is distributed in the hope that it will be useful, but
  12  * WITHOUT ANY WARRANTY; without even the implied warranty of MERCHANTABILITY
  13  * or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU Lesser General Public
  14  * License for more details.
  15  *
  16  * Authors:
  17  *      Paulo Cesar Pereira de Andrade
  18  */
  19
  20 #define jit_arg_reg_p(i)                ((i) >= 0 && (i) < 8)
  21 #if !_CALL_SYSV
  22 #  define jit_arg_f_reg_p(i)            ((i) >= 0 && (i) < 13)
  23 #else
  24 #  define jit_arg_f_reg_p(i)            ((i) >= 0 && (i) < 8)
  25 #  if __WORDSIZE == 32
  26 #    define va_gp_shift                 2
  27 #  else
  28 #    define va_gp_shift                 3
  29 #  endif
  30 #  define va_gp_increment               sizeof(jit_word_t)
  31 #  define first_gp_argument             r3
  32 #  define first_gp_offset               offsetof(jit_va_list_t,         \
  33                                                  first_gp_argument)
  34 #  define va_fp_increment               sizeof(jit_float64_t)
  35 #  define first_fp_argument             f1
  36 #  define first_fp_offset               offsetof(jit_va_list_t,         \
  37                                                  first_fp_argument)
  38 #endif
  39 #if __BYTE_ORDER == __LITTLE_ENDIAN
  40 #  define C_DISP                        0
  41 #  define S_DISP                        0
  42 #  define I_DISP                        0
  43 #  define F_DISP                        0
  44 #else
  45 #  define C_DISP                        (__WORDSIZE >> 3) - sizeof(jit_int8_t)
  46 #  define S_DISP                        (__WORDSIZE >> 3) - sizeof(jit_int16_t)
  47 #  define I_DISP                        (__WORDSIZE >> 3) - sizeof(jit_int32_t)
  48 #  define F_DISP                        (__WORDSIZE >> 3) - sizeof(jit_float32_t)
  49 #endif
  50 #define CVT_OFFSET                      _jitc->function->cvt_offset
  51 #define CHECK_CVT_OFFSET()                                              \
  52     do {                                                                \
  53         if (!_jitc->function->cvt_offset) {                             \
  54             _jitc->again = 1;                                           \
  55             _jitc->function->cvt_offset =                               \
  56                  jit_allocai(sizeof(jit_float64_t));                    \
  57         }                                                               \
  58     } while (0)
  59
  60 /*
  61  * Types
  62  */
  63 #if _CALL_SYSV
  64 typedef struct jit_va_list {
  65     jit_uint8_t         ngpr;
  66     jit_uint8_t         nfpr;
  67     jit_uint16_t        _pad;
  68 #  if __WORDSIZE == 64
  69     jit_uint32_t        _pad2;
  70 #  endif
  71     jit_pointer_t       over;
  72     jit_pointer_t       save;
  73 #  if __WORDSIZE == 32
  74     jit_word_t          _pad2;
  75 #  endif
  76     jit_word_t          r3;
  77     jit_word_t          r4;
  78     jit_word_t          r5;
  79     jit_word_t          r6;
  80     jit_word_t          r7;
  81     jit_word_t          r8;
  82     jit_word_t          r9;
  83     jit_word_t          r10;
  84     jit_float64_t       f1;
  85     jit_float64_t       f2;
  86     jit_float64_t       f3;
  87     jit_float64_t       f4;
  88     jit_float64_t       f5;
  89     jit_float64_t       f6;
  90     jit_float64_t       f7;
  91     jit_float64_t       f8;
  92 } jit_va_list_t;
  93 #else
  94 typedef jit_pointer_t jit_va_list_t;
  95 #endif
  96
  97 /*
  98  * Prototypes
  99  */
 100 #define patch(instr, node)              _patch(_jit, instr, node)
 101 static void _patch(jit_state_t*,jit_word_t,jit_node_t*);
 102
 103 /* libgcc */
 104 extern void __clear_cache(void *, void *);
 105
 106 #define PROTO                           1
 107 #  include "jit_ppc-cpu.c"
 108 #  include "jit_ppc-fpu.c"
 109 #  include "jit_fallback.c"
 110 #undef PROTO
 111
 112 /*
 113  * Initialization
 114  */
 115 jit_register_t          _rvs[] = {
 116     { rc(sav) | 0,                      "r0" },
 117     { rc(sav) | 11,                     "r11" },        /* env */
 118     { rc(sav) | 12,                     "r12" },        /* exception */
 119     { rc(sav) | 13,                     "r13" },        /* thread */
 120     { rc(sav) | 2,                      "r2" },         /* toc */
 121     { rc(sav) | rc(gpr) | 14,           "r14" },
 122     { rc(sav) | rc(gpr) | 15,           "r15" },
 123     { rc(sav) | rc(gpr) | 16,           "r16" },
 124     { rc(sav) | rc(gpr) | 17,           "r17" },
 125     { rc(sav) | rc(gpr) | 18,           "r18" },
 126     { rc(sav) | rc(gpr) | 19,           "r19" },
 127     { rc(sav) | rc(gpr) | 20,           "r20" },
 128     { rc(sav) | rc(gpr) | 21,           "r21" },
 129     { rc(sav) | rc(gpr) | 22,           "r22" },
 130     { rc(sav) | rc(gpr) | 23,           "r23" },
 131     { rc(sav) | rc(gpr) | 24,           "r24" },
 132     { rc(sav) | rc(gpr) | 25,           "r25" },
 133     { rc(sav) | rc(gpr) | 26,           "r26" },
 134     { rc(sav) | rc(gpr) | 27,           "r27" },
 135     { rc(sav) | rc(gpr) | 28,           "r28" },
 136     { rc(sav) | rc(gpr) | 29,           "r29" },
 137     { rc(sav) | rc(gpr) | 30,           "r30" },
 138     { rc(sav) | 1,                      "r1" },
 139     { rc(sav) | 31,                     "r31" },
 140     { rc(arg) | rc(gpr) | 10,           "r10" },
 141     { rc(arg) | rc(gpr) | 9,            "r9" },
 142     { rc(arg) | rc(gpr) | 8,            "r8" },
 143     { rc(arg) | rc(gpr) | 7,            "r7" },
 144     { rc(arg) | rc(gpr) | 6,            "r6" },
 145     { rc(arg) | rc(gpr) | 5,            "r5" },
 146     { rc(arg) | rc(gpr) | 4,            "r4" },
 147     { rc(arg) | rc(gpr) | 3,            "r3" },
 148     { rc(fpr) | 0,                      "f0" },
 149     { rc(sav) | rc(fpr) | 14,           "f14" },
 150     { rc(sav) | rc(fpr) | 15,           "f15" },
 151     { rc(sav) | rc(fpr) | 16,           "f16" },
 152     { rc(sav) | rc(fpr) | 17,           "f17" },
 153     { rc(sav) | rc(fpr) | 18,           "f18" },
 154     { rc(sav) | rc(fpr) | 19,           "f19" },
 155     { rc(sav) | rc(fpr) | 20,           "f20" },
 156     { rc(sav) | rc(fpr) | 21,           "f21" },
 157     { rc(sav) | rc(fpr) | 22,           "f22" },
 158     { rc(sav) | rc(fpr) | 23,           "f23" },
 159     { rc(sav) | rc(fpr) | 24,           "f24" },
 160     { rc(sav) | rc(fpr) | 25,           "f25" },
 161     { rc(sav) | rc(fpr) | 26,           "f26" },
 162     { rc(sav) | rc(fpr) | 27,           "f27" },
 163     { rc(sav) | rc(fpr) | 28,           "f28" },
 164     { rc(sav) | rc(fpr) | 29,           "f29" },
 165     { rc(sav) | rc(fpr) | 30,           "f30" },
 166     { rc(sav) | rc(fpr) | 31,           "f31" },
 167 #if !_CALL_SYSV
 168     { rc(arg) | rc(fpr) | 13,           "f13" },
 169     { rc(arg) | rc(fpr) | 12,           "f12" },
 170     { rc(arg) | rc(fpr) | 11,           "f11" },
 171     { rc(arg) | rc(fpr) | 10,           "f10" },
 172     { rc(arg) | rc(fpr) | 9,            "f9" },
 173 #else
 174     { rc(fpr) | 13,                     "f13" },
 175     { rc(fpr) | 12,                     "f12" },
 176     { rc(fpr) | 11,                     "f11" },
 177     { rc(fpr) | 10,                     "f10" },
 178     { rc(fpr) | 9,                      "f9" },
 179 #endif
 180     { rc(arg) | rc(fpr) | 8,            "f8" },
 181     { rc(arg) | rc(fpr) | 7,            "f7" },
 182     { rc(arg) | rc(fpr) | 6,            "f6" },
 183     { rc(arg) | rc(fpr) | 5,            "f5" },
 184     { rc(arg) | rc(fpr) | 4,            "f4" },
 185     { rc(arg) | rc(fpr) | 3,            "f3" },
 186     { rc(arg) | rc(fpr) | 2,            "f2" },
 187     { rc(arg) | rc(fpr) | 1,            "f1" },
 188     { _NOREG,                           "<none>" },
 189 };
 190
 191 static jit_int32_t iregs[] = {
 192     _R14, _R15, _R16, _R17, _R18, _R19, _R20, _R21, _R22,
 193     _R23, _R24, _R25, _R26, _R27, _R28, _R29, _R30
 194 };
 195
 196 static jit_int32_t fregs[] = {
 197     _F14, _F15, _F16, _F17, _F18, _F19, _F20, _F21,
 198 };
 199
 200 /*
 201  * Implementation
 202  */
 203 void
 204 jit_get_cpu(void)
 205 {
 206 }
 207
 208 void
 209 _jit_init(jit_state_t *_jit)
 210 {
 211     _jitc->reglen = jit_size(_rvs) - 1;
 212 }
 213
 214 void
 215 _jit_prolog(jit_state_t *_jit)
 216 {
 217     jit_int32_t          offset;
 218
 219     if (_jitc->function)
 220         jit_epilog();
 221     assert(jit_regset_cmp_ui(&_jitc->regarg, 0) == 0);
 222     jit_regset_set_ui(&_jitc->regsav, 0);
 223     offset = _jitc->functions.offset;
 224     if (offset >= _jitc->functions.length) {
 225         jit_realloc((jit_pointer_t *)&_jitc->functions.ptr,
 226                     _jitc->functions.length * sizeof(jit_function_t),
 227                     (_jitc->functions.length + 16) * sizeof(jit_function_t));
 228         _jitc->functions.length += 16;
 229     }
 230     _jitc->function = _jitc->functions.ptr + _jitc->functions.offset++;
 231     _jitc->function->self.size = params_offset;
 232     _jitc->function->self.argi = _jitc->function->self.argf =
 233         _jitc->function->self.alen = 0;
 234     /* float conversion */
 235     _jitc->function->cvt_offset = 0;
 236     _jitc->function->self.aoff = alloca_offset - 8;
 237     _jitc->function->self.call = jit_call_default;
 238     jit_alloc((jit_pointer_t *)&_jitc->function->regoff,
 239               _jitc->reglen * sizeof(jit_int32_t));
 240
 241     /* _no_link here does not mean the jit_link() call can be removed
 242      * by rewriting as:
 243      * _jitc->function->prolog = jit_new_node(jit_code_prolog);
 244      */
 245     _jitc->function->prolog = jit_new_node_no_link(jit_code_prolog);
 246     jit_link(_jitc->function->prolog);
 247     _jitc->function->prolog->w.w = offset;
 248     _jitc->function->epilog = jit_new_node_no_link(jit_code_epilog);
 249     /*  u:      label value
 250      *  v:      offset in blocks vector
 251      *  w:      offset in functions vector
 252      */
 253     _jitc->function->epilog->w.w = offset;
 254
 255     jit_regset_new(&_jitc->function->regset);
 256 }
 257
 258 jit_int32_t
 259 _jit_allocai(jit_state_t *_jit, jit_int32_t length)
 260 {
 261     assert(_jitc->function);
 262     jit_check_frame();
 263     switch (length) {
 264         case 0: case 1:                                         break;
 265         case 2:         _jitc->function->self.aoff &= -2;       break;
 266         case 3: case 4: _jitc->function->self.aoff &= -4;       break;
 267         default:        _jitc->function->self.aoff &= -8;       break;
 268     }
 269     _jitc->function->self.aoff -= length;
 270     if (!_jitc->realize) {
 271         jit_inc_synth_ww(allocai, _jitc->function->self.aoff, length);
 272         jit_dec_synth();
 273     }
 274     return (_jitc->function->self.aoff);
 275 }
 276
 277 void
 278 _jit_allocar(jit_state_t *_jit, jit_int32_t u, jit_int32_t v)
 279 {
 280     jit_int32_t          r0, r1;
 281     assert(_jitc->function);
 282     jit_inc_synth_ww(allocar, u, v);
 283     if (!_jitc->function->allocar) {
 284         _jitc->function->aoffoff = jit_allocai(sizeof(jit_int32_t));
 285         _jitc->function->allocar = 1;
 286     }
 287     r0 = jit_get_reg(jit_class_gpr);
 288     r1 = jit_get_reg(jit_class_gpr);
 289     jit_ldr(r0, JIT_SP);
 290     jit_negr(r1, v);
 291     jit_andi(r1, r1, -16);
 292     jit_ldxi_i(u, JIT_FP, _jitc->function->aoffoff);
 293     jit_addr(u, u, r1);
 294     jit_addr(JIT_SP, JIT_SP, r1);
 295     jit_stxi_i(_jitc->function->aoffoff, JIT_FP, u);
 296     jit_str(JIT_SP, r0);
 297     jit_unget_reg(r1);
 298     jit_unget_reg(r0);
 299     jit_dec_synth();
 300 }
 301
 302 void
 303 _jit_ret(jit_state_t *_jit)
 304 {
 305     jit_node_t          *instr;
 306     assert(_jitc->function);
 307     jit_inc_synth(ret);
 308     /* jump to epilog */
 309     instr = jit_jmpi();
 310     jit_patch_at(instr, _jitc->function->epilog);
 311     jit_dec_synth();
 312 }
 313
 314 void
 315 _jit_retr(jit_state_t *_jit, jit_int32_t u, jit_code_t code)
 316 {
 317     jit_code_inc_synth_w(code, u);
 318     jit_movr(JIT_RET, u);
 319     jit_ret();
 320     jit_dec_synth();
 321 }
 322
 323 void
 324 _jit_reti(jit_state_t *_jit, jit_word_t u, jit_code_t code)
 325 {
 326     jit_code_inc_synth_w(code, u);
 327     jit_movi(JIT_RET, u);
 328     jit_ret();
 329     jit_dec_synth();
 330 }
 331
 332 void
 333 _jit_retr_f(jit_state_t *_jit, jit_int32_t u)
 334 {
 335     jit_inc_synth_w(retr_f, u);
 336     if (JIT_RET != u)
 337         jit_movr_f(JIT_FRET, u);
 338     else
 339         jit_live(JIT_FRET);
 340     jit_ret();
 341     jit_dec_synth();
 342 }
 343
 344 void
 345 _jit_reti_f(jit_state_t *_jit, jit_float32_t u)
 346 {
 347     jit_inc_synth_f(reti_f, u);
 348     jit_movi_f(JIT_FRET, u);
 349     jit_ret();
 350     jit_dec_synth();
 351 }
 352
 353 void
 354 _jit_retr_d(jit_state_t *_jit, jit_int32_t u)
 355 {
 356     jit_inc_synth_w(retr_d, u);
 357     if (JIT_FRET != u)
 358         jit_movr_d(JIT_FRET, u);
 359     else
 360         jit_live(JIT_FRET);
 361     jit_ret();
 362     jit_dec_synth();
 363 }
 364
 365 void
 366 _jit_reti_d(jit_state_t *_jit, jit_float64_t u)
 367 {
 368     jit_inc_synth_d(reti_d, u);
 369     jit_movi_d(JIT_FRET, u);
 370     jit_ret();
 371     jit_dec_synth();
 372 }
 373
 374 void
 375 _jit_epilog(jit_state_t *_jit)
 376 {
 377     assert(_jitc->function);
 378     assert(_jitc->function->epilog->next == NULL);
 379     jit_link(_jitc->function->epilog);
 380     _jitc->function = NULL;
 381 }
 382
 383 jit_bool_t
 384 _jit_arg_register_p(jit_state_t *_jit, jit_node_t *u)
 385 {
 386     if (u->code >= jit_code_arg_c && u->code <= jit_code_arg)
 387         return (jit_arg_reg_p(u->u.w));
 388     assert(u->code == jit_code_arg_f || u->code == jit_code_arg_d);
 389     return (jit_arg_f_reg_p(u->u.w));
 390 }
 391
 392 void
 393 _jit_ellipsis(jit_state_t *_jit)
 394 {
 395     jit_inc_synth(ellipsis);
 396     if (_jitc->prepare) {
 397         jit_link_prepare();
 398         assert(!(_jitc->function->call.call & jit_call_varargs));
 399         _jitc->function->call.call |= jit_call_varargs;
 400     }
 401     else {
 402         jit_link_prolog();
 403         assert(!(_jitc->function->self.call & jit_call_varargs));
 404         _jitc->function->self.call |= jit_call_varargs;
 405 #if _CALL_SYSV
 406         /* Allocate va_list like object in the stack.
 407          * If applicable, with enough space to save all argument
 408          * registers, and use fixed offsets for them. */
 409         _jitc->function->vaoff = jit_allocai(sizeof(jit_va_list_t));
 410 #endif
 411         _jitc->function->vagp = _jitc->function->self.argi;
 412         _jitc->function->vafp = _jitc->function->self.argf;
 413     }
 414     jit_dec_synth();
 415 }
 416
 417 void
 418 _jit_va_push(jit_state_t *_jit, jit_int32_t u)
 419 {
 420     jit_inc_synth_w(va_push, u);
 421     jit_pushargr(u);
 422     jit_dec_synth();
 423 }
 424
 425 jit_node_t *
 426 _jit_arg(jit_state_t *_jit, jit_code_t code)
 427 {
 428     jit_node_t          *node;
 429     jit_int32_t          offset;
 430     jit_bool_t           incr = 1;
 431     assert(_jitc->function);
 432     assert(!(_jitc->function->self.call & jit_call_varargs));
 433 #if STRONG_TYPE_CHECKING
 434     assert(code >= jit_code_arg_c && code <= jit_code_arg);
 435 #endif
 436     if (jit_arg_reg_p(_jitc->function->self.argi)) {
 437         offset = _jitc->function->self.argi++;
 438 #if _CALL_SYSV
 439         incr = 0;
 440 #endif
 441     }
 442     else
 443         offset = _jitc->function->self.size;
 444     if (incr)
 445         _jitc->function->self.size += sizeof(jit_word_t);
 446     node = jit_new_node_ww(code, offset,
 447                            ++_jitc->function->self.argn);
 448     jit_link_prolog();
 449     return (node);
 450 }
 451
 452 jit_node_t *
 453 _jit_arg_f(jit_state_t *_jit)
 454 {
 455     jit_node_t          *node;
 456     jit_int32_t          offset;
 457     jit_bool_t           incr = 1;
 458     assert(_jitc->function);
 459     if (jit_arg_f_reg_p(_jitc->function->self.argf)) {
 460         offset = _jitc->function->self.argf++;
 461 #if _CALL_SYSV
 462         incr = 0;
 463 #endif
 464     }
 465     else
 466         offset = _jitc->function->self.size + F_DISP;
 467 #if !_CALL_SYSV
 468     if (jit_arg_reg_p(_jitc->function->self.argi)) {
 469 #  if __WORDSIZE == 32
 470         _jitc->function->self.argi += 2;
 471 #  else
 472         _jitc->function->self.argi++;
 473 #  endif
 474     }
 475 #endif
 476     if (incr)
 477         _jitc->function->self.size += sizeof(jit_word_t);
 478     node = jit_new_node_ww(jit_code_arg_f, offset,
 479                            ++_jitc->function->self.argn);
 480     jit_link_prolog();
 481     return (node);
 482 }
 483
 484 jit_node_t *
 485 _jit_arg_d(jit_state_t *_jit)
 486 {
 487     jit_node_t          *node;
 488     jit_int32_t          offset;
 489     jit_bool_t           incr = 1;
 490     assert(_jitc->function);
 491     if (jit_arg_f_reg_p(_jitc->function->self.argf)) {
 492         offset = _jitc->function->self.argf++;
 493 #if _CALL_SYSV
 494         incr = 0;
 495 #endif
 496     }
 497     else {
 498 #if _CALL_SYSV
 499         if (_jitc->function->self.size & 7)
 500             _jitc->function->self.size += 4;
 501 #endif
 502         offset = _jitc->function->self.size;
 503     }
 504 #if !_CALL_SYSV
 505     if (jit_arg_reg_p(_jitc->function->self.argi)) {
 506 #  if __WORDSIZE == 32
 507         _jitc->function->self.argi += 2;
 508 #  else
 509         _jitc->function->self.argi++;
 510 #  endif
 511     }
 512 #endif
 513     if (incr)
 514         _jitc->function->self.size += sizeof(jit_float64_t);
 515     node = jit_new_node_ww(jit_code_arg_d, offset,
 516                            ++_jitc->function->self.argn);
 517     jit_link_prolog();
 518     return (node);
 519 }
 520
 521 void
 522 _jit_getarg_c(jit_state_t *_jit, jit_int32_t u, jit_node_t *v)
 523 {
 524     assert_arg_type(v->code, jit_code_arg_c);
 525     jit_inc_synth_wp(getarg_c, u, v);
 526     if (jit_arg_reg_p(v->u.w))
 527         jit_extr_c(u, JIT_RA0 - v->u.w);
 528     else {
 529         jit_check_frame();
 530         jit_ldxi_c(u, JIT_FP, v->u.w + C_DISP);
 531     }
 532     jit_dec_synth();
 533 }
 534
 535 void
 536 _jit_getarg_uc(jit_state_t *_jit, jit_int32_t u, jit_node_t *v)
 537 {
 538     assert_arg_type(v->code, jit_code_arg_c);
 539     jit_inc_synth_wp(getarg_uc, u, v);
 540     if (jit_arg_reg_p(v->u.w))
 541         jit_extr_uc(u, JIT_RA0 - v->u.w);
 542     else {
 543         jit_check_frame();
 544         jit_ldxi_uc(u, JIT_FP, v->u.w + C_DISP);
 545     }
 546     jit_dec_synth();
 547 }
 548
 549 void
 550 _jit_getarg_s(jit_state_t *_jit, jit_int32_t u, jit_node_t *v)
 551 {
 552     assert_arg_type(v->code, jit_code_arg_s);
 553     jit_inc_synth_wp(getarg_s, u, v);
 554     if (jit_arg_reg_p(v->u.w))
 555         jit_extr_s(u, JIT_RA0 - v->u.w);
 556     else {
 557         jit_check_frame();
 558         jit_ldxi_s(u, JIT_FP, v->u.w + S_DISP);
 559     }
 560     jit_dec_synth();
 561 }
 562
 563 void
 564 _jit_getarg_us(jit_state_t *_jit, jit_int32_t u, jit_node_t *v)
 565 {
 566     assert_arg_type(v->code, jit_code_arg_s);
 567     jit_inc_synth_wp(getarg_us, u, v);
 568     if (jit_arg_reg_p(v->u.w))
 569         jit_extr_us(u, JIT_RA0 - v->u.w);
 570     else {
 571         jit_check_frame();
 572         jit_ldxi_us(u, JIT_FP, v->u.w + S_DISP);
 573     }
 574     jit_dec_synth();
 575 }
 576
 577 void
 578 _jit_getarg_i(jit_state_t *_jit, jit_int32_t u, jit_node_t *v)
 579 {
 580     assert_arg_type(v->code, jit_code_arg_i);
 581     jit_inc_synth_wp(getarg_i, u, v);
 582     if (jit_arg_reg_p(v->u.w)) {
 583 #if __WORDSIZE == 32
 584         jit_movr(u, JIT_RA0 - v->u.w);
 585 #else
 586         jit_extr_i(u, JIT_RA0 - v->u.w);
 587 #endif
 588     }
 589     else {
 590         jit_check_frame();
 591         jit_ldxi_i(u, JIT_FP, v->u.w + I_DISP);
 592     }
 593     jit_dec_synth();
 594 }
 595
 596 #if __WORDSIZE == 64
 597 void
 598 _jit_getarg_ui(jit_state_t *_jit, jit_int32_t u, jit_node_t *v)
 599 {
 600     assert_arg_type(v->code, jit_code_arg_i);
 601     jit_inc_synth_wp(getarg_ui, u, v);
 602     if (jit_arg_reg_p(v->u.w))
 603         jit_extr_ui(u, JIT_RA0 - v->u.w);
 604     else {
 605         jit_check_frame();
 606         jit_ldxi_ui(u, JIT_FP, v->u.w + I_DISP);
 607     }
 608     jit_dec_synth();
 609 }
 610
 611 void
 612 _jit_getarg_l(jit_state_t *_jit, jit_int32_t u, jit_node_t *v)
 613 {
 614     assert_arg_type(v->code, jit_code_arg_l);
 615     jit_inc_synth_wp(getarg_l, u, v);
 616     if (jit_arg_reg_p(v->u.w))
 617         jit_movr(u, JIT_RA0 - v->u.w);
 618     else {
 619         jit_check_frame();
 620         jit_ldxi_l(u, JIT_FP, v->u.w);
 621     }
 622     jit_dec_synth();
 623 }
 624 #endif
 625
 626 void
 627 _jit_putargr(jit_state_t *_jit, jit_int32_t u, jit_node_t *v, jit_code_t code)
 628 {
 629     assert_putarg_type(code, v->code);
 630     jit_code_inc_synth_wp(code, u, v);
 631     if (jit_arg_reg_p(v->u.w))
 632         jit_movr(JIT_RA0 - v->u.w, u);
 633     else {
 634         jit_check_frame();
 635         jit_stxi(v->u.w, JIT_FP, u);
 636     }
 637     jit_dec_synth();
 638 }
 639
 640 void
 641 _jit_putargi(jit_state_t *_jit, jit_word_t u, jit_node_t *v, jit_code_t code)
 642 {
 643     jit_int32_t         regno;
 644     assert_putarg_type(code, v->code);
 645     jit_code_inc_synth_wp(code, u, v);
 646     if (jit_arg_reg_p(v->u.w))
 647         jit_movi(JIT_RA0 - v->u.w, u);
 648     else {
 649         jit_check_frame();
 650         regno = jit_get_reg(jit_class_gpr);
 651         jit_movi(regno, u);
 652         jit_stxi(v->u.w, JIT_FP, regno);
 653         jit_unget_reg(regno);
 654     }
 655     jit_dec_synth();
 656 }
 657
 658 void
 659 _jit_getarg_f(jit_state_t *_jit, jit_int32_t u, jit_node_t *v)
 660 {
 661     assert(v->code == jit_code_arg_f);
 662     jit_inc_synth_wp(getarg_f, u, v);
 663     if (jit_arg_f_reg_p(v->u.w))
 664         jit_movr_d(u, JIT_FA0 - v->u.w);
 665     else {
 666         jit_check_frame();
 667         jit_ldxi_f(u, JIT_FP, v->u.w);
 668     }
 669     jit_dec_synth();
 670 }
 671
 672 void
 673 _jit_putargr_f(jit_state_t *_jit, jit_int32_t u, jit_node_t *v)
 674 {
 675     assert(v->code == jit_code_arg_f);
 676     jit_inc_synth_wp(putargr_f, u, v);
 677     if (jit_arg_f_reg_p(v->u.w))
 678         jit_movr_d(JIT_FA0 - v->u.w, u);
 679     else {
 680         jit_check_frame();
 681         jit_stxi_f(v->u.w, JIT_FP, u);
 682     }
 683     jit_dec_synth();
 684 }
 685
 686 void
 687 _jit_putargi_f(jit_state_t *_jit, jit_float32_t u, jit_node_t *v)
 688 {
 689     jit_int32_t         regno;
 690     assert(v->code == jit_code_arg_f);
 691     jit_inc_synth_fp(putargi_f, u, v);
 692     if (jit_arg_f_reg_p(v->u.w))
 693         jit_movi_d(JIT_FA0 - v->u.w, u);
 694     else {
 695         jit_check_frame();
 696         regno = jit_get_reg(jit_class_fpr);
 697         jit_movi_d(regno, u);
 698         jit_stxi_f(v->u.w, JIT_FP, regno);
 699         jit_unget_reg(regno);
 700     }
 701     jit_dec_synth();
 702 }
 703
 704 void
 705 _jit_getarg_d(jit_state_t *_jit, jit_int32_t u, jit_node_t *v)
 706 {
 707     assert(v->code == jit_code_arg_d);
 708     jit_inc_synth_wp(getarg_d, u, v);
 709     if (jit_arg_f_reg_p(v->u.w))
 710         jit_movr_d(u, JIT_FA0 - v->u.w);
 711     else {
 712         jit_check_frame();
 713         jit_ldxi_d(u, JIT_FP, v->u.w);
 714     }
 715     jit_dec_synth();
 716 }
 717
 718 void
 719 _jit_putargr_d(jit_state_t *_jit, jit_int32_t u, jit_node_t *v)
 720 {
 721     assert(v->code == jit_code_arg_d);
 722     jit_inc_synth_wp(putargr_d, u, v);
 723     if (jit_arg_f_reg_p(v->u.w))
 724         jit_movr_d(JIT_FA0 - v->u.w, u);
 725     else {
 726         jit_check_frame();
 727         jit_stxi_d(v->u.w, JIT_FP, u);
 728     }
 729     jit_dec_synth();
 730 }
 731
 732 void
 733 _jit_putargi_d(jit_state_t *_jit, jit_float64_t u, jit_node_t *v)
 734 {
 735     jit_int32_t         regno;
 736     assert(v->code == jit_code_arg_d);
 737     jit_inc_synth_dp(putargi_d, u, v);
 738     if (jit_arg_f_reg_p(v->u.w))
 739         jit_movi_d(JIT_FA0 - v->u.w, u);
 740     else {
 741         jit_check_frame();
 742         regno = jit_get_reg(jit_class_fpr);
 743         jit_movi_d(regno, u);
 744         jit_stxi_d(v->u.w, JIT_FP, regno);
 745         jit_unget_reg(regno);
 746     }
 747     jit_dec_synth();
 748 }
 749
 750 void
 751 _jit_pushargr(jit_state_t *_jit, jit_int32_t u, jit_code_t code)
 752 {
 753     jit_bool_t          incr = 1;
 754     assert(_jitc->function);
 755     jit_code_inc_synth_w(code, u);
 756     jit_link_prepare();
 757     if (jit_arg_reg_p(_jitc->function->call.argi)) {
 758         jit_movr(JIT_RA0 - _jitc->function->call.argi, u);
 759         ++_jitc->function->call.argi;
 760 #if _CALL_SYSV
 761         incr = 0;
 762 #endif
 763     }
 764     else
 765         jit_stxi(_jitc->function->call.size + params_offset, JIT_SP, u);
 766     if (incr)
 767         _jitc->function->call.size += sizeof(jit_word_t);
 768     jit_dec_synth();
 769 }
 770
 771 void
 772 _jit_pushargi(jit_state_t *_jit, jit_word_t u, jit_code_t code)
 773 {
 774     jit_int32_t          regno;
 775     jit_bool_t           incr = 1;
 776     assert(_jitc->function);
 777     jit_code_inc_synth_w(code, u);
 778     jit_link_prepare();
 779     if (jit_arg_reg_p(_jitc->function->call.argi)) {
 780         jit_movi(JIT_RA0 - _jitc->function->call.argi, u);
 781         ++_jitc->function->call.argi;
 782 #if _CALL_SYSV
 783         incr = 0;
 784 #endif
 785     }
 786     else {
 787         regno = jit_get_reg(jit_class_gpr);
 788         jit_movi(regno, u);
 789         jit_stxi(_jitc->function->call.size + params_offset, JIT_SP, regno);
 790         jit_unget_reg(regno);
 791     }
 792     if (incr)
 793         _jitc->function->call.size += sizeof(jit_word_t);
 794     jit_dec_synth();
 795 }
 796
 797 void
 798 _jit_pushargr_f(jit_state_t *_jit, jit_int32_t u)
 799 {
 800     jit_bool_t          incr = 1;
 801     assert(_jitc->function);
 802     jit_inc_synth_w(pushargr_f, u);
 803     jit_link_prepare();
 804     if (jit_arg_f_reg_p(_jitc->function->call.argf)
 805 #if !_CALL_SYSV
 806         && !(_jitc->function->call.call & jit_call_varargs)
 807 #endif
 808         ) {
 809         jit_movr_d(JIT_FA0 - _jitc->function->call.argf, u);
 810         ++_jitc->function->call.argf;
 811 #if !_CALL_SYSV
 812         /* in case of excess arguments */
 813         if (jit_arg_reg_p(_jitc->function->call.argi)) {
 814 #  if __WORDSIZE == 32
 815             _jitc->function->call.argi += 2;
 816             if (!jit_arg_reg_p(_jitc->function->call.argi - 1))
 817                 --_jitc->function->call.argi;
 818 #  else
 819             _jitc->function->call.argi++;
 820 #  endif
 821         }
 822 #elif _CALL_SYSV
 823         incr = 0;
 824 #endif
 825     }
 826 #if !_CALL_SYSV
 827     else if (jit_arg_reg_p(_jitc->function->call.argi
 828 #  if __WORDSIZE == 32
 829                           + 1
 830 #  endif
 831                            )) {
 832         CHECK_CVT_OFFSET();
 833         jit_check_frame();
 834         jit_stxi_d(CVT_OFFSET, JIT_FP, u);
 835         jit_ldxi(JIT_RA0 - _jitc->function->call.argi, JIT_FP, CVT_OFFSET);
 836         _jitc->function->call.argi++;
 837 #  if __WORDSIZE == 32
 838         jit_ldxi(JIT_RA0 - _jitc->function->call.argi, JIT_FP,
 839                  CVT_OFFSET + 4);
 840         _jitc->function->call.argi++;
 841 #  endif
 842     }
 843 #endif
 844     else
 845         jit_stxi_f(_jitc->function->call.size + params_offset + F_DISP,
 846                    JIT_SP, u);
 847     if (incr)
 848         _jitc->function->call.size += sizeof(jit_word_t);
 849     jit_dec_synth();
 850 }
 851
 852 void
 853 _jit_pushargi_f(jit_state_t *_jit, jit_float32_t u)
 854 {
 855     jit_bool_t           incr = 1;
 856     jit_int32_t          regno;
 857     assert(_jitc->function);
 858     jit_inc_synth_f(pushargi_f, u);
 859     jit_link_prepare();
 860     if (jit_arg_f_reg_p(_jitc->function->call.argf)
 861 #if !_CALL_SYSV
 862         && !(_jitc->function->call.call & jit_call_varargs)
 863 #endif
 864         ) {
 865         jit_movi_d(JIT_FA0 - _jitc->function->call.argf, u);
 866         ++_jitc->function->call.argf;
 867 #if !_CALL_SYSV
 868             /* in case of excess arguments */
 869 #  if __WORDSIZE == 32
 870         _jitc->function->call.argi += 2;
 871         if (!jit_arg_reg_p(_jitc->function->call.argi - 1))
 872             --_jitc->function->call.argi;
 873 #  else
 874         _jitc->function->call.argi++;
 875 #  endif
 876 #elif _CALL_SYSV
 877         incr = 0;
 878 #endif
 879     }
 880     else {
 881         regno = jit_get_reg(jit_class_fpr);
 882         jit_movi_f(regno, u);
 883 #if !_CALL_SYSV
 884         if (jit_arg_reg_p(_jitc->function->call.argi
 885 #  if __WORDSIZE == 32
 886                           + 1
 887 #  endif
 888                           )) {
 889             CHECK_CVT_OFFSET();
 890             jit_check_frame();
 891             jit_stxi_d(CVT_OFFSET, JIT_FP, regno);
 892             jit_ldxi(JIT_RA0 - _jitc->function->call.argi, JIT_FP,
 893                      CVT_OFFSET);
 894             _jitc->function->call.argi++;
 895 #  if __WORDSIZE == 32
 896             jit_ldxi(JIT_RA0 - _jitc->function->call.argi, JIT_FP,
 897                      CVT_OFFSET + 4);
 898             _jitc->function->call.argi++;
 899 #  endif
 900         }
 901         else
 902 #endif
 903             jit_stxi_f(_jitc->function->call.size + params_offset + F_DISP,
 904                        JIT_SP, regno);
 905         jit_unget_reg(regno);
 906     }
 907     if (incr)
 908         _jitc->function->call.size += sizeof(jit_word_t);
 909     jit_dec_synth();
 910 }
 911
 912 void
 913 _jit_pushargr_d(jit_state_t *_jit, jit_int32_t u)
 914 {
 915     jit_bool_t          incr = 1;
 916     assert(_jitc->function);
 917     jit_inc_synth_w(pushargr_d, u);
 918     jit_link_prepare();
 919     if (jit_arg_f_reg_p(_jitc->function->call.argf)
 920 #if !_CALL_SYSV
 921         && !(_jitc->function->call.call & jit_call_varargs)
 922 #endif
 923         ) {
 924         jit_movr_d(JIT_FA0 - _jitc->function->call.argf, u);
 925         ++_jitc->function->call.argf;
 926 #if !_CALL_SYSV
 927             /* in case of excess arguments */
 928 #  if __WORDSIZE == 32
 929         _jitc->function->call.argi += 2;
 930         if (!jit_arg_reg_p(_jitc->function->call.argi - 1))
 931             --_jitc->function->call.argi;
 932 #  else
 933         _jitc->function->call.argi++;
 934 #  endif
 935 #else /* _CALL_SYSV */
 936         incr = 0;
 937 #endif
 938     }
 939 #if !_CALL_SYSV
 940     else if (jit_arg_reg_p(_jitc->function->call.argi
 941 #  if __WORDSIZE == 32
 942                           + 1
 943 #  endif
 944                            )) {
 945         CHECK_CVT_OFFSET();
 946         jit_check_frame();
 947         jit_stxi_d(CVT_OFFSET, JIT_FP, u);
 948         jit_ldxi(JIT_RA0 - _jitc->function->call.argi, JIT_FP, CVT_OFFSET);
 949         _jitc->function->call.argi++;
 950 #  if __WORDSIZE == 32
 951         jit_ldxi(JIT_RA0 - _jitc->function->call.argi, JIT_FP, CVT_OFFSET + 4);
 952         _jitc->function->call.argi++;
 953 #  endif
 954     }
 955     else
 956 #endif /* !_CALL_SYSV */
 957     {
 958 #if _CALL_SYSV
 959         if (_jitc->function->call.size & 7)
 960             _jitc->function->call.size += 4;
 961 #endif
 962         jit_stxi_d(_jitc->function->call.size + params_offset, JIT_SP, u);
 963 #if !_CALL_SYSV && __WORDSIZE == 32
 964         if (jit_arg_reg_p(_jitc->function->call.argi)) {
 965             jit_ldxi(JIT_RA0 - _jitc->function->call.argi, JIT_SP,
 966                      _jitc->function->call.size + params_offset);
 967             _jitc->function->call.argi++;
 968         }
 969 #endif
 970     }
 971     if (incr)
 972         _jitc->function->call.size += sizeof(jit_float64_t);
 973     jit_dec_synth();
 974 }
 975
 976 void
 977 _jit_pushargi_d(jit_state_t *_jit, jit_float64_t u)
 978 {
 979     jit_int32_t          regno;
 980     jit_bool_t           incr = 1;
 981     assert(_jitc->function);
 982     jit_inc_synth_d(pushargi_d, u);
 983     jit_link_prepare();
 984     if (jit_arg_f_reg_p(_jitc->function->call.argf)
 985 #if !_CALL_SYSV
 986         && !(_jitc->function->call.call & jit_call_varargs)
 987 #endif
 988         ) {
 989         jit_movi_d(JIT_FA0 - _jitc->function->call.argf, u);
 990         ++_jitc->function->call.argf;
 991 #if !_CALL_SYSV
 992         /* in case of excess arguments */
 993         if (jit_arg_reg_p(_jitc->function->call.argi)) {
 994 #  if __WORDSIZE == 32
 995             _jitc->function->call.argi += 2;
 996             if (!jit_arg_reg_p(_jitc->function->call.argi - 1))
 997                 --_jitc->function->call.argi;
 998 #  else
 999             _jitc->function->call.argi++;
1000 #  endif
1001         }
1002 #else /* _CALL_SYSV */
1003             incr = 0;
1004 #endif
1005     }
1006     else {
1007         regno = jit_get_reg(jit_class_fpr);
1008         jit_movi_d(regno, u);
1009 #if !_CALL_SYSV
1010         if (jit_arg_reg_p(_jitc->function->call.argi
1011 #  if __WORDSIZE == 32
1012                           + 1
1013 #  endif
1014                           )) {
1015             CHECK_CVT_OFFSET();
1016             jit_check_frame();
1017             jit_stxi_d(CVT_OFFSET, JIT_FP, regno);
1018             jit_ldxi(JIT_RA0 - _jitc->function->call.argi, JIT_FP,
1019                      CVT_OFFSET);
1020             _jitc->function->call.argi++;
1021 #  if __WORDSIZE == 32
1022             jit_ldxi(JIT_RA0 - _jitc->function->call.argi, JIT_FP,
1023                      CVT_OFFSET + 4);
1024             _jitc->function->call.argi++;
1025 #  endif
1026         }
1027         else
1028 #endif /* !_CALL_SYSV */
1029         {
1030 #if _CALL_SYSV
1031             if (_jitc->function->call.size & 7)
1032                 _jitc->function->call.size += 4;
1033 #endif
1034             jit_stxi_d(_jitc->function->call.size + params_offset,
1035                        JIT_SP, regno);
1036 #if !_CALL_SYSV && __WORDSIZE == 32
1037             if (jit_arg_reg_p(_jitc->function->call.argi)) {
1038                 jit_ldxi(JIT_RA0 - _jitc->function->call.argi, JIT_SP,
1039                          _jitc->function->call.size + params_offset);
1040                 _jitc->function->call.argi++;
1041             }
1042 #endif
1043         }
1044         jit_unget_reg(regno);
1045     }
1046     if (incr)
1047         _jitc->function->call.size += sizeof(jit_float64_t);
1048     jit_dec_synth();
1049 }
1050
1051 jit_bool_t
1052 _jit_regarg_p(jit_state_t *_jit, jit_node_t *node, jit_int32_t regno)
1053 {
1054     jit_int32_t         spec;
1055     spec = jit_class(_rvs[regno].spec);
1056     if (spec & jit_class_arg) {
1057         if (spec & jit_class_gpr) {
1058             regno = JIT_RA0 - regno;
1059             if (regno >= 0 && regno < node->v.w)
1060                 return (1);
1061         }
1062         else if (spec & jit_class_fpr) {
1063             regno = JIT_FA0 - regno;
1064             if (regno >= 0 && regno < node->w.w)
1065                 return (1);
1066         }
1067     }
1068     return (0);
1069 }
1070
1071 void
1072 _jit_finishr(jit_state_t *_jit, jit_int32_t r0)
1073 {
1074     jit_node_t          *call;
1075     assert(_jitc->function);
1076     jit_inc_synth_w(finishr, r0);
1077     if (_jitc->function->self.alen < _jitc->function->call.size)
1078         _jitc->function->self.alen = _jitc->function->call.size;
1079     call = jit_callr(r0);
1080     call->v.w = _jitc->function->call.argi;
1081     call->w.w = _jitc->function->call.argf;
1082 #if _CALL_SYSV
1083     /* If passing float arguments in registers */
1084     if ((_jitc->function->call.call & jit_call_varargs) && call->w.w)
1085         call->flag |= jit_flag_varargs;
1086 #endif
1087     _jitc->function->call.argi = _jitc->function->call.argf = 0;
1088     _jitc->prepare = 0;
1089     jit_dec_synth();
1090 }
1091
1092 jit_node_t *
1093 _jit_finishi(jit_state_t *_jit, jit_pointer_t i0)
1094 {
1095     jit_node_t          *node;
1096     assert(_jitc->function);
1097     jit_inc_synth_w(finishi, (jit_word_t)i0);
1098     if (_jitc->function->self.alen < _jitc->function->call.size)
1099         _jitc->function->self.alen = _jitc->function->call.size;
1100     node = jit_calli(i0);
1101     node->v.w = _jitc->function->call.argi;
1102     node->w.w = _jitc->function->call.argf;
1103 #if _CALL_SYSV
1104     if ((_jitc->function->call.call & jit_call_varargs) && node->w.w)
1105         node->flag |= jit_flag_varargs;
1106 #endif
1107     _jitc->function->call.argi = _jitc->function->call.argf = 0;
1108     _jitc->prepare = 0;
1109     jit_dec_synth();
1110     return (node);
1111 }
1112
1113 void
1114 _jit_retval_c(jit_state_t *_jit, jit_int32_t r0)
1115 {
1116     jit_inc_synth(retval_c);
1117     jit_extr_c(r0, JIT_RET);
1118     jit_dec_synth();
1119 }
1120
1121 void
1122 _jit_retval_uc(jit_state_t *_jit, jit_int32_t r0)
1123 {
1124     jit_inc_synth(retval_uc);
1125     jit_extr_uc(r0, JIT_RET);
1126     jit_dec_synth();
1127 }
1128
1129 void
1130 _jit_retval_s(jit_state_t *_jit, jit_int32_t r0)
1131 {
1132     jit_inc_synth(retval_s);
1133     jit_extr_s(r0, JIT_RET);
1134     jit_dec_synth();
1135 }
1136
1137 void
1138 _jit_retval_us(jit_state_t *_jit, jit_int32_t r0)
1139 {
1140     jit_inc_synth(retval_us);
1141     jit_extr_us(r0, JIT_RET);
1142     jit_dec_synth();
1143 }
1144
1145 void
1146 _jit_retval_i(jit_state_t *_jit, jit_int32_t r0)
1147 {
1148     jit_inc_synth(retval_i);
1149 #if __WORDSIZE == 32
1150     if (r0 != JIT_RET)
1151         jit_movr(r0, JIT_RET);
1152 #else
1153     jit_extr_i(r0, JIT_RET);
1154 #endif
1155     jit_dec_synth();
1156 }
1157
1158 #if __WORDSIZE == 64
1159 void
1160 _jit_retval_ui(jit_state_t *_jit, jit_int32_t r0)
1161 {
1162     jit_inc_synth(retval_ui);
1163     jit_extr_ui(r0, JIT_RET);
1164     jit_dec_synth();
1165 }
1166
1167 void
1168 _jit_retval_l(jit_state_t *_jit, jit_int32_t r0)
1169 {
1170     jit_inc_synth(retval_l);
1171     if (r0 != JIT_RET)
1172         jit_movr(r0, JIT_RET);
1173     jit_dec_synth();
1174 }
1175 #endif
1176
1177 void
1178 _jit_retval_f(jit_state_t *_jit, jit_int32_t r0)
1179 {
1180     jit_inc_synth(retval_f);
1181     jit_retval_d(r0);
1182     jit_dec_synth();
1183 }
1184
1185 void
1186 _jit_retval_d(jit_state_t *_jit, jit_int32_t r0)
1187 {
1188     jit_inc_synth(retval_d);
1189     if (r0 != JIT_FRET)
1190         jit_movr_d(r0, JIT_FRET);
1191     jit_dec_synth();
1192 }
1193
1194 jit_pointer_t
1195 _emit_code(jit_state_t *_jit)
1196 {
1197     jit_node_t          *node;
1198     jit_node_t          *temp;
1199     jit_word_t           word;
1200     jit_int32_t          value;
1201     jit_int32_t          offset;
1202     jit_bool_t       no_flag = 0;       /* Set if previous instruction is
1203                                          * *not* a jump target. */
1204     struct {
1205         jit_node_t      *node;
1206         jit_word_t       word;
1207         jit_function_t   func;
1208 #if DEVEL_DISASSEMBLER
1209         jit_word_t       prevw;
1210 #endif
1211         jit_word_t       patch_offset;
1212 #if _CALL_AIXDESC
1213         jit_word_t       prolog_offset;
1214 #endif
1215     } undo;
1216 #if DEVEL_DISASSEMBLER
1217     jit_word_t           prevw;
1218 #endif
1219
1220     _jitc->function = NULL;
1221
1222     jit_reglive_setup();
1223
1224     undo.word = 0;
1225     undo.node = NULL;
1226     undo.patch_offset = 0;
1227
1228 #if DEVEL_DISASSEMBLER
1229     prevw = _jit->pc.w;
1230 #endif
1231 #if _CALL_AIXDESC
1232     undo.prolog_offset = 0;
1233     for (node = _jitc->head; node; node = node->next)
1234         if (node->code != jit_code_label &&
1235             node->code != jit_code_note &&
1236             node->code != jit_code_name)
1237             break;
1238     if (node && (node->code != jit_code_prolog ||
1239                  !(_jitc->functions.ptr + node->w.w)->assume_frame)) {
1240         /* code may start with a jump so add an initial function descriptor */
1241         word = _jit->pc.w + sizeof(void*) * 3;
1242         iw(word);                       /* addr */
1243         iw(0);                          /* toc */
1244         iw(0);                          /* env */
1245     }
1246 #endif
1247
1248 #define case_rr(name, type)                                             \
1249             case jit_code_##name##r##type:                              \
1250                 name##r##type(rn(node->u.w), rn(node->v.w));            \
1251                 break
1252 #define case_rw(name, type)                                             \
1253             case jit_code_##name##i##type:                              \
1254                 name##i##type(rn(node->u.w), node->v.w);                \
1255                 break
1256 #define case_wr(name, type)                                             \
1257             case jit_code_##name##i##type:                              \
1258                 name##i##type(node->u.w, rn(node->v.w));                \
1259                 break
1260 #define case_rrr(name, type)                                            \
1261             case jit_code_##name##r##type:                              \
1262                 name##r##type(rn(node->u.w),                            \
1263                               rn(node->v.w), rn(node->w.w));            \
1264                 break
1265 #define case_rrrr(name, type)                                           \
1266             case jit_code_##name##r##type:                              \
1267                 name##r##type(rn(node->u.q.l), rn(node->u.q.h),         \
1268                               rn(node->v.w), rn(node->w.w));            \
1269                 break
1270 #define case_rqr(name, type)                                            \
1271             case jit_code_##name##r##type:                              \
1272                 name##r##type(rn(node->u.w), rn(node->v.q.l),           \
1273                               rn(node->v.q.h), rn(node->w.w));          \
1274             case jit_code_##name##i##type:                              \
1275                 break;
1276 #define case_rrw(name, type)                                            \
1277             case jit_code_##name##i##type:                              \
1278                 name##i##type(rn(node->u.w), rn(node->v.w), node->w.w); \
1279                 break
1280 #define case_rrrw(name, type)                                           \
1281             case jit_code_##name##i##type:                              \
1282                 name##i##type(rn(node->u.q.l), rn(node->u.q.h),         \
1283                               rn(node->v.w), node->w.w);                \
1284                 break
1285 #define case_rrf(name, type, size)                                      \
1286             case jit_code_##name##i##type:                              \
1287                 assert(node->flag & jit_flag_data);                     \
1288                 name##i##type(rn(node->u.w), rn(node->v.w),             \
1289                               (jit_float##size##_t *)node->w.n->u.w);   \
1290                 break
1291 #define case_wrr(name, type)                                            \
1292             case jit_code_##name##i##type:                              \
1293                 name##i##type(node->u.w, rn(node->v.w), rn(node->w.w)); \
1294                 break
1295 #define case_brr(name, type)                                            \
1296             case jit_code_##name##r##type:                              \
1297                 temp = node->u.n;                                       \
1298                 assert(temp->code == jit_code_label ||                  \
1299                        temp->code == jit_code_epilog);                  \
1300                 if (temp->flag & jit_flag_patch)                        \
1301                     name##r##type(temp->u.w, rn(node->v.w),             \
1302                                   rn(node->w.w));                       \
1303                 else {                                                  \
1304                     word = name##r##type(_jit->pc.w,                    \
1305                                          rn(node->v.w), rn(node->w.w)); \
1306                     patch(word, node);                                  \
1307                 }                                                       \
1308                 break
1309 #define case_brw(name, type)                                            \
1310             case jit_code_##name##i##type:                              \
1311                 temp = node->u.n;                                       \
1312                 assert(temp->code == jit_code_label ||                  \
1313                        temp->code == jit_code_epilog);                  \
1314                 if (temp->flag & jit_flag_patch)                        \
1315                     name##i##type(temp->u.w,                            \
1316                                   rn(node->v.w), node->w.w);            \
1317                 else {                                                  \
1318                     word = name##i##type(_jit->pc.w,                    \
1319                                          rn(node->v.w), node->w.w);     \
1320                     patch(word, node);                                  \
1321                 }                                                       \
1322                 break
1323 #define case_brf(name, type, size)                                      \
1324             case jit_code_##name##i##type:                              \
1325                 temp = node->u.n;                                       \
1326                 assert(temp->code == jit_code_label ||                  \
1327                        temp->code == jit_code_epilog);                  \
1328                 if (temp->flag & jit_flag_patch)                        \
1329                     name##i##type(temp->u.w, rn(node->v.w),             \
1330                                 (jit_float##size##_t *)node->w.n->u.w); \
1331                 else {                                                  \
1332                     word = name##i##type(_jit->pc.w, rn(node->v.w),     \
1333                                 (jit_float##size##_t *)node->w.n->u.w); \
1334                     patch(word, node);                                  \
1335                 }                                                       \
1336                 break
1337     for (node = _jitc->head; node; node = node->next) {
1338         if (_jit->pc.uc >= _jitc->code.end)
1339             return (NULL);
1340
1341 #if DEVEL_DISASSEMBLER
1342         node->offset = (jit_uword_t)_jit->pc.w - (jit_uword_t)prevw;
1343         prevw = _jit->pc.w;
1344 #endif
1345         value = jit_classify(node->code);
1346         jit_regarg_set(node, value);
1347         switch (node->code) {
1348             case jit_code_align:
1349                 /* Must align to a power of two */
1350                 assert(!(node->u.w & (node->u.w - 1)));
1351                 if ((word = _jit->pc.w & (node->u.w - 1)))
1352                     nop(node->u.w - word);
1353                 break;
1354             case jit_code_skip:
1355                 nop((node->u.w + 3) & ~3);
1356                 break;
1357             case jit_code_note:         case jit_code_name:
1358                 node->u.w = _jit->pc.w;
1359                 break;
1360             case jit_code_label:
1361                 /* remember label is defined */
1362                 node->flag |= jit_flag_patch;
1363                 node->u.w = _jit->pc.w;
1364                 break;
1365                 case_rrr(add,);
1366                 case_rrw(add,);
1367                 case_rrr(addc,);
1368                 case_rrw(addc,);
1369                 case_rrr(addx,);
1370                 case_rrw(addx,);
1371                 case_rrr(sub,);
1372                 case_rrw(sub,);
1373                 case_rrr(subc,);
1374                 case_rrw(subc,);
1375                 case_rrr(subx,);
1376                 case_rrw(subx,);
1377                 case_rrw(rsb,);
1378                 case_rrr(mul,);
1379                 case_rrw(mul,);
1380                 case_rrr(hmul,);
1381                 case_rrw(hmul,);
1382                 case_rrr(hmul, _u);
1383                 case_rrw(hmul, _u);
1384                 case_rrrr(qmul,);
1385                 case_rrrw(qmul,);
1386                 case_rrrr(qmul, _u);
1387                 case_rrrw(qmul, _u);
1388                 case_rrr(div,);
1389                 case_rrw(div,);
1390                 case_rrr(div, _u);
1391                 case_rrw(div, _u);
1392                 case_rrrr(qdiv,);
1393                 case_rrrw(qdiv,);
1394                 case_rrrr(qdiv, _u);
1395                 case_rrrw(qdiv, _u);
1396                 case_rrr(rem,);
1397                 case_rrw(rem,);
1398                 case_rrr(rem, _u);
1399                 case_rrw(rem, _u);
1400                 case_rrr(and,);
1401                 case_rrw(and,);
1402                 case_rrr(or,);
1403                 case_rrw(or,);
1404                 case_rrr(xor,);
1405                 case_rrw(xor,);
1406                 case_rrr(lsh,);
1407                 case_rrw(lsh,);
1408                 case_rrrr(qlsh,);
1409                 case_rrrw(qlsh,);
1410                 case_rrrr(qlsh, _u);
1411                 case_rrrw(qlsh, _u);
1412                 case_rrr(rsh,);
1413                 case_rrw(rsh,);
1414                 case_rrr(rsh, _u);
1415                 case_rrw(rsh, _u);
1416                 case_rrrr(qrsh,);
1417                 case_rrrw(qrsh,);
1418                 case_rrrr(qrsh, _u);
1419                 case_rrrw(qrsh, _u);
1420                 case_rrr(lrot,);
1421                 case_rrw(lrot,);
1422                 case_rrr(rrot,);
1423                 case_rrw(rrot,);
1424             case jit_code_extr:
1425                 extr(rn(node->u.w), rn(node->v.w), node->w.q.l, node->w.q.h);
1426                 break;
1427             case jit_code_extr_u:
1428                 extr_u(rn(node->u.w), rn(node->v.w), node->w.q.l, node->w.q.h);
1429                 break;
1430             case jit_code_depr:
1431                 depr(rn(node->u.w), rn(node->v.w), node->w.q.l, node->w.q.h);
1432                 break;
1433             case jit_code_depi:
1434                 depi(rn(node->u.w), node->v.w, node->w.q.l, node->w.q.h);
1435                 break;
1436                 case_rr(ext, _c);
1437                 case_rr(ext, _uc);
1438                 case_rr(ext, _s);
1439                 case_rr(ext, _us);
1440 #  if __WORDSIZE == 64
1441                 case_rr(ext, _i);
1442                 case_rr(ext, _ui);
1443 #  endif
1444                 case_rr(hton, _us);
1445                 case_rr(hton, _ui);
1446 #  if __WORDSIZE == 64
1447                 case_rr(hton, _ul);
1448 #  endif
1449             case jit_code_bswapr_us:
1450                 bswapr_us_lh(rn(node->u.w), rn(node->v.w), no_flag);
1451                 break;
1452             case jit_code_bswapr_ui:
1453                 bswapr_ui_lw(rn(node->u.w), rn(node->v.w), no_flag);
1454                 break;
1455 #  if __WORDSIZE == 64
1456                 case_rr(bswap, _ul);
1457 #  endif
1458                 case_rr(neg,);
1459                 case_rr(com,);
1460                 case_rr(clo,);
1461                 case_rr(clz,);
1462                 case_rr(cto,);
1463                 case_rr(ctz,);
1464 #define rbitr(r0, r1)   fallback_rbit(r0, r1)
1465                 case_rr(rbit,);
1466                 case_rr(popcnt,);
1467             case jit_code_casr:
1468                 casr(rn(node->u.w), rn(node->v.w),
1469                      rn(node->w.q.l), rn(node->w.q.h));
1470                 break;
1471             case jit_code_casi:
1472                 casi(rn(node->u.w), node->v.w,
1473                      rn(node->w.q.l), rn(node->w.q.h));
1474                 break;
1475                 case_rrr(movn,);
1476                 case_rrr(movz,);
1477                 case_rr(mov,);
1478             case jit_code_movi:
1479                 if (node->flag & jit_flag_node) {
1480                     temp = node->v.n;
1481                     if (temp->code == jit_code_data ||
1482                         (temp->code == jit_code_label &&
1483                          (temp->flag & jit_flag_patch)))
1484                         movi(rn(node->u.w), temp->u.w);
1485                     else {
1486                         assert(temp->code == jit_code_label ||
1487                                temp->code == jit_code_epilog);
1488                         word = movi_p(rn(node->u.w), node->v.w);
1489                         patch(word, node);
1490                     }
1491                 }
1492                 else
1493                     movi(rn(node->u.w), node->v.w);
1494                 break;
1495                 case_rr(trunc, _f_i);
1496                 case_rr(trunc, _d_i);
1497 #  if __WORDSIZE == 64
1498                 case_rr(trunc, _f_l);
1499                 case_rr(trunc, _d_l);
1500 #  endif
1501                 case_rrr(lt,);
1502                 case_rrw(lt,);
1503                 case_rrr(lt, _u);
1504                 case_rrw(lt, _u);
1505                 case_rrr(le,);
1506                 case_rrw(le,);
1507                 case_rrr(le, _u);
1508                 case_rrw(le, _u);
1509                 case_rrr(eq,);
1510                 case_rrw(eq,);
1511                 case_rrr(ge,);
1512                 case_rrw(ge,);
1513                 case_rrr(ge, _u);
1514                 case_rrw(ge, _u);
1515                 case_rrr(gt,);
1516                 case_rrw(gt,);
1517                 case_rrr(gt, _u);
1518                 case_rrw(gt, _u);
1519                 case_rrr(ne,);
1520                 case_rrw(ne,);
1521                 case_rr(ld, _c);
1522                 case_rw(ld, _c);
1523                 case_brr(blt,);
1524                 case_brw(blt,);
1525                 case_brr(blt, _u);
1526                 case_brw(blt, _u);
1527                 case_brr(ble,);
1528                 case_brw(ble,);
1529                 case_brr(ble, _u);
1530                 case_brw(ble, _u);
1531                 case_brr(beq,);
1532                 case_brw(beq,);
1533                 case_brr(bge,);
1534                 case_brw(bge,);
1535                 case_brr(bge, _u);
1536                 case_brw(bge, _u);
1537                 case_brr(bgt,);
1538                 case_brw(bgt,);
1539                 case_brr(bgt, _u);
1540                 case_brw(bgt, _u);
1541                 case_brr(bne,);
1542                 case_brw(bne,);
1543                 case_brr(bms,);
1544                 case_brw(bms,);
1545                 case_brr(bmc,);
1546                 case_brw(bmc,);
1547                 case_brr(boadd,);
1548                 case_brw(boadd,);
1549                 case_brr(boadd, _u);
1550                 case_brw(boadd, _u);
1551                 case_brr(bxadd,);
1552                 case_brw(bxadd,);
1553                 case_brr(bxadd, _u);
1554                 case_brw(bxadd, _u);
1555                 case_brr(bosub,);
1556                 case_brw(bosub,);
1557                 case_brr(bosub, _u);
1558                 case_brw(bosub, _u);
1559                 case_brr(bxsub,);
1560                 case_brw(bxsub,);
1561                 case_brr(bxsub, _u);
1562                 case_brw(bxsub, _u);
1563                 case_rrr(ldx, _c);
1564                 case_rrw(ldx, _c);
1565                 case_rr(ld, _uc);
1566                 case_rw(ld, _uc);
1567                 case_rrr(ldx, _uc);
1568                 case_rrw(ldx, _uc);
1569                 case_rr(ld, _s);
1570                 case_rw(ld, _s);
1571                 case_rrr(ldx, _s);
1572                 case_rrw(ldx, _s);
1573                 case_rr(ld, _us);
1574                 case_rw(ld, _us);
1575                 case_rrr(ldx, _us);
1576                 case_rrw(ldx, _us);
1577                 case_rr(ld, _i);
1578                 case_rw(ld, _i);
1579                 case_rrr(ldx, _i);
1580                 case_rrw(ldx, _i);
1581 #if __WORDSIZE == 64
1582                 case_rr(ld, _ui);
1583                 case_rw(ld, _ui);
1584                 case_rrr(ldx, _ui);
1585                 case_rrw(ldx, _ui);
1586                 case_rr(ld, _l);
1587                 case_rw(ld, _l);
1588                 case_rrr(ldx, _l);
1589                 case_rrw(ldx, _l);
1590 #endif
1591 #define unldr(r0, r1, i0)       fallback_unldr(r0, r1, i0)
1592             case jit_code_unldr:
1593                 unldr(rn(node->u.w), rn(node->v.w), node->w.w);
1594                 break;
1595 #define unldi(r0, i0, i1)       fallback_unldi(r0, i0, i1)
1596             case jit_code_unldi:
1597                 unldi(rn(node->u.w), node->v.w, node->w.w);
1598                 break;
1599 #define unldr_u(r0, r1, i0)     fallback_unldr_u(r0, r1, i0)
1600             case jit_code_unldr_u:
1601                 unldr_u(rn(node->u.w), rn(node->v.w), node->w.w);
1602                 break;
1603 #define unldi_u(r0, i0, i1)     fallback_unldi_u(r0, i0, i1)
1604             case jit_code_unldi_u:
1605                 unldi_u(rn(node->u.w), node->v.w, node->w.w);
1606                 break;
1607                 case_rr(st, _c);
1608                 case_wr(st, _c);
1609                 case_rrr(stx, _c);
1610                 case_wrr(stx, _c);
1611                 case_rr(st, _s);
1612                 case_wr(st, _s);
1613                 case_rrr(stx, _s);
1614                 case_wrr(stx, _s);
1615                 case_rr(st, _i);
1616                 case_wr(st, _i);
1617                 case_rrr(stx, _i);
1618                 case_wrr(stx, _i);
1619 #if __WORDSIZE == 64
1620                 case_rr(st, _l);
1621                 case_wr(st, _l);
1622                 case_rrr(stx, _l);
1623                 case_wrr(stx, _l);
1624 #endif
1625 #define unstr(r0, r1, i0)       fallback_unstr(r0, r1, i0)
1626             case jit_code_unstr:
1627                 unstr(rn(node->u.w), rn(node->v.w), node->w.w);
1628                 break;
1629 #define unsti(i0, r0, i1)       fallback_unsti(i0, r0, i1)
1630             case jit_code_unsti:
1631                 unsti(node->u.w, rn(node->v.w), node->w.w);
1632                 break;
1633                 case_rr(mov, _f);
1634             case jit_code_movi_f:
1635                 assert(node->flag & jit_flag_data);
1636                 movi_f(rn(node->u.w), (jit_float32_t *)node->v.n->u.w);
1637                 break;
1638                 case_rr(ext, _f);
1639                 case_rr(ext, _d_f);
1640                 case_rr(abs, _f);
1641                 case_rr(neg, _f);
1642                 case_rr(sqrt, _f);
1643                 case_rqr(fma, _f);
1644                 case_rqr(fms, _f);
1645                 case_rqr(fnma, _f);
1646                 case_rqr(fnms, _f);
1647                 case_rrr(add, _f);
1648                 case_rrf(add, _f, 32);
1649                 case_rrr(sub, _f);
1650                 case_rrf(sub, _f, 32);
1651                 case_rrf(rsb, _f, 32);
1652                 case_rrr(mul, _f);
1653                 case_rrf(mul, _f, 32);
1654                 case_rrr(div, _f);
1655                 case_rrf(div, _f, 32);
1656                 case_rrr(lt, _f);
1657                 case_rrf(lt, _f, 32);
1658                 case_rrr(le, _f);
1659                 case_rrf(le, _f, 32);
1660                 case_rrr(eq, _f);
1661                 case_rrf(eq, _f, 32);
1662                 case_rrr(ge, _f);
1663                 case_rrf(ge, _f, 32);
1664                 case_rrr(gt, _f);
1665                 case_rrf(gt, _f, 32);
1666                 case_rrr(ne, _f);
1667                 case_rrf(ne, _f, 32);
1668                 case_rrr(unlt, _f);
1669                 case_rrf(unlt, _f, 32);
1670                 case_rrr(unle, _f);
1671                 case_rrf(unle, _f, 32);
1672                 case_rrr(uneq, _f);
1673                 case_rrf(uneq, _f, 32);
1674                 case_rrr(unge, _f);
1675                 case_rrf(unge, _f, 32);
1676                 case_rrr(ungt, _f);
1677                 case_rrf(ungt, _f, 32);
1678                 case_rrr(ltgt, _f);
1679                 case_rrf(ltgt, _f, 32);
1680                 case_rrr(ord, _f);
1681                 case_rrf(ord, _f, 32);
1682                 case_rrr(unord, _f);
1683                 case_rrf(unord, _f, 32);
1684                 case_brr(blt, _f);
1685                 case_brf(blt, _f, 32);
1686                 case_brr(ble, _f);
1687                 case_brf(ble, _f, 32);
1688                 case_brr(beq, _f);
1689                 case_brf(beq, _f, 32);
1690                 case_brr(bge, _f);
1691                 case_brf(bge, _f, 32);
1692                 case_brr(bgt, _f);
1693                 case_brf(bgt, _f, 32);
1694                 case_brr(bne, _f);
1695                 case_brf(bne, _f, 32);
1696                 case_brr(bunlt, _f);
1697                 case_brf(bunlt, _f, 32);
1698                 case_brr(bunle, _f);
1699                 case_brf(bunle, _f, 32);
1700                 case_brr(buneq, _f);
1701                 case_brf(buneq, _f, 32);
1702                 case_brr(bunge, _f);
1703                 case_brf(bunge, _f, 32);
1704                 case_brr(bungt, _f);
1705                 case_brf(bungt, _f, 32);
1706                 case_brr(bltgt, _f);
1707                 case_brf(bltgt, _f, 32);
1708                 case_brr(bord, _f);
1709                 case_brf(bord, _f, 32);
1710                 case_brr(bunord, _f);
1711                 case_brf(bunord, _f, 32);
1712                 case_rr(ld, _f);
1713                 case_rw(ld, _f);
1714                 case_rrr(ldx, _f);
1715                 case_rrw(ldx, _f);
1716 #define unldr_x(r0, r1, i0)     fallback_unldr_x(r0, r1, i0)
1717             case jit_code_unldr_x:
1718                 unldr_x(rn(node->u.w), rn(node->v.w), node->w.w);
1719                 break;
1720 #define unldi_x(r0, i0, i1)     fallback_unldi_x(r0, i0, i1)
1721             case jit_code_unldi_x:
1722                 unldi_x(rn(node->u.w), node->v.w, node->w.w);
1723                 break;
1724                 case_rr(st, _f);
1725                 case_wr(st, _f);
1726                 case_rrr(stx, _f);
1727                 case_wrr(stx, _f);
1728 #define unstr_x(r0, r1, i0)     fallback_unstr_x(r0, r1, i0)
1729             case jit_code_unstr_x:
1730                 unstr_x(rn(node->u.w), rn(node->v.w), node->w.w);
1731                 break;
1732 #define unsti_x(i0, r0, i1)     fallback_unsti_x(i0, r0, i1)
1733             case jit_code_unsti_x:
1734                 unsti_x(node->u.w, rn(node->v.w), node->w.w);
1735                 break;
1736                 case_rr(mov, _d);
1737             case jit_code_movi_d:
1738                 assert(node->flag & jit_flag_data);
1739                 movi_d(rn(node->u.w), (jit_float64_t *)node->v.n->u.w);
1740                 break;
1741                 case_rr(ext, _d);
1742                 case_rr(ext, _f_d);
1743                 case_rr(abs, _d);
1744                 case_rr(neg, _d);
1745                 case_rr(sqrt, _d);
1746                 case_rqr(fma, _d);
1747                 case_rqr(fms, _d);
1748                 case_rqr(fnma, _d);
1749                 case_rqr(fnms, _d);
1750                 case_rrr(add, _d);
1751                 case_rrf(add, _d, 64);
1752                 case_rrr(sub, _d);
1753                 case_rrf(sub, _d, 64);
1754                 case_rrf(rsb, _d, 64);
1755                 case_rrr(mul, _d);
1756                 case_rrf(mul, _d, 64);
1757                 case_rrr(div, _d);
1758                 case_rrf(div, _d, 64);
1759                 case_rrr(lt, _d);
1760                 case_rrf(lt, _d, 64);
1761                 case_rrr(le, _d);
1762                 case_rrf(le, _d, 64);
1763                 case_rrr(eq, _d);
1764                 case_rrf(eq, _d, 64);
1765                 case_rrr(ge, _d);
1766                 case_rrf(ge, _d, 64);
1767                 case_rrr(gt, _d);
1768                 case_rrf(gt, _d, 64);
1769                 case_rrr(ne, _d);
1770                 case_rrf(ne, _d, 64);
1771                 case_rrr(unlt, _d);
1772                 case_rrf(unlt, _d, 64);
1773                 case_rrr(unle, _d);
1774                 case_rrf(unle, _d, 64);
1775                 case_rrr(uneq, _d);
1776                 case_rrf(uneq, _d, 64);
1777                 case_rrr(unge, _d);
1778                 case_rrf(unge, _d, 64);
1779                 case_rrr(ungt, _d);
1780                 case_rrf(ungt, _d, 64);
1781                 case_rrr(ltgt, _d);
1782                 case_rrf(ltgt, _d, 64);
1783                 case_rrr(ord, _d);
1784                 case_rrf(ord, _d, 64);
1785                 case_rrr(unord, _d);
1786                 case_rrf(unord, _d, 64);
1787                 case_brr(blt, _d);
1788                 case_brf(blt, _d, 64);
1789                 case_brr(ble, _d);
1790                 case_brf(ble, _d, 64);
1791                 case_brr(beq, _d);
1792                 case_brf(beq, _d, 64);
1793                 case_brr(bge, _d);
1794                 case_brf(bge, _d, 64);
1795                 case_brr(bgt, _d);
1796                 case_brf(bgt, _d, 64);
1797                 case_brr(bne, _d);
1798                 case_brf(bne, _d, 64);
1799                 case_brr(bunlt, _d);
1800                 case_brf(bunlt, _d, 64);
1801                 case_brr(bunle, _d);
1802                 case_brf(bunle, _d, 64);
1803                 case_brr(buneq, _d);
1804                 case_brf(buneq, _d, 64);
1805                 case_brr(bunge, _d);
1806                 case_brf(bunge, _d, 64);
1807                 case_brr(bungt, _d);
1808                 case_brf(bungt, _d, 64);
1809                 case_brr(bltgt, _d);
1810                 case_brf(bltgt, _d, 64);
1811                 case_brr(bord, _d);
1812                 case_brf(bord, _d, 64);
1813                 case_brr(bunord, _d);
1814                 case_brf(bunord, _d, 64);
1815                 case_rr(ld, _d);
1816                 case_rw(ld, _d);
1817                 case_rrr(ldx, _d);
1818                 case_rrw(ldx, _d);
1819                 case_rr(st, _d);
1820                 case_wr(st, _d);
1821                 case_rrr(stx, _d);
1822                 case_wrr(stx, _d);
1823             case jit_code_jmpr:
1824                 jit_check_frame();
1825                 jmpr(rn(node->u.w));
1826                 break;
1827             case jit_code_jmpi:
1828                 if (node->flag & jit_flag_node) {
1829 #if _CALL_AIXDESC
1830                     if (_jit->pc.uc == _jit->code.ptr + sizeof(void*) * 3)
1831                         _jitc->jump = 1;
1832 #endif
1833                     temp = node->u.n;
1834                     assert(temp->code == jit_code_label ||
1835                            temp->code == jit_code_epilog);
1836                     if (temp->flag & jit_flag_patch)
1837                         jmpi(temp->u.w);
1838                     else {
1839                         word = _jit->code.length -
1840                             (_jit->pc.uc - _jit->code.ptr);
1841                         if (can_sign_extend_jump_p(word))
1842                             word = jmpi(_jit->pc.w);
1843                         else
1844                             word = jmpi_p(_jit->pc.w);
1845                         patch(word, node);
1846                     }
1847                 }
1848                 else {
1849                     jit_check_frame();
1850                     jmpi(node->u.w);
1851                 }
1852                 break;
1853             case jit_code_callr:
1854 #if _CALL_SYSV
1855 #  define xcallr(u, v)          callr(u, v)
1856 #  define xcalli_p(u, v)        calli_p(u, v)
1857 #  define xcalli(u, v)          calli(u, v)
1858 #else
1859 #  define xcallr(u, v)          callr(u)
1860 #  define xcalli_p(u, v)        calli_p(u)
1861 #  define xcalli(u, v)          calli(u)
1862 #endif
1863                 jit_check_frame();
1864                 xcallr(rn(node->u.w), !!(node->flag & jit_flag_varargs));
1865                 break;
1866             case jit_code_calli:
1867                 value = !!(node->flag & jit_flag_varargs);
1868                 if (node->flag & jit_flag_node) {
1869                     _jitc->function->need_return = 1;
1870                     temp = node->u.n;
1871                     assert(temp->code == jit_code_label ||
1872                            temp->code == jit_code_epilog);
1873                     if (temp->flag & jit_flag_patch)
1874                         xcalli(temp->u.w, value);
1875                     else {
1876                         word = _jit->code.length -
1877                             (_jit->pc.uc - _jit->code.ptr);
1878 #if _CALL_SYSV
1879                         if (can_sign_extend_jump_p(word + value * 4))
1880                             word = xcalli(_jit->pc.w, value);
1881                         else
1882 #endif
1883                             word = xcalli_p(_jit->pc.w, value);
1884                         patch(word, node);
1885                     }
1886                 }
1887                 else {
1888                     jit_check_frame();
1889                     xcalli(node->u.w, value);
1890                 }
1891                 break;
1892             case jit_code_prolog:
1893                 _jitc->function = _jitc->functions.ptr + node->w.w;
1894                 undo.node = node;
1895                 undo.word = _jit->pc.w;
1896                 memcpy(&undo.func, _jitc->function, sizeof(undo.func));
1897 #if DEVEL_DISASSEMBLER
1898                 undo.prevw = prevw;
1899 #endif
1900                 undo.patch_offset = _jitc->patches.offset;
1901 #if _CALL_AIXDESC
1902                 undo.prolog_offset = _jitc->prolog.offset;
1903 #endif
1904             restart_function:
1905                 _jitc->again = 0;
1906 #if _CALL_AIXDESC
1907                 if (_jitc->jump && !_jitc->function->assume_frame) {
1908                     /* remember prolog to hide offset adjustment for a jump
1909                      * to the start of a function, what is expected to be
1910                      * a common practice as first jit instruction */
1911                     if (_jitc->prolog.offset >= _jitc->prolog.length) {
1912                         _jitc->prolog.length += 16;
1913                         jit_realloc((jit_pointer_t *)&_jitc->prolog.ptr,
1914                                     (_jitc->prolog.length - 16) *
1915                                     sizeof(jit_word_t),
1916                                     _jitc->prolog.length * sizeof(jit_word_t));
1917                     }
1918                     _jitc->prolog.ptr[_jitc->prolog.offset++] = _jit->pc.w;
1919                     /* function descriptor */
1920                     word = _jit->pc.w + sizeof(void*) * 3;
1921                     iw(word);                           /* addr */
1922                     iw(0);                              /* toc */
1923                     iw(0);                              /* env */
1924                 }
1925 #endif
1926                 prolog(node);
1927                 break;
1928             case jit_code_epilog:
1929                 assert(_jitc->function == _jitc->functions.ptr + node->w.w);
1930                 if (_jitc->again) {
1931                     for (temp = undo.node->next;
1932                          temp != node; temp = temp->next) {
1933                         if (temp->code == jit_code_label ||
1934                             temp->code == jit_code_epilog)
1935                             temp->flag &= ~jit_flag_patch;
1936                     }
1937                     temp->flag &= ~jit_flag_patch;
1938                     node = undo.node;
1939                     _jit->pc.w = undo.word;
1940                     /* undo.func.self.aoff and undo.func.regset should not
1941                      * be undone, as they will be further updated, and are
1942                      * the reason of the undo. */
1943                     undo.func.self.aoff = _jitc->function->frame +
1944                         _jitc->function->self.aoff;
1945                     undo.func.need_frame = _jitc->function->need_frame;
1946                     undo.func.need_stack = _jitc->function->need_stack;
1947                     undo.func.need_return = _jitc->function->need_return;
1948                     jit_regset_set(&undo.func.regset, &_jitc->function->regset);
1949                     /* allocar information also does not need to be undone */
1950                     undo.func.aoffoff = _jitc->function->aoffoff;
1951                     undo.func.allocar = _jitc->function->allocar;
1952                     /* cvt_offset must also not be undone */
1953                     undo.func.cvt_offset = _jitc->function->cvt_offset;
1954                     memcpy(_jitc->function, &undo.func, sizeof(undo.func));
1955 #if DEVEL_DISASSEMBLER
1956                     prevw = undo.prevw;
1957 #endif
1958                     _jitc->patches.offset = undo.patch_offset;
1959 #if _CALL_AIXDESC
1960                     _jitc->prolog.offset = undo.prolog_offset;
1961 #endif
1962                     goto restart_function;
1963                 }
1964                 /* remember label is defined */
1965                 node->flag |= jit_flag_patch;
1966                 node->u.w = _jit->pc.w;
1967                 epilog(node);
1968                 _jitc->function = NULL;
1969                 break;
1970             case jit_code_movr_w_f:
1971                 movr_w_f(rn(node->u.w), rn(node->v.w));
1972                 break;
1973             case jit_code_movr_f_w:
1974                 movr_f_w(rn(node->u.w), rn(node->v.w));
1975                 break;
1976             case jit_code_movi_f_w:
1977                 assert(node->flag & jit_flag_data);
1978                 movi_f_w(rn(node->u.w), *(jit_float32_t *)node->v.n->u.w);
1979                 break;
1980             case jit_code_movi_w_f:
1981                 movi_w_f(rn(node->u.w), node->v.w);
1982                 break;
1983 #if __WORDSIZE == 64
1984             case jit_code_movr_d_w:
1985                 movr_d_w(rn(node->u.w), rn(node->v.w));
1986                 break;
1987             case jit_code_movi_d_w:
1988                 assert(node->flag & jit_flag_data);
1989                 movi_d_w(rn(node->u.w), *(jit_float64_t *)node->v.n->u.w);
1990                 break;
1991             case jit_code_movr_w_d:
1992                 movr_w_d(rn(node->u.w), rn(node->v.w));
1993                 break;
1994             case jit_code_movi_w_d:
1995                 movi_w_d(rn(node->u.w), node->v.w);
1996                 break;
1997 #else
1998             case jit_code_movr_ww_d:
1999                 movr_ww_d(rn(node->u.w), rn(node->v.w), rn(node->w.w));
2000                 break;
2001             case jit_code_movr_d_ww:
2002                 movr_d_ww(rn(node->u.w), rn(node->v.w), rn(node->w.w));
2003                 break;
2004             case jit_code_movi_d_ww:
2005                 assert(node->flag & jit_flag_data);
2006                 movi_d_ww(rn(node->u.w), rn(node->v.w),
2007                           *(jit_float64_t *)node->w.n->u.w);
2008                 break;
2009             case jit_code_movi_ww_d:
2010                 movi_ww_d(rn(node->u.w), node->v.w, node->w.w);
2011                 break;
2012 #endif
2013             case jit_code_va_start:
2014                 vastart(rn(node->u.w));
2015                 break;
2016             case jit_code_va_arg:
2017                 vaarg(rn(node->u.w), rn(node->v.w));
2018                 break;
2019             case jit_code_va_arg_d:
2020                 vaarg_d(rn(node->u.w), rn(node->v.w));
2021                 break;
2022             case jit_code_live:                 case jit_code_ellipsis:
2023             case jit_code_va_push:
2024             case jit_code_allocai:              case jit_code_allocar:
2025             case jit_code_arg_c:                case jit_code_arg_s:
2026             case jit_code_arg_i:
2027 #  if __WORDSIZE == 64
2028             case jit_code_arg_l:
2029 #  endif
2030             case jit_code_arg_f:                case jit_code_arg_d:
2031             case jit_code_va_end:
2032             case jit_code_ret:
2033             case jit_code_retr_c:               case jit_code_reti_c:
2034             case jit_code_retr_uc:              case jit_code_reti_uc:
2035             case jit_code_retr_s:               case jit_code_reti_s:
2036             case jit_code_retr_us:              case jit_code_reti_us:
2037             case jit_code_retr_i:               case jit_code_reti_i:
2038 #if __WORDSIZE == 64
2039             case jit_code_retr_ui:              case jit_code_reti_ui:
2040             case jit_code_retr_l:               case jit_code_reti_l:
2041 #endif
2042             case jit_code_retr_f:               case jit_code_reti_f:
2043             case jit_code_retr_d:               case jit_code_reti_d:
2044             case jit_code_getarg_c:             case jit_code_getarg_uc:
2045             case jit_code_getarg_s:             case jit_code_getarg_us:
2046             case jit_code_getarg_i:
2047 #if __WORDSIZE == 64
2048             case jit_code_getarg_ui:            case jit_code_getarg_l:
2049 #endif
2050             case jit_code_getarg_f:             case jit_code_getarg_d:
2051             case jit_code_putargr_c:            case jit_code_putargi_c:
2052             case jit_code_putargr_uc:           case jit_code_putargi_uc:
2053             case jit_code_putargr_s:            case jit_code_putargi_s:
2054             case jit_code_putargr_us:           case jit_code_putargi_us:
2055             case jit_code_putargr_i:            case jit_code_putargi_i:
2056 #if __WORDSIZE == 64
2057             case jit_code_putargr_ui:           case jit_code_putargi_ui:
2058             case jit_code_putargr_l:            case jit_code_putargi_l:
2059 #endif
2060             case jit_code_putargr_f:            case jit_code_putargi_f:
2061             case jit_code_putargr_d:            case jit_code_putargi_d:
2062             case jit_code_pushargr_c:           case jit_code_pushargi_c:
2063             case jit_code_pushargr_uc:          case jit_code_pushargi_uc:
2064             case jit_code_pushargr_s:           case jit_code_pushargi_s:
2065             case jit_code_pushargr_us:          case jit_code_pushargi_us:
2066             case jit_code_pushargr_i:           case jit_code_pushargi_i:
2067 #if __WORDSIZE == 64
2068             case jit_code_pushargr_ui:          case jit_code_pushargi_ui:
2069             case jit_code_pushargr_l:           case jit_code_pushargi_l:
2070 #endif
2071             case jit_code_pushargr_f:           case jit_code_pushargi_f:
2072             case jit_code_pushargr_d:           case jit_code_pushargi_d:
2073             case jit_code_retval_c:             case jit_code_retval_uc:
2074             case jit_code_retval_s:             case jit_code_retval_us:
2075             case jit_code_retval_i:
2076 #if __WORDSIZE == 64
2077             case jit_code_retval_ui:            case jit_code_retval_l:
2078 #endif
2079             case jit_code_retval_f:             case jit_code_retval_d:
2080             case jit_code_prepare:
2081             case jit_code_finishr:              case jit_code_finishi:
2082             case jit_code_negi_f:               case jit_code_absi_f:
2083             case jit_code_sqrti_f:              case jit_code_negi_d:
2084             case jit_code_absi_d:               case jit_code_sqrti_d:
2085                 break;
2086             case jit_code_negi:
2087                 negi(rn(node->u.w), node->v.w);
2088                 break;
2089             case jit_code_comi:
2090                 comi(rn(node->u.w), node->v.w);
2091                 break;
2092             case jit_code_exti_c:
2093                 exti_c(rn(node->u.w), node->v.w);
2094                 break;
2095             case jit_code_exti_uc:
2096                 exti_uc(rn(node->u.w), node->v.w);
2097                 break;
2098             case jit_code_exti_s:
2099                 exti_s(rn(node->u.w), node->v.w);
2100                 break;
2101             case jit_code_exti_us:
2102                 exti_us(rn(node->u.w), node->v.w);
2103                 break;
2104             case jit_code_bswapi_us:
2105                 bswapi_us(rn(node->u.w), node->v.w);
2106                 break;
2107             case jit_code_bswapi_ui:
2108                 bswapi_ui(rn(node->u.w), node->v.w);
2109                 break;
2110             case jit_code_htoni_us:
2111                 htoni_us(rn(node->u.w), node->v.w);
2112                 break;
2113             case jit_code_htoni_ui:
2114                 htoni_ui(rn(node->u.w), node->v.w);
2115                 break;
2116 #if __WORDSIZE == 64
2117             case jit_code_exti_i:
2118                 exti_i(rn(node->u.w), node->v.w);
2119                 break;
2120             case jit_code_exti_ui:
2121                 exti_ui(rn(node->u.w), node->v.w);
2122                 break;
2123             case jit_code_bswapi_ul:
2124                 bswapi_ul(rn(node->u.w), node->v.w);
2125                 break;
2126             case jit_code_htoni_ul:
2127                 htoni_ul(rn(node->u.w), node->v.w);
2128                 break;
2129 #endif
2130             case jit_code_cloi:
2131                 cloi(rn(node->u.w), node->v.w);
2132                 break;
2133             case jit_code_clzi:
2134                 clzi(rn(node->u.w), node->v.w);
2135                 break;
2136             case jit_code_ctoi:
2137                 ctoi(rn(node->u.w), node->v.w);
2138                 break;
2139             case jit_code_ctzi:
2140                 ctzi(rn(node->u.w), node->v.w);
2141                 break;
2142             case jit_code_rbiti:
2143                 rbiti(rn(node->u.w), node->v.w);
2144                 break;
2145             case jit_code_popcnti:
2146                 popcnti(rn(node->u.w), node->v.w);
2147                 break;
2148             case jit_code_exti:
2149                 exti(rn(node->u.w), node->v.w, node->w.q.l, node->w.q.h);
2150                 break;
2151             case jit_code_exti_u:
2152                 exti_u(rn(node->u.w), node->v.w, node->w.q.l, node->w.q.h);
2153                 break;
2154             default:
2155                 abort();
2156         }
2157         jit_regarg_clr(node, value);
2158         assert(_jitc->regarg == 0 && _jitc->synth == 0);
2159         /* update register live state */
2160         jit_reglive(node);
2161
2162         no_flag = !(node->flag & jit_flag_patch);
2163     }
2164 #undef case_brf
2165 #undef case_brw
2166 #undef case_brr
2167 #undef case_wrr
2168 #undef case_rrf
2169 #undef case_rrw
2170 #undef case_rrr
2171 #undef case_wr
2172 #undef case_rw
2173 #undef case_rr
2174
2175     for (offset = 0; offset < _jitc->patches.offset; offset++) {
2176         node = _jitc->patches.ptr[offset].node;
2177         word = node->code == jit_code_movi ? node->v.n->u.w : node->u.n->u.w;
2178         patch_at(_jitc->patches.ptr[offset].inst, word);
2179     }
2180
2181     jit_flush(_jit->code.ptr, _jit->pc.uc);
2182
2183     return (_jit->code.ptr);
2184 }
2185
2186 #define CODE                            1
2187 #  include "jit_ppc-cpu.c"
2188 #  include "jit_ppc-fpu.c"
2189 #  include "jit_fallback.c"
2190 #undef CODE
2191
2192 void
2193 jit_flush(void *fptr, void *tptr)
2194 {
2195 #if defined(__GNUC__)
2196     jit_word_t          f, t, s;
2197
2198     s = sysconf(_SC_PAGE_SIZE);
2199     f = (jit_word_t)fptr & -s;
2200     t = (((jit_word_t)tptr) + s - 1) & -s;
2201     __clear_cache((void *)f, (void *)t);
2202 #endif
2203 }
2204
2205 void
2206 _emit_ldxi(jit_state_t *_jit, jit_int32_t r0, jit_int32_t r1, jit_word_t i0)
2207 {
2208 #if __WORDSIZE == 32
2209     ldxi_i(rn(r0), rn(r1), i0);
2210 #else
2211     ldxi_l(rn(r0), rn(r1), i0);
2212 #endif
2213 }
2214
2215 void
2216 _emit_stxi(jit_state_t *_jit, jit_word_t i0, jit_int32_t r0, jit_int32_t r1)
2217 {
2218 #if __WORDSIZE == 32
2219     stxi_i(i0, rn(r0), rn(r1));
2220 #else
2221     stxi_l(i0, rn(r0), rn(r1));
2222 #endif
2223 }
2224
2225 void
2226 _emit_ldxi_d(jit_state_t *_jit, jit_int32_t r0, jit_int32_t r1, jit_word_t i0)
2227 {
2228     ldxi_d(rn(r0), rn(r1), i0);
2229 }
2230
2231 void
2232 _emit_stxi_d(jit_state_t *_jit, jit_word_t i0, jit_int32_t r0, jit_int32_t r1)
2233 {
2234     stxi_d(i0, rn(r0), rn(r1));
2235 }
2236
2237 static void
2238 _patch(jit_state_t *_jit, jit_word_t instr, jit_node_t *node)
2239 {
2240     jit_int32_t          flag;
2241
2242     assert(node->flag & jit_flag_node);
2243     if (node->code == jit_code_movi)
2244         flag = node->v.n->flag;
2245     else
2246         flag = node->u.n->flag;
2247     assert(!(flag & jit_flag_patch));
2248     if (_jitc->patches.offset >= _jitc->patches.length) {
2249         jit_realloc((jit_pointer_t *)&_jitc->patches.ptr,
2250                     _jitc->patches.length * sizeof(jit_patch_t),
2251                     (_jitc->patches.length + 1024) * sizeof(jit_patch_t));
2252         _jitc->patches.length += 1024;
2253     }
2254     _jitc->patches.ptr[_jitc->patches.offset].inst = instr;
2255     _jitc->patches.ptr[_jitc->patches.offset].node = node;
2256     ++_jitc->patches.offset;
2257 }