deps/lightning/lib/jit_ppc.c

   1 /*
   2  * Copyright (C) 2012-2023  Free Software Foundation, Inc.
   3  *
   4  * This file is part of GNU lightning.
   5  *
   6  * GNU lightning is free software; you can redistribute it and/or modify it
   7  * under the terms of the GNU Lesser General Public License as published
   8  * by the Free Software Foundation; either version 3, or (at your option)
   9  * any later version.
  10  *
  11  * GNU lightning is distributed in the hope that it will be useful, but
  12  * WITHOUT ANY WARRANTY; without even the implied warranty of MERCHANTABILITY
  13  * or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU Lesser General Public
  14  * License for more details.
  15  *
  16  * Authors:
  17  *      Paulo Cesar Pereira de Andrade
  18  */
  19
  20 #define jit_arg_reg_p(i)                ((i) >= 0 && (i) < 8)
  21 #if !_CALL_SYSV
  22 #  define jit_arg_f_reg_p(i)            ((i) >= 0 && (i) < 13)
  23 #else
  24 #  define jit_arg_f_reg_p(i)            ((i) >= 0 && (i) < 8)
  25 #  if __WORDSIZE == 32
  26 #    define va_gp_shift                 2
  27 #  else
  28 #    define va_gp_shift                 3
  29 #  endif
  30 #  define va_gp_increment               sizeof(jit_word_t)
  31 #  define first_gp_argument             r3
  32 #  define first_gp_offset               offsetof(jit_va_list_t,         \
  33                                                  first_gp_argument)
  34 #  define va_fp_increment               sizeof(jit_float64_t)
  35 #  define first_fp_argument             f1
  36 #  define first_fp_offset               offsetof(jit_va_list_t,         \
  37                                                  first_fp_argument)
  38 #endif
  39 #if __BYTE_ORDER == __LITTLE_ENDIAN
  40 #  define C_DISP                        0
  41 #  define S_DISP                        0
  42 #  define I_DISP                        0
  43 #  define F_DISP                        0
  44 #else
  45 #  define C_DISP                        (__WORDSIZE >> 3) - sizeof(jit_int8_t)
  46 #  define S_DISP                        (__WORDSIZE >> 3) - sizeof(jit_int16_t)
  47 #  define I_DISP                        (__WORDSIZE >> 3) - sizeof(jit_int32_t)
  48 #  define F_DISP                        (__WORDSIZE >> 3) - sizeof(jit_float32_t)
  49 #endif
  50
  51 /*
  52  * Types
  53  */
  54 #if _CALL_SYSV
  55 typedef struct jit_va_list {
  56     jit_uint8_t         ngpr;
  57     jit_uint8_t         nfpr;
  58     jit_uint16_t        _pad;
  59 #  if __WORDSIZE == 64
  60     jit_uint32_t        _pad2;
  61 #  endif
  62     jit_pointer_t       over;
  63     jit_pointer_t       save;
  64 #  if __WORDSIZE == 32
  65     jit_word_t          _pad2;
  66 #  endif
  67     jit_word_t          r3;
  68     jit_word_t          r4;
  69     jit_word_t          r5;
  70     jit_word_t          r6;
  71     jit_word_t          r7;
  72     jit_word_t          r8;
  73     jit_word_t          r9;
  74     jit_word_t          r10;
  75     jit_float64_t       f1;
  76     jit_float64_t       f2;
  77     jit_float64_t       f3;
  78     jit_float64_t       f4;
  79     jit_float64_t       f5;
  80     jit_float64_t       f6;
  81     jit_float64_t       f7;
  82     jit_float64_t       f8;
  83 } jit_va_list_t;
  84 #else
  85 typedef jit_pointer_t jit_va_list_t;
  86 #endif
  87
  88 /*
  89  * Prototypes
  90  */
  91 #define patch(instr, node)              _patch(_jit, instr, node)
  92 static void _patch(jit_state_t*,jit_word_t,jit_node_t*);
  93
  94 /* libgcc */
  95 extern void __clear_cache(void *, void *);
  96
  97 #define PROTO                           1
  98 #  include "jit_ppc-cpu.c"
  99 #  include "jit_ppc-fpu.c"
 100 #undef PROTO
 101
 102 /*
 103  * Initialization
 104  */
 105 jit_register_t          _rvs[] = {
 106     { rc(sav) | 0,                      "r0" },
 107     { rc(sav) | 11,                     "r11" },        /* env */
 108     { rc(sav) | 12,                     "r12" },        /* exception */
 109     { rc(sav) | 13,                     "r13" },        /* thread */
 110     { rc(sav) | 2,                      "r2" },         /* toc */
 111     { rc(sav) | rc(gpr) | 14,           "r14" },
 112     { rc(sav) | rc(gpr) | 15,           "r15" },
 113     { rc(sav) | rc(gpr) | 16,           "r16" },
 114     { rc(sav) | rc(gpr) | 17,           "r17" },
 115     { rc(sav) | rc(gpr) | 18,           "r18" },
 116     { rc(sav) | rc(gpr) | 19,           "r19" },
 117     { rc(sav) | rc(gpr) | 20,           "r20" },
 118     { rc(sav) | rc(gpr) | 21,           "r21" },
 119     { rc(sav) | rc(gpr) | 22,           "r22" },
 120     { rc(sav) | rc(gpr) | 23,           "r23" },
 121     { rc(sav) | rc(gpr) | 24,           "r24" },
 122     { rc(sav) | rc(gpr) | 25,           "r25" },
 123     { rc(sav) | rc(gpr) | 26,           "r26" },
 124     { rc(sav) | rc(gpr) | 27,           "r27" },
 125     { rc(sav) | rc(gpr) | 28,           "r28" },
 126     { rc(sav) | rc(gpr) | 29,           "r29" },
 127     { rc(sav) | rc(gpr) | 30,           "r30" },
 128     { rc(sav) | 1,                      "r1" },
 129     { rc(sav) | 31,                     "r31" },
 130     { rc(arg) | rc(gpr) | 10,           "r10" },
 131     { rc(arg) | rc(gpr) | 9,            "r9" },
 132     { rc(arg) | rc(gpr) | 8,            "r8" },
 133     { rc(arg) | rc(gpr) | 7,            "r7" },
 134     { rc(arg) | rc(gpr) | 6,            "r6" },
 135     { rc(arg) | rc(gpr) | 5,            "r5" },
 136     { rc(arg) | rc(gpr) | 4,            "r4" },
 137     { rc(arg) | rc(gpr) | 3,            "r3" },
 138     { rc(fpr) | 0,                      "f0" },
 139     { rc(sav) | rc(fpr) | 14,           "f14" },
 140     { rc(sav) | rc(fpr) | 15,           "f15" },
 141     { rc(sav) | rc(fpr) | 16,           "f16" },
 142     { rc(sav) | rc(fpr) | 17,           "f17" },
 143     { rc(sav) | rc(fpr) | 18,           "f18" },
 144     { rc(sav) | rc(fpr) | 19,           "f19" },
 145     { rc(sav) | rc(fpr) | 20,           "f20" },
 146     { rc(sav) | rc(fpr) | 21,           "f21" },
 147     { rc(sav) | rc(fpr) | 22,           "f22" },
 148     { rc(sav) | rc(fpr) | 23,           "f23" },
 149     { rc(sav) | rc(fpr) | 24,           "f24" },
 150     { rc(sav) | rc(fpr) | 25,           "f25" },
 151     { rc(sav) | rc(fpr) | 26,           "f26" },
 152     { rc(sav) | rc(fpr) | 27,           "f27" },
 153     { rc(sav) | rc(fpr) | 28,           "f28" },
 154     { rc(sav) | rc(fpr) | 29,           "f29" },
 155     { rc(sav) | rc(fpr) | 30,           "f30" },
 156     { rc(sav) | rc(fpr) | 31,           "f31" },
 157 #if !_CALL_SYSV
 158     { rc(arg) | rc(fpr) | 13,           "f13" },
 159     { rc(arg) | rc(fpr) | 12,           "f12" },
 160     { rc(arg) | rc(fpr) | 11,           "f11" },
 161     { rc(arg) | rc(fpr) | 10,           "f10" },
 162     { rc(arg) | rc(fpr) | 9,            "f9" },
 163 #else
 164     { rc(fpr) | 13,                     "f13" },
 165     { rc(fpr) | 12,                     "f12" },
 166     { rc(fpr) | 11,                     "f11" },
 167     { rc(fpr) | 10,                     "f10" },
 168     { rc(fpr) | 9,                      "f9" },
 169 #endif
 170     { rc(arg) | rc(fpr) | 8,            "f8" },
 171     { rc(arg) | rc(fpr) | 7,            "f7" },
 172     { rc(arg) | rc(fpr) | 6,            "f6" },
 173     { rc(arg) | rc(fpr) | 5,            "f5" },
 174     { rc(arg) | rc(fpr) | 4,            "f4" },
 175     { rc(arg) | rc(fpr) | 3,            "f3" },
 176     { rc(arg) | rc(fpr) | 2,            "f2" },
 177     { rc(arg) | rc(fpr) | 1,            "f1" },
 178     { _NOREG,                           "<none>" },
 179 };
 180
 181 /*
 182  * Implementation
 183  */
 184 void
 185 jit_get_cpu(void)
 186 {
 187 }
 188
 189 void
 190 _jit_init(jit_state_t *_jit)
 191 {
 192     _jitc->reglen = jit_size(_rvs) - 1;
 193 }
 194
 195 void
 196 _jit_prolog(jit_state_t *_jit)
 197 {
 198     jit_int32_t          offset;
 199
 200     if (_jitc->function)
 201         jit_epilog();
 202     assert(jit_regset_cmp_ui(&_jitc->regarg, 0) == 0);
 203     jit_regset_set_ui(&_jitc->regsav, 0);
 204     offset = _jitc->functions.offset;
 205     if (offset >= _jitc->functions.length) {
 206         jit_realloc((jit_pointer_t *)&_jitc->functions.ptr,
 207                     _jitc->functions.length * sizeof(jit_function_t),
 208                     (_jitc->functions.length + 16) * sizeof(jit_function_t));
 209         _jitc->functions.length += 16;
 210     }
 211     _jitc->function = _jitc->functions.ptr + _jitc->functions.offset++;
 212     _jitc->function->self.size = params_offset;
 213     _jitc->function->self.argi = _jitc->function->self.argf =
 214         _jitc->function->self.alen = 0;
 215     /* float conversion */
 216     _jitc->function->self.aoff = alloca_offset - 8;
 217     _jitc->function->self.call = jit_call_default;
 218     jit_alloc((jit_pointer_t *)&_jitc->function->regoff,
 219               _jitc->reglen * sizeof(jit_int32_t));
 220
 221     /* _no_link here does not mean the jit_link() call can be removed
 222      * by rewriting as:
 223      * _jitc->function->prolog = jit_new_node(jit_code_prolog);
 224      */
 225     _jitc->function->prolog = jit_new_node_no_link(jit_code_prolog);
 226     jit_link(_jitc->function->prolog);
 227     _jitc->function->prolog->w.w = offset;
 228     _jitc->function->epilog = jit_new_node_no_link(jit_code_epilog);
 229     /*  u:      label value
 230      *  v:      offset in blocks vector
 231      *  w:      offset in functions vector
 232      */
 233     _jitc->function->epilog->w.w = offset;
 234
 235     jit_regset_new(&_jitc->function->regset);
 236 }
 237
 238 jit_int32_t
 239 _jit_allocai(jit_state_t *_jit, jit_int32_t length)
 240 {
 241     assert(_jitc->function);
 242     switch (length) {
 243         case 0: case 1:                                         break;
 244         case 2:         _jitc->function->self.aoff &= -2;       break;
 245         case 3: case 4: _jitc->function->self.aoff &= -4;       break;
 246         default:        _jitc->function->self.aoff &= -8;       break;
 247     }
 248     _jitc->function->self.aoff -= length;
 249     if (!_jitc->realize) {
 250         jit_inc_synth_ww(allocai, _jitc->function->self.aoff, length);
 251         jit_dec_synth();
 252     }
 253     return (_jitc->function->self.aoff);
 254 }
 255
 256 void
 257 _jit_allocar(jit_state_t *_jit, jit_int32_t u, jit_int32_t v)
 258 {
 259     jit_int32_t          r0, r1;
 260     assert(_jitc->function);
 261     jit_inc_synth_ww(allocar, u, v);
 262     if (!_jitc->function->allocar) {
 263         _jitc->function->aoffoff = jit_allocai(sizeof(jit_int32_t));
 264         _jitc->function->allocar = 1;
 265     }
 266     r0 = jit_get_reg(jit_class_gpr);
 267     r1 = jit_get_reg(jit_class_gpr);
 268     jit_ldr(r0, JIT_SP);
 269     jit_negr(r1, v);
 270     jit_andi(r1, r1, -16);
 271     jit_ldxi_i(u, JIT_FP, _jitc->function->aoffoff);
 272     jit_addr(u, u, r1);
 273     jit_addr(JIT_SP, JIT_SP, r1);
 274     jit_stxi_i(_jitc->function->aoffoff, JIT_FP, u);
 275     jit_str(JIT_SP, r0);
 276     jit_unget_reg(r1);
 277     jit_unget_reg(r0);
 278     jit_dec_synth();
 279 }
 280
 281 void
 282 _jit_ret(jit_state_t *_jit)
 283 {
 284     jit_node_t          *instr;
 285     assert(_jitc->function);
 286     jit_inc_synth(ret);
 287     /* jump to epilog */
 288     instr = jit_jmpi();
 289     jit_patch_at(instr, _jitc->function->epilog);
 290     jit_dec_synth();
 291 }
 292
 293 void
 294 _jit_retr(jit_state_t *_jit, jit_int32_t u, jit_code_t code)
 295 {
 296     jit_code_inc_synth_w(code, u);
 297     jit_movr(JIT_RET, u);
 298     jit_ret();
 299     jit_dec_synth();
 300 }
 301
 302 void
 303 _jit_reti(jit_state_t *_jit, jit_word_t u, jit_code_t code)
 304 {
 305     jit_code_inc_synth_w(code, u);
 306     jit_movi(JIT_RET, u);
 307     jit_ret();
 308     jit_dec_synth();
 309 }
 310
 311 void
 312 _jit_retr_f(jit_state_t *_jit, jit_int32_t u)
 313 {
 314     jit_inc_synth_w(retr_f, u);
 315     if (JIT_RET != u)
 316         jit_movr_f(JIT_FRET, u);
 317     else
 318         jit_live(JIT_FRET);
 319     jit_ret();
 320     jit_dec_synth();
 321 }
 322
 323 void
 324 _jit_reti_f(jit_state_t *_jit, jit_float32_t u)
 325 {
 326     jit_inc_synth_f(reti_f, u);
 327     jit_movi_f(JIT_FRET, u);
 328     jit_ret();
 329     jit_dec_synth();
 330 }
 331
 332 void
 333 _jit_retr_d(jit_state_t *_jit, jit_int32_t u)
 334 {
 335     jit_inc_synth_w(retr_d, u);
 336     if (JIT_FRET != u)
 337         jit_movr_d(JIT_FRET, u);
 338     else
 339         jit_live(JIT_FRET);
 340     jit_ret();
 341     jit_dec_synth();
 342 }
 343
 344 void
 345 _jit_reti_d(jit_state_t *_jit, jit_float64_t u)
 346 {
 347     jit_inc_synth_d(reti_d, u);
 348     jit_movi_d(JIT_FRET, u);
 349     jit_ret();
 350     jit_dec_synth();
 351 }
 352
 353 void
 354 _jit_epilog(jit_state_t *_jit)
 355 {
 356     assert(_jitc->function);
 357     assert(_jitc->function->epilog->next == NULL);
 358     jit_link(_jitc->function->epilog);
 359     _jitc->function = NULL;
 360 }
 361
 362 jit_bool_t
 363 _jit_arg_register_p(jit_state_t *_jit, jit_node_t *u)
 364 {
 365     if (u->code >= jit_code_arg_c && u->code <= jit_code_arg)
 366         return (jit_arg_reg_p(u->u.w));
 367     assert(u->code == jit_code_arg_f || u->code == jit_code_arg_d);
 368     return (jit_arg_f_reg_p(u->u.w));
 369 }
 370
 371 void
 372 _jit_ellipsis(jit_state_t *_jit)
 373 {
 374     jit_inc_synth(ellipsis);
 375     if (_jitc->prepare) {
 376         jit_link_prepare();
 377         assert(!(_jitc->function->call.call & jit_call_varargs));
 378         _jitc->function->call.call |= jit_call_varargs;
 379     }
 380     else {
 381         jit_link_prolog();
 382         assert(!(_jitc->function->self.call & jit_call_varargs));
 383         _jitc->function->self.call |= jit_call_varargs;
 384 #if _CALL_SYSV
 385         /* Allocate va_list like object in the stack.
 386          * If applicable, with enough space to save all argument
 387          * registers, and use fixed offsets for them. */
 388         _jitc->function->vaoff = jit_allocai(sizeof(jit_va_list_t));
 389 #endif
 390         _jitc->function->vagp = _jitc->function->self.argi;
 391         _jitc->function->vafp = _jitc->function->self.argf;
 392     }
 393     jit_dec_synth();
 394 }
 395
 396 void
 397 _jit_va_push(jit_state_t *_jit, jit_int32_t u)
 398 {
 399     jit_inc_synth_w(va_push, u);
 400     jit_pushargr(u);
 401     jit_dec_synth();
 402 }
 403
 404 jit_node_t *
 405 _jit_arg(jit_state_t *_jit, jit_code_t code)
 406 {
 407     jit_node_t          *node;
 408     jit_int32_t          offset;
 409     jit_bool_t           incr = 1;
 410     assert(_jitc->function);
 411     assert(!(_jitc->function->self.call & jit_call_varargs));
 412 #if STRONG_TYPE_CHECKING
 413     assert(code >= jit_code_arg_c && code <= jit_code_arg);
 414 #endif
 415     if (jit_arg_reg_p(_jitc->function->self.argi)) {
 416         offset = _jitc->function->self.argi++;
 417 #if _CALL_SYSV
 418         incr = 0;
 419 #endif
 420     }
 421     else
 422         offset = _jitc->function->self.size;
 423     if (incr)
 424         _jitc->function->self.size += sizeof(jit_word_t);
 425     node = jit_new_node_ww(code, offset,
 426                            ++_jitc->function->self.argn);
 427     jit_link_prolog();
 428     return (node);
 429 }
 430
 431 jit_node_t *
 432 _jit_arg_f(jit_state_t *_jit)
 433 {
 434     jit_node_t          *node;
 435     jit_int32_t          offset;
 436     jit_bool_t           incr = 1;
 437     assert(_jitc->function);
 438     if (jit_arg_f_reg_p(_jitc->function->self.argf)) {
 439         offset = _jitc->function->self.argf++;
 440 #if _CALL_SYSV
 441         incr = 0;
 442 #endif
 443     }
 444     else
 445         offset = _jitc->function->self.size + F_DISP;
 446 #if !_CALL_SYSV
 447     if (jit_arg_reg_p(_jitc->function->self.argi)) {
 448 #  if __WORDSIZE == 32
 449         _jitc->function->self.argi += 2;
 450 #  else
 451         _jitc->function->self.argi++;
 452 #  endif
 453     }
 454 #endif
 455     if (incr)
 456         _jitc->function->self.size += sizeof(jit_word_t);
 457     node = jit_new_node_ww(jit_code_arg_f, offset,
 458                            ++_jitc->function->self.argn);
 459     jit_link_prolog();
 460     return (node);
 461 }
 462
 463 jit_node_t *
 464 _jit_arg_d(jit_state_t *_jit)
 465 {
 466     jit_node_t          *node;
 467     jit_int32_t          offset;
 468     jit_bool_t           incr = 1;
 469     assert(_jitc->function);
 470     if (jit_arg_f_reg_p(_jitc->function->self.argf)) {
 471         offset = _jitc->function->self.argf++;
 472 #if _CALL_SYSV
 473         incr = 0;
 474 #endif
 475     }
 476     else {
 477 #if _CALL_SYSV
 478         if (_jitc->function->self.size & 7)
 479             _jitc->function->self.size += 4;
 480 #endif
 481         offset = _jitc->function->self.size;
 482     }
 483 #if !_CALL_SYSV
 484     if (jit_arg_reg_p(_jitc->function->self.argi)) {
 485 #  if __WORDSIZE == 32
 486         _jitc->function->self.argi += 2;
 487 #  else
 488         _jitc->function->self.argi++;
 489 #  endif
 490     }
 491 #endif
 492     if (incr)
 493         _jitc->function->self.size += sizeof(jit_float64_t);
 494     node = jit_new_node_ww(jit_code_arg_d, offset,
 495                            ++_jitc->function->self.argn);
 496     jit_link_prolog();
 497     return (node);
 498 }
 499
 500 void
 501 _jit_getarg_c(jit_state_t *_jit, jit_int32_t u, jit_node_t *v)
 502 {
 503     assert_arg_type(v->code, jit_code_arg_c);
 504     jit_inc_synth_wp(getarg_c, u, v);
 505     if (jit_arg_reg_p(v->u.w))
 506         jit_extr_c(u, JIT_RA0 - v->u.w);
 507     else
 508         jit_ldxi_c(u, JIT_FP, v->u.w + C_DISP);
 509     jit_dec_synth();
 510 }
 511
 512 void
 513 _jit_getarg_uc(jit_state_t *_jit, jit_int32_t u, jit_node_t *v)
 514 {
 515     assert_arg_type(v->code, jit_code_arg_c);
 516     jit_inc_synth_wp(getarg_uc, u, v);
 517     if (jit_arg_reg_p(v->u.w))
 518         jit_extr_uc(u, JIT_RA0 - v->u.w);
 519     else
 520         jit_ldxi_uc(u, JIT_FP, v->u.w + C_DISP);
 521     jit_dec_synth();
 522 }
 523
 524 void
 525 _jit_getarg_s(jit_state_t *_jit, jit_int32_t u, jit_node_t *v)
 526 {
 527     assert_arg_type(v->code, jit_code_arg_s);
 528     jit_inc_synth_wp(getarg_s, u, v);
 529     if (jit_arg_reg_p(v->u.w))
 530         jit_extr_s(u, JIT_RA0 - v->u.w);
 531     else
 532         jit_ldxi_s(u, JIT_FP, v->u.w + S_DISP);
 533     jit_dec_synth();
 534 }
 535
 536 void
 537 _jit_getarg_us(jit_state_t *_jit, jit_int32_t u, jit_node_t *v)
 538 {
 539     assert_arg_type(v->code, jit_code_arg_s);
 540     jit_inc_synth_wp(getarg_us, u, v);
 541     if (jit_arg_reg_p(v->u.w))
 542         jit_extr_us(u, JIT_RA0 - v->u.w);
 543     else
 544         jit_ldxi_us(u, JIT_FP, v->u.w + S_DISP);
 545     jit_dec_synth();
 546 }
 547
 548 void
 549 _jit_getarg_i(jit_state_t *_jit, jit_int32_t u, jit_node_t *v)
 550 {
 551     assert_arg_type(v->code, jit_code_arg_i);
 552     jit_inc_synth_wp(getarg_i, u, v);
 553     if (jit_arg_reg_p(v->u.w)) {
 554 #if __WORDSIZE == 32
 555         jit_movr(u, JIT_RA0 - v->u.w);
 556 #else
 557         jit_extr_i(u, JIT_RA0 - v->u.w);
 558 #endif
 559     }
 560     else
 561         jit_ldxi_i(u, JIT_FP, v->u.w + I_DISP);
 562     jit_dec_synth();
 563 }
 564
 565 #if __WORDSIZE == 64
 566 void
 567 _jit_getarg_ui(jit_state_t *_jit, jit_int32_t u, jit_node_t *v)
 568 {
 569     assert_arg_type(v->code, jit_code_arg_i);
 570     jit_inc_synth_wp(getarg_ui, u, v);
 571     if (jit_arg_reg_p(v->u.w))
 572         jit_extr_ui(u, JIT_RA0 - v->u.w);
 573     else
 574         jit_ldxi_ui(u, JIT_FP, v->u.w + I_DISP);
 575     jit_dec_synth();
 576 }
 577
 578 void
 579 _jit_getarg_l(jit_state_t *_jit, jit_int32_t u, jit_node_t *v)
 580 {
 581     assert_arg_type(v->code, jit_code_arg_l);
 582     jit_inc_synth_wp(getarg_l, u, v);
 583     if (jit_arg_reg_p(v->u.w))
 584         jit_movr(u, JIT_RA0 - v->u.w);
 585     else
 586         jit_ldxi_l(u, JIT_FP, v->u.w);
 587     jit_dec_synth();
 588 }
 589 #endif
 590
 591 void
 592 _jit_putargr(jit_state_t *_jit, jit_int32_t u, jit_node_t *v, jit_code_t code)
 593 {
 594     assert_putarg_type(code, v->code);
 595     jit_code_inc_synth_wp(code, u, v);
 596     if (jit_arg_reg_p(v->u.w))
 597         jit_movr(JIT_RA0 - v->u.w, u);
 598     else
 599         jit_stxi(v->u.w, JIT_FP, u);
 600     jit_dec_synth();
 601 }
 602
 603 void
 604 _jit_putargi(jit_state_t *_jit, jit_word_t u, jit_node_t *v, jit_code_t code)
 605 {
 606     jit_int32_t         regno;
 607     assert_putarg_type(code, v->code);
 608     jit_code_inc_synth_wp(code, u, v);
 609     if (jit_arg_reg_p(v->u.w))
 610         jit_movi(JIT_RA0 - v->u.w, u);
 611     else {
 612         regno = jit_get_reg(jit_class_gpr);
 613         jit_movi(regno, u);
 614         jit_stxi(v->u.w, JIT_FP, regno);
 615         jit_unget_reg(regno);
 616     }
 617     jit_dec_synth();
 618 }
 619
 620 void
 621 _jit_getarg_f(jit_state_t *_jit, jit_int32_t u, jit_node_t *v)
 622 {
 623     assert(v->code == jit_code_arg_f);
 624     jit_inc_synth_wp(getarg_f, u, v);
 625     if (jit_arg_f_reg_p(v->u.w))
 626         jit_movr_d(u, JIT_FA0 - v->u.w);
 627     else
 628         jit_ldxi_f(u, JIT_FP, v->u.w);
 629     jit_dec_synth();
 630 }
 631
 632 void
 633 _jit_putargr_f(jit_state_t *_jit, jit_int32_t u, jit_node_t *v)
 634 {
 635     assert(v->code == jit_code_arg_f);
 636     jit_inc_synth_wp(putargr_f, u, v);
 637     if (jit_arg_f_reg_p(v->u.w))
 638         jit_movr_d(JIT_FA0 - v->u.w, u);
 639     else
 640         jit_stxi_f(v->u.w, JIT_FP, u);
 641     jit_dec_synth();
 642 }
 643
 644 void
 645 _jit_putargi_f(jit_state_t *_jit, jit_float32_t u, jit_node_t *v)
 646 {
 647     jit_int32_t         regno;
 648     assert(v->code == jit_code_arg_f);
 649     jit_inc_synth_fp(putargi_f, u, v);
 650     if (jit_arg_f_reg_p(v->u.w))
 651         jit_movi_d(JIT_FA0 - v->u.w, u);
 652     else {
 653         regno = jit_get_reg(jit_class_fpr);
 654         jit_movi_d(regno, u);
 655         jit_stxi_f(v->u.w, JIT_FP, regno);
 656         jit_unget_reg(regno);
 657     }
 658     jit_dec_synth();
 659 }
 660
 661 void
 662 _jit_getarg_d(jit_state_t *_jit, jit_int32_t u, jit_node_t *v)
 663 {
 664     assert(v->code == jit_code_arg_d);
 665     jit_inc_synth_wp(getarg_d, u, v);
 666     if (jit_arg_f_reg_p(v->u.w))
 667         jit_movr_d(u, JIT_FA0 - v->u.w);
 668     else
 669         jit_ldxi_d(u, JIT_FP, v->u.w);
 670     jit_dec_synth();
 671 }
 672
 673 void
 674 _jit_putargr_d(jit_state_t *_jit, jit_int32_t u, jit_node_t *v)
 675 {
 676     assert(v->code == jit_code_arg_d);
 677     jit_inc_synth_wp(putargr_d, u, v);
 678     if (jit_arg_f_reg_p(v->u.w))
 679         jit_movr_d(JIT_FA0 - v->u.w, u);
 680     else
 681         jit_stxi_d(v->u.w, JIT_FP, u);
 682     jit_dec_synth();
 683 }
 684
 685 void
 686 _jit_putargi_d(jit_state_t *_jit, jit_float64_t u, jit_node_t *v)
 687 {
 688     jit_int32_t         regno;
 689     assert(v->code == jit_code_arg_d);
 690     jit_inc_synth_dp(putargi_d, u, v);
 691     if (jit_arg_f_reg_p(v->u.w))
 692         jit_movi_d(JIT_FA0 - v->u.w, u);
 693     else {
 694         regno = jit_get_reg(jit_class_fpr);
 695         jit_movi_d(regno, u);
 696         jit_stxi_d(v->u.w, JIT_FP, regno);
 697         jit_unget_reg(regno);
 698     }
 699     jit_dec_synth();
 700 }
 701
 702 void
 703 _jit_pushargr(jit_state_t *_jit, jit_int32_t u, jit_code_t code)
 704 {
 705     jit_bool_t          incr = 1;
 706     assert(_jitc->function);
 707     jit_code_inc_synth_w(code, u);
 708     jit_link_prepare();
 709     if (jit_arg_reg_p(_jitc->function->call.argi)) {
 710         jit_movr(JIT_RA0 - _jitc->function->call.argi, u);
 711         ++_jitc->function->call.argi;
 712 #if _CALL_SYSV
 713         incr = 0;
 714 #endif
 715     }
 716     else
 717         jit_stxi(_jitc->function->call.size + params_offset, JIT_SP, u);
 718     if (incr)
 719         _jitc->function->call.size += sizeof(jit_word_t);
 720     jit_dec_synth();
 721 }
 722
 723 void
 724 _jit_pushargi(jit_state_t *_jit, jit_word_t u, jit_code_t code)
 725 {
 726     jit_int32_t          regno;
 727     jit_bool_t           incr = 1;
 728     assert(_jitc->function);
 729     jit_code_inc_synth_w(code, u);
 730     jit_link_prepare();
 731     if (jit_arg_reg_p(_jitc->function->call.argi)) {
 732         jit_movi(JIT_RA0 - _jitc->function->call.argi, u);
 733         ++_jitc->function->call.argi;
 734 #if _CALL_SYSV
 735         incr = 0;
 736 #endif
 737     }
 738     else {
 739         regno = jit_get_reg(jit_class_gpr);
 740         jit_movi(regno, u);
 741         jit_stxi(_jitc->function->call.size + params_offset, JIT_SP, regno);
 742         jit_unget_reg(regno);
 743     }
 744     if (incr)
 745         _jitc->function->call.size += sizeof(jit_word_t);
 746     jit_dec_synth();
 747 }
 748
 749 void
 750 _jit_pushargr_f(jit_state_t *_jit, jit_int32_t u)
 751 {
 752     jit_bool_t          incr = 1;
 753     assert(_jitc->function);
 754     jit_inc_synth_w(pushargr_f, u);
 755     jit_link_prepare();
 756     if (jit_arg_f_reg_p(_jitc->function->call.argf)
 757 #if !_CALL_SYSV
 758         && !(_jitc->function->call.call & jit_call_varargs)
 759 #endif
 760         ) {
 761         jit_movr_d(JIT_FA0 - _jitc->function->call.argf, u);
 762         ++_jitc->function->call.argf;
 763 #if !_CALL_SYSV
 764         /* in case of excess arguments */
 765         if (jit_arg_reg_p(_jitc->function->call.argi)) {
 766 #  if __WORDSIZE == 32
 767             _jitc->function->call.argi += 2;
 768             if (!jit_arg_reg_p(_jitc->function->call.argi - 1))
 769                 --_jitc->function->call.argi;
 770 #  else
 771             _jitc->function->call.argi++;
 772 #  endif
 773         }
 774 #elif _CALL_SYSV
 775         incr = 0;
 776 #endif
 777     }
 778 #if !_CALL_SYSV
 779     else if (jit_arg_reg_p(_jitc->function->call.argi
 780 #  if __WORDSIZE == 32
 781                           + 1
 782 #  endif
 783                            )) {
 784         /* use reserved 8 bytes area */
 785         jit_stxi_d(alloca_offset - 8, JIT_FP, u);
 786         jit_ldxi(JIT_RA0 - _jitc->function->call.argi, JIT_FP,
 787                  alloca_offset - 8);
 788         _jitc->function->call.argi++;
 789 #  if __WORDSIZE == 32
 790         jit_ldxi(JIT_RA0 - _jitc->function->call.argi, JIT_FP,
 791                  alloca_offset - 4);
 792         _jitc->function->call.argi++;
 793 #  endif
 794     }
 795 #endif
 796     else
 797         jit_stxi_f(_jitc->function->call.size + params_offset + F_DISP,
 798                    JIT_SP, u);
 799     if (incr)
 800         _jitc->function->call.size += sizeof(jit_word_t);
 801     jit_dec_synth();
 802 }
 803
 804 void
 805 _jit_pushargi_f(jit_state_t *_jit, jit_float32_t u)
 806 {
 807     jit_bool_t           incr = 1;
 808     jit_int32_t          regno;
 809     assert(_jitc->function);
 810     jit_inc_synth_f(pushargi_f, u);
 811     jit_link_prepare();
 812     if (jit_arg_f_reg_p(_jitc->function->call.argf)
 813 #if !_CALL_SYSV
 814         && !(_jitc->function->call.call & jit_call_varargs)
 815 #endif
 816         ) {
 817         jit_movi_d(JIT_FA0 - _jitc->function->call.argf, u);
 818         ++_jitc->function->call.argf;
 819 #if !_CALL_SYSV
 820             /* in case of excess arguments */
 821 #  if __WORDSIZE == 32
 822         _jitc->function->call.argi += 2;
 823         if (!jit_arg_reg_p(_jitc->function->call.argi - 1))
 824             --_jitc->function->call.argi;
 825 #  else
 826         _jitc->function->call.argi++;
 827 #  endif
 828 #elif _CALL_SYSV
 829         incr = 0;
 830 #endif
 831     }
 832     else {
 833         regno = jit_get_reg(jit_class_fpr);
 834         jit_movi_f(regno, u);
 835 #if !_CALL_SYSV
 836         if (jit_arg_reg_p(_jitc->function->call.argi
 837 #  if __WORDSIZE == 32
 838                           + 1
 839 #  endif
 840                           )) {
 841             /* use reserved 8 bytes area */
 842             jit_stxi_d(alloca_offset - 8, JIT_FP, regno);
 843             jit_ldxi(JIT_RA0 - _jitc->function->call.argi, JIT_FP,
 844                      alloca_offset - 8);
 845             _jitc->function->call.argi++;
 846 #  if __WORDSIZE == 32
 847             jit_ldxi(JIT_RA0 - _jitc->function->call.argi, JIT_FP,
 848                      alloca_offset - 4);
 849             _jitc->function->call.argi++;
 850 #  endif
 851         }
 852         else
 853 #endif
 854             jit_stxi_f(_jitc->function->call.size + params_offset + F_DISP,
 855                        JIT_SP, regno);
 856         jit_unget_reg(regno);
 857     }
 858     if (incr)
 859         _jitc->function->call.size += sizeof(jit_word_t);
 860     jit_dec_synth();
 861 }
 862
 863 void
 864 _jit_pushargr_d(jit_state_t *_jit, jit_int32_t u)
 865 {
 866     jit_bool_t          incr = 1;
 867     assert(_jitc->function);
 868     jit_inc_synth_w(pushargr_d, u);
 869     jit_link_prepare();
 870     if (jit_arg_f_reg_p(_jitc->function->call.argf)
 871 #if !_CALL_SYSV
 872         && !(_jitc->function->call.call & jit_call_varargs)
 873 #endif
 874         ) {
 875         jit_movr_d(JIT_FA0 - _jitc->function->call.argf, u);
 876         ++_jitc->function->call.argf;
 877 #if !_CALL_SYSV
 878             /* in case of excess arguments */
 879 #  if __WORDSIZE == 32
 880         _jitc->function->call.argi += 2;
 881         if (!jit_arg_reg_p(_jitc->function->call.argi - 1))
 882             --_jitc->function->call.argi;
 883 #  else
 884         _jitc->function->call.argi++;
 885 #  endif
 886 #else /* _CALL_SYSV */
 887         incr = 0;
 888 #endif
 889     }
 890 #if !_CALL_SYSV
 891     else if (jit_arg_reg_p(_jitc->function->call.argi
 892 #  if __WORDSIZE == 32
 893                           + 1
 894 #  endif
 895                            )) {
 896         /* use reserved 8 bytes area */
 897         jit_stxi_d(alloca_offset - 8, JIT_FP, u);
 898         jit_ldxi(JIT_RA0 - _jitc->function->call.argi, JIT_FP,
 899                  alloca_offset - 8);
 900         _jitc->function->call.argi++;
 901 #  if __WORDSIZE == 32
 902         jit_ldxi(JIT_RA0 - _jitc->function->call.argi, JIT_FP,
 903                  alloca_offset - 4);
 904         _jitc->function->call.argi++;
 905 #  endif
 906     }
 907     else
 908 #endif /* !_CALL_SYSV */
 909     {
 910 #if _CALL_SYSV
 911         if (_jitc->function->call.size & 7)
 912             _jitc->function->call.size += 4;
 913 #endif
 914         jit_stxi_d(_jitc->function->call.size + params_offset, JIT_SP, u);
 915 #if !_CALL_SYSV && __WORDSIZE == 32
 916         if (jit_arg_reg_p(_jitc->function->call.argi)) {
 917             jit_ldxi(JIT_RA0 - _jitc->function->call.argi, JIT_SP,
 918                      _jitc->function->call.size + params_offset);
 919             _jitc->function->call.argi++;
 920         }
 921 #endif
 922     }
 923     if (incr)
 924         _jitc->function->call.size += sizeof(jit_float64_t);
 925     jit_dec_synth();
 926 }
 927
 928 void
 929 _jit_pushargi_d(jit_state_t *_jit, jit_float64_t u)
 930 {
 931     jit_int32_t          regno;
 932     jit_bool_t           incr = 1;
 933     assert(_jitc->function);
 934     jit_inc_synth_d(pushargi_d, u);
 935     jit_link_prepare();
 936     if (jit_arg_f_reg_p(_jitc->function->call.argf)
 937 #if !_CALL_SYSV
 938         && !(_jitc->function->call.call & jit_call_varargs)
 939 #endif
 940         ) {
 941         jit_movi_d(JIT_FA0 - _jitc->function->call.argf, u);
 942         ++_jitc->function->call.argf;
 943 #if !_CALL_SYSV
 944         /* in case of excess arguments */
 945         if (jit_arg_reg_p(_jitc->function->call.argi)) {
 946 #  if __WORDSIZE == 32
 947             _jitc->function->call.argi += 2;
 948             if (!jit_arg_reg_p(_jitc->function->call.argi - 1))
 949                 --_jitc->function->call.argi;
 950 #  else
 951             _jitc->function->call.argi++;
 952 #  endif
 953         }
 954 #else /* _CALL_SYSV */
 955             incr = 0;
 956 #endif
 957     }
 958     else {
 959         regno = jit_get_reg(jit_class_fpr);
 960         jit_movi_d(regno, u);
 961 #if !_CALL_SYSV
 962         if (jit_arg_reg_p(_jitc->function->call.argi
 963 #  if __WORDSIZE == 32
 964                           + 1
 965 #  endif
 966                           )) {
 967             /* use reserved 8 bytes area */
 968             jit_stxi_d(alloca_offset - 8, JIT_FP, regno);
 969             jit_ldxi(JIT_RA0 - _jitc->function->call.argi, JIT_FP,
 970                      alloca_offset - 8);
 971             _jitc->function->call.argi++;
 972 #  if __WORDSIZE == 32
 973             jit_ldxi(JIT_RA0 - _jitc->function->call.argi, JIT_FP,
 974                      alloca_offset - 4);
 975             _jitc->function->call.argi++;
 976 #  endif
 977         }
 978         else
 979 #endif /* !_CALL_SYSV */
 980         {
 981 #if _CALL_SYSV
 982             if (_jitc->function->call.size & 7)
 983                 _jitc->function->call.size += 4;
 984 #endif
 985             jit_stxi_d(_jitc->function->call.size + params_offset,
 986                        JIT_SP, regno);
 987 #if !_CALL_SYSV && __WORDSIZE == 32
 988             if (jit_arg_reg_p(_jitc->function->call.argi)) {
 989                 jit_ldxi(JIT_RA0 - _jitc->function->call.argi, JIT_SP,
 990                          _jitc->function->call.size + params_offset);
 991                 _jitc->function->call.argi++;
 992             }
 993 #endif
 994         }
 995         jit_unget_reg(regno);
 996     }
 997     if (incr)
 998         _jitc->function->call.size += sizeof(jit_float64_t);
 999     jit_dec_synth();
1000 }
1001
1002 jit_bool_t
1003 _jit_regarg_p(jit_state_t *_jit, jit_node_t *node, jit_int32_t regno)
1004 {
1005     jit_int32_t         spec;
1006     spec = jit_class(_rvs[regno].spec);
1007     if (spec & jit_class_arg) {
1008         if (spec & jit_class_gpr) {
1009             regno = JIT_RA0 - regno;
1010             if (regno >= 0 && regno < node->v.w)
1011                 return (1);
1012         }
1013         else if (spec & jit_class_fpr) {
1014             regno = JIT_FA0 - regno;
1015             if (regno >= 0 && regno < node->w.w)
1016                 return (1);
1017         }
1018     }
1019     return (0);
1020 }
1021
1022 void
1023 _jit_finishr(jit_state_t *_jit, jit_int32_t r0)
1024 {
1025     jit_node_t          *call;
1026     assert(_jitc->function);
1027     jit_inc_synth_w(finishr, r0);
1028     if (_jitc->function->self.alen < _jitc->function->call.size)
1029         _jitc->function->self.alen = _jitc->function->call.size;
1030     call = jit_callr(r0);
1031     call->v.w = _jitc->function->call.argi;
1032     call->w.w = _jitc->function->call.argf;
1033 #if _CALL_SYSV
1034     /* If passing float arguments in registers */
1035     if ((_jitc->function->call.call & jit_call_varargs) && call->w.w)
1036         call->flag |= jit_flag_varargs;
1037 #endif
1038     _jitc->function->call.argi = _jitc->function->call.argf = 0;
1039     _jitc->prepare = 0;
1040     jit_dec_synth();
1041 }
1042
1043 jit_node_t *
1044 _jit_finishi(jit_state_t *_jit, jit_pointer_t i0)
1045 {
1046     jit_node_t          *node;
1047     assert(_jitc->function);
1048     jit_inc_synth_w(finishi, (jit_word_t)i0);
1049     if (_jitc->function->self.alen < _jitc->function->call.size)
1050         _jitc->function->self.alen = _jitc->function->call.size;
1051     node = jit_calli(i0);
1052     node->v.w = _jitc->function->call.argi;
1053     node->w.w = _jitc->function->call.argf;
1054 #if _CALL_SYSV
1055     if ((_jitc->function->call.call & jit_call_varargs) && node->w.w)
1056         node->flag |= jit_flag_varargs;
1057 #endif
1058     _jitc->function->call.argi = _jitc->function->call.argf = 0;
1059     _jitc->prepare = 0;
1060     jit_dec_synth();
1061     return (node);
1062 }
1063
1064 void
1065 _jit_retval_c(jit_state_t *_jit, jit_int32_t r0)
1066 {
1067     jit_inc_synth(retval_c);
1068     jit_extr_c(r0, JIT_RET);
1069     jit_dec_synth();
1070 }
1071
1072 void
1073 _jit_retval_uc(jit_state_t *_jit, jit_int32_t r0)
1074 {
1075     jit_inc_synth(retval_uc);
1076     jit_extr_uc(r0, JIT_RET);
1077     jit_dec_synth();
1078 }
1079
1080 void
1081 _jit_retval_s(jit_state_t *_jit, jit_int32_t r0)
1082 {
1083     jit_inc_synth(retval_s);
1084     jit_extr_s(r0, JIT_RET);
1085     jit_dec_synth();
1086 }
1087
1088 void
1089 _jit_retval_us(jit_state_t *_jit, jit_int32_t r0)
1090 {
1091     jit_inc_synth(retval_us);
1092     jit_extr_us(r0, JIT_RET);
1093     jit_dec_synth();
1094 }
1095
1096 void
1097 _jit_retval_i(jit_state_t *_jit, jit_int32_t r0)
1098 {
1099     jit_inc_synth(retval_i);
1100 #if __WORDSIZE == 32
1101     if (r0 != JIT_RET)
1102         jit_movr(r0, JIT_RET);
1103 #else
1104     jit_extr_i(r0, JIT_RET);
1105 #endif
1106     jit_dec_synth();
1107 }
1108
1109 #if __WORDSIZE == 64
1110 void
1111 _jit_retval_ui(jit_state_t *_jit, jit_int32_t r0)
1112 {
1113     jit_inc_synth(retval_ui);
1114     jit_extr_ui(r0, JIT_RET);
1115     jit_dec_synth();
1116 }
1117
1118 void
1119 _jit_retval_l(jit_state_t *_jit, jit_int32_t r0)
1120 {
1121     jit_inc_synth(retval_l);
1122     if (r0 != JIT_RET)
1123         jit_movr(r0, JIT_RET);
1124     jit_dec_synth();
1125 }
1126 #endif
1127
1128 void
1129 _jit_retval_f(jit_state_t *_jit, jit_int32_t r0)
1130 {
1131     jit_inc_synth(retval_f);
1132     jit_retval_d(r0);
1133     jit_dec_synth();
1134 }
1135
1136 void
1137 _jit_retval_d(jit_state_t *_jit, jit_int32_t r0)
1138 {
1139     jit_inc_synth(retval_d);
1140     if (r0 != JIT_FRET)
1141         jit_movr_d(r0, JIT_FRET);
1142     jit_dec_synth();
1143 }
1144
1145 jit_pointer_t
1146 _emit_code(jit_state_t *_jit)
1147 {
1148     jit_node_t          *node;
1149     jit_node_t          *temp;
1150     jit_word_t           word;
1151     jit_int32_t          value;
1152     jit_int32_t          offset;
1153     jit_bool_t       no_flag = 0;       /* Set if previous instruction is
1154                                          * *not* a jump target. */
1155     struct {
1156         jit_node_t      *node;
1157         jit_word_t       word;
1158         jit_function_t   func;
1159 #if DEVEL_DISASSEMBLER
1160         jit_word_t       prevw;
1161 #endif
1162         jit_word_t       patch_offset;
1163 #if _CALL_AIXDESC
1164         jit_word_t       prolog_offset;
1165 #endif
1166     } undo;
1167 #if DEVEL_DISASSEMBLER
1168     jit_word_t           prevw;
1169 #endif
1170
1171     _jitc->function = NULL;
1172
1173     jit_reglive_setup();
1174
1175     undo.word = 0;
1176     undo.node = NULL;
1177     undo.patch_offset = 0;
1178
1179 #if DEVEL_DISASSEMBLER
1180     prevw = _jit->pc.w;
1181 #endif
1182 #if _CALL_AIXDESC
1183     undo.prolog_offset = 0;
1184     for (node = _jitc->head; node; node = node->next)
1185         if (node->code != jit_code_label &&
1186             node->code != jit_code_note &&
1187             node->code != jit_code_name)
1188             break;
1189     if (node && (node->code != jit_code_prolog ||
1190                  !(_jitc->functions.ptr + node->w.w)->assume_frame)) {
1191         /* code may start with a jump so add an initial function descriptor */
1192         word = _jit->pc.w + sizeof(void*) * 3;
1193         iw(word);                       /* addr */
1194         iw(0);                          /* toc */
1195         iw(0);                          /* env */
1196     }
1197 #endif
1198
1199 #define case_rr(name, type)                                             \
1200             case jit_code_##name##r##type:                              \
1201                 name##r##type(rn(node->u.w), rn(node->v.w));            \
1202                 break
1203 #define case_rw(name, type)                                             \
1204             case jit_code_##name##i##type:                              \
1205                 name##i##type(rn(node->u.w), node->v.w);                \
1206                 break
1207 #define case_wr(name, type)                                             \
1208             case jit_code_##name##i##type:                              \
1209                 name##i##type(node->u.w, rn(node->v.w));                \
1210                 break
1211 #define case_rrr(name, type)                                            \
1212             case jit_code_##name##r##type:                              \
1213                 name##r##type(rn(node->u.w),                            \
1214                               rn(node->v.w), rn(node->w.w));            \
1215                 break
1216 #define case_rrrr(name, type)                                           \
1217             case jit_code_##name##r##type:                              \
1218                 name##r##type(rn(node->u.q.l), rn(node->u.q.h),         \
1219                               rn(node->v.w), rn(node->w.w));            \
1220                 break
1221 #define case_rrw(name, type)                                            \
1222             case jit_code_##name##i##type:                              \
1223                 name##i##type(rn(node->u.w), rn(node->v.w), node->w.w); \
1224                 break
1225 #define case_rrrw(name, type)                                           \
1226             case jit_code_##name##i##type:                              \
1227                 name##i##type(rn(node->u.q.l), rn(node->u.q.h),         \
1228                               rn(node->v.w), node->w.w);                \
1229                 break
1230 #define case_rrf(name, type, size)                                      \
1231             case jit_code_##name##i##type:                              \
1232                 assert(node->flag & jit_flag_data);                     \
1233                 name##i##type(rn(node->u.w), rn(node->v.w),             \
1234                               (jit_float##size##_t *)node->w.n->u.w);   \
1235                 break
1236 #define case_wrr(name, type)                                            \
1237             case jit_code_##name##i##type:                              \
1238                 name##i##type(node->u.w, rn(node->v.w), rn(node->w.w)); \
1239                 break
1240 #define case_brr(name, type)                                            \
1241             case jit_code_##name##r##type:                              \
1242                 temp = node->u.n;                                       \
1243                 assert(temp->code == jit_code_label ||                  \
1244                        temp->code == jit_code_epilog);                  \
1245                 if (temp->flag & jit_flag_patch)                        \
1246                     name##r##type(temp->u.w, rn(node->v.w),             \
1247                                   rn(node->w.w));                       \
1248                 else {                                                  \
1249                     word = name##r##type(_jit->pc.w,                    \
1250                                          rn(node->v.w), rn(node->w.w)); \
1251                     patch(word, node);                                  \
1252                 }                                                       \
1253                 break
1254 #define case_brw(name, type)                                            \
1255             case jit_code_##name##i##type:                              \
1256                 temp = node->u.n;                                       \
1257                 assert(temp->code == jit_code_label ||                  \
1258                        temp->code == jit_code_epilog);                  \
1259                 if (temp->flag & jit_flag_patch)                        \
1260                     name##i##type(temp->u.w,                            \
1261                                   rn(node->v.w), node->w.w);            \
1262                 else {                                                  \
1263                     word = name##i##type(_jit->pc.w,                    \
1264                                          rn(node->v.w), node->w.w);     \
1265                     patch(word, node);                                  \
1266                 }                                                       \
1267                 break
1268 #define case_brf(name, type, size)                                      \
1269             case jit_code_##name##i##type:                              \
1270                 temp = node->u.n;                                       \
1271                 assert(temp->code == jit_code_label ||                  \
1272                        temp->code == jit_code_epilog);                  \
1273                 if (temp->flag & jit_flag_patch)                        \
1274                     name##i##type(temp->u.w, rn(node->v.w),             \
1275                                 (jit_float##size##_t *)node->w.n->u.w); \
1276                 else {                                                  \
1277                     word = name##i##type(_jit->pc.w, rn(node->v.w),     \
1278                                 (jit_float##size##_t *)node->w.n->u.w); \
1279                     patch(word, node);                                  \
1280                 }                                                       \
1281                 break
1282     for (node = _jitc->head; node; node = node->next) {
1283         if (_jit->pc.uc >= _jitc->code.end)
1284             return (NULL);
1285
1286 #if DEVEL_DISASSEMBLER
1287         node->offset = (jit_uword_t)_jit->pc.w - (jit_uword_t)prevw;
1288         prevw = _jit->pc.w;
1289 #endif
1290         value = jit_classify(node->code);
1291         jit_regarg_set(node, value);
1292         switch (node->code) {
1293             case jit_code_align:
1294                 /* Must align to a power of two */
1295                 assert(!(node->u.w & (node->u.w - 1)));
1296                 if ((word = _jit->pc.w & (node->u.w - 1)))
1297                     nop(node->u.w - word);
1298                 break;
1299             case jit_code_skip:
1300                 nop((node->u.w + 3) & ~3);
1301                 break;
1302             case jit_code_note:         case jit_code_name:
1303                 node->u.w = _jit->pc.w;
1304                 break;
1305             case jit_code_label:
1306                 /* remember label is defined */
1307                 node->flag |= jit_flag_patch;
1308                 node->u.w = _jit->pc.w;
1309                 break;
1310                 case_rrr(add,);
1311                 case_rrw(add,);
1312                 case_rrr(addc,);
1313                 case_rrw(addc,);
1314                 case_rrr(addx,);
1315                 case_rrw(addx,);
1316                 case_rrr(sub,);
1317                 case_rrw(sub,);
1318                 case_rrr(subc,);
1319                 case_rrw(subc,);
1320                 case_rrr(subx,);
1321                 case_rrw(subx,);
1322                 case_rrw(rsb,);
1323                 case_rrr(mul,);
1324                 case_rrw(mul,);
1325                 case_rrrr(qmul,);
1326                 case_rrrw(qmul,);
1327                 case_rrrr(qmul, _u);
1328                 case_rrrw(qmul, _u);
1329                 case_rrr(div,);
1330                 case_rrw(div,);
1331                 case_rrr(div, _u);
1332                 case_rrw(div, _u);
1333                 case_rrrr(qdiv,);
1334                 case_rrrw(qdiv,);
1335                 case_rrrr(qdiv, _u);
1336                 case_rrrw(qdiv, _u);
1337                 case_rrr(rem,);
1338                 case_rrw(rem,);
1339                 case_rrr(rem, _u);
1340                 case_rrw(rem, _u);
1341                 case_rrr(and,);
1342                 case_rrw(and,);
1343                 case_rrr(or,);
1344                 case_rrw(or,);
1345                 case_rrr(xor,);
1346                 case_rrw(xor,);
1347                 case_rrr(lsh,);
1348                 case_rrw(lsh,);
1349                 case_rrr(rsh,);
1350                 case_rrw(rsh,);
1351                 case_rrr(rsh, _u);
1352                 case_rrw(rsh, _u);
1353                 case_rr(ext, _c);
1354                 case_rr(ext, _uc);
1355                 case_rr(ext, _s);
1356                 case_rr(ext, _us);
1357 #  if __WORDSIZE == 64
1358                 case_rr(ext, _i);
1359                 case_rr(ext, _ui);
1360 #  endif
1361                 case_rr(hton, _us);
1362                 case_rr(hton, _ui);
1363 #  if __WORDSIZE == 64
1364                 case_rr(hton, _ul);
1365 #  endif
1366             case jit_code_bswapr_us:
1367                 bswapr_us_lh(rn(node->u.w), rn(node->v.w), no_flag);
1368                 break;
1369             case jit_code_bswapr_ui:
1370                 bswapr_ui_lw(rn(node->u.w), rn(node->v.w), no_flag);
1371                 break;
1372 #  if __WORDSIZE == 64
1373                 case_rr(bswap, _ul);
1374 #  endif
1375                 case_rr(neg,);
1376                 case_rr(com,);
1377                 case_rr(clo,);
1378                 case_rr(clz,);
1379                 case_rr(cto,);
1380                 case_rr(ctz,);
1381             case jit_code_casr:
1382                 casr(rn(node->u.w), rn(node->v.w),
1383                      rn(node->w.q.l), rn(node->w.q.h));
1384                 break;
1385             case jit_code_casi:
1386                 casi(rn(node->u.w), node->v.w,
1387                      rn(node->w.q.l), rn(node->w.q.h));
1388                 break;
1389                 case_rrr(movn,);
1390                 case_rrr(movz,);
1391                 case_rr(mov,);
1392             case jit_code_movi:
1393                 if (node->flag & jit_flag_node) {
1394                     temp = node->v.n;
1395                     if (temp->code == jit_code_data ||
1396                         (temp->code == jit_code_label &&
1397                          (temp->flag & jit_flag_patch)))
1398                         movi(rn(node->u.w), temp->u.w);
1399                     else {
1400                         assert(temp->code == jit_code_label ||
1401                                temp->code == jit_code_epilog);
1402                         word = movi_p(rn(node->u.w), node->v.w);
1403                         patch(word, node);
1404                     }
1405                 }
1406                 else
1407                     movi(rn(node->u.w), node->v.w);
1408                 break;
1409                 case_rr(trunc, _f_i);
1410                 case_rr(trunc, _d_i);
1411 #  if __WORDSIZE == 64
1412                 case_rr(trunc, _f_l);
1413                 case_rr(trunc, _d_l);
1414 #  endif
1415                 case_rrr(lt,);
1416                 case_rrw(lt,);
1417                 case_rrr(lt, _u);
1418                 case_rrw(lt, _u);
1419                 case_rrr(le,);
1420                 case_rrw(le,);
1421                 case_rrr(le, _u);
1422                 case_rrw(le, _u);
1423                 case_rrr(eq,);
1424                 case_rrw(eq,);
1425                 case_rrr(ge,);
1426                 case_rrw(ge,);
1427                 case_rrr(ge, _u);
1428                 case_rrw(ge, _u);
1429                 case_rrr(gt,);
1430                 case_rrw(gt,);
1431                 case_rrr(gt, _u);
1432                 case_rrw(gt, _u);
1433                 case_rrr(ne,);
1434                 case_rrw(ne,);
1435                 case_rr(ld, _c);
1436                 case_rw(ld, _c);
1437                 case_brr(blt,);
1438                 case_brw(blt,);
1439                 case_brr(blt, _u);
1440                 case_brw(blt, _u);
1441                 case_brr(ble,);
1442                 case_brw(ble,);
1443                 case_brr(ble, _u);
1444                 case_brw(ble, _u);
1445                 case_brr(beq,);
1446                 case_brw(beq,);
1447                 case_brr(bge,);
1448                 case_brw(bge,);
1449                 case_brr(bge, _u);
1450                 case_brw(bge, _u);
1451                 case_brr(bgt,);
1452                 case_brw(bgt,);
1453                 case_brr(bgt, _u);
1454                 case_brw(bgt, _u);
1455                 case_brr(bne,);
1456                 case_brw(bne,);
1457                 case_brr(bms,);
1458                 case_brw(bms,);
1459                 case_brr(bmc,);
1460                 case_brw(bmc,);
1461                 case_brr(boadd,);
1462                 case_brw(boadd,);
1463                 case_brr(boadd, _u);
1464                 case_brw(boadd, _u);
1465                 case_brr(bxadd,);
1466                 case_brw(bxadd,);
1467                 case_brr(bxadd, _u);
1468                 case_brw(bxadd, _u);
1469                 case_brr(bosub,);
1470                 case_brw(bosub,);
1471                 case_brr(bosub, _u);
1472                 case_brw(bosub, _u);
1473                 case_brr(bxsub,);
1474                 case_brw(bxsub,);
1475                 case_brr(bxsub, _u);
1476                 case_brw(bxsub, _u);
1477                 case_rrr(ldx, _c);
1478                 case_rrw(ldx, _c);
1479                 case_rr(ld, _uc);
1480                 case_rw(ld, _uc);
1481                 case_rrr(ldx, _uc);
1482                 case_rrw(ldx, _uc);
1483                 case_rr(ld, _s);
1484                 case_rw(ld, _s);
1485                 case_rrr(ldx, _s);
1486                 case_rrw(ldx, _s);
1487                 case_rr(ld, _us);
1488                 case_rw(ld, _us);
1489                 case_rrr(ldx, _us);
1490                 case_rrw(ldx, _us);
1491                 case_rr(ld, _i);
1492                 case_rw(ld, _i);
1493                 case_rrr(ldx, _i);
1494                 case_rrw(ldx, _i);
1495 #if __WORDSIZE == 64
1496                 case_rr(ld, _ui);
1497                 case_rw(ld, _ui);
1498                 case_rrr(ldx, _ui);
1499                 case_rrw(ldx, _ui);
1500                 case_rr(ld, _l);
1501                 case_rw(ld, _l);
1502                 case_rrr(ldx, _l);
1503                 case_rrw(ldx, _l);
1504 #endif
1505                 case_rr(st, _c);
1506                 case_wr(st, _c);
1507                 case_rrr(stx, _c);
1508                 case_wrr(stx, _c);
1509                 case_rr(st, _s);
1510                 case_wr(st, _s);
1511                 case_rrr(stx, _s);
1512                 case_wrr(stx, _s);
1513                 case_rr(st, _i);
1514                 case_wr(st, _i);
1515                 case_rrr(stx, _i);
1516                 case_wrr(stx, _i);
1517 #if __WORDSIZE == 64
1518                 case_rr(st, _l);
1519                 case_wr(st, _l);
1520                 case_rrr(stx, _l);
1521                 case_wrr(stx, _l);
1522 #endif
1523                 case_rr(mov, _f);
1524             case jit_code_movi_f:
1525                 assert(node->flag & jit_flag_data);
1526                 movi_f(rn(node->u.w), (jit_float32_t *)node->v.n->u.w);
1527                 break;
1528                 case_rr(ext, _f);
1529                 case_rr(ext, _d_f);
1530                 case_rr(abs, _f);
1531                 case_rr(neg, _f);
1532                 case_rr(sqrt, _f);
1533                 case_rrr(add, _f);
1534                 case_rrf(add, _f, 32);
1535                 case_rrr(sub, _f);
1536                 case_rrf(sub, _f, 32);
1537                 case_rrf(rsb, _f, 32);
1538                 case_rrr(mul, _f);
1539                 case_rrf(mul, _f, 32);
1540                 case_rrr(div, _f);
1541                 case_rrf(div, _f, 32);
1542                 case_rrr(lt, _f);
1543                 case_rrf(lt, _f, 32);
1544                 case_rrr(le, _f);
1545                 case_rrf(le, _f, 32);
1546                 case_rrr(eq, _f);
1547                 case_rrf(eq, _f, 32);
1548                 case_rrr(ge, _f);
1549                 case_rrf(ge, _f, 32);
1550                 case_rrr(gt, _f);
1551                 case_rrf(gt, _f, 32);
1552                 case_rrr(ne, _f);
1553                 case_rrf(ne, _f, 32);
1554                 case_rrr(unlt, _f);
1555                 case_rrf(unlt, _f, 32);
1556                 case_rrr(unle, _f);
1557                 case_rrf(unle, _f, 32);
1558                 case_rrr(uneq, _f);
1559                 case_rrf(uneq, _f, 32);
1560                 case_rrr(unge, _f);
1561                 case_rrf(unge, _f, 32);
1562                 case_rrr(ungt, _f);
1563                 case_rrf(ungt, _f, 32);
1564                 case_rrr(ltgt, _f);
1565                 case_rrf(ltgt, _f, 32);
1566                 case_rrr(ord, _f);
1567                 case_rrf(ord, _f, 32);
1568                 case_rrr(unord, _f);
1569                 case_rrf(unord, _f, 32);
1570                 case_brr(blt, _f);
1571                 case_brf(blt, _f, 32);
1572                 case_brr(ble, _f);
1573                 case_brf(ble, _f, 32);
1574                 case_brr(beq, _f);
1575                 case_brf(beq, _f, 32);
1576                 case_brr(bge, _f);
1577                 case_brf(bge, _f, 32);
1578                 case_brr(bgt, _f);
1579                 case_brf(bgt, _f, 32);
1580                 case_brr(bne, _f);
1581                 case_brf(bne, _f, 32);
1582                 case_brr(bunlt, _f);
1583                 case_brf(bunlt, _f, 32);
1584                 case_brr(bunle, _f);
1585                 case_brf(bunle, _f, 32);
1586                 case_brr(buneq, _f);
1587                 case_brf(buneq, _f, 32);
1588                 case_brr(bunge, _f);
1589                 case_brf(bunge, _f, 32);
1590                 case_brr(bungt, _f);
1591                 case_brf(bungt, _f, 32);
1592                 case_brr(bltgt, _f);
1593                 case_brf(bltgt, _f, 32);
1594                 case_brr(bord, _f);
1595                 case_brf(bord, _f, 32);
1596                 case_brr(bunord, _f);
1597                 case_brf(bunord, _f, 32);
1598                 case_rr(ld, _f);
1599                 case_rw(ld, _f);
1600                 case_rrr(ldx, _f);
1601                 case_rrw(ldx, _f);
1602                 case_rr(st, _f);
1603                 case_wr(st, _f);
1604                 case_rrr(stx, _f);
1605                 case_wrr(stx, _f);
1606                 case_rr(mov, _d);
1607             case jit_code_movi_d:
1608                 assert(node->flag & jit_flag_data);
1609                 movi_d(rn(node->u.w), (jit_float64_t *)node->v.n->u.w);
1610                 break;
1611                 case_rr(ext, _d);
1612                 case_rr(ext, _f_d);
1613                 case_rr(abs, _d);
1614                 case_rr(neg, _d);
1615                 case_rr(sqrt, _d);
1616                 case_rrr(add, _d);
1617                 case_rrf(add, _d, 64);
1618                 case_rrr(sub, _d);
1619                 case_rrf(sub, _d, 64);
1620                 case_rrf(rsb, _d, 64);
1621                 case_rrr(mul, _d);
1622                 case_rrf(mul, _d, 64);
1623                 case_rrr(div, _d);
1624                 case_rrf(div, _d, 64);
1625                 case_rrr(lt, _d);
1626                 case_rrf(lt, _d, 64);
1627                 case_rrr(le, _d);
1628                 case_rrf(le, _d, 64);
1629                 case_rrr(eq, _d);
1630                 case_rrf(eq, _d, 64);
1631                 case_rrr(ge, _d);
1632                 case_rrf(ge, _d, 64);
1633                 case_rrr(gt, _d);
1634                 case_rrf(gt, _d, 64);
1635                 case_rrr(ne, _d);
1636                 case_rrf(ne, _d, 64);
1637                 case_rrr(unlt, _d);
1638                 case_rrf(unlt, _d, 64);
1639                 case_rrr(unle, _d);
1640                 case_rrf(unle, _d, 64);
1641                 case_rrr(uneq, _d);
1642                 case_rrf(uneq, _d, 64);
1643                 case_rrr(unge, _d);
1644                 case_rrf(unge, _d, 64);
1645                 case_rrr(ungt, _d);
1646                 case_rrf(ungt, _d, 64);
1647                 case_rrr(ltgt, _d);
1648                 case_rrf(ltgt, _d, 64);
1649                 case_rrr(ord, _d);
1650                 case_rrf(ord, _d, 64);
1651                 case_rrr(unord, _d);
1652                 case_rrf(unord, _d, 64);
1653                 case_brr(blt, _d);
1654                 case_brf(blt, _d, 64);
1655                 case_brr(ble, _d);
1656                 case_brf(ble, _d, 64);
1657                 case_brr(beq, _d);
1658                 case_brf(beq, _d, 64);
1659                 case_brr(bge, _d);
1660                 case_brf(bge, _d, 64);
1661                 case_brr(bgt, _d);
1662                 case_brf(bgt, _d, 64);
1663                 case_brr(bne, _d);
1664                 case_brf(bne, _d, 64);
1665                 case_brr(bunlt, _d);
1666                 case_brf(bunlt, _d, 64);
1667                 case_brr(bunle, _d);
1668                 case_brf(bunle, _d, 64);
1669                 case_brr(buneq, _d);
1670                 case_brf(buneq, _d, 64);
1671                 case_brr(bunge, _d);
1672                 case_brf(bunge, _d, 64);
1673                 case_brr(bungt, _d);
1674                 case_brf(bungt, _d, 64);
1675                 case_brr(bltgt, _d);
1676                 case_brf(bltgt, _d, 64);
1677                 case_brr(bord, _d);
1678                 case_brf(bord, _d, 64);
1679                 case_brr(bunord, _d);
1680                 case_brf(bunord, _d, 64);
1681                 case_rr(ld, _d);
1682                 case_rw(ld, _d);
1683                 case_rrr(ldx, _d);
1684                 case_rrw(ldx, _d);
1685                 case_rr(st, _d);
1686                 case_wr(st, _d);
1687                 case_rrr(stx, _d);
1688                 case_wrr(stx, _d);
1689             case jit_code_jmpr:
1690                 jmpr(rn(node->u.w));
1691                 break;
1692             case jit_code_jmpi:
1693                 if (node->flag & jit_flag_node) {
1694 #if _CALL_AIXDESC
1695                     if (_jit->pc.uc == _jit->code.ptr + sizeof(void*) * 3)
1696                         _jitc->jump = 1;
1697 #endif
1698                     temp = node->u.n;
1699                     assert(temp->code == jit_code_label ||
1700                            temp->code == jit_code_epilog);
1701                     if (temp->flag & jit_flag_patch)
1702                         jmpi(temp->u.w);
1703                     else {
1704                         word = _jit->code.length -
1705                             (_jit->pc.uc - _jit->code.ptr);
1706                         if (can_sign_extend_jump_p(word))
1707                             word = jmpi(_jit->pc.w);
1708                         else
1709                             word = jmpi_p(_jit->pc.w);
1710                         patch(word, node);
1711                     }
1712                 }
1713                 else
1714                     jmpi(node->u.w);
1715                 break;
1716             case jit_code_callr:
1717 #if _CALL_SYSV
1718 #  define xcallr(u, v)          callr(u, v)
1719 #  define xcalli_p(u, v)        calli_p(u, v)
1720 #  define xcalli(u, v)          calli(u, v)
1721 #else
1722 #  define xcallr(u, v)          callr(u)
1723 #  define xcalli_p(u, v)        calli_p(u)
1724 #  define xcalli(u, v)          calli(u)
1725 #endif
1726                 xcallr(rn(node->u.w), !!(node->flag & jit_flag_varargs));
1727                 break;
1728             case jit_code_calli:
1729                 value = !!(node->flag & jit_flag_varargs);
1730                 if (node->flag & jit_flag_node) {
1731                     temp = node->u.n;
1732                     assert(temp->code == jit_code_label ||
1733                            temp->code == jit_code_epilog);
1734                     if (temp->flag & jit_flag_patch)
1735                         xcalli(temp->u.w, value);
1736                     else {
1737                         word = _jit->code.length -
1738                             (_jit->pc.uc - _jit->code.ptr);
1739 #if _CALL_SYSV
1740                         if (can_sign_extend_jump_p(word + value * 4))
1741                             word = xcalli(_jit->pc.w, value);
1742                         else
1743 #endif
1744                             word = xcalli_p(_jit->pc.w, value);
1745                         patch(word, node);
1746                     }
1747                 }
1748                 else
1749                     xcalli(node->u.w, value);
1750                 break;
1751             case jit_code_prolog:
1752                 _jitc->function = _jitc->functions.ptr + node->w.w;
1753                 undo.node = node;
1754                 undo.word = _jit->pc.w;
1755                 memcpy(&undo.func, _jitc->function, sizeof(undo.func));
1756 #if DEVEL_DISASSEMBLER
1757                 undo.prevw = prevw;
1758 #endif
1759                 undo.patch_offset = _jitc->patches.offset;
1760 #if _CALL_AIXDESC
1761                 undo.prolog_offset = _jitc->prolog.offset;
1762 #endif
1763             restart_function:
1764                 _jitc->again = 0;
1765 #if _CALL_AIXDESC
1766                 if (_jitc->jump && !_jitc->function->assume_frame) {
1767                     /* remember prolog to hide offset adjustment for a jump
1768                      * to the start of a function, what is expected to be
1769                      * a common practice as first jit instruction */
1770                     if (_jitc->prolog.offset >= _jitc->prolog.length) {
1771                         _jitc->prolog.length += 16;
1772                         jit_realloc((jit_pointer_t *)&_jitc->prolog.ptr,
1773                                     (_jitc->prolog.length - 16) *
1774                                     sizeof(jit_word_t),
1775                                     _jitc->prolog.length * sizeof(jit_word_t));
1776                     }
1777                     _jitc->prolog.ptr[_jitc->prolog.offset++] = _jit->pc.w;
1778                     /* function descriptor */
1779                     word = _jit->pc.w + sizeof(void*) * 3;
1780                     iw(word);                           /* addr */
1781                     iw(0);                              /* toc */
1782                     iw(0);                              /* env */
1783                 }
1784 #endif
1785                 prolog(node);
1786                 break;
1787             case jit_code_epilog:
1788                 assert(_jitc->function == _jitc->functions.ptr + node->w.w);
1789                 if (_jitc->again) {
1790                     for (temp = undo.node->next;
1791                          temp != node; temp = temp->next) {
1792                         if (temp->code == jit_code_label ||
1793                             temp->code == jit_code_epilog)
1794                             temp->flag &= ~jit_flag_patch;
1795                     }
1796                     temp->flag &= ~jit_flag_patch;
1797                     node = undo.node;
1798                     _jit->pc.w = undo.word;
1799                     /* undo.func.self.aoff and undo.func.regset should not
1800                      * be undone, as they will be further updated, and are
1801                      * the reason of the undo. */
1802                     undo.func.self.aoff = _jitc->function->frame +
1803                         _jitc->function->self.aoff;
1804                     jit_regset_set(&undo.func.regset, &_jitc->function->regset);
1805                     /* allocar information also does not need to be undone */
1806                     undo.func.aoffoff = _jitc->function->aoffoff;
1807                     undo.func.allocar = _jitc->function->allocar;
1808                     memcpy(_jitc->function, &undo.func, sizeof(undo.func));
1809 #if DEVEL_DISASSEMBLER
1810                     prevw = undo.prevw;
1811 #endif
1812                     _jitc->patches.offset = undo.patch_offset;
1813 #if _CALL_AIXDESC
1814                     _jitc->prolog.offset = undo.prolog_offset;
1815 #endif
1816                     goto restart_function;
1817                 }
1818                 /* remember label is defined */
1819                 node->flag |= jit_flag_patch;
1820                 node->u.w = _jit->pc.w;
1821                 epilog(node);
1822                 _jitc->function = NULL;
1823                 break;
1824             case jit_code_va_start:
1825                 vastart(rn(node->u.w));
1826                 break;
1827             case jit_code_va_arg:
1828                 vaarg(rn(node->u.w), rn(node->v.w));
1829                 break;
1830             case jit_code_va_arg_d:
1831                 vaarg_d(rn(node->u.w), rn(node->v.w));
1832                 break;
1833             case jit_code_live:                 case jit_code_ellipsis:
1834             case jit_code_va_push:
1835             case jit_code_allocai:              case jit_code_allocar:
1836             case jit_code_arg_c:                case jit_code_arg_s:
1837             case jit_code_arg_i:
1838 #  if __WORDSIZE == 64
1839             case jit_code_arg_l:
1840 #  endif
1841             case jit_code_arg_f:                case jit_code_arg_d:
1842             case jit_code_va_end:
1843             case jit_code_ret:
1844             case jit_code_retr_c:               case jit_code_reti_c:
1845             case jit_code_retr_uc:              case jit_code_reti_uc:
1846             case jit_code_retr_s:               case jit_code_reti_s:
1847             case jit_code_retr_us:              case jit_code_reti_us:
1848             case jit_code_retr_i:               case jit_code_reti_i:
1849 #if __WORDSIZE == 64
1850             case jit_code_retr_ui:              case jit_code_reti_ui:
1851             case jit_code_retr_l:               case jit_code_reti_l:
1852 #endif
1853             case jit_code_retr_f:               case jit_code_reti_f:
1854             case jit_code_retr_d:               case jit_code_reti_d:
1855             case jit_code_getarg_c:             case jit_code_getarg_uc:
1856             case jit_code_getarg_s:             case jit_code_getarg_us:
1857             case jit_code_getarg_i:
1858 #if __WORDSIZE == 64
1859             case jit_code_getarg_ui:            case jit_code_getarg_l:
1860 #endif
1861             case jit_code_getarg_f:             case jit_code_getarg_d:
1862             case jit_code_putargr_c:            case jit_code_putargi_c:
1863             case jit_code_putargr_uc:           case jit_code_putargi_uc:
1864             case jit_code_putargr_s:            case jit_code_putargi_s:
1865             case jit_code_putargr_us:           case jit_code_putargi_us:
1866             case jit_code_putargr_i:            case jit_code_putargi_i:
1867 #if __WORDSIZE == 64
1868             case jit_code_putargr_ui:           case jit_code_putargi_ui:
1869             case jit_code_putargr_l:            case jit_code_putargi_l:
1870 #endif
1871             case jit_code_putargr_f:            case jit_code_putargi_f:
1872             case jit_code_putargr_d:            case jit_code_putargi_d:
1873             case jit_code_pushargr_c:           case jit_code_pushargi_c:
1874             case jit_code_pushargr_uc:          case jit_code_pushargi_uc:
1875             case jit_code_pushargr_s:           case jit_code_pushargi_s:
1876             case jit_code_pushargr_us:          case jit_code_pushargi_us:
1877             case jit_code_pushargr_i:           case jit_code_pushargi_i:
1878 #if __WORDSIZE == 64
1879             case jit_code_pushargr_ui:          case jit_code_pushargi_ui:
1880             case jit_code_pushargr_l:           case jit_code_pushargi_l:
1881 #endif
1882             case jit_code_pushargr_f:           case jit_code_pushargi_f:
1883             case jit_code_pushargr_d:           case jit_code_pushargi_d:
1884             case jit_code_retval_c:             case jit_code_retval_uc:
1885             case jit_code_retval_s:             case jit_code_retval_us:
1886             case jit_code_retval_i:
1887 #if __WORDSIZE == 64
1888             case jit_code_retval_ui:            case jit_code_retval_l:
1889 #endif
1890             case jit_code_retval_f:             case jit_code_retval_d:
1891             case jit_code_prepare:
1892             case jit_code_finishr:              case jit_code_finishi:
1893                 break;
1894             default:
1895                 abort();
1896         }
1897         jit_regarg_clr(node, value);
1898         assert(_jitc->regarg == 0 && _jitc->synth == 0);
1899         /* update register live state */
1900         jit_reglive(node);
1901
1902         no_flag = !(node->flag & jit_flag_patch);
1903     }
1904 #undef case_brf
1905 #undef case_brw
1906 #undef case_brr
1907 #undef case_wrr
1908 #undef case_rrf
1909 #undef case_rrw
1910 #undef case_rrr
1911 #undef case_wr
1912 #undef case_rw
1913 #undef case_rr
1914
1915     for (offset = 0; offset < _jitc->patches.offset; offset++) {
1916         node = _jitc->patches.ptr[offset].node;
1917         word = node->code == jit_code_movi ? node->v.n->u.w : node->u.n->u.w;
1918         patch_at(_jitc->patches.ptr[offset].inst, word);
1919     }
1920
1921     jit_flush(_jit->code.ptr, _jit->pc.uc);
1922
1923     return (_jit->code.ptr);
1924 }
1925
1926 #define CODE                            1
1927 #  include "jit_ppc-cpu.c"
1928 #  include "jit_ppc-fpu.c"
1929 #undef CODE
1930
1931 void
1932 jit_flush(void *fptr, void *tptr)
1933 {
1934 #if defined(__GNUC__)
1935     jit_word_t          f, t, s;
1936
1937     s = sysconf(_SC_PAGE_SIZE);
1938     f = (jit_word_t)fptr & -s;
1939     t = (((jit_word_t)tptr) + s - 1) & -s;
1940     __clear_cache((void *)f, (void *)t);
1941 #endif
1942 }
1943
1944 void
1945 _emit_ldxi(jit_state_t *_jit, jit_int32_t r0, jit_int32_t r1, jit_word_t i0)
1946 {
1947 #if __WORDSIZE == 32
1948     ldxi_i(rn(r0), rn(r1), i0);
1949 #else
1950     ldxi_l(rn(r0), rn(r1), i0);
1951 #endif
1952 }
1953
1954 void
1955 _emit_stxi(jit_state_t *_jit, jit_word_t i0, jit_int32_t r0, jit_int32_t r1)
1956 {
1957 #if __WORDSIZE == 32
1958     stxi_i(i0, rn(r0), rn(r1));
1959 #else
1960     stxi_l(i0, rn(r0), rn(r1));
1961 #endif
1962 }
1963
1964 void
1965 _emit_ldxi_d(jit_state_t *_jit, jit_int32_t r0, jit_int32_t r1, jit_word_t i0)
1966 {
1967     ldxi_d(rn(r0), rn(r1), i0);
1968 }
1969
1970 void
1971 _emit_stxi_d(jit_state_t *_jit, jit_word_t i0, jit_int32_t r0, jit_int32_t r1)
1972 {
1973     stxi_d(i0, rn(r0), rn(r1));
1974 }
1975
1976 static void
1977 _patch(jit_state_t *_jit, jit_word_t instr, jit_node_t *node)
1978 {
1979     jit_int32_t          flag;
1980
1981     assert(node->flag & jit_flag_node);
1982     if (node->code == jit_code_movi)
1983         flag = node->v.n->flag;
1984     else
1985         flag = node->u.n->flag;
1986     assert(!(flag & jit_flag_patch));
1987     if (_jitc->patches.offset >= _jitc->patches.length) {
1988         jit_realloc((jit_pointer_t *)&_jitc->patches.ptr,
1989                     _jitc->patches.length * sizeof(jit_patch_t),
1990                     (_jitc->patches.length + 1024) * sizeof(jit_patch_t));
1991         _jitc->patches.length += 1024;
1992     }
1993     _jitc->patches.ptr[_jitc->patches.offset].inst = instr;
1994     _jitc->patches.ptr[_jitc->patches.offset].node = node;
1995     ++_jitc->patches.offset;
1996 }