deps/lightning/lib/jit_ppc.c

   1 /*
   2  * Copyright (C) 2012-2019  Free Software Foundation, Inc.
   3  *
   4  * This file is part of GNU lightning.
   5  *
   6  * GNU lightning is free software; you can redistribute it and/or modify it
   7  * under the terms of the GNU Lesser General Public License as published
   8  * by the Free Software Foundation; either version 3, or (at your option)
   9  * any later version.
  10  *
  11  * GNU lightning is distributed in the hope that it will be useful, but
  12  * WITHOUT ANY WARRANTY; without even the implied warranty of MERCHANTABILITY
  13  * or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU Lesser General Public
  14  * License for more details.
  15  *
  16  * Authors:
  17  *      Paulo Cesar Pereira de Andrade
  18  */
  19
  20 #define jit_arg_reg_p(i)                ((i) >= 0 && (i) < 8)
  21 #if !_CALL_SYSV
  22 #  define jit_arg_f_reg_p(i)            ((i) >= 0 && (i) < 13)
  23 #else
  24 #  define jit_arg_f_reg_p(i)            ((i) >= 0 && (i) < 8)
  25 #  if __WORDSIZE == 32
  26 #    define va_gp_shift                 2
  27 #  else
  28 #    define va_gp_shift                 3
  29 #  endif
  30 #  define va_gp_increment               sizeof(jit_word_t)
  31 #  define first_gp_argument             r3
  32 #  define first_gp_offset               offsetof(jit_va_list_t,         \
  33                                                  first_gp_argument)
  34 #  define va_fp_increment               sizeof(jit_float64_t)
  35 #  define first_fp_argument             f1
  36 #  define first_fp_offset               offsetof(jit_va_list_t,         \
  37                                                  first_fp_argument)
  38 #endif
  39 #if __BYTE_ORDER == __LITTLE_ENDIAN
  40 #  define C_DISP                        0
  41 #  define S_DISP                        0
  42 #  define I_DISP                        0
  43 #  define F_DISP                        0
  44 #else
  45 #  define C_DISP                        (__WORDSIZE >> 3) - sizeof(jit_int8_t)
  46 #  define S_DISP                        (__WORDSIZE >> 3) - sizeof(jit_int16_t)
  47 #  define I_DISP                        (__WORDSIZE >> 3) - sizeof(jit_int32_t)
  48 #  define F_DISP                        (__WORDSIZE >> 3) - sizeof(jit_float32_t)
  49 #endif
  50
  51 /*
  52  * Types
  53  */
  54 #if _CALL_SYSV
  55 typedef struct jit_va_list {
  56     jit_uint8_t         ngpr;
  57     jit_uint8_t         nfpr;
  58     jit_uint16_t        _pad;
  59 #  if __WORDSIZE == 64
  60     jit_uint32_t        _pad2;
  61 #  endif
  62     jit_pointer_t       over;
  63     jit_pointer_t       save;
  64 #  if __WORDSIZE == 32
  65     jit_word_t          _pad2;
  66 #  endif
  67     jit_word_t          r3;
  68     jit_word_t          r4;
  69     jit_word_t          r5;
  70     jit_word_t          r6;
  71     jit_word_t          r7;
  72     jit_word_t          r8;
  73     jit_word_t          r9;
  74     jit_word_t          r10;
  75     jit_float64_t       f1;
  76     jit_float64_t       f2;
  77     jit_float64_t       f3;
  78     jit_float64_t       f4;
  79     jit_float64_t       f5;
  80     jit_float64_t       f6;
  81     jit_float64_t       f7;
  82     jit_float64_t       f8;
  83 } jit_va_list_t;
  84 #else
  85 typedef jit_pointer_t jit_va_list_t;
  86 #endif
  87
  88 /*
  89  * Prototypes
  90  */
  91 #define patch(instr, node)              _patch(_jit, instr, node)
  92 static void _patch(jit_state_t*,jit_word_t,jit_node_t*);
  93
  94 /* libgcc */
  95 extern void __clear_cache(void *, void *);
  96
  97 #define PROTO                           1
  98 #  include "jit_ppc-cpu.c"
  99 #  include "jit_ppc-fpu.c"
 100 #undef PROTO
 101
 102 /*
 103  * Initialization
 104  */
 105 jit_register_t          _rvs[] = {
 106     { rc(sav) | 0,                      "r0" },
 107     { rc(sav) | 11,                     "r11" },        /* env */
 108     { rc(sav) | 12,                     "r12" },        /* exception */
 109     { rc(sav) | 13,                     "r13" },        /* thread */
 110     { rc(sav) | 2,                      "r2" },         /* toc */
 111     { rc(sav) | rc(gpr) | 14,           "r14" },
 112     { rc(sav) | rc(gpr) | 15,           "r15" },
 113     { rc(sav) | rc(gpr) | 16,           "r16" },
 114     { rc(sav) | rc(gpr) | 17,           "r17" },
 115     { rc(sav) | rc(gpr) | 18,           "r18" },
 116     { rc(sav) | rc(gpr) | 19,           "r19" },
 117     { rc(sav) | rc(gpr) | 20,           "r20" },
 118     { rc(sav) | rc(gpr) | 21,           "r21" },
 119     { rc(sav) | rc(gpr) | 22,           "r22" },
 120     { rc(sav) | rc(gpr) | 23,           "r23" },
 121     { rc(sav) | rc(gpr) | 24,           "r24" },
 122     { rc(sav) | rc(gpr) | 25,           "r25" },
 123     { rc(sav) | rc(gpr) | 26,           "r26" },
 124     { rc(sav) | rc(gpr) | 27,           "r27" },
 125     { rc(sav) | rc(gpr) | 28,           "r28" },
 126     { rc(sav) | rc(gpr) | 29,           "r29" },
 127     { rc(sav) | rc(gpr) | 30,           "r30" },
 128     { rc(sav) | 1,                      "r1" },
 129     { rc(sav) | 31,                     "r31" },
 130     { rc(arg) | rc(gpr) | 10,           "r10" },
 131     { rc(arg) | rc(gpr) | 9,            "r9" },
 132     { rc(arg) | rc(gpr) | 8,            "r8" },
 133     { rc(arg) | rc(gpr) | 7,            "r7" },
 134     { rc(arg) | rc(gpr) | 6,            "r6" },
 135     { rc(arg) | rc(gpr) | 5,            "r5" },
 136     { rc(arg) | rc(gpr) | 4,            "r4" },
 137     { rc(arg) | rc(gpr) | 3,            "r3" },
 138     { rc(fpr) | 0,                      "f0" },
 139     { rc(sav) | rc(fpr) | 14,           "f14" },
 140     { rc(sav) | rc(fpr) | 15,           "f15" },
 141     { rc(sav) | rc(fpr) | 16,           "f16" },
 142     { rc(sav) | rc(fpr) | 17,           "f17" },
 143     { rc(sav) | rc(fpr) | 18,           "f18" },
 144     { rc(sav) | rc(fpr) | 19,           "f19" },
 145     { rc(sav) | rc(fpr) | 20,           "f20" },
 146     { rc(sav) | rc(fpr) | 21,           "f21" },
 147     { rc(sav) | rc(fpr) | 22,           "f22" },
 148     { rc(sav) | rc(fpr) | 23,           "f23" },
 149     { rc(sav) | rc(fpr) | 24,           "f24" },
 150     { rc(sav) | rc(fpr) | 25,           "f25" },
 151     { rc(sav) | rc(fpr) | 26,           "f26" },
 152     { rc(sav) | rc(fpr) | 27,           "f27" },
 153     { rc(sav) | rc(fpr) | 28,           "f28" },
 154     { rc(sav) | rc(fpr) | 29,           "f29" },
 155     { rc(sav) | rc(fpr) | 30,           "f30" },
 156     { rc(sav) | rc(fpr) | 31,           "f31" },
 157 #if !_CALL_SYSV
 158     { rc(arg) | rc(fpr) | 13,           "f13" },
 159     { rc(arg) | rc(fpr) | 12,           "f12" },
 160     { rc(arg) | rc(fpr) | 11,           "f11" },
 161     { rc(arg) | rc(fpr) | 10,           "f10" },
 162     { rc(arg) | rc(fpr) | 9,            "f9" },
 163 #else
 164     { rc(fpr) | 13,                     "f13" },
 165     { rc(fpr) | 12,                     "f12" },
 166     { rc(fpr) | 11,                     "f11" },
 167     { rc(fpr) | 10,                     "f10" },
 168     { rc(fpr) | 9,                      "f9" },
 169 #endif
 170     { rc(arg) | rc(fpr) | 8,            "f8" },
 171     { rc(arg) | rc(fpr) | 7,            "f7" },
 172     { rc(arg) | rc(fpr) | 6,            "f6" },
 173     { rc(arg) | rc(fpr) | 5,            "f5" },
 174     { rc(arg) | rc(fpr) | 4,            "f4" },
 175     { rc(arg) | rc(fpr) | 3,            "f3" },
 176     { rc(arg) | rc(fpr) | 2,            "f2" },
 177     { rc(arg) | rc(fpr) | 1,            "f1" },
 178     { _NOREG,                           "<none>" },
 179 };
 180
 181 /*
 182  * Implementation
 183  */
 184 void
 185 jit_get_cpu(void)
 186 {
 187 }
 188
 189 void
 190 _jit_init(jit_state_t *_jit)
 191 {
 192     _jitc->reglen = jit_size(_rvs) - 1;
 193 }
 194
 195 void
 196 _jit_prolog(jit_state_t *_jit)
 197 {
 198     jit_int32_t          offset;
 199
 200     if (_jitc->function)
 201         jit_epilog();
 202     assert(jit_regset_cmp_ui(&_jitc->regarg, 0) == 0);
 203     jit_regset_set_ui(&_jitc->regsav, 0);
 204     offset = _jitc->functions.offset;
 205     if (offset >= _jitc->functions.length) {
 206         jit_realloc((jit_pointer_t *)&_jitc->functions.ptr,
 207                     _jitc->functions.length * sizeof(jit_function_t),
 208                     (_jitc->functions.length + 16) * sizeof(jit_function_t));
 209         _jitc->functions.length += 16;
 210     }
 211     _jitc->function = _jitc->functions.ptr + _jitc->functions.offset++;
 212     _jitc->function->self.size = params_offset;
 213     _jitc->function->self.argi = _jitc->function->self.argf =
 214         _jitc->function->self.alen = 0;
 215     /* float conversion */
 216     _jitc->function->self.aoff = alloca_offset - 8;
 217     _jitc->function->self.call = jit_call_default;
 218     jit_alloc((jit_pointer_t *)&_jitc->function->regoff,
 219               _jitc->reglen * sizeof(jit_int32_t));
 220
 221     /* _no_link here does not mean the jit_link() call can be removed
 222      * by rewriting as:
 223      * _jitc->function->prolog = jit_new_node(jit_code_prolog);
 224      */
 225     _jitc->function->prolog = jit_new_node_no_link(jit_code_prolog);
 226     jit_link(_jitc->function->prolog);
 227     _jitc->function->prolog->w.w = offset;
 228     _jitc->function->epilog = jit_new_node_no_link(jit_code_epilog);
 229     /*  u:      label value
 230      *  v:      offset in blocks vector
 231      *  w:      offset in functions vector
 232      */
 233     _jitc->function->epilog->w.w = offset;
 234
 235     jit_regset_new(&_jitc->function->regset);
 236 }
 237
 238 jit_int32_t
 239 _jit_allocai(jit_state_t *_jit, jit_int32_t length)
 240 {
 241     assert(_jitc->function);
 242     switch (length) {
 243         case 0: case 1:                                         break;
 244         case 2:         _jitc->function->self.aoff &= -2;       break;
 245         case 3: case 4: _jitc->function->self.aoff &= -4;       break;
 246         default:        _jitc->function->self.aoff &= -8;       break;
 247     }
 248     _jitc->function->self.aoff -= length;
 249     if (!_jitc->realize) {
 250         jit_inc_synth_ww(allocai, _jitc->function->self.aoff, length);
 251         jit_dec_synth();
 252     }
 253     return (_jitc->function->self.aoff);
 254 }
 255
 256 void
 257 _jit_allocar(jit_state_t *_jit, jit_int32_t u, jit_int32_t v)
 258 {
 259     jit_int32_t          r0, r1;
 260     assert(_jitc->function);
 261     jit_inc_synth_ww(allocar, u, v);
 262     if (!_jitc->function->allocar) {
 263         _jitc->function->aoffoff = jit_allocai(sizeof(jit_int32_t));
 264         _jitc->function->allocar = 1;
 265     }
 266     r0 = jit_get_reg(jit_class_gpr);
 267     r1 = jit_get_reg(jit_class_gpr);
 268     jit_ldr(r0, JIT_SP);
 269     jit_negr(r1, v);
 270     jit_andi(r1, r1, -16);
 271     jit_ldxi_i(u, JIT_FP, _jitc->function->aoffoff);
 272     jit_addr(u, u, r1);
 273     jit_addr(JIT_SP, JIT_SP, r1);
 274     jit_stxi_i(_jitc->function->aoffoff, JIT_FP, u);
 275     jit_str(JIT_SP, r0);
 276     jit_unget_reg(r1);
 277     jit_unget_reg(r0);
 278     jit_dec_synth();
 279 }
 280
 281 void
 282 _jit_ret(jit_state_t *_jit)
 283 {
 284     jit_node_t          *instr;
 285     assert(_jitc->function);
 286     jit_inc_synth(ret);
 287     /* jump to epilog */
 288     instr = jit_jmpi();
 289     jit_patch_at(instr, _jitc->function->epilog);
 290     jit_dec_synth();
 291 }
 292
 293 void
 294 _jit_retr(jit_state_t *_jit, jit_int32_t u)
 295 {
 296     jit_inc_synth_w(retr, u);
 297     if (JIT_RET != u)
 298         jit_movr(JIT_RET, u);
 299     jit_live(JIT_RET);
 300     jit_ret();
 301     jit_dec_synth();
 302 }
 303
 304 void
 305 _jit_reti(jit_state_t *_jit, jit_word_t u)
 306 {
 307     jit_inc_synth_w(reti, u);
 308     jit_movi(JIT_RET, u);
 309     jit_ret();
 310     jit_dec_synth();
 311 }
 312
 313 void
 314 _jit_retr_f(jit_state_t *_jit, jit_int32_t u)
 315 {
 316     jit_inc_synth_w(retr_f, u);
 317     if (JIT_RET != u)
 318         jit_movr_f(JIT_FRET, u);
 319     else
 320         jit_live(JIT_FRET);
 321     jit_ret();
 322     jit_dec_synth();
 323 }
 324
 325 void
 326 _jit_reti_f(jit_state_t *_jit, jit_float32_t u)
 327 {
 328     jit_inc_synth_f(reti_f, u);
 329     jit_movi_f(JIT_FRET, u);
 330     jit_ret();
 331     jit_dec_synth();
 332 }
 333
 334 void
 335 _jit_retr_d(jit_state_t *_jit, jit_int32_t u)
 336 {
 337     jit_inc_synth_w(retr_d, u);
 338     if (JIT_FRET != u)
 339         jit_movr_d(JIT_FRET, u);
 340     else
 341         jit_live(JIT_FRET);
 342     jit_ret();
 343     jit_dec_synth();
 344 }
 345
 346 void
 347 _jit_reti_d(jit_state_t *_jit, jit_float64_t u)
 348 {
 349     jit_inc_synth_d(reti_d, u);
 350     jit_movi_d(JIT_FRET, u);
 351     jit_ret();
 352     jit_dec_synth();
 353 }
 354
 355 void
 356 _jit_epilog(jit_state_t *_jit)
 357 {
 358     assert(_jitc->function);
 359     assert(_jitc->function->epilog->next == NULL);
 360     jit_link(_jitc->function->epilog);
 361     _jitc->function = NULL;
 362 }
 363
 364 jit_bool_t
 365 _jit_arg_register_p(jit_state_t *_jit, jit_node_t *u)
 366 {
 367     if (u->code == jit_code_arg)
 368         return (jit_arg_reg_p(u->u.w));
 369     assert(u->code == jit_code_arg_f || u->code == jit_code_arg_d);
 370     return (jit_arg_f_reg_p(u->u.w));
 371 }
 372
 373 void
 374 _jit_ellipsis(jit_state_t *_jit)
 375 {
 376     jit_inc_synth(ellipsis);
 377     if (_jitc->prepare) {
 378         jit_link_prepare();
 379         assert(!(_jitc->function->call.call & jit_call_varargs));
 380         _jitc->function->call.call |= jit_call_varargs;
 381     }
 382     else {
 383         jit_link_prolog();
 384         assert(!(_jitc->function->self.call & jit_call_varargs));
 385         _jitc->function->self.call |= jit_call_varargs;
 386 #if _CALL_SYSV
 387         /* Allocate va_list like object in the stack.
 388          * If applicable, with enough space to save all argument
 389          * registers, and use fixed offsets for them. */
 390         _jitc->function->vaoff = jit_allocai(sizeof(jit_va_list_t));
 391 #endif
 392         _jitc->function->vagp = _jitc->function->self.argi;
 393         _jitc->function->vafp = _jitc->function->self.argf;
 394     }
 395     jit_dec_synth();
 396 }
 397
 398 void
 399 _jit_va_push(jit_state_t *_jit, jit_int32_t u)
 400 {
 401     jit_inc_synth_w(va_push, u);
 402     jit_pushargr(u);
 403     jit_dec_synth();
 404 }
 405
 406 jit_node_t *
 407 _jit_arg(jit_state_t *_jit)
 408 {
 409     jit_node_t          *node;
 410     jit_int32_t          offset;
 411     jit_bool_t           incr = 1;
 412     assert(_jitc->function);
 413     if (jit_arg_reg_p(_jitc->function->self.argi)) {
 414         offset = _jitc->function->self.argi++;
 415 #if _CALL_SYSV
 416         incr = 0;
 417 #endif
 418     }
 419     else
 420         offset = _jitc->function->self.size;
 421     if (incr)
 422         _jitc->function->self.size += sizeof(jit_word_t);
 423     node = jit_new_node_ww(jit_code_arg, offset,
 424                            ++_jitc->function->self.argn);
 425     jit_link_prolog();
 426     return (node);
 427 }
 428
 429 jit_node_t *
 430 _jit_arg_f(jit_state_t *_jit)
 431 {
 432     jit_node_t          *node;
 433     jit_int32_t          offset;
 434     jit_bool_t           incr = 1;
 435     assert(_jitc->function);
 436     if (jit_arg_f_reg_p(_jitc->function->self.argf)) {
 437         offset = _jitc->function->self.argf++;
 438 #if _CALL_SYSV
 439         incr = 0;
 440 #endif
 441     }
 442     else
 443         offset = _jitc->function->self.size + F_DISP;
 444 #if !_CALL_SYSV
 445     if (jit_arg_reg_p(_jitc->function->self.argi)) {
 446 #  if __WORDSIZE == 32
 447         _jitc->function->self.argi += 2;
 448 #  else
 449         _jitc->function->self.argi++;
 450 #  endif
 451     }
 452 #endif
 453     if (incr)
 454         _jitc->function->self.size += sizeof(jit_word_t);
 455     node = jit_new_node_ww(jit_code_arg_f, offset,
 456                            ++_jitc->function->self.argn);
 457     jit_link_prolog();
 458     return (node);
 459 }
 460
 461 jit_node_t *
 462 _jit_arg_d(jit_state_t *_jit)
 463 {
 464     jit_node_t          *node;
 465     jit_int32_t          offset;
 466     jit_bool_t           incr = 1;
 467     assert(_jitc->function);
 468     if (jit_arg_f_reg_p(_jitc->function->self.argf)) {
 469         offset = _jitc->function->self.argf++;
 470 #if _CALL_SYSV
 471         incr = 0;
 472 #endif
 473     }
 474     else {
 475 #if _CALL_SYSV
 476         if (_jitc->function->self.size & 7)
 477             _jitc->function->self.size += 4;
 478 #endif
 479         offset = _jitc->function->self.size;
 480     }
 481 #if !_CALL_SYSV
 482     if (jit_arg_reg_p(_jitc->function->self.argi)) {
 483 #  if __WORDSIZE == 32
 484         _jitc->function->self.argi += 2;
 485 #  else
 486         _jitc->function->self.argi++;
 487 #  endif
 488     }
 489 #endif
 490     if (incr)
 491         _jitc->function->self.size += sizeof(jit_float64_t);
 492     node = jit_new_node_ww(jit_code_arg_d, offset,
 493                            ++_jitc->function->self.argn);
 494     jit_link_prolog();
 495     return (node);
 496 }
 497
 498 void
 499 _jit_getarg_c(jit_state_t *_jit, jit_int32_t u, jit_node_t *v)
 500 {
 501     assert(v->code == jit_code_arg);
 502     jit_inc_synth_wp(getarg_c, u, v);
 503     if (jit_arg_reg_p(v->u.w))
 504         jit_extr_c(u, JIT_RA0 - v->u.w);
 505     else
 506         jit_ldxi_c(u, JIT_FP, v->u.w + C_DISP);
 507     jit_dec_synth();
 508 }
 509
 510 void
 511 _jit_getarg_uc(jit_state_t *_jit, jit_int32_t u, jit_node_t *v)
 512 {
 513     assert(v->code == jit_code_arg);
 514     jit_inc_synth_wp(getarg_uc, u, v);
 515     if (jit_arg_reg_p(v->u.w))
 516         jit_extr_uc(u, JIT_RA0 - v->u.w);
 517     else
 518         jit_ldxi_uc(u, JIT_FP, v->u.w + C_DISP);
 519     jit_dec_synth();
 520 }
 521
 522 void
 523 _jit_getarg_s(jit_state_t *_jit, jit_int32_t u, jit_node_t *v)
 524 {
 525     assert(v->code == jit_code_arg);
 526     jit_inc_synth_wp(getarg_s, u, v);
 527     if (jit_arg_reg_p(v->u.w))
 528         jit_extr_s(u, JIT_RA0 - v->u.w);
 529     else
 530         jit_ldxi_s(u, JIT_FP, v->u.w + S_DISP);
 531     jit_dec_synth();
 532 }
 533
 534 void
 535 _jit_getarg_us(jit_state_t *_jit, jit_int32_t u, jit_node_t *v)
 536 {
 537     assert(v->code == jit_code_arg);
 538     jit_inc_synth_wp(getarg_us, u, v);
 539     if (jit_arg_reg_p(v->u.w))
 540         jit_extr_us(u, JIT_RA0 - v->u.w);
 541     else
 542         jit_ldxi_us(u, JIT_FP, v->u.w + S_DISP);
 543     jit_dec_synth();
 544 }
 545
 546 void
 547 _jit_getarg_i(jit_state_t *_jit, jit_int32_t u, jit_node_t *v)
 548 {
 549     assert(v->code == jit_code_arg);
 550     jit_inc_synth_wp(getarg_i, u, v);
 551     if (jit_arg_reg_p(v->u.w)) {
 552 #if __WORDSIZE == 32
 553         jit_movr(u, JIT_RA0 - v->u.w);
 554 #else
 555         jit_extr_i(u, JIT_RA0 - v->u.w);
 556 #endif
 557     }
 558     else
 559         jit_ldxi_i(u, JIT_FP, v->u.w + I_DISP);
 560     jit_dec_synth();
 561 }
 562
 563 #if __WORDSIZE == 64
 564 void
 565 _jit_getarg_ui(jit_state_t *_jit, jit_int32_t u, jit_node_t *v)
 566 {
 567     assert(v->code == jit_code_arg);
 568     jit_inc_synth_wp(getarg_ui, u, v);
 569     if (jit_arg_reg_p(v->u.w))
 570         jit_extr_ui(u, JIT_RA0 - v->u.w);
 571     else
 572         jit_ldxi_ui(u, JIT_FP, v->u.w + I_DISP);
 573     jit_dec_synth();
 574 }
 575
 576 void
 577 _jit_getarg_l(jit_state_t *_jit, jit_int32_t u, jit_node_t *v)
 578 {
 579     assert(v->code == jit_code_arg);
 580     jit_inc_synth_wp(getarg_l, u, v);
 581     if (jit_arg_reg_p(v->u.w))
 582         jit_movr(u, JIT_RA0 - v->u.w);
 583     else
 584         jit_ldxi_l(u, JIT_FP, v->u.w);
 585     jit_dec_synth();
 586 }
 587 #endif
 588
 589 void
 590 _jit_putargr(jit_state_t *_jit, jit_int32_t u, jit_node_t *v)
 591 {
 592     assert(v->code == jit_code_arg);
 593     jit_inc_synth_wp(putargr, u, v);
 594     if (jit_arg_reg_p(v->u.w))
 595         jit_movr(JIT_RA0 - v->u.w, u);
 596     else
 597         jit_stxi(v->u.w, JIT_FP, u);
 598     jit_dec_synth();
 599 }
 600
 601 void
 602 _jit_putargi(jit_state_t *_jit, jit_word_t u, jit_node_t *v)
 603 {
 604     jit_int32_t         regno;
 605     jit_inc_synth_wp(putargi, u, v);
 606     assert(v->code == jit_code_arg);
 607     if (jit_arg_reg_p(v->u.w))
 608         jit_movi(JIT_RA0 - v->u.w, u);
 609     else {
 610         regno = jit_get_reg(jit_class_gpr);
 611         jit_movi(regno, u);
 612         jit_stxi(v->u.w, JIT_FP, regno);
 613         jit_unget_reg(regno);
 614     }
 615     jit_dec_synth();
 616 }
 617
 618 void
 619 _jit_getarg_f(jit_state_t *_jit, jit_int32_t u, jit_node_t *v)
 620 {
 621     assert(v->code == jit_code_arg_f);
 622     jit_inc_synth_wp(getarg_f, u, v);
 623     if (jit_arg_f_reg_p(v->u.w))
 624         jit_movr_d(u, JIT_FA0 - v->u.w);
 625     else
 626         jit_ldxi_f(u, JIT_FP, v->u.w);
 627     jit_dec_synth();
 628 }
 629
 630 void
 631 _jit_putargr_f(jit_state_t *_jit, jit_int32_t u, jit_node_t *v)
 632 {
 633     assert(v->code == jit_code_arg_f);
 634     jit_inc_synth_wp(putargr_f, u, v);
 635     if (jit_arg_f_reg_p(v->u.w))
 636         jit_movr_d(JIT_FA0 - v->u.w, u);
 637     else
 638         jit_stxi_f(v->u.w, JIT_FP, u);
 639     jit_dec_synth();
 640 }
 641
 642 void
 643 _jit_putargi_f(jit_state_t *_jit, jit_float32_t u, jit_node_t *v)
 644 {
 645     jit_int32_t         regno;
 646     assert(v->code == jit_code_arg_f);
 647     jit_inc_synth_fp(putargi_f, u, v);
 648     if (jit_arg_f_reg_p(v->u.w))
 649         jit_movi_d(JIT_FA0 - v->u.w, u);
 650     else {
 651         regno = jit_get_reg(jit_class_fpr);
 652         jit_movi_d(regno, u);
 653         jit_stxi_f(v->u.w, JIT_FP, regno);
 654         jit_unget_reg(regno);
 655     }
 656     jit_dec_synth();
 657 }
 658
 659 void
 660 _jit_getarg_d(jit_state_t *_jit, jit_int32_t u, jit_node_t *v)
 661 {
 662     assert(v->code == jit_code_arg_d);
 663     jit_inc_synth_wp(getarg_d, u, v);
 664     if (jit_arg_f_reg_p(v->u.w))
 665         jit_movr_d(u, JIT_FA0 - v->u.w);
 666     else
 667         jit_ldxi_d(u, JIT_FP, v->u.w);
 668     jit_dec_synth();
 669 }
 670
 671 void
 672 _jit_putargr_d(jit_state_t *_jit, jit_int32_t u, jit_node_t *v)
 673 {
 674     assert(v->code == jit_code_arg_d);
 675     jit_inc_synth_wp(putargr_d, u, v);
 676     if (jit_arg_f_reg_p(v->u.w))
 677         jit_movr_d(JIT_FA0 - v->u.w, u);
 678     else
 679         jit_stxi_d(v->u.w, JIT_FP, u);
 680     jit_dec_synth();
 681 }
 682
 683 void
 684 _jit_putargi_d(jit_state_t *_jit, jit_float64_t u, jit_node_t *v)
 685 {
 686     jit_int32_t         regno;
 687     assert(v->code == jit_code_arg_d);
 688     jit_inc_synth_dp(putargi_d, u, v);
 689     if (jit_arg_f_reg_p(v->u.w))
 690         jit_movi_d(JIT_FA0 - v->u.w, u);
 691     else {
 692         regno = jit_get_reg(jit_class_fpr);
 693         jit_movi_d(regno, u);
 694         jit_stxi_d(v->u.w, JIT_FP, regno);
 695         jit_unget_reg(regno);
 696     }
 697     jit_dec_synth();
 698 }
 699
 700 void
 701 _jit_pushargr(jit_state_t *_jit, jit_int32_t u)
 702 {
 703     jit_bool_t          incr = 1;
 704     assert(_jitc->function);
 705     jit_inc_synth_w(pushargr, u);
 706     jit_link_prepare();
 707     if (jit_arg_reg_p(_jitc->function->call.argi)) {
 708         jit_movr(JIT_RA0 - _jitc->function->call.argi, u);
 709         ++_jitc->function->call.argi;
 710 #if _CALL_SYSV
 711         incr = 0;
 712 #endif
 713     }
 714     else
 715         jit_stxi(_jitc->function->call.size + params_offset, JIT_SP, u);
 716     if (incr)
 717         _jitc->function->call.size += sizeof(jit_word_t);
 718     jit_dec_synth();
 719 }
 720
 721 void
 722 _jit_pushargi(jit_state_t *_jit, jit_word_t u)
 723 {
 724     jit_int32_t          regno;
 725     jit_bool_t           incr = 1;
 726     assert(_jitc->function);
 727     jit_inc_synth_w(pushargi, u);
 728     jit_link_prepare();
 729     if (jit_arg_reg_p(_jitc->function->call.argi)) {
 730         jit_movi(JIT_RA0 - _jitc->function->call.argi, u);
 731         ++_jitc->function->call.argi;
 732 #if _CALL_SYSV
 733         incr = 0;
 734 #endif
 735     }
 736     else {
 737         regno = jit_get_reg(jit_class_gpr);
 738         jit_movi(regno, u);
 739         jit_stxi(_jitc->function->call.size + params_offset, JIT_SP, regno);
 740         jit_unget_reg(regno);
 741     }
 742     if (incr)
 743         _jitc->function->call.size += sizeof(jit_word_t);
 744     jit_dec_synth();
 745 }
 746
 747 void
 748 _jit_pushargr_f(jit_state_t *_jit, jit_int32_t u)
 749 {
 750     jit_bool_t          incr = 1;
 751     assert(_jitc->function);
 752     jit_inc_synth_w(pushargr_f, u);
 753     jit_link_prepare();
 754     if (jit_arg_f_reg_p(_jitc->function->call.argf)
 755 #if !_CALL_SYSV
 756         && !(_jitc->function->call.call & jit_call_varargs)
 757 #endif
 758         ) {
 759         jit_movr_d(JIT_FA0 - _jitc->function->call.argf, u);
 760         ++_jitc->function->call.argf;
 761 #if !_CALL_SYSV
 762         /* in case of excess arguments */
 763         if (jit_arg_reg_p(_jitc->function->call.argi)) {
 764 #  if __WORDSIZE == 32
 765             _jitc->function->call.argi += 2;
 766             if (!jit_arg_reg_p(_jitc->function->call.argi - 1))
 767                 --_jitc->function->call.argi;
 768 #  else
 769             _jitc->function->call.argi++;
 770 #  endif
 771         }
 772 #elif _CALL_SYSV
 773         incr = 0;
 774 #endif
 775     }
 776 #if !_CALL_SYSV
 777     else if (jit_arg_reg_p(_jitc->function->call.argi
 778 #  if __WORDSIZE == 32
 779                           + 1
 780 #  endif
 781                            )) {
 782         /* use reserved 8 bytes area */
 783         jit_stxi_d(alloca_offset - 8, JIT_FP, u);
 784         jit_ldxi(JIT_RA0 - _jitc->function->call.argi, JIT_FP,
 785                  alloca_offset - 8);
 786         _jitc->function->call.argi++;
 787 #  if __WORDSIZE == 32
 788         jit_ldxi(JIT_RA0 - _jitc->function->call.argi, JIT_FP,
 789                  alloca_offset - 4);
 790         _jitc->function->call.argi++;
 791 #  endif
 792     }
 793 #endif
 794     else
 795         jit_stxi_f(_jitc->function->call.size + params_offset + F_DISP,
 796                    JIT_SP, u);
 797     if (incr)
 798         _jitc->function->call.size += sizeof(jit_word_t);
 799     jit_dec_synth();
 800 }
 801
 802 void
 803 _jit_pushargi_f(jit_state_t *_jit, jit_float32_t u)
 804 {
 805     jit_bool_t           incr = 1;
 806     jit_int32_t          regno;
 807     assert(_jitc->function);
 808     jit_inc_synth_f(pushargi_f, u);
 809     jit_link_prepare();
 810     if (jit_arg_f_reg_p(_jitc->function->call.argf)
 811 #if !_CALL_SYSV
 812         && !(_jitc->function->call.call & jit_call_varargs)
 813 #endif
 814         ) {
 815         jit_movi_d(JIT_FA0 - _jitc->function->call.argf, u);
 816         ++_jitc->function->call.argf;
 817 #if !_CALL_SYSV
 818             /* in case of excess arguments */
 819 #  if __WORDSIZE == 32
 820         _jitc->function->call.argi += 2;
 821         if (!jit_arg_reg_p(_jitc->function->call.argi - 1))
 822             --_jitc->function->call.argi;
 823 #  else
 824         _jitc->function->call.argi++;
 825 #  endif
 826 #elif _CALL_SYSV
 827         incr = 0;
 828 #endif
 829     }
 830     else {
 831         regno = jit_get_reg(jit_class_fpr);
 832         jit_movi_f(regno, u);
 833 #if !_CALL_SYSV
 834         if (jit_arg_reg_p(_jitc->function->call.argi
 835 #  if __WORDSIZE == 32
 836                           + 1
 837 #  endif
 838                           )) {
 839             /* use reserved 8 bytes area */
 840             jit_stxi_d(alloca_offset - 8, JIT_FP, regno);
 841             jit_ldxi(JIT_RA0 - _jitc->function->call.argi, JIT_FP,
 842                      alloca_offset - 8);
 843             _jitc->function->call.argi++;
 844 #  if __WORDSIZE == 32
 845             jit_ldxi(JIT_RA0 - _jitc->function->call.argi, JIT_FP,
 846                      alloca_offset - 4);
 847             _jitc->function->call.argi++;
 848 #  endif
 849         }
 850         else
 851 #endif
 852             jit_stxi_f(_jitc->function->call.size + params_offset + F_DISP,
 853                        JIT_SP, regno);
 854         jit_unget_reg(regno);
 855     }
 856     if (incr)
 857         _jitc->function->call.size += sizeof(jit_word_t);
 858     jit_dec_synth();
 859 }
 860
 861 void
 862 _jit_pushargr_d(jit_state_t *_jit, jit_int32_t u)
 863 {
 864     jit_bool_t          incr = 1;
 865     assert(_jitc->function);
 866     jit_inc_synth_w(pushargr_d, u);
 867     jit_link_prepare();
 868     if (jit_arg_f_reg_p(_jitc->function->call.argf)
 869 #if !_CALL_SYSV
 870         && !(_jitc->function->call.call & jit_call_varargs)
 871 #endif
 872         ) {
 873         jit_movr_d(JIT_FA0 - _jitc->function->call.argf, u);
 874         ++_jitc->function->call.argf;
 875 #if !_CALL_SYSV
 876             /* in case of excess arguments */
 877 #  if __WORDSIZE == 32
 878         _jitc->function->call.argi += 2;
 879         if (!jit_arg_reg_p(_jitc->function->call.argi - 1))
 880             --_jitc->function->call.argi;
 881 #  else
 882         _jitc->function->call.argi++;
 883 #  endif
 884 #else /* _CALL_SYSV */
 885         incr = 0;
 886 #endif
 887     }
 888 #if !_CALL_SYSV
 889     else if (jit_arg_reg_p(_jitc->function->call.argi
 890 #  if __WORDSIZE == 32
 891                           + 1
 892 #  endif
 893                            )) {
 894         /* use reserved 8 bytes area */
 895         jit_stxi_d(alloca_offset - 8, JIT_FP, u);
 896         jit_ldxi(JIT_RA0 - _jitc->function->call.argi, JIT_FP,
 897                  alloca_offset - 8);
 898         _jitc->function->call.argi++;
 899 #  if __WORDSIZE == 32
 900         jit_ldxi(JIT_RA0 - _jitc->function->call.argi, JIT_FP,
 901                  alloca_offset - 4);
 902         _jitc->function->call.argi++;
 903 #  endif
 904     }
 905     else
 906 #endif /* !_CALL_SYSV */
 907     {
 908 #if _CALL_SYSV
 909         if (_jitc->function->call.size & 7)
 910             _jitc->function->call.size += 4;
 911 #endif
 912         jit_stxi_d(_jitc->function->call.size + params_offset, JIT_SP, u);
 913 #if !_CALL_SYSV && __WORDSIZE == 32
 914         if (jit_arg_reg_p(_jitc->function->call.argi)) {
 915             jit_ldxi(JIT_RA0 - _jitc->function->call.argi, JIT_SP,
 916                      _jitc->function->call.size + params_offset);
 917             _jitc->function->call.argi++;
 918         }
 919 #endif
 920     }
 921     if (incr)
 922         _jitc->function->call.size += sizeof(jit_float64_t);
 923     jit_dec_synth();
 924 }
 925
 926 void
 927 _jit_pushargi_d(jit_state_t *_jit, jit_float64_t u)
 928 {
 929     jit_int32_t          regno;
 930     jit_bool_t           incr = 1;
 931     assert(_jitc->function);
 932     jit_inc_synth_d(pushargi_d, u);
 933     jit_link_prepare();
 934     if (jit_arg_f_reg_p(_jitc->function->call.argf)
 935 #if !_CALL_SYSV
 936         && !(_jitc->function->call.call & jit_call_varargs)
 937 #endif
 938         ) {
 939         jit_movi_d(JIT_FA0 - _jitc->function->call.argf, u);
 940         ++_jitc->function->call.argf;
 941 #if !_CALL_SYSV
 942         /* in case of excess arguments */
 943         if (jit_arg_reg_p(_jitc->function->call.argi)) {
 944 #  if __WORDSIZE == 32
 945             _jitc->function->call.argi += 2;
 946             if (!jit_arg_reg_p(_jitc->function->call.argi - 1))
 947                 --_jitc->function->call.argi;
 948 #  else
 949             _jitc->function->call.argi++;
 950 #  endif
 951         }
 952 #else /* _CALL_SYSV */
 953             incr = 0;
 954 #endif
 955     }
 956     else {
 957         regno = jit_get_reg(jit_class_fpr);
 958         jit_movi_d(regno, u);
 959 #if !_CALL_SYSV
 960         if (jit_arg_reg_p(_jitc->function->call.argi
 961 #  if __WORDSIZE == 32
 962                           + 1
 963 #  endif
 964                           )) {
 965             /* use reserved 8 bytes area */
 966             jit_stxi_d(alloca_offset - 8, JIT_FP, regno);
 967             jit_ldxi(JIT_RA0 - _jitc->function->call.argi, JIT_FP,
 968                      alloca_offset - 8);
 969             _jitc->function->call.argi++;
 970 #  if __WORDSIZE == 32
 971             jit_ldxi(JIT_RA0 - _jitc->function->call.argi, JIT_FP,
 972                      alloca_offset - 4);
 973             _jitc->function->call.argi++;
 974 #  endif
 975         }
 976         else
 977 #endif /* !_CALL_SYSV */
 978         {
 979 #if _CALL_SYSV
 980             if (_jitc->function->call.size & 7)
 981                 _jitc->function->call.size += 4;
 982 #endif
 983             jit_stxi_d(_jitc->function->call.size + params_offset,
 984                        JIT_SP, regno);
 985 #if !_CALL_SYSV && __WORDSIZE == 32
 986             if (jit_arg_reg_p(_jitc->function->call.argi)) {
 987                 jit_ldxi(JIT_RA0 - _jitc->function->call.argi, JIT_SP,
 988                          _jitc->function->call.size + params_offset);
 989                 _jitc->function->call.argi++;
 990             }
 991 #endif
 992         }
 993         jit_unget_reg(regno);
 994     }
 995     if (incr)
 996         _jitc->function->call.size += sizeof(jit_float64_t);
 997     jit_dec_synth();
 998 }
 999
1000 jit_bool_t
1001 _jit_regarg_p(jit_state_t *_jit, jit_node_t *node, jit_int32_t regno)
1002 {
1003     jit_int32_t         spec;
1004     spec = jit_class(_rvs[regno].spec);
1005     if (spec & jit_class_arg) {
1006         if (spec & jit_class_gpr) {
1007             regno = JIT_RA0 - regno;
1008             if (regno >= 0 && regno < node->v.w)
1009                 return (1);
1010         }
1011         else if (spec & jit_class_fpr) {
1012             regno = JIT_FA0 - regno;
1013             if (regno >= 0 && regno < node->w.w)
1014                 return (1);
1015         }
1016     }
1017     return (0);
1018 }
1019
1020 void
1021 _jit_finishr(jit_state_t *_jit, jit_int32_t r0)
1022 {
1023     jit_node_t          *call;
1024     assert(_jitc->function);
1025     jit_inc_synth_w(finishr, r0);
1026     if (_jitc->function->self.alen < _jitc->function->call.size)
1027         _jitc->function->self.alen = _jitc->function->call.size;
1028     call = jit_callr(r0);
1029     call->v.w = _jitc->function->call.argi;
1030     call->w.w = _jitc->function->call.argf;
1031 #if _CALL_SYSV
1032     /* If passing float arguments in registers */
1033     if ((_jitc->function->call.call & jit_call_varargs) && call->w.w)
1034         call->flag |= jit_flag_varargs;
1035 #endif
1036     _jitc->function->call.argi = _jitc->function->call.argf = 0;
1037     _jitc->prepare = 0;
1038     jit_dec_synth();
1039 }
1040
1041 jit_node_t *
1042 _jit_finishi(jit_state_t *_jit, jit_pointer_t i0)
1043 {
1044     jit_node_t          *node;
1045     assert(_jitc->function);
1046     jit_inc_synth_w(finishi, (jit_word_t)i0);
1047     if (_jitc->function->self.alen < _jitc->function->call.size)
1048         _jitc->function->self.alen = _jitc->function->call.size;
1049     node = jit_calli(i0);
1050     node->v.w = _jitc->function->call.argi;
1051     node->w.w = _jitc->function->call.argf;
1052 #if _CALL_SYSV
1053     if ((_jitc->function->call.call & jit_call_varargs) && node->w.w)
1054         node->flag |= jit_flag_varargs;
1055 #endif
1056     _jitc->function->call.argi = _jitc->function->call.argf = 0;
1057     _jitc->prepare = 0;
1058     jit_dec_synth();
1059     return (node);
1060 }
1061
1062 void
1063 _jit_retval_c(jit_state_t *_jit, jit_int32_t r0)
1064 {
1065     jit_inc_synth(retval_c);
1066     jit_extr_c(r0, JIT_RET);
1067     jit_dec_synth();
1068 }
1069
1070 void
1071 _jit_retval_uc(jit_state_t *_jit, jit_int32_t r0)
1072 {
1073     jit_inc_synth(retval_uc);
1074     jit_extr_uc(r0, JIT_RET);
1075     jit_dec_synth();
1076 }
1077
1078 void
1079 _jit_retval_s(jit_state_t *_jit, jit_int32_t r0)
1080 {
1081     jit_inc_synth(retval_s);
1082     jit_extr_s(r0, JIT_RET);
1083     jit_dec_synth();
1084 }
1085
1086 void
1087 _jit_retval_us(jit_state_t *_jit, jit_int32_t r0)
1088 {
1089     jit_inc_synth(retval_us);
1090     jit_extr_us(r0, JIT_RET);
1091     jit_dec_synth();
1092 }
1093
1094 void
1095 _jit_retval_i(jit_state_t *_jit, jit_int32_t r0)
1096 {
1097     jit_inc_synth(retval_i);
1098 #if __WORDSIZE == 32
1099     if (r0 != JIT_RET)
1100         jit_movr(r0, JIT_RET);
1101 #else
1102     jit_extr_i(r0, JIT_RET);
1103 #endif
1104     jit_dec_synth();
1105 }
1106
1107 #if __WORDSIZE == 64
1108 void
1109 _jit_retval_ui(jit_state_t *_jit, jit_int32_t r0)
1110 {
1111     jit_inc_synth(retval_ui);
1112     jit_extr_ui(r0, JIT_RET);
1113     jit_dec_synth();
1114 }
1115
1116 void
1117 _jit_retval_l(jit_state_t *_jit, jit_int32_t r0)
1118 {
1119     jit_inc_synth(retval_l);
1120     if (r0 != JIT_RET)
1121         jit_movr(r0, JIT_RET);
1122     jit_dec_synth();
1123 }
1124 #endif
1125
1126 void
1127 _jit_retval_f(jit_state_t *_jit, jit_int32_t r0)
1128 {
1129     jit_inc_synth(retval_f);
1130     jit_retval_d(r0);
1131     jit_dec_synth();
1132 }
1133
1134 void
1135 _jit_retval_d(jit_state_t *_jit, jit_int32_t r0)
1136 {
1137     jit_inc_synth(retval_d);
1138     if (r0 != JIT_FRET)
1139         jit_movr_d(r0, JIT_FRET);
1140     jit_dec_synth();
1141 }
1142
1143 jit_pointer_t
1144 _emit_code(jit_state_t *_jit)
1145 {
1146     jit_node_t          *node;
1147     jit_node_t          *temp;
1148     jit_word_t           word;
1149     jit_int32_t          value;
1150     jit_int32_t          offset;
1151     jit_bool_t       no_flag = 0;       /* Set if previous instruction is
1152                                          * *not* a jump target. */
1153     struct {
1154         jit_node_t      *node;
1155         jit_word_t       word;
1156 #if DEVEL_DISASSEMBLER
1157         jit_word_t       prevw;
1158 #endif
1159         jit_word_t       patch_offset;
1160 #if _CALL_AIXDESC
1161         jit_word_t       prolog_offset;
1162 #endif
1163     } undo;
1164 #if DEVEL_DISASSEMBLER
1165     jit_word_t           prevw;
1166 #endif
1167
1168     _jitc->function = NULL;
1169
1170     jit_reglive_setup();
1171
1172     undo.word = 0;
1173     undo.node = NULL;
1174     undo.patch_offset = 0;
1175
1176 #if DEVEL_DISASSEMBLER
1177     prevw = _jit->pc.w;
1178 #endif
1179 #if _CALL_AIXDESC
1180     undo.prolog_offset = 0;
1181     for (node = _jitc->head; node; node = node->next)
1182         if (node->code != jit_code_label &&
1183             node->code != jit_code_note &&
1184             node->code != jit_code_name)
1185             break;
1186     if (node && (node->code != jit_code_prolog ||
1187                  !(_jitc->functions.ptr + node->w.w)->assume_frame)) {
1188         /* code may start with a jump so add an initial function descriptor */
1189         word = _jit->pc.w + sizeof(void*) * 3;
1190         iw(word);                       /* addr */
1191         iw(0);                          /* toc */
1192         iw(0);                          /* env */
1193     }
1194 #endif
1195
1196 #define case_rr(name, type)                                             \
1197             case jit_code_##name##r##type:                              \
1198                 name##r##type(rn(node->u.w), rn(node->v.w));            \
1199                 break
1200 #define case_rw(name, type)                                             \
1201             case jit_code_##name##i##type:                              \
1202                 name##i##type(rn(node->u.w), node->v.w);                \
1203                 break
1204 #define case_wr(name, type)                                             \
1205             case jit_code_##name##i##type:                              \
1206                 name##i##type(node->u.w, rn(node->v.w));                \
1207                 break
1208 #define case_rrr(name, type)                                            \
1209             case jit_code_##name##r##type:                              \
1210                 name##r##type(rn(node->u.w),                            \
1211                               rn(node->v.w), rn(node->w.w));            \
1212                 break
1213 #define case_rrrr(name, type)                                           \
1214             case jit_code_##name##r##type:                              \
1215                 name##r##type(rn(node->u.q.l), rn(node->u.q.h),         \
1216                               rn(node->v.w), rn(node->w.w));            \
1217                 break
1218 #define case_rrw(name, type)                                            \
1219             case jit_code_##name##i##type:                              \
1220                 name##i##type(rn(node->u.w), rn(node->v.w), node->w.w); \
1221                 break
1222 #define case_rrrw(name, type)                                           \
1223             case jit_code_##name##i##type:                              \
1224                 name##i##type(rn(node->u.q.l), rn(node->u.q.h),         \
1225                               rn(node->v.w), node->w.w);                \
1226                 break
1227 #define case_rrf(name, type, size)                                      \
1228             case jit_code_##name##i##type:                              \
1229                 assert(node->flag & jit_flag_data);                     \
1230                 name##i##type(rn(node->u.w), rn(node->v.w),             \
1231                               (jit_float##size##_t *)node->w.n->u.w);   \
1232                 break
1233 #define case_wrr(name, type)                                            \
1234             case jit_code_##name##i##type:                              \
1235                 name##i##type(node->u.w, rn(node->v.w), rn(node->w.w)); \
1236                 break
1237 #define case_brr(name, type)                                            \
1238             case jit_code_##name##r##type:                              \
1239                 temp = node->u.n;                                       \
1240                 assert(temp->code == jit_code_label ||                  \
1241                        temp->code == jit_code_epilog);                  \
1242                 if (temp->flag & jit_flag_patch)                        \
1243                     name##r##type(temp->u.w, rn(node->v.w),             \
1244                                   rn(node->w.w));                       \
1245                 else {                                                  \
1246                     word = name##r##type(_jit->pc.w,                    \
1247                                          rn(node->v.w), rn(node->w.w)); \
1248                     patch(word, node);                                  \
1249                 }                                                       \
1250                 break
1251 #define case_brw(name, type)                                            \
1252             case jit_code_##name##i##type:                              \
1253                 temp = node->u.n;                                       \
1254                 assert(temp->code == jit_code_label ||                  \
1255                        temp->code == jit_code_epilog);                  \
1256                 if (temp->flag & jit_flag_patch)                        \
1257                     name##i##type(temp->u.w,                            \
1258                                   rn(node->v.w), node->w.w);            \
1259                 else {                                                  \
1260                     word = name##i##type(_jit->pc.w,                    \
1261                                          rn(node->v.w), node->w.w);     \
1262                     patch(word, node);                                  \
1263                 }                                                       \
1264                 break
1265 #define case_brf(name, type, size)                                      \
1266             case jit_code_##name##i##type:                              \
1267                 temp = node->u.n;                                       \
1268                 assert(temp->code == jit_code_label ||                  \
1269                        temp->code == jit_code_epilog);                  \
1270                 if (temp->flag & jit_flag_patch)                        \
1271                     name##i##type(temp->u.w, rn(node->v.w),             \
1272                                 (jit_float##size##_t *)node->w.n->u.w); \
1273                 else {                                                  \
1274                     word = name##i##type(_jit->pc.w, rn(node->v.w),     \
1275                                 (jit_float##size##_t *)node->w.n->u.w); \
1276                     patch(word, node);                                  \
1277                 }                                                       \
1278                 break
1279     for (node = _jitc->head; node; node = node->next) {
1280         if (_jit->pc.uc >= _jitc->code.end)
1281             return (NULL);
1282
1283 #if DEVEL_DISASSEMBLER
1284         node->offset = (jit_uword_t)_jit->pc.w - (jit_uword_t)prevw;
1285         prevw = _jit->pc.w;
1286 #endif
1287         value = jit_classify(node->code);
1288         jit_regarg_set(node, value);
1289         switch (node->code) {
1290             case jit_code_align:
1291                 assert(!(node->u.w & (node->u.w - 1)) &&
1292                        node->u.w <= sizeof(jit_word_t));
1293                 if (node->u.w == sizeof(jit_word_t) &&
1294                     (word = _jit->pc.w & (sizeof(jit_word_t) - 1)))
1295                     nop(sizeof(jit_word_t) - word);
1296                 break;
1297             case jit_code_note:         case jit_code_name:
1298                 node->u.w = _jit->pc.w;
1299                 break;
1300             case jit_code_label:
1301                 /* remember label is defined */
1302                 node->flag |= jit_flag_patch;
1303                 node->u.w = _jit->pc.w;
1304                 break;
1305                 case_rrr(add,);
1306                 case_rrw(add,);
1307                 case_rrr(addc,);
1308                 case_rrw(addc,);
1309                 case_rrr(addx,);
1310                 case_rrw(addx,);
1311                 case_rrr(sub,);
1312                 case_rrw(sub,);
1313                 case_rrr(subc,);
1314                 case_rrw(subc,);
1315                 case_rrr(subx,);
1316                 case_rrw(subx,);
1317                 case_rrw(rsb,);
1318                 case_rrr(mul,);
1319                 case_rrw(mul,);
1320                 case_rrrr(qmul,);
1321                 case_rrrw(qmul,);
1322                 case_rrrr(qmul, _u);
1323                 case_rrrw(qmul, _u);
1324                 case_rrr(div,);
1325                 case_rrw(div,);
1326                 case_rrr(div, _u);
1327                 case_rrw(div, _u);
1328                 case_rrrr(qdiv,);
1329                 case_rrrw(qdiv,);
1330                 case_rrrr(qdiv, _u);
1331                 case_rrrw(qdiv, _u);
1332                 case_rrr(rem,);
1333                 case_rrw(rem,);
1334                 case_rrr(rem, _u);
1335                 case_rrw(rem, _u);
1336                 case_rrr(and,);
1337                 case_rrw(and,);
1338                 case_rrr(or,);
1339                 case_rrw(or,);
1340                 case_rrr(xor,);
1341                 case_rrw(xor,);
1342                 case_rrr(lsh,);
1343                 case_rrw(lsh,);
1344                 case_rrr(rsh,);
1345                 case_rrw(rsh,);
1346                 case_rrr(rsh, _u);
1347                 case_rrw(rsh, _u);
1348                 case_rr(ext, _c);
1349                 case_rr(ext, _uc);
1350                 case_rr(ext, _s);
1351                 case_rr(ext, _us);
1352 #  if __WORDSIZE == 64
1353                 case_rr(ext, _i);
1354                 case_rr(ext, _ui);
1355 #  endif
1356                 case_rr(hton, _us);
1357                 case_rr(hton, _ui);
1358 #  if __WORDSIZE == 64
1359                 case_rr(hton, _ul);
1360 #  endif
1361             case jit_code_bswapr_us:
1362                 bswapr_us_lh(rn(node->u.w), rn(node->v.w), no_flag);
1363                 break;
1364             case jit_code_bswapr_ui:
1365                 bswapr_ui_lw(rn(node->u.w), rn(node->v.w), no_flag);
1366                 break;
1367 #  if __WORDSIZE == 64
1368                 case_rr(bswap, _ul);
1369 #  endif
1370                 case_rr(neg,);
1371                 case_rr(com,);
1372             case jit_code_casr:
1373                 casr(rn(node->u.w), rn(node->v.w),
1374                      rn(node->w.q.l), rn(node->w.q.h));
1375                 break;
1376             case jit_code_casi:
1377                 casi(rn(node->u.w), node->v.w,
1378                      rn(node->w.q.l), rn(node->w.q.h));
1379                 break;
1380                 case_rrr(movn,);
1381                 case_rrr(movz,);
1382                 case_rr(mov,);
1383             case jit_code_movi:
1384                 if (node->flag & jit_flag_node) {
1385                     temp = node->v.n;
1386                     if (temp->code == jit_code_data ||
1387                         (temp->code == jit_code_label &&
1388                          (temp->flag & jit_flag_patch)))
1389                         movi(rn(node->u.w), temp->u.w);
1390                     else {
1391                         assert(temp->code == jit_code_label ||
1392                                temp->code == jit_code_epilog);
1393                         word = movi_p(rn(node->u.w), node->v.w);
1394                         patch(word, node);
1395                     }
1396                 }
1397                 else
1398                     movi(rn(node->u.w), node->v.w);
1399                 break;
1400                 case_rr(trunc, _f_i);
1401                 case_rr(trunc, _d_i);
1402 #  if __WORDSIZE == 64
1403                 case_rr(trunc, _f_l);
1404                 case_rr(trunc, _d_l);
1405 #  endif
1406                 case_rrr(lt,);
1407                 case_rrw(lt,);
1408                 case_rrr(lt, _u);
1409                 case_rrw(lt, _u);
1410                 case_rrr(le,);
1411                 case_rrw(le,);
1412                 case_rrr(le, _u);
1413                 case_rrw(le, _u);
1414                 case_rrr(eq,);
1415                 case_rrw(eq,);
1416                 case_rrr(ge,);
1417                 case_rrw(ge,);
1418                 case_rrr(ge, _u);
1419                 case_rrw(ge, _u);
1420                 case_rrr(gt,);
1421                 case_rrw(gt,);
1422                 case_rrr(gt, _u);
1423                 case_rrw(gt, _u);
1424                 case_rrr(ne,);
1425                 case_rrw(ne,);
1426                 case_rr(ld, _c);
1427                 case_rw(ld, _c);
1428                 case_brr(blt,);
1429                 case_brw(blt,);
1430                 case_brr(blt, _u);
1431                 case_brw(blt, _u);
1432                 case_brr(ble,);
1433                 case_brw(ble,);
1434                 case_brr(ble, _u);
1435                 case_brw(ble, _u);
1436                 case_brr(beq,);
1437                 case_brw(beq,);
1438                 case_brr(bge,);
1439                 case_brw(bge,);
1440                 case_brr(bge, _u);
1441                 case_brw(bge, _u);
1442                 case_brr(bgt,);
1443                 case_brw(bgt,);
1444                 case_brr(bgt, _u);
1445                 case_brw(bgt, _u);
1446                 case_brr(bne,);
1447                 case_brw(bne,);
1448                 case_brr(bms,);
1449                 case_brw(bms,);
1450                 case_brr(bmc,);
1451                 case_brw(bmc,);
1452                 case_brr(boadd,);
1453                 case_brw(boadd,);
1454                 case_brr(boadd, _u);
1455                 case_brw(boadd, _u);
1456                 case_brr(bxadd,);
1457                 case_brw(bxadd,);
1458                 case_brr(bxadd, _u);
1459                 case_brw(bxadd, _u);
1460                 case_brr(bosub,);
1461                 case_brw(bosub,);
1462                 case_brr(bosub, _u);
1463                 case_brw(bosub, _u);
1464                 case_brr(bxsub,);
1465                 case_brw(bxsub,);
1466                 case_brr(bxsub, _u);
1467                 case_brw(bxsub, _u);
1468                 case_rrr(ldx, _c);
1469                 case_rrw(ldx, _c);
1470                 case_rr(ld, _uc);
1471                 case_rw(ld, _uc);
1472                 case_rrr(ldx, _uc);
1473                 case_rrw(ldx, _uc);
1474                 case_rr(ld, _s);
1475                 case_rw(ld, _s);
1476                 case_rrr(ldx, _s);
1477                 case_rrw(ldx, _s);
1478                 case_rr(ld, _us);
1479                 case_rw(ld, _us);
1480                 case_rrr(ldx, _us);
1481                 case_rrw(ldx, _us);
1482                 case_rr(ld, _i);
1483                 case_rw(ld, _i);
1484                 case_rrr(ldx, _i);
1485                 case_rrw(ldx, _i);
1486 #if __WORDSIZE == 64
1487                 case_rr(ld, _ui);
1488                 case_rw(ld, _ui);
1489                 case_rrr(ldx, _ui);
1490                 case_rrw(ldx, _ui);
1491                 case_rr(ld, _l);
1492                 case_rw(ld, _l);
1493                 case_rrr(ldx, _l);
1494                 case_rrw(ldx, _l);
1495 #endif
1496                 case_rr(st, _c);
1497                 case_wr(st, _c);
1498                 case_rrr(stx, _c);
1499                 case_wrr(stx, _c);
1500                 case_rr(st, _s);
1501                 case_wr(st, _s);
1502                 case_rrr(stx, _s);
1503                 case_wrr(stx, _s);
1504                 case_rr(st, _i);
1505                 case_wr(st, _i);
1506                 case_rrr(stx, _i);
1507                 case_wrr(stx, _i);
1508 #if __WORDSIZE == 64
1509                 case_rr(st, _l);
1510                 case_wr(st, _l);
1511                 case_rrr(stx, _l);
1512                 case_wrr(stx, _l);
1513 #endif
1514                 case_rr(mov, _f);
1515             case jit_code_movi_f:
1516                 assert(node->flag & jit_flag_data);
1517                 movi_f(rn(node->u.w), (jit_float32_t *)node->v.n->u.w);
1518                 break;
1519                 case_rr(ext, _f);
1520                 case_rr(ext, _d_f);
1521                 case_rr(abs, _f);
1522                 case_rr(neg, _f);
1523                 case_rr(sqrt, _f);
1524                 case_rrr(add, _f);
1525                 case_rrf(add, _f, 32);
1526                 case_rrr(sub, _f);
1527                 case_rrf(sub, _f, 32);
1528                 case_rrf(rsb, _f, 32);
1529                 case_rrr(mul, _f);
1530                 case_rrf(mul, _f, 32);
1531                 case_rrr(div, _f);
1532                 case_rrf(div, _f, 32);
1533                 case_rrr(lt, _f);
1534                 case_rrf(lt, _f, 32);
1535                 case_rrr(le, _f);
1536                 case_rrf(le, _f, 32);
1537                 case_rrr(eq, _f);
1538                 case_rrf(eq, _f, 32);
1539                 case_rrr(ge, _f);
1540                 case_rrf(ge, _f, 32);
1541                 case_rrr(gt, _f);
1542                 case_rrf(gt, _f, 32);
1543                 case_rrr(ne, _f);
1544                 case_rrf(ne, _f, 32);
1545                 case_rrr(unlt, _f);
1546                 case_rrf(unlt, _f, 32);
1547                 case_rrr(unle, _f);
1548                 case_rrf(unle, _f, 32);
1549                 case_rrr(uneq, _f);
1550                 case_rrf(uneq, _f, 32);
1551                 case_rrr(unge, _f);
1552                 case_rrf(unge, _f, 32);
1553                 case_rrr(ungt, _f);
1554                 case_rrf(ungt, _f, 32);
1555                 case_rrr(ltgt, _f);
1556                 case_rrf(ltgt, _f, 32);
1557                 case_rrr(ord, _f);
1558                 case_rrf(ord, _f, 32);
1559                 case_rrr(unord, _f);
1560                 case_rrf(unord, _f, 32);
1561                 case_brr(blt, _f);
1562                 case_brf(blt, _f, 32);
1563                 case_brr(ble, _f);
1564                 case_brf(ble, _f, 32);
1565                 case_brr(beq, _f);
1566                 case_brf(beq, _f, 32);
1567                 case_brr(bge, _f);
1568                 case_brf(bge, _f, 32);
1569                 case_brr(bgt, _f);
1570                 case_brf(bgt, _f, 32);
1571                 case_brr(bne, _f);
1572                 case_brf(bne, _f, 32);
1573                 case_brr(bunlt, _f);
1574                 case_brf(bunlt, _f, 32);
1575                 case_brr(bunle, _f);
1576                 case_brf(bunle, _f, 32);
1577                 case_brr(buneq, _f);
1578                 case_brf(buneq, _f, 32);
1579                 case_brr(bunge, _f);
1580                 case_brf(bunge, _f, 32);
1581                 case_brr(bungt, _f);
1582                 case_brf(bungt, _f, 32);
1583                 case_brr(bltgt, _f);
1584                 case_brf(bltgt, _f, 32);
1585                 case_brr(bord, _f);
1586                 case_brf(bord, _f, 32);
1587                 case_brr(bunord, _f);
1588                 case_brf(bunord, _f, 32);
1589                 case_rr(ld, _f);
1590                 case_rw(ld, _f);
1591                 case_rrr(ldx, _f);
1592                 case_rrw(ldx, _f);
1593                 case_rr(st, _f);
1594                 case_wr(st, _f);
1595                 case_rrr(stx, _f);
1596                 case_wrr(stx, _f);
1597                 case_rr(mov, _d);
1598             case jit_code_movi_d:
1599                 assert(node->flag & jit_flag_data);
1600                 movi_d(rn(node->u.w), (jit_float64_t *)node->v.n->u.w);
1601                 break;
1602                 case_rr(ext, _d);
1603                 case_rr(ext, _f_d);
1604                 case_rr(abs, _d);
1605                 case_rr(neg, _d);
1606                 case_rr(sqrt, _d);
1607                 case_rrr(add, _d);
1608                 case_rrf(add, _d, 64);
1609                 case_rrr(sub, _d);
1610                 case_rrf(sub, _d, 64);
1611                 case_rrf(rsb, _d, 64);
1612                 case_rrr(mul, _d);
1613                 case_rrf(mul, _d, 64);
1614                 case_rrr(div, _d);
1615                 case_rrf(div, _d, 64);
1616                 case_rrr(lt, _d);
1617                 case_rrf(lt, _d, 64);
1618                 case_rrr(le, _d);
1619                 case_rrf(le, _d, 64);
1620                 case_rrr(eq, _d);
1621                 case_rrf(eq, _d, 64);
1622                 case_rrr(ge, _d);
1623                 case_rrf(ge, _d, 64);
1624                 case_rrr(gt, _d);
1625                 case_rrf(gt, _d, 64);
1626                 case_rrr(ne, _d);
1627                 case_rrf(ne, _d, 64);
1628                 case_rrr(unlt, _d);
1629                 case_rrf(unlt, _d, 64);
1630                 case_rrr(unle, _d);
1631                 case_rrf(unle, _d, 64);
1632                 case_rrr(uneq, _d);
1633                 case_rrf(uneq, _d, 64);
1634                 case_rrr(unge, _d);
1635                 case_rrf(unge, _d, 64);
1636                 case_rrr(ungt, _d);
1637                 case_rrf(ungt, _d, 64);
1638                 case_rrr(ltgt, _d);
1639                 case_rrf(ltgt, _d, 64);
1640                 case_rrr(ord, _d);
1641                 case_rrf(ord, _d, 64);
1642                 case_rrr(unord, _d);
1643                 case_rrf(unord, _d, 64);
1644                 case_brr(blt, _d);
1645                 case_brf(blt, _d, 64);
1646                 case_brr(ble, _d);
1647                 case_brf(ble, _d, 64);
1648                 case_brr(beq, _d);
1649                 case_brf(beq, _d, 64);
1650                 case_brr(bge, _d);
1651                 case_brf(bge, _d, 64);
1652                 case_brr(bgt, _d);
1653                 case_brf(bgt, _d, 64);
1654                 case_brr(bne, _d);
1655                 case_brf(bne, _d, 64);
1656                 case_brr(bunlt, _d);
1657                 case_brf(bunlt, _d, 64);
1658                 case_brr(bunle, _d);
1659                 case_brf(bunle, _d, 64);
1660                 case_brr(buneq, _d);
1661                 case_brf(buneq, _d, 64);
1662                 case_brr(bunge, _d);
1663                 case_brf(bunge, _d, 64);
1664                 case_brr(bungt, _d);
1665                 case_brf(bungt, _d, 64);
1666                 case_brr(bltgt, _d);
1667                 case_brf(bltgt, _d, 64);
1668                 case_brr(bord, _d);
1669                 case_brf(bord, _d, 64);
1670                 case_brr(bunord, _d);
1671                 case_brf(bunord, _d, 64);
1672                 case_rr(ld, _d);
1673                 case_rw(ld, _d);
1674                 case_rrr(ldx, _d);
1675                 case_rrw(ldx, _d);
1676                 case_rr(st, _d);
1677                 case_wr(st, _d);
1678                 case_rrr(stx, _d);
1679                 case_wrr(stx, _d);
1680             case jit_code_jmpr:
1681                 jmpr(rn(node->u.w));
1682                 break;
1683             case jit_code_jmpi:
1684                 if (node->flag & jit_flag_node) {
1685 #if _CALL_AIXDESC
1686                     if (_jit->pc.uc == _jit->code.ptr + sizeof(void*) * 3)
1687                         _jitc->jump = 1;
1688 #endif
1689                     temp = node->u.n;
1690                     assert(temp->code == jit_code_label ||
1691                            temp->code == jit_code_epilog);
1692                     if (temp->flag & jit_flag_patch)
1693                         jmpi(temp->u.w);
1694                     else {
1695                         word = jmpi(_jit->pc.w);
1696                         patch(word, node);
1697                     }
1698                 }
1699                 else
1700                     jmpi(node->u.w);
1701                 break;
1702             case jit_code_callr:
1703                 callr(rn(node->u.w)
1704 #if _CALL_SYSV
1705                       , !!(node->flag & jit_flag_varargs)
1706 #endif
1707                       );
1708                 break;
1709             case jit_code_calli:
1710                 if (node->flag & jit_flag_node) {
1711                     temp = node->u.n;
1712                     assert(temp->code == jit_code_label ||
1713                            temp->code == jit_code_epilog);
1714                     word = calli_p(temp->u.w
1715 #if _CALL_SYSV
1716                                    , !!(node->flag & jit_flag_varargs)
1717 #endif
1718                                    );
1719                     if (!(temp->flag & jit_flag_patch))
1720                         patch(word, node);
1721                 }
1722                 else
1723                     calli(node->u.w
1724 #if _CALL_SYSV
1725                           , !!(node->flag & jit_flag_varargs)
1726 #endif
1727                           );
1728                 break;
1729             case jit_code_prolog:
1730                 _jitc->function = _jitc->functions.ptr + node->w.w;
1731                 undo.node = node;
1732                 undo.word = _jit->pc.w;
1733 #if DEVEL_DISASSEMBLER
1734                 undo.prevw = prevw;
1735 #endif
1736                 undo.patch_offset = _jitc->patches.offset;
1737 #if _CALL_AIXDESC
1738                 undo.prolog_offset = _jitc->prolog.offset;
1739 #endif
1740             restart_function:
1741                 _jitc->again = 0;
1742 #if _CALL_AIXDESC
1743                 if (_jitc->jump && !_jitc->function->assume_frame) {
1744                     /* remember prolog to hide offset adjustment for a jump
1745                      * to the start of a function, what is expected to be
1746                      * a common practice as first jit instruction */
1747                     if (_jitc->prolog.offset >= _jitc->prolog.length) {
1748                         _jitc->prolog.length += 16;
1749                         jit_realloc((jit_pointer_t *)&_jitc->prolog.ptr,
1750                                     (_jitc->prolog.length - 16) *
1751                                     sizeof(jit_word_t),
1752                                     _jitc->prolog.length * sizeof(jit_word_t));
1753                     }
1754                     _jitc->prolog.ptr[_jitc->prolog.offset++] = _jit->pc.w;
1755                     /* function descriptor */
1756                     word = _jit->pc.w + sizeof(void*) * 3;
1757                     iw(word);                           /* addr */
1758                     iw(0);                              /* toc */
1759                     iw(0);                              /* env */
1760                 }
1761 #endif
1762                 prolog(node);
1763                 break;
1764             case jit_code_epilog:
1765                 assert(_jitc->function == _jitc->functions.ptr + node->w.w);
1766                 if (_jitc->again) {
1767                     for (temp = undo.node->next;
1768                          temp != node; temp = temp->next) {
1769                         if (temp->code == jit_code_label ||
1770                             temp->code == jit_code_epilog)
1771                             temp->flag &= ~jit_flag_patch;
1772                     }
1773                     temp->flag &= ~jit_flag_patch;
1774                     node = undo.node;
1775                     _jit->pc.w = undo.word;
1776 #if DEVEL_DISASSEMBLER
1777                     prevw = undo.prevw;
1778 #endif
1779                     _jitc->patches.offset = undo.patch_offset;
1780 #if _CALL_AIXDESC
1781                     _jitc->prolog.offset = undo.prolog_offset;
1782 #endif
1783                     goto restart_function;
1784                 }
1785                 /* remember label is defined */
1786                 node->flag |= jit_flag_patch;
1787                 node->u.w = _jit->pc.w;
1788                 epilog(node);
1789                 _jitc->function = NULL;
1790                 break;
1791             case jit_code_va_start:
1792                 vastart(rn(node->u.w));
1793                 break;
1794             case jit_code_va_arg:
1795                 vaarg(rn(node->u.w), rn(node->v.w));
1796                 break;
1797             case jit_code_va_arg_d:
1798                 vaarg_d(rn(node->u.w), rn(node->v.w));
1799                 break;
1800             case jit_code_live:
1801             case jit_code_arg:                  case jit_code_ellipsis:
1802             case jit_code_va_push:
1803             case jit_code_allocai:              case jit_code_allocar:
1804             case jit_code_arg_f:                case jit_code_arg_d:
1805             case jit_code_va_end:
1806             case jit_code_ret:
1807             case jit_code_retr:                 case jit_code_reti:
1808             case jit_code_retr_f:               case jit_code_reti_f:
1809             case jit_code_retr_d:               case jit_code_reti_d:
1810             case jit_code_getarg_c:             case jit_code_getarg_uc:
1811             case jit_code_getarg_s:             case jit_code_getarg_us:
1812             case jit_code_getarg_i:
1813 #if __WORDSIZE == 64
1814             case jit_code_getarg_ui:            case jit_code_getarg_l:
1815 #endif
1816             case jit_code_getarg_f:             case jit_code_getarg_d:
1817             case jit_code_putargr:              case jit_code_putargi:
1818             case jit_code_putargr_f:            case jit_code_putargi_f:
1819             case jit_code_putargr_d:            case jit_code_putargi_d:
1820             case jit_code_pushargr:             case jit_code_pushargi:
1821             case jit_code_pushargr_f:           case jit_code_pushargi_f:
1822             case jit_code_pushargr_d:           case jit_code_pushargi_d:
1823             case jit_code_retval_c:             case jit_code_retval_uc:
1824             case jit_code_retval_s:             case jit_code_retval_us:
1825             case jit_code_retval_i:
1826 #if __WORDSIZE == 64
1827             case jit_code_retval_ui:            case jit_code_retval_l:
1828 #endif
1829             case jit_code_retval_f:             case jit_code_retval_d:
1830             case jit_code_prepare:
1831             case jit_code_finishr:              case jit_code_finishi:
1832                 break;
1833             default:
1834                 abort();
1835         }
1836         jit_regarg_clr(node, value);
1837         assert(_jitc->regarg == 0 && _jitc->synth == 0);
1838         /* update register live state */
1839         jit_reglive(node);
1840
1841         no_flag = !(node->flag & jit_flag_patch);
1842     }
1843 #undef case_brf
1844 #undef case_brw
1845 #undef case_brr
1846 #undef case_wrr
1847 #undef case_rrf
1848 #undef case_rrw
1849 #undef case_rrr
1850 #undef case_wr
1851 #undef case_rw
1852 #undef case_rr
1853
1854     for (offset = 0; offset < _jitc->patches.offset; offset++) {
1855         node = _jitc->patches.ptr[offset].node;
1856         word = node->code == jit_code_movi ? node->v.n->u.w : node->u.n->u.w;
1857         patch_at(_jitc->patches.ptr[offset].inst, word);
1858     }
1859
1860     jit_flush(_jit->code.ptr, _jit->pc.uc);
1861
1862     return (_jit->code.ptr);
1863 }
1864
1865 #define CODE                            1
1866 #  include "jit_ppc-cpu.c"
1867 #  include "jit_ppc-fpu.c"
1868 #undef CODE
1869
1870 void
1871 jit_flush(void *fptr, void *tptr)
1872 {
1873 #if defined(__GNUC__)
1874     jit_word_t          f, t, s;
1875
1876     s = sysconf(_SC_PAGE_SIZE);
1877     f = (jit_word_t)fptr & -s;
1878     t = (((jit_word_t)tptr) + s - 1) & -s;
1879     __clear_cache((void *)f, (void *)t);
1880 #endif
1881 }
1882
1883 void
1884 _emit_ldxi(jit_state_t *_jit, jit_int32_t r0, jit_int32_t r1, jit_word_t i0)
1885 {
1886 #if __WORDSIZE == 32
1887     ldxi_i(rn(r0), rn(r1), i0);
1888 #else
1889     ldxi_l(rn(r0), rn(r1), i0);
1890 #endif
1891 }
1892
1893 void
1894 _emit_stxi(jit_state_t *_jit, jit_word_t i0, jit_int32_t r0, jit_int32_t r1)
1895 {
1896 #if __WORDSIZE == 32
1897     stxi_i(i0, rn(r0), rn(r1));
1898 #else
1899     stxi_l(i0, rn(r0), rn(r1));
1900 #endif
1901 }
1902
1903 void
1904 _emit_ldxi_d(jit_state_t *_jit, jit_int32_t r0, jit_int32_t r1, jit_word_t i0)
1905 {
1906     ldxi_d(rn(r0), rn(r1), i0);
1907 }
1908
1909 void
1910 _emit_stxi_d(jit_state_t *_jit, jit_word_t i0, jit_int32_t r0, jit_int32_t r1)
1911 {
1912     stxi_d(i0, rn(r0), rn(r1));
1913 }
1914
1915 static void
1916 _patch(jit_state_t *_jit, jit_word_t instr, jit_node_t *node)
1917 {
1918     jit_int32_t          flag;
1919
1920     assert(node->flag & jit_flag_node);
1921     if (node->code == jit_code_movi)
1922         flag = node->v.n->flag;
1923     else
1924         flag = node->u.n->flag;
1925     assert(!(flag & jit_flag_patch));
1926     if (_jitc->patches.offset >= _jitc->patches.length) {
1927         jit_realloc((jit_pointer_t *)&_jitc->patches.ptr,
1928                     _jitc->patches.length * sizeof(jit_patch_t),
1929                     (_jitc->patches.length + 1024) * sizeof(jit_patch_t));
1930         _jitc->patches.length += 1024;
1931     }
1932     _jitc->patches.ptr[_jitc->patches.offset].inst = instr;
1933     _jitc->patches.ptr[_jitc->patches.offset].node = node;
1934     ++_jitc->patches.offset;
1935 }