deps/lightning/lib/jit_s390.c

   1 /*
   2  * Copyright (C) 2013-2019  Free Software Foundation, Inc.
   3  *
   4  * This file is part of GNU lightning.
   5  *
   6  * GNU lightning is free software; you can redistribute it and/or modify it
   7  * under the terms of the GNU Lesser General Public License as published
   8  * by the Free Software Foundation; either version 3, or (at your option)
   9  * any later version.
  10  *
  11  * GNU lightning is distributed in the hope that it will be useful, but
  12  * WITHOUT ANY WARRANTY; without even the implied warranty of MERCHANTABILITY
  13  * or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU Lesser General Public
  14  * License for more details.
  15  *
  16  * Authors:
  17  *      Paulo Cesar Pereira de Andrade
  18  */
  19
  20 #include <lightning.h>
  21 #include <lightning/jit_private.h>
  22
  23 #if __WORDSIZE == 32
  24 #  define NUM_FLOAT_REG_ARGS            2
  25 #else
  26 #  define NUM_FLOAT_REG_ARGS            4
  27 #endif
  28 #define jit_arg_reg_p(i)                ((i) >= 0 && (i) < 5)
  29 #define jit_arg_f_reg_p(i)              ((i) >= 0 && (i) < NUM_FLOAT_REG_ARGS)
  30
  31 /*
  32  * Types
  33  */
  34 typedef struct jit_va_list {
  35     /* The offsets are "1" based, as addresses are fixed in the
  36      * standard stack frame format. */
  37     jit_word_t          gpoff;
  38     jit_word_t          fpoff;
  39
  40     /* Easier when there is an explicitly defined type...
  41 (gdb) ptype ap
  42 type = struct __va_list_tag {
  43     long __gpr;
  44     long __fpr;
  45     void *__overflow_arg_area;
  46     void *__reg_save_area;
  47
  48     Note that gopff (__gpr) and fpoff (__fpr) are jit_word_t equivalent
  49     and, again, "1" (unit) based, so must be adjusted at va_arg time.
  50  */
  51     jit_pointer_t       over;
  52     jit_pointer_t       save;
  53
  54     /* For variadic functions, always allocate space to save callee
  55      * save fpr registers.
  56      * Note that s390 has a standard stack frame format that lightning
  57      * does not fully comply with, but for variadic functions it must,
  58      * for those (variadic) do not use the "empty" spaces for any
  59      * callee save fpr register, but save them after the va_list
  60      * space; and use the standard stack frame format, as required
  61      * by variadic functions (and have a compatible va_list pointer). */
  62     jit_float64_t       f8;
  63     jit_float64_t       f9;
  64     jit_float64_t       f10;
  65     jit_float64_t       f11;
  66     jit_float64_t       f12;
  67     jit_float64_t       f13;
  68     jit_float64_t       f14;
  69     jit_float64_t       f15;
  70 } jit_va_list_t;
  71
  72 /*
  73  * Prototypes
  74  */
  75 #define jit_get_reg_pair()              _jit_get_reg_pair(_jit)
  76 static jit_int32_t _jit_get_reg_pair(jit_state_t*);
  77 #define jit_unget_reg_pair(regno)       _jit_unget_reg_pair(_jit,regno)
  78 static void _jit_unget_reg_pair(jit_state_t*,jit_int32_t);
  79 #define jit_get_reg_but_zero(flags)     _jit_get_reg_but_zero(_jit,flags)
  80 static jit_int32_t _jit_get_reg_but_zero(jit_state_t*,jit_int32_t);
  81 #define jit_unget_reg_but_zero(reg)     jit_unget_reg(reg)
  82 #define patch(instr, node)              _patch(_jit, instr, node)
  83 static void _patch(jit_state_t*,jit_word_t,jit_node_t*);
  84
  85 /* libgcc */
  86 extern void __clear_cache(void *, void *);
  87
  88 #define PROTO                           1
  89 #  include "jit_s390-cpu.c"
  90 #  include "jit_s390-fpu.c"
  91 #  include "jit_fallback.c"
  92 #undef PROTO
  93
  94 /*
  95  * Initialization
  96  */
  97 jit_register_t          _rvs[] = {
  98     { rc(gpr) | 0x0,                    "%r0" },
  99     { rc(gpr) | 0x1,                    "%r1" },
 100     { rc(gpr) | rc(sav) | 0xc,          "%r12" },
 101     { rc(gpr) | rc(sav) | 0xb,          "%r11" },
 102     { rc(gpr) | rc(sav) | 0xa,          "%r10" },
 103     { rc(gpr) | rc(sav) | 0x9,          "%r9" },
 104     { rc(gpr) | rc(sav) | 0x8,          "%r8" },
 105     { rc(gpr) | rc(sav) | 0x7,          "%r7" },
 106     { rc(gpr) | rc(arg) | rc(sav) | 0x6,"%r6" },
 107     { rc(gpr) | rc(arg) | 0x5,          "%r5" },
 108     { rc(gpr) | rc(arg) | 0x4,          "%r4" },
 109     { rc(gpr) | rc(arg) | 0x3,          "%r3" },
 110     { rc(gpr) | rc(arg) | 0x2,          "%r2" },
 111     { rc(sav) | 0xd,                    "%r13" },       /* used as JIT_FP */
 112     { 0xe,                              "%r14" },
 113     { rc(sav) | 0xf,                    "%r15" },
 114     { rc(fpr) | 0x1,                    "%f1" },
 115     { rc(fpr) | 0x3,                    "%f3" },
 116     { rc(fpr) | 0x5,                    "%f5" },
 117     { rc(fpr) | 0x7,                    "%f7" },
 118     { rc(fpr) | rc(sav) | 0xe,          "%f14" },
 119     /* Do not use as temporary to simplify stack layout */
 120     { 0xf,                              "%f15" },
 121     { rc(fpr) | rc(sav) | 0x8,          "%f8" },
 122     { rc(fpr) | rc(sav) | 0x9,          "%f9" },
 123     { rc(fpr) | rc(sav) | 0xa,          "%f10" },
 124     { rc(fpr) | rc(sav) | 0xb,          "%f11" },
 125     { rc(fpr) | rc(sav) | 0xc,          "%f12" },
 126     { rc(fpr) | rc(sav) | 0xd,          "%f13" },
 127     { rc(fpr) | rc(arg) | 0x6,          "%f6" },
 128     { rc(fpr) | rc(arg) | 0x4,          "%f4" },
 129     { rc(fpr) | rc(arg) | 0x2,          "%f2" },
 130     { rc(fpr) | rc(arg) | 0x0,          "%f0" },
 131     { _NOREG,                           "<none>" },
 132 };
 133
 134 /*
 135  * Implementation
 136  */
 137 void
 138 jit_get_cpu(void)
 139 {
 140 }
 141
 142 void
 143 _jit_init(jit_state_t *_jit)
 144 {
 145     _jitc->reglen = jit_size(_rvs) - 1;
 146 }
 147
 148 void
 149 _jit_prolog(jit_state_t *_jit)
 150 {
 151     jit_int32_t         offset;
 152
 153     if (_jitc->function)
 154         jit_epilog();
 155     assert(jit_regset_cmp_ui(&_jitc->regarg, 0) == 0);
 156     jit_regset_set_ui(&_jitc->regsav, 0);
 157     offset = _jitc->functions.offset;
 158     if (offset >= _jitc->functions.length) {
 159         jit_realloc((jit_pointer_t *)&_jitc->functions.ptr,
 160                     _jitc->functions.length * sizeof(jit_function_t),
 161                     (_jitc->functions.length + 16) * sizeof(jit_function_t));
 162         _jitc->functions.length += 16;
 163     }
 164     _jitc->function = _jitc->functions.ptr + _jitc->functions.offset++;
 165     _jitc->function->self.size = stack_framesize;
 166     _jitc->function->self.argi = _jitc->function->self.argf =
 167         _jitc->function->self.aoff = _jitc->function->self.alen = 0;
 168     /* preallocate 8 bytes if not using a constant data buffer */
 169     if (_jitc->no_data)
 170         _jitc->function->self.aoff = -8;
 171     _jitc->function->self.call = jit_call_default;
 172     jit_alloc((jit_pointer_t *)&_jitc->function->regoff,
 173               _jitc->reglen * sizeof(jit_int32_t));
 174
 175     /* _no_link here does not mean the jit_link() call can be removed
 176      * by rewriting as:
 177      * _jitc->function->prolog = jit_new_node(jit_code_prolog);
 178      */
 179     _jitc->function->prolog = jit_new_node_no_link(jit_code_prolog);
 180     jit_link(_jitc->function->prolog);
 181     _jitc->function->prolog->w.w = offset;
 182     _jitc->function->epilog = jit_new_node_no_link(jit_code_epilog);
 183     /*  u:      label value
 184      *  v:      offset in blocks vector
 185      *  w:      offset in functions vector
 186      */
 187     _jitc->function->epilog->w.w = offset;
 188
 189     jit_regset_new(&_jitc->function->regset);
 190 }
 191
 192 jit_int32_t
 193 _jit_allocai(jit_state_t *_jit, jit_int32_t length)
 194 {
 195     assert(_jitc->function);
 196     switch (length) {
 197         case 0: case 1:                                         break;
 198         case 2:         _jitc->function->self.aoff &= -2;       break;
 199         case 3: case 4: _jitc->function->self.aoff &= -4;       break;
 200         default:        _jitc->function->self.aoff &= -8;       break;
 201     }
 202     _jitc->function->self.aoff -= length;
 203     if (!_jitc->realize) {
 204         jit_inc_synth_ww(allocai, _jitc->function->self.aoff, length);
 205         jit_dec_synth();
 206     }
 207     return (_jitc->function->self.aoff);
 208 }
 209
 210 void
 211 _jit_allocar(jit_state_t *_jit, jit_int32_t u, jit_int32_t v)
 212 {
 213     jit_int32_t          reg;
 214     assert(_jitc->function);
 215     jit_inc_synth_ww(allocar, u, v);
 216     if (!_jitc->function->allocar) {
 217         _jitc->function->aoffoff = jit_allocai(sizeof(jit_int32_t));
 218         _jitc->function->allocar = 1;
 219     }
 220     reg = jit_get_reg(jit_class_gpr);
 221     jit_negr(reg, v);
 222     jit_andi(reg, reg, -8);
 223     jit_ldxi_i(u, JIT_FP, _jitc->function->aoffoff);
 224     jit_addr(u, u, reg);
 225     jit_addr(JIT_SP, JIT_SP, reg);
 226     jit_stxi_i(_jitc->function->aoffoff, JIT_FP, u);
 227     jit_unget_reg(reg);
 228     jit_dec_synth();
 229 }
 230
 231 void
 232 _jit_ret(jit_state_t *_jit)
 233 {
 234     jit_node_t          *instr;
 235     assert(_jitc->function);
 236     jit_inc_synth(ret);
 237     /* jump to epilog */
 238     instr = jit_jmpi();
 239     jit_patch_at(instr, _jitc->function->epilog);
 240     jit_dec_synth();
 241 }
 242
 243 void
 244 _jit_retr(jit_state_t *_jit, jit_int32_t u)
 245 {
 246     jit_inc_synth_w(retr, u);
 247     jit_movr(JIT_RET, u);
 248     jit_ret();
 249     jit_dec_synth();
 250 }
 251
 252 void
 253 _jit_reti(jit_state_t *_jit, jit_word_t u)
 254 {
 255     jit_inc_synth_w(reti, u);
 256     jit_movi(JIT_RET, u);
 257     jit_ret();
 258     jit_dec_synth();
 259 }
 260
 261 void
 262 _jit_retr_f(jit_state_t *_jit, jit_int32_t u)
 263 {
 264     jit_inc_synth_w(retr_f, u);
 265     jit_movr_f(JIT_FRET, u);
 266     jit_ret();
 267     jit_dec_synth();
 268 }
 269
 270 void
 271 _jit_reti_f(jit_state_t *_jit, jit_float32_t u)
 272 {
 273     jit_inc_synth_f(reti_f, u);
 274     jit_movi_f(JIT_FRET, u);
 275     jit_ret();
 276     jit_dec_synth();
 277 }
 278
 279 void
 280 _jit_retr_d(jit_state_t *_jit, jit_int32_t u)
 281 {
 282     jit_inc_synth_w(retr_d, u);
 283     jit_movr_d(JIT_FRET, u);
 284     jit_ret();
 285     jit_dec_synth();
 286 }
 287
 288 void
 289 _jit_reti_d(jit_state_t *_jit, jit_float64_t u)
 290 {
 291     jit_inc_synth_d(reti_d, u);
 292     jit_movi_d(JIT_FRET, u);
 293     jit_ret();
 294     jit_dec_synth();
 295 }
 296
 297 void
 298 _jit_epilog(jit_state_t *_jit)
 299 {
 300     assert(_jitc->function);
 301     assert(_jitc->function->epilog->next == NULL);
 302     jit_link(_jitc->function->epilog);
 303     _jitc->function = NULL;
 304 }
 305
 306 jit_bool_t
 307 _jit_arg_register_p(jit_state_t *_jit, jit_node_t *u)
 308 {
 309     if (u->code == jit_code_arg)
 310         return (jit_arg_reg_p(u->u.w));
 311     assert(u->code == jit_code_arg_f || u->code == jit_code_arg_d);
 312     return (jit_arg_f_reg_p(u->u.w));
 313 }
 314
 315 void
 316 _jit_ellipsis(jit_state_t *_jit)
 317 {
 318     jit_inc_synth(ellipsis);
 319     if (_jitc->prepare) {
 320         jit_link_prepare();
 321         assert(!(_jitc->function->call.call & jit_call_varargs));
 322         _jitc->function->call.call |= jit_call_varargs;
 323     }
 324     else {
 325         jit_link_prolog();
 326         assert(!(_jitc->function->self.call & jit_call_varargs));
 327         _jitc->function->self.call |= jit_call_varargs;
 328
 329         /* Allocate va_list like object in the stack. */
 330         _jitc->function->vaoff = jit_allocai(sizeof(jit_va_list_t));
 331
 332         /* Initialize gp offset in save area. */
 333         if (jit_arg_reg_p(_jitc->function->self.argi))
 334             _jitc->function->vagp = _jitc->function->self.argi;
 335         else
 336             _jitc->function->vagp = 5;
 337
 338         /* Initialize fp offset in save area. */
 339         if (jit_arg_f_reg_p(_jitc->function->self.argf))
 340             _jitc->function->vafp = _jitc->function->self.argf;
 341         else
 342             _jitc->function->vafp = NUM_FLOAT_REG_ARGS;
 343     }
 344     jit_dec_synth();
 345 }
 346
 347 void
 348 _jit_va_push(jit_state_t *_jit, jit_int32_t u)
 349 {
 350     jit_inc_synth_w(va_push, u);
 351     jit_pushargr(u);
 352     jit_dec_synth();
 353 }
 354
 355 jit_node_t *
 356 _jit_arg(jit_state_t *_jit)
 357 {
 358     jit_node_t          *node;
 359     jit_int32_t          offset;
 360     assert(_jitc->function);
 361     if (jit_arg_reg_p(_jitc->function->self.argi))
 362         offset = _jitc->function->self.argi++;
 363     else {
 364         offset = _jitc->function->self.size;
 365         _jitc->function->self.size += sizeof(jit_word_t);
 366     }
 367     node = jit_new_node_ww(jit_code_arg, offset,
 368                            ++_jitc->function->self.argn);
 369     jit_link_prolog();
 370     return (node);
 371 }
 372
 373 jit_node_t *
 374 _jit_arg_f(jit_state_t *_jit)
 375 {
 376     jit_node_t          *node;
 377     jit_int32_t          offset;
 378     assert(_jitc->function);
 379     if (jit_arg_f_reg_p(_jitc->function->self.argf))
 380         offset = _jitc->function->self.argf++;
 381     else {
 382         offset = _jitc->function->self.size;
 383         _jitc->function->self.size += sizeof(jit_word_t);
 384     }
 385     node = jit_new_node_ww(jit_code_arg_f, offset,
 386                            ++_jitc->function->self.argn);
 387     jit_link_prolog();
 388     return (node);
 389 }
 390
 391 jit_node_t *
 392 _jit_arg_d(jit_state_t *_jit)
 393 {
 394     jit_node_t          *node;
 395     jit_int32_t          offset;
 396     assert(_jitc->function);
 397     if (jit_arg_f_reg_p(_jitc->function->self.argf))
 398         offset = _jitc->function->self.argf++;
 399     else {
 400         offset = _jitc->function->self.size;
 401         _jitc->function->self.size += sizeof(jit_float64_t);
 402     }
 403     node = jit_new_node_ww(jit_code_arg_d, offset,
 404                            ++_jitc->function->self.argn);
 405     jit_link_prolog();
 406     return (node);
 407 }
 408
 409 void
 410 _jit_getarg_c(jit_state_t *_jit, jit_int32_t u, jit_node_t *v)
 411 {
 412     assert(v->code == jit_code_arg);
 413     jit_inc_synth_wp(getarg_c, u, v);
 414     if (jit_arg_reg_p(v->u.w))
 415         jit_extr_c(u, _R2 - v->u.w);
 416     else
 417         jit_ldxi_c(u, JIT_FP,
 418                    v->u.w + (__WORDSIZE >> 3) - sizeof(jit_int8_t));
 419     jit_dec_synth();
 420 }
 421
 422 void
 423 _jit_getarg_uc(jit_state_t *_jit, jit_int32_t u, jit_node_t *v)
 424 {
 425     assert(v->code == jit_code_arg);
 426     jit_inc_synth_wp(getarg_uc, u, v);
 427     if (jit_arg_reg_p(v->u.w))
 428         jit_extr_uc(u, _R2 - v->u.w);
 429     else
 430         jit_ldxi_uc(u, JIT_FP,
 431                     v->u.w + (__WORDSIZE >> 3) - sizeof(jit_uint8_t));
 432     jit_dec_synth();
 433 }
 434
 435 void
 436 _jit_getarg_s(jit_state_t *_jit, jit_int32_t u, jit_node_t *v)
 437 {
 438     assert(v->code == jit_code_arg);
 439     jit_inc_synth_wp(getarg_s, u, v);
 440     if (jit_arg_reg_p(v->u.w))
 441         jit_extr_s(u, _R2 - v->u.w);
 442     else
 443         jit_ldxi_s(u, JIT_FP,
 444                    v->u.w + (__WORDSIZE >> 3) - sizeof(jit_int16_t));
 445     jit_dec_synth();
 446 }
 447
 448 void
 449 _jit_getarg_us(jit_state_t *_jit, jit_int32_t u, jit_node_t *v)
 450 {
 451     assert(v->code == jit_code_arg);
 452     jit_inc_synth_wp(getarg_us, u, v);
 453     if (jit_arg_reg_p(v->u.w))
 454         jit_extr_us(u, _R2 - v->u.w);
 455     else
 456         jit_ldxi_us(u, JIT_FP,
 457                     v->u.w + (__WORDSIZE >> 3) - sizeof(jit_uint16_t));
 458     jit_dec_synth();
 459 }
 460
 461 void
 462 _jit_getarg_i(jit_state_t *_jit, jit_int32_t u, jit_node_t *v)
 463 {
 464     assert(v->code == jit_code_arg);
 465     jit_inc_synth_wp(getarg_i, u, v);
 466     if (jit_arg_reg_p(v->u.w)) {
 467 #if __WORDSIZE == 32
 468         jit_movr(u, _R2 - v->u.w);
 469 #else
 470         jit_extr_i(u, _R2 - v->u.w);
 471 #endif
 472     }
 473     else
 474         jit_ldxi_i(u, JIT_FP,
 475                    v->u.w + (__WORDSIZE >> 3) - sizeof(jit_int32_t));
 476     jit_dec_synth();
 477 }
 478
 479 #if __WORDSIZE == 64
 480 void
 481 _jit_getarg_ui(jit_state_t *_jit, jit_int32_t u, jit_node_t *v)
 482 {
 483     assert(v->code == jit_code_arg);
 484     jit_inc_synth_wp(getarg_ui, u, v);
 485     if (jit_arg_reg_p(v->u.w))
 486         jit_extr_ui(u, _R2 - v->u.w);
 487     else
 488         jit_ldxi_ui(u, JIT_FP,
 489                     v->u.w + (__WORDSIZE >> 3) - sizeof(jit_uint32_t));
 490     jit_dec_synth();
 491 }
 492
 493 void
 494 _jit_getarg_l(jit_state_t *_jit, jit_int32_t u, jit_node_t *v)
 495 {
 496     assert(v->code == jit_code_arg);
 497     jit_inc_synth_wp(getarg_l, u, v);
 498     if (jit_arg_reg_p(v->u.w))
 499         jit_movr(u, _R2 - v->u.w);
 500     else
 501         jit_ldxi_l(u, JIT_FP, v->u.w);
 502     jit_dec_synth();
 503 }
 504 #endif
 505
 506 void
 507 _jit_putargr(jit_state_t *_jit, jit_int32_t u, jit_node_t *v)
 508 {
 509     assert(v->code == jit_code_arg);
 510     jit_inc_synth_wp(putargr, u, v);
 511     if (jit_arg_reg_p(v->u.w))
 512         jit_movr(_R2 - v->u.w, u);
 513     else
 514         jit_stxi(v->u.w, JIT_FP, u);
 515     jit_dec_synth();
 516 }
 517
 518 void
 519 _jit_putargi(jit_state_t *_jit, jit_word_t u, jit_node_t *v)
 520 {
 521     jit_int32_t         regno;
 522     assert(v->code == jit_code_arg);
 523     jit_inc_synth_wp(putargi, u, v);
 524     if (jit_arg_reg_p(v->u.w))
 525         jit_movi(_R2 - v->u.w, u);
 526     else {
 527         regno = jit_get_reg(jit_class_gpr);
 528         jit_movi(regno, u);
 529         jit_stxi(v->u.w, JIT_FP, regno);
 530         jit_unget_reg(regno);
 531     }
 532     jit_dec_synth();
 533 }
 534
 535 void
 536 _jit_getarg_f(jit_state_t *_jit, jit_int32_t u, jit_node_t *v)
 537 {
 538     assert(v->code == jit_code_arg_f);
 539     jit_inc_synth_wp(getarg_f, u, v);
 540     if (jit_arg_f_reg_p(v->u.w))
 541         jit_movr_f(u, _F0 - v->u.w);
 542     else
 543         jit_ldxi_f(u, JIT_FP,
 544                    v->u.w
 545 #if __WORDSIZE == 64
 546                    + (__WORDSIZE >> 3) - sizeof(jit_float32_t)
 547 #endif
 548                    );
 549     jit_dec_synth();
 550 }
 551
 552 void
 553 _jit_putargr_f(jit_state_t *_jit, jit_int32_t u, jit_node_t *v)
 554 {
 555     assert(v->code == jit_code_arg_f);
 556     jit_inc_synth_wp(putargr_f, u, v);
 557     if (jit_arg_f_reg_p(v->u.w))
 558         jit_movr_f(_F0 - v->u.w, u);
 559     else
 560         jit_stxi_f(v->u.w
 561 #if __WORDSIZE == 64
 562                    + (__WORDSIZE >> 3) - sizeof(jit_float32_t)
 563 #endif
 564                    , JIT_FP, u);
 565     jit_dec_synth();
 566 }
 567
 568 void
 569 _jit_putargi_f(jit_state_t *_jit, jit_float32_t u, jit_node_t *v)
 570 {
 571     jit_int32_t         regno;
 572     assert(v->code == jit_code_arg_f);
 573     jit_inc_synth_fp(putargi_f, u, v);
 574     if (jit_arg_f_reg_p(v->u.w))
 575         jit_movi_f(_F0 - v->u.w, u);
 576     else {
 577         regno = jit_get_reg(jit_class_fpr);
 578         jit_movi_f(regno, u);
 579         jit_stxi_f(v->u.w
 580 #if __WORDSIZE == 64
 581                    + (__WORDSIZE >> 3) - sizeof(jit_float32_t)
 582 #endif
 583                    , JIT_FP, regno);
 584         jit_unget_reg(regno);
 585     }
 586     jit_dec_synth();
 587 }
 588
 589 void
 590 _jit_getarg_d(jit_state_t *_jit, jit_int32_t u, jit_node_t *v)
 591 {
 592     assert(v->code == jit_code_arg_d);
 593     jit_inc_synth_wp(getarg_d, u, v);
 594     if (jit_arg_f_reg_p(v->u.w))
 595         jit_movr_d(u, _F0 - v->u.w);
 596     else
 597         jit_ldxi_d(u, JIT_FP, v->u.w);
 598     jit_dec_synth();
 599 }
 600
 601 void
 602 _jit_putargr_d(jit_state_t *_jit, jit_int32_t u, jit_node_t *v)
 603 {
 604     assert(v->code == jit_code_arg_d);
 605     jit_inc_synth_wp(putargr_d, u, v);
 606     if (jit_arg_f_reg_p(v->u.w))
 607         jit_movr_d(_F0 - v->u.w, u);
 608     else
 609         jit_stxi_d(v->u.w, JIT_FP, u);
 610     jit_dec_synth();
 611 }
 612
 613 void
 614 _jit_putargi_d(jit_state_t *_jit, jit_float64_t u, jit_node_t *v)
 615 {
 616     jit_int32_t         regno;
 617     assert(v->code == jit_code_arg_d);
 618     jit_inc_synth_dp(putargi_d, u, v);
 619     if (jit_arg_f_reg_p(v->u.w))
 620         jit_movi_d(_F0 - v->u.w, u);
 621     else {
 622         regno = jit_get_reg(jit_class_fpr);
 623         jit_movi_d(regno, u);
 624         jit_stxi_d(v->u.w, JIT_FP, regno);
 625         jit_unget_reg(regno);
 626     }
 627     jit_dec_synth();
 628 }
 629
 630 void
 631 _jit_pushargr(jit_state_t *_jit, jit_int32_t u)
 632 {
 633     assert(_jitc->function);
 634     jit_inc_synth_w(pushargr, u);
 635     jit_link_prepare();
 636     if (jit_arg_reg_p(_jitc->function->call.argi)) {
 637         jit_movr(_R2 - _jitc->function->call.argi, u);
 638         ++_jitc->function->call.argi;
 639     }
 640     else {
 641         jit_stxi(_jitc->function->call.size + stack_framesize, JIT_SP, u);
 642         _jitc->function->call.size += sizeof(jit_word_t);
 643     }
 644     jit_dec_synth();
 645 }
 646
 647 void
 648 _jit_pushargi(jit_state_t *_jit, jit_word_t u)
 649 {
 650     jit_int32_t          regno;
 651     assert(_jitc->function);
 652     jit_inc_synth_w(pushargi, u);
 653     jit_link_prepare();
 654     if (jit_arg_reg_p(_jitc->function->call.argi)) {
 655         jit_movi(_R2 - _jitc->function->call.argi, u);
 656         ++_jitc->function->call.argi;
 657     }
 658     else {
 659         regno = jit_get_reg(jit_class_gpr);
 660         jit_movi(regno, u);
 661         jit_stxi(_jitc->function->call.size + stack_framesize, JIT_SP, regno);
 662         jit_unget_reg(regno);
 663         _jitc->function->call.size += sizeof(jit_word_t);
 664     }
 665     jit_dec_synth();
 666 }
 667
 668 void
 669 _jit_pushargr_f(jit_state_t *_jit, jit_int32_t u)
 670 {
 671     assert(_jitc->function);
 672     jit_inc_synth_w(pushargr_f, u);
 673     jit_link_prepare();
 674     if (jit_arg_f_reg_p(_jitc->function->call.argf)) {
 675         jit_movr_f(_F0 - _jitc->function->call.argf, u);
 676         ++_jitc->function->call.argf;
 677     }
 678     else {
 679         jit_stxi_f(_jitc->function->call.size + stack_framesize
 680 #if __WORDSIZE == 64
 681                    + (__WORDSIZE >> 3) - sizeof(jit_float32_t)
 682 #endif
 683                    , JIT_SP, u);
 684         _jitc->function->call.size += sizeof(jit_word_t);
 685     }
 686     jit_dec_synth();
 687 }
 688
 689 void
 690 _jit_pushargi_f(jit_state_t *_jit, jit_float32_t u)
 691 {
 692     jit_int32_t         regno;
 693     assert(_jitc->function);
 694     jit_inc_synth_f(pushargi_f, u);
 695     jit_link_prepare();
 696     if (jit_arg_f_reg_p(_jitc->function->call.argf)) {
 697         jit_movi_f(_F0 - _jitc->function->call.argf, u);
 698         ++_jitc->function->call.argf;
 699     }
 700     else {
 701         regno = jit_get_reg(jit_class_fpr);
 702         jit_movi_f(regno, u);
 703         jit_stxi_f(_jitc->function->call.size + stack_framesize
 704 #if __WORDSIZE == 64
 705                    + (__WORDSIZE >> 3) - sizeof(jit_float32_t)
 706 #endif
 707                    , JIT_SP, regno);
 708         jit_unget_reg(regno);
 709         _jitc->function->call.size += sizeof(jit_word_t);
 710     }
 711     jit_dec_synth();
 712 }
 713
 714 void
 715 _jit_pushargr_d(jit_state_t *_jit, jit_int32_t u)
 716 {
 717     assert(_jitc->function);
 718     jit_inc_synth_w(pushargr_d, u);
 719     jit_link_prepare();
 720     if (jit_arg_f_reg_p(_jitc->function->call.argf)) {
 721         jit_movr_d(_F0 - _jitc->function->call.argf, u);
 722         ++_jitc->function->call.argf;
 723     }
 724     else {
 725         jit_stxi_d(_jitc->function->call.size + stack_framesize, JIT_SP, u);
 726         _jitc->function->call.size += sizeof(jit_float64_t);
 727     }
 728     jit_dec_synth();
 729 }
 730
 731 void
 732 _jit_pushargi_d(jit_state_t *_jit, jit_float64_t u)
 733 {
 734     jit_int32_t         regno;
 735     assert(_jitc->function);
 736     jit_inc_synth_d(pushargi_d, u);
 737     jit_link_prepare();
 738     if (jit_arg_f_reg_p(_jitc->function->call.argf)) {
 739         jit_movi_d(_F0 - _jitc->function->call.argf, u);
 740         ++_jitc->function->call.argf;
 741     }
 742     else {
 743         regno = jit_get_reg(jit_class_fpr);
 744         jit_movi_d(regno, u);
 745         jit_stxi_d(_jitc->function->call.size + stack_framesize, JIT_SP, regno);
 746         jit_unget_reg(regno);
 747         _jitc->function->call.size += sizeof(jit_float64_t);
 748     }
 749     jit_dec_synth();
 750 }
 751
 752 jit_bool_t
 753 _jit_regarg_p(jit_state_t *_jit, jit_node_t *node, jit_int32_t regno)
 754 {
 755     jit_int32_t         spec;
 756     spec = jit_class(_rvs[regno].spec);
 757     if (spec & jit_class_arg) {
 758         regno = _R2 - regno;
 759         if (regno >= 0 && regno < node->v.w)
 760             return (1);
 761         if (spec & jit_class_fpr) {
 762             regno = _F0 - regno;
 763             if (regno >= 0 && regno < node->w.w)
 764                 return (1);
 765         }
 766     }
 767     return (0);
 768 }
 769
 770 void
 771 _jit_finishr(jit_state_t *_jit, jit_int32_t r0)
 772 {
 773     jit_node_t          *call;
 774     assert(_jitc->function);
 775     jit_inc_synth_w(finishr, r0);
 776     if (_jitc->function->self.alen < _jitc->function->call.size)
 777         _jitc->function->self.alen = _jitc->function->call.size;
 778     call = jit_callr(r0);
 779     call->v.w = _jitc->function->call.argi;
 780     call->w.w = _jitc->function->call.argf;
 781     _jitc->function->call.argi = _jitc->function->call.argf =
 782         _jitc->function->call.size = 0;
 783     _jitc->prepare = 0;
 784     jit_dec_synth();
 785 }
 786
 787 jit_node_t *
 788 _jit_finishi(jit_state_t *_jit, jit_pointer_t i0)
 789 {
 790     jit_node_t          *node;
 791     assert(_jitc->function);
 792     jit_inc_synth_w(finishi, (jit_word_t)i0);
 793     if (_jitc->function->self.alen < _jitc->function->call.size)
 794         _jitc->function->self.alen = _jitc->function->call.size;
 795     node = jit_calli(i0);
 796     node->v.w = _jitc->function->call.argi;
 797     node->w.w = _jitc->function->call.argf;
 798     _jitc->function->call.argi = _jitc->function->call.argf =
 799         _jitc->function->call.size = 0;
 800     _jitc->prepare = 0;
 801     jit_dec_synth();
 802     return (node);
 803 }
 804
 805 void
 806 _jit_retval_c(jit_state_t *_jit, jit_int32_t r0)
 807 {
 808     jit_inc_synth_w(retval_c, r0);
 809     jit_extr_c(r0, JIT_RET);
 810     jit_dec_synth();
 811 }
 812
 813 void
 814 _jit_retval_uc(jit_state_t *_jit, jit_int32_t r0)
 815 {
 816     jit_inc_synth_w(retval_uc, r0);
 817     jit_extr_uc(r0, JIT_RET);
 818     jit_dec_synth();
 819 }
 820
 821 void
 822 _jit_retval_s(jit_state_t *_jit, jit_int32_t r0)
 823 {
 824     jit_inc_synth_w(retval_s, r0);
 825     jit_extr_s(r0, JIT_RET);
 826     jit_dec_synth();
 827 }
 828
 829 void
 830 _jit_retval_us(jit_state_t *_jit, jit_int32_t r0)
 831 {
 832     jit_inc_synth_w(retval_us, r0);
 833     jit_extr_us(r0, JIT_RET);
 834     jit_dec_synth();
 835 }
 836
 837 void
 838 _jit_retval_i(jit_state_t *_jit, jit_int32_t r0)
 839 {
 840     jit_inc_synth_w(retval_i, r0);
 841 #if __WORDSIZE == 64
 842     jit_extr_i(r0, JIT_RET);
 843 #else
 844     jit_movr(r0, JIT_RET);
 845 #endif
 846     jit_dec_synth();
 847 }
 848
 849 #if __WORDSIZE == 64
 850 void
 851 _jit_retval_ui(jit_state_t *_jit, jit_int32_t r0)
 852 {
 853     jit_inc_synth_w(retval_ui, r0);
 854     jit_extr_ui(r0, JIT_RET);
 855     jit_dec_synth();
 856 }
 857
 858 void
 859 _jit_retval_l(jit_state_t *_jit, jit_int32_t r0)
 860 {
 861     jit_inc_synth_w(retval_l, r0);
 862     jit_movr(r0, JIT_RET);
 863     jit_dec_synth();
 864 }
 865 #endif
 866
 867 void
 868 _jit_retval_f(jit_state_t *_jit, jit_int32_t r0)
 869 {
 870     jit_inc_synth_w(retval_f, r0);
 871     jit_movr_f(r0, JIT_FRET);
 872     jit_dec_synth();
 873 }
 874
 875 void
 876 _jit_retval_d(jit_state_t *_jit, jit_int32_t r0)
 877 {
 878     jit_inc_synth_w(retval_d, r0);
 879     jit_movr_d(r0, JIT_FRET);
 880     jit_dec_synth();
 881 }
 882
 883 jit_pointer_t
 884 _emit_code(jit_state_t *_jit)
 885 {
 886     jit_node_t          *node;
 887     jit_node_t          *temp;
 888     jit_word_t           word;
 889     jit_int32_t          value;
 890     jit_int32_t          offset;
 891     struct {
 892         jit_node_t      *node;
 893         jit_word_t       word;
 894 #if DEVEL_DISASSEMBLER
 895         jit_word_t       prevw;
 896 #endif
 897         jit_int32_t      patch_offset;
 898     } undo;
 899 #if DEVEL_DISASSEMBLER
 900     jit_word_t           prevw;
 901 #endif
 902
 903     _jitc->function = NULL;
 904
 905     jit_reglive_setup();
 906
 907     undo.word = 0;
 908     undo.node = NULL;
 909     undo.patch_offset = 0;
 910
 911 #define assert_data(node)               /**/
 912 #define case_rr(name, type)                                             \
 913             case jit_code_##name##r##type:                              \
 914                 name##r##type(rn(node->u.w), rn(node->v.w));            \
 915                 break
 916 #define case_rw(name, type)                                             \
 917             case jit_code_##name##i##type:                              \
 918                 name##i##type(rn(node->u.w), node->v.w);                \
 919                 break
 920 #define case_wr(name, type)                                             \
 921             case jit_code_##name##i##type:                              \
 922                 name##i##type(node->u.w, rn(node->v.w));                \
 923                 break
 924 #define case_rrr(name, type)                                            \
 925             case jit_code_##name##r##type:                              \
 926                 name##r##type(rn(node->u.w),                            \
 927                               rn(node->v.w), rn(node->w.w));            \
 928                 break
 929 #define case_rrrr(name, type)                                           \
 930             case jit_code_##name##r##type:                              \
 931                 name##r##type(rn(node->u.q.l), rn(node->u.q.h),         \
 932                               rn(node->v.w), rn(node->w.w));            \
 933                 break
 934 #define case_rrw(name, type)                                            \
 935             case jit_code_##name##i##type:                              \
 936                 name##i##type(rn(node->u.w), rn(node->v.w), node->w.w); \
 937                 break
 938 #define case_rrrw(name, type)                                           \
 939             case jit_code_##name##i##type:                              \
 940                 name##i##type(rn(node->u.q.l), rn(node->u.q.h),         \
 941                               rn(node->v.w), node->w.w);                \
 942                 break
 943 #define case_rrf(name)                                                  \
 944             case jit_code_##name##i_f:                                  \
 945                 assert_data(node);                                      \
 946                 name##i_f(rn(node->u.w), rn(node->v.w),                 \
 947                           (jit_float32_t *)node->w.n->u.w);             \
 948                 break
 949 #define case_rrd(name)                                                  \
 950             case jit_code_##name##i_d:                                  \
 951                 assert_data(node);                                      \
 952                 name##i_d(rn(node->u.w), rn(node->v.w),                 \
 953                           (jit_float64_t *)node->w.n->u.w);             \
 954                 break
 955 #define case_wrr(name, type)                                            \
 956             case jit_code_##name##i##type:                              \
 957                 name##i##type(node->u.w, rn(node->v.w), rn(node->w.w)); \
 958                 break
 959 #define case_brr(name, type)                                            \
 960             case jit_code_##name##r##type:                              \
 961                 temp = node->u.n;                                       \
 962                 assert(temp->code == jit_code_label ||                  \
 963                        temp->code == jit_code_epilog);                  \
 964                 if (temp->flag & jit_flag_patch)                        \
 965                     name##r##type(temp->u.w, rn(node->v.w),             \
 966                                   rn(node->w.w));                       \
 967                 else {                                                  \
 968                     word = name##r##type##_p(_jit->pc.w,                \
 969                                              rn(node->v.w),             \
 970                                              rn(node->w.w));            \
 971                     patch(word, node);                                  \
 972                 }                                                       \
 973                 break
 974 #define case_brw(name, type)                                            \
 975             case jit_code_##name##i##type:                              \
 976                 temp = node->u.n;                                       \
 977                 assert(temp->code == jit_code_label ||                  \
 978                        temp->code == jit_code_epilog);                  \
 979                 if (temp->flag & jit_flag_patch)                        \
 980                     name##i##type(temp->u.w,                            \
 981                                   rn(node->v.w), node->w.w);            \
 982                 else {                                                  \
 983                     word = name##i##type##_p(_jit->pc.w,                \
 984                                              rn(node->v.w), node->w.w); \
 985                     patch(word, node);                                  \
 986                 }                                                       \
 987                 break;
 988 #define case_brf(name)                                                  \
 989             case jit_code_##name##i_f:                                  \
 990                 temp = node->u.n;                                       \
 991                 assert(temp->code == jit_code_label ||                  \
 992                        temp->code == jit_code_epilog);                  \
 993                 if (temp->flag & jit_flag_patch)                        \
 994                     name##i_f(temp->u.w, rn(node->v.w),                 \
 995                               (jit_float32_t *)node->w.n->u.w);         \
 996                 else {                                                  \
 997                     word = name##i_f_p(_jit->pc.w, rn(node->v.w),       \
 998                                        (jit_float32_t *)node->w.n->u.w);\
 999                     patch(word, node);                                  \
1000                 }                                                       \
1001                 break
1002 #define case_brd(name)                                                  \
1003             case jit_code_##name##i_d:                                  \
1004                 temp = node->u.n;                                       \
1005                 assert(temp->code == jit_code_label ||                  \
1006                        temp->code == jit_code_epilog);                  \
1007                 if (temp->flag & jit_flag_patch)                        \
1008                     name##i_d(temp->u.w, rn(node->v.w),                 \
1009                               (jit_float64_t *)node->w.n->u.w);         \
1010                 else {                                                  \
1011                     word = name##i_d_p(_jit->pc.w, rn(node->v.w),       \
1012                                        (jit_float64_t *)node->w.n->u.w);\
1013                     patch(word, node);                                  \
1014                 }                                                       \
1015                 break
1016 #if DEVEL_DISASSEMBLER
1017     prevw = _jit->pc.w;
1018 #endif
1019     for (node = _jitc->head; node; node = node->next) {
1020         if (_jit->pc.uc >= _jitc->code.end)
1021             return (NULL);
1022
1023 #if DEVEL_DISASSEMBLER
1024         node->offset = (jit_uword_t)_jit->pc.w - (jit_uword_t)prevw;
1025         prevw = _jit->pc.w;
1026 #endif
1027         value = jit_classify(node->code);
1028         jit_regarg_set(node, value);
1029         switch (node->code) {
1030             case jit_code_align:
1031                 assert(!(node->u.w & (node->u.w - 1)) &&
1032                        node->u.w <= sizeof(jit_word_t));
1033                 if (node->u.w == sizeof(jit_word_t) &&
1034                     (word = _jit->pc.w & (sizeof(jit_word_t) - 1)))
1035                     nop(sizeof(jit_word_t) - word);
1036                 break;
1037             case jit_code_note:         case jit_code_name:
1038                 node->u.w = _jit->pc.w;
1039                 break;
1040             case jit_code_label:
1041                 if ((node->link || (node->flag & jit_flag_use)) &&
1042                     (word = _jit->pc.w & 3))
1043                     nop(4 - word);
1044                 /* remember label is defined */
1045                 node->flag |= jit_flag_patch;
1046                 node->u.w = _jit->pc.w;
1047                 break;
1048                 case_rrr(add,);
1049                 case_rrw(add,);
1050                 case_rrr(addc,);
1051                 case_rrw(addc,);
1052                 case_rrr(addx,);
1053                 case_rrw(addx,);
1054                 case_rrr(sub,);
1055                 case_rrw(sub,);
1056                 case_rrr(subc,);
1057                 case_rrw(subc,);
1058                 case_rrr(subx,);
1059                 case_rrw(subx,);
1060                 case_rrw(rsb,);
1061                 case_rrr(mul,);
1062                 case_rrw(mul,);
1063                 case_rrrr(qmul,);
1064                 case_rrrw(qmul,);
1065                 case_rrrr(qmul, _u);
1066                 case_rrrw(qmul, _u);
1067                 case_rrr(div,);
1068                 case_rrw(div,);
1069                 case_rrr(div, _u);
1070                 case_rrw(div, _u);
1071                 case_rrr(rem,);
1072                 case_rrw(rem,);
1073                 case_rrr(rem, _u);
1074                 case_rrw(rem, _u);
1075                 case_rrrr(qdiv,);
1076                 case_rrrw(qdiv,);
1077                 case_rrrr(qdiv, _u);
1078                 case_rrrw(qdiv, _u);
1079                 case_rrr(lsh,);
1080                 case_rrw(lsh,);
1081                 case_rrr(rsh,);
1082                 case_rrw(rsh,);
1083                 case_rrr(rsh, _u);
1084                 case_rrw(rsh, _u);
1085                 case_rr(neg,);
1086                 case_rr(com,);
1087                 case_rrr(and,);
1088                 case_rrw(and,);
1089                 case_rrr(or,);
1090                 case_rrw(or,);
1091                 case_rrr(xor,);
1092                 case_rrw(xor,);
1093                 case_rr(trunc, _f_i);
1094                 case_rr(trunc, _d_i);
1095 #if __WORDSIZE == 64
1096                 case_rr(trunc, _f_l);
1097                 case_rr(trunc, _d_l);
1098 #endif
1099                 case_rr(ld, _c);
1100                 case_rw(ld, _c);
1101                 case_rr(ld, _uc);
1102                 case_rw(ld, _uc);
1103                 case_rr(ld, _s);
1104                 case_rw(ld, _s);
1105                 case_rr(ld, _us);
1106                 case_rw(ld, _us);
1107                 case_rr(ld, _i);
1108                 case_rw(ld, _i);
1109 #if __WORDSIZE == 64
1110                 case_rr(ld, _ui);
1111                 case_rw(ld, _ui);
1112                 case_rr(ld, _l);
1113                 case_rw(ld, _l);
1114 #endif
1115                 case_rrr(ldx, _c);
1116                 case_rrw(ldx, _c);
1117                 case_rrr(ldx, _uc);
1118                 case_rrw(ldx, _uc);
1119                 case_rrr(ldx, _s);
1120                 case_rrw(ldx, _s);
1121                 case_rrr(ldx, _us);
1122                 case_rrw(ldx, _us);
1123                 case_rrr(ldx, _i);
1124                 case_rrw(ldx, _i);
1125 #if __WORDSIZE == 64
1126                 case_rrr(ldx, _ui);
1127                 case_rrw(ldx, _ui);
1128                 case_rrr(ldx, _l);
1129                 case_rrw(ldx, _l);
1130 #endif
1131                 case_rr(st, _c);
1132                 case_wr(st, _c);
1133                 case_rr(st, _s);
1134                 case_wr(st, _s);
1135                 case_rr(st, _i);
1136                 case_wr(st, _i);
1137 #if __WORDSIZE == 64
1138                 case_rr(st, _l);
1139                 case_wr(st, _l);
1140 #endif
1141                 case_rrr(stx, _c);
1142                 case_wrr(stx, _c);
1143                 case_rrr(stx, _s);
1144                 case_wrr(stx, _s);
1145                 case_rrr(stx, _i);
1146                 case_wrr(stx, _i);
1147 #if __WORDSIZE == 64
1148                 case_rrr(stx, _l);
1149                 case_wrr(stx, _l);
1150 #endif
1151                 case_rr(hton, _us);
1152                 case_rr(hton, _ui);
1153 #if __WORDSIZE == 64
1154                 case_rr(hton, _ul);
1155 #endif
1156                 case_rr(bswap, _us);
1157                 case_rr(bswap, _ui);
1158 #if __WORDSIZE == 64
1159                 case_rr(bswap, _ul);
1160 #endif
1161                 case_rr(ext, _c);
1162                 case_rr(ext, _uc);
1163                 case_rr(ext, _s);
1164                 case_rr(ext, _us);
1165 #if __WORDSIZE == 64
1166                 case_rr(ext, _i);
1167                 case_rr(ext, _ui);
1168 #endif
1169             case jit_code_casr:
1170                 casr(rn(node->u.w), rn(node->v.w),
1171                      rn(node->w.q.l), rn(node->w.q.h));
1172                 break;
1173             case jit_code_casi:
1174                 casi(rn(node->u.w), node->v.w,
1175                      rn(node->w.q.l), rn(node->w.q.h));
1176                 break;
1177                 case_rrr(movn,);
1178                 case_rrr(movz,);
1179                 case_rr(mov,);
1180             case jit_code_movi:
1181                 if (node->flag & jit_flag_node) {
1182                     temp = node->v.n;
1183                     if (temp->code == jit_code_data ||
1184                         (temp->code == jit_code_label &&
1185                          (temp->flag & jit_flag_patch)))
1186                         movi(rn(node->u.w), temp->u.w);
1187                     else {
1188                         assert(temp->code == jit_code_label ||
1189                                temp->code == jit_code_epilog);
1190                         word = movi_p(rn(node->u.w), temp->u.w);
1191                         patch(word, node);
1192                     }
1193                 }
1194                 else
1195                     movi(rn(node->u.w), node->v.w);
1196                 break;
1197                 case_rrr(lt,);
1198                 case_rrw(lt,);
1199                 case_rrr(lt, _u);
1200                 case_rrw(lt, _u);
1201                 case_rrr(le,);
1202                 case_rrw(le,);
1203                 case_rrr(le, _u);
1204                 case_rrw(le, _u);
1205                 case_rrr(eq,);
1206                 case_rrw(eq,);
1207                 case_rrr(ge,);
1208                 case_rrw(ge,);
1209                 case_rrr(ge, _u);
1210                 case_rrw(ge, _u);
1211                 case_rrr(gt,);
1212                 case_rrw(gt,);
1213                 case_rrr(gt, _u);
1214                 case_rrw(gt, _u);
1215                 case_rrr(ne,);
1216                 case_rrw(ne,);
1217                 case_brr(blt,);
1218                 case_brw(blt,);
1219                 case_brr(blt, _u);
1220                 case_brw(blt, _u);
1221                 case_brr(ble,);
1222                 case_brw(ble,);
1223                 case_brr(ble, _u);
1224                 case_brw(ble, _u);
1225                 case_brr(beq,);
1226                 case_brw(beq,);
1227                 case_brr(bge,);
1228                 case_brw(bge,);
1229                 case_brr(bge, _u);
1230                 case_brw(bge, _u);
1231                 case_brr(bgt,);
1232                 case_brw(bgt,);
1233                 case_brr(bgt, _u);
1234                 case_brw(bgt, _u);
1235                 case_brr(bne,);
1236                 case_brw(bne,);
1237                 case_brr(boadd,);
1238                 case_brw(boadd,);
1239                 case_brr(boadd, _u);
1240                 case_brw(boadd, _u);
1241                 case_brr(bxadd,);
1242                 case_brw(bxadd,);
1243                 case_brr(bxadd, _u);
1244                 case_brw(bxadd, _u);
1245                 case_brr(bosub,);
1246                 case_brw(bosub,);
1247                 case_brr(bosub, _u);
1248                 case_brw(bosub, _u);
1249                 case_brr(bxsub,);
1250                 case_brw(bxsub,);
1251                 case_brr(bxsub, _u);
1252                 case_brw(bxsub, _u);
1253                 case_brr(bms,);
1254                 case_brw(bms,);
1255                 case_brr(bmc,);
1256                 case_brw(bmc,);
1257                 case_rrr(add, _f);
1258                 case_rrf(add);
1259                 case_rrr(sub, _f);
1260                 case_rrf(sub);
1261                 case_rrf(rsb);
1262                 case_rrr(mul, _f);
1263                 case_rrf(mul);
1264                 case_rrr(div, _f);
1265                 case_rrf(div);
1266                 case_rr(abs, _f);
1267                 case_rr(neg, _f);
1268                 case_rr(sqrt, _f);
1269                 case_rr(ext, _f);
1270                 case_rr(ld, _f);
1271                 case_rw(ld, _f);
1272                 case_rrr(ldx, _f);
1273                 case_rrw(ldx, _f);
1274                 case_rr(st, _f);
1275                 case_wr(st, _f);
1276                 case_rrr(stx, _f);
1277                 case_wrr(stx, _f);
1278                 case_rr(mov, _f);
1279             case jit_code_movi_f:
1280                 assert_data(node);
1281                 movi_f(rn(node->u.w), (jit_float32_t *)node->v.n->u.w);
1282                 break;
1283                 case_rr(ext, _d_f);
1284                 case_rrr(lt, _f);
1285                 case_rrf(lt);
1286                 case_rrr(le, _f);
1287                 case_rrf(le);
1288                 case_rrr(eq, _f);
1289                 case_rrf(eq);
1290                 case_rrr(ge, _f);
1291                 case_rrf(ge);
1292                 case_rrr(gt, _f);
1293                 case_rrf(gt);
1294                 case_rrr(ne, _f);
1295                 case_rrf(ne);
1296                 case_rrr(unlt, _f);
1297                 case_rrf(unlt);
1298                 case_rrr(unle, _f);
1299                 case_rrf(unle);
1300                 case_rrr(uneq, _f);
1301                 case_rrf(uneq);
1302                 case_rrr(unge, _f);
1303                 case_rrf(unge);
1304                 case_rrr(ungt, _f);
1305                 case_rrf(ungt);
1306                 case_rrr(ltgt, _f);
1307                 case_rrf(ltgt);
1308                 case_rrr(ord, _f);
1309                 case_rrf(ord);
1310                 case_rrr(unord, _f);
1311                 case_rrf(unord);
1312                 case_brr(blt, _f);
1313                 case_brf(blt);
1314                 case_brr(ble, _f);
1315                 case_brf(ble);
1316                 case_brr(beq, _f);
1317                 case_brf(beq);
1318                 case_brr(bge, _f);
1319                 case_brf(bge);
1320                 case_brr(bgt, _f);
1321                 case_brf(bgt);
1322                 case_brr(bne, _f);
1323                 case_brf(bne);
1324                 case_brr(bunlt, _f);
1325                 case_brf(bunlt);
1326                 case_brr(bunle, _f);
1327                 case_brf(bunle);
1328                 case_brr(buneq, _f);
1329                 case_brf(buneq);
1330                 case_brr(bunge, _f);
1331                 case_brf(bunge);
1332                 case_brr(bungt, _f);
1333                 case_brf(bungt);
1334                 case_brr(bltgt, _f);
1335                 case_brf(bltgt);
1336                 case_brr(bord, _f);
1337                 case_brf(bord);
1338                 case_brr(bunord, _f);
1339                 case_brf(bunord);
1340                 case_rrr(add, _d);
1341                 case_rrd(add);
1342                 case_rrr(sub, _d);
1343                 case_rrd(sub);
1344                 case_rrd(rsb);
1345                 case_rrr(mul, _d);
1346                 case_rrd(mul);
1347                 case_rrr(div, _d);
1348                 case_rrd(div);
1349                 case_rr(abs, _d);
1350                 case_rr(neg, _d);
1351                 case_rr(sqrt, _d);
1352                 case_rr(ext, _d);
1353                 case_rr(ld, _d);
1354                 case_rw(ld, _d);
1355                 case_rrr(ldx, _d);
1356                 case_rrw(ldx, _d);
1357                 case_rr(st, _d);
1358                 case_wr(st, _d);
1359                 case_rrr(stx, _d);
1360                 case_wrr(stx, _d);
1361                 case_rr(mov, _d);
1362             case jit_code_movi_d:
1363                 assert_data(node);
1364                 movi_d(rn(node->u.w), (jit_float64_t *)node->v.n->u.w);
1365                 break;
1366                 case_rr(ext, _f_d);
1367                 case_rrr(lt, _d);
1368                 case_rrd(lt);
1369                 case_rrr(le, _d);
1370                 case_rrd(le);
1371                 case_rrr(eq, _d);
1372                 case_rrd(eq);
1373                 case_rrr(ge, _d);
1374                 case_rrd(ge);
1375                 case_rrr(gt, _d);
1376                 case_rrd(gt);
1377                 case_rrr(ne, _d);
1378                 case_rrd(ne);
1379                 case_rrr(unlt, _d);
1380                 case_rrd(unlt);
1381                 case_rrr(unle, _d);
1382                 case_rrd(unle);
1383                 case_rrr(uneq, _d);
1384                 case_rrd(uneq);
1385                 case_rrr(unge, _d);
1386                 case_rrd(unge);
1387                 case_rrr(ungt, _d);
1388                 case_rrd(ungt);
1389                 case_rrr(ltgt, _d);
1390                 case_rrd(ltgt);
1391                 case_rrr(ord, _d);
1392                 case_rrd(ord);
1393                 case_rrr(unord, _d);
1394                 case_rrd(unord);
1395                 case_brr(blt, _d);
1396                 case_brd(blt);
1397                 case_brr(ble, _d);
1398                 case_brd(ble);
1399                 case_brr(beq, _d);
1400                 case_brd(beq);
1401                 case_brr(bge, _d);
1402                 case_brd(bge);
1403                 case_brr(bgt, _d);
1404                 case_brd(bgt);
1405                 case_brr(bne, _d);
1406                 case_brd(bne);
1407                 case_brr(bunlt, _d);
1408                 case_brd(bunlt);
1409                 case_brr(bunle, _d);
1410                 case_brd(bunle);
1411                 case_brr(buneq, _d);
1412                 case_brd(buneq);
1413                 case_brr(bunge, _d);
1414                 case_brd(bunge);
1415                 case_brr(bungt, _d);
1416                 case_brd(bungt);
1417                 case_brr(bltgt, _d);
1418                 case_brd(bltgt);
1419                 case_brr(bord, _d);
1420                 case_brd(bord);
1421                 case_brr(bunord, _d);
1422                 case_brd(bunord);
1423             case jit_code_jmpr:
1424                 jmpr(rn(node->u.w));
1425                 break;
1426             case jit_code_jmpi:
1427                 if (node->flag & jit_flag_node) {
1428                     temp = node->u.n;
1429                     assert(temp->code == jit_code_label ||
1430                            temp->code == jit_code_epilog);
1431                     if (temp->flag & jit_flag_patch)
1432                         jmpi(temp->u.w);
1433                     else {
1434                         word = jmpi_p(_jit->pc.w);
1435                         patch(word, node);
1436                     }
1437                 }
1438                 else
1439                     jmpi(node->u.w);
1440                 break;
1441             case jit_code_callr:
1442                 callr(rn(node->u.w));
1443                 break;
1444             case jit_code_calli:
1445                 if (node->flag & jit_flag_node) {
1446                     temp = node->u.n;
1447                     assert(temp->code == jit_code_label ||
1448                            temp->code == jit_code_epilog);
1449                     if (temp->flag & jit_flag_patch)
1450                         calli(temp->u.w);
1451                     else {
1452                         word = calli_p(_jit->pc.w);
1453                         patch(word, node);
1454                     }
1455                 }
1456                 else
1457                     calli(node->u.w);
1458                 break;
1459             case jit_code_prolog:
1460                 _jitc->function = _jitc->functions.ptr + node->w.w;
1461                 undo.node = node;
1462                 undo.word = _jit->pc.w;
1463 #if DEVEL_DISASSEMBLER
1464                 undo.prevw = prevw;
1465 #endif
1466                 undo.patch_offset = _jitc->patches.offset;
1467             restart_function:
1468                 _jitc->again = 0;
1469                 prolog(node);
1470                 break;
1471             case jit_code_epilog:
1472                 assert(_jitc->function == _jitc->functions.ptr + node->w.w);
1473                 if (_jitc->again) {
1474                     for (temp = undo.node->next;
1475                          temp != node; temp = temp->next) {
1476                         if (temp->code == jit_code_label ||
1477                             temp->code == jit_code_epilog)
1478                             temp->flag &= ~jit_flag_patch;
1479                     }
1480                     temp->flag &= ~jit_flag_patch;
1481                     node = undo.node;
1482                     _jit->pc.w = undo.word;
1483 #if DEVEL_DISASSEMBLER
1484                     prevw = undo.prevw;
1485 #endif
1486                     _jitc->patches.offset = undo.patch_offset;
1487                     goto restart_function;
1488                 }
1489                 if (node->link && (word = _jit->pc.w & 3))
1490                     nop(4 - word);
1491                 /* remember label is defined */
1492                 node->flag |= jit_flag_patch;
1493                 node->u.w = _jit->pc.w;
1494                 epilog(node);
1495                 _jitc->function = NULL;
1496                 break;
1497             case jit_code_va_start:
1498                 vastart(rn(node->u.w));
1499                 break;
1500             case jit_code_va_arg:
1501                 vaarg(rn(node->u.w), rn(node->v.w));
1502                 break;
1503             case jit_code_va_arg_d:
1504                 vaarg_d(rn(node->u.w), rn(node->v.w));
1505                 break;
1506             case jit_code_live:                 case jit_code_ellipsis:
1507             case jit_code_va_push:
1508             case jit_code_allocai:              case jit_code_allocar:
1509             case jit_code_arg:
1510             case jit_code_arg_f:                case jit_code_arg_d:
1511             case jit_code_va_end:
1512             case jit_code_ret:
1513             case jit_code_retr:                 case jit_code_reti:
1514             case jit_code_retr_f:               case jit_code_reti_f:
1515             case jit_code_retr_d:               case jit_code_reti_d:
1516             case jit_code_getarg_c:             case jit_code_getarg_uc:
1517             case jit_code_getarg_s:             case jit_code_getarg_us:
1518             case jit_code_getarg_i:
1519 #if __WORDSIZE == 64
1520             case jit_code_getarg_ui:            case jit_code_getarg_l:
1521 #endif
1522             case jit_code_getarg_f:             case jit_code_getarg_d:
1523             case jit_code_putargr:              case jit_code_putargi:
1524             case jit_code_putargr_f:            case jit_code_putargi_f:
1525             case jit_code_putargr_d:            case jit_code_putargi_d:
1526             case jit_code_pushargr:             case jit_code_pushargi:
1527             case jit_code_pushargr_f:           case jit_code_pushargi_f:
1528             case jit_code_pushargr_d:           case jit_code_pushargi_d:
1529             case jit_code_retval_c:             case jit_code_retval_uc:
1530             case jit_code_retval_s:             case jit_code_retval_us:
1531             case jit_code_retval_i:
1532 #if __WORDSIZE == 64
1533             case jit_code_retval_ui:            case jit_code_retval_l:
1534 #endif
1535             case jit_code_retval_f:             case jit_code_retval_d:
1536             case jit_code_prepare:
1537             case jit_code_finishr:              case jit_code_finishi:
1538                 break;
1539             default:
1540                 abort();
1541         }
1542         jit_regarg_clr(node, value);
1543         assert(_jitc->regarg == 0 && _jitc->synth == 0);
1544         /* update register live state */
1545         jit_reglive(node);
1546     }
1547 #undef case_brw
1548 #undef case_brr
1549 #undef case_wrr
1550 #undef case_rrw
1551 #undef case_rrr
1552 #undef case_wr
1553 #undef case_rw
1554 #undef case_rr
1555
1556     for (offset = 0; offset < _jitc->patches.offset; offset++) {
1557         node = _jitc->patches.ptr[offset].node;
1558         word = node->code == jit_code_movi ? node->v.n->u.w : node->u.n->u.w;
1559         patch_at(_jitc->patches.ptr[offset].inst, word);
1560     }
1561
1562     jit_flush(_jit->code.ptr, _jit->pc.uc);
1563
1564     return (_jit->code.ptr);
1565 }
1566
1567 #define CODE                            1
1568 #  include "jit_s390-cpu.c"
1569 #  include "jit_s390-fpu.c"
1570 #  include "jit_fallback.c"
1571 #undef CODE
1572
1573 void
1574 jit_flush(void *fptr, void *tptr)
1575 {
1576 #if defined(__GNUC__)
1577     jit_word_t          f, t, s;
1578
1579     s = sysconf(_SC_PAGE_SIZE);
1580     f = (jit_word_t)fptr & -s;
1581     t = (((jit_word_t)tptr) + s - 1) & -s;
1582     __clear_cache((void *)f, (void *)t);
1583 #endif
1584 }
1585
1586 void
1587 _emit_ldxi(jit_state_t *_jit, jit_gpr_t r0, jit_gpr_t r1, jit_word_t i0)
1588 {
1589     ldxi(rn(r0), rn(r1), i0);
1590 }
1591
1592 void
1593 _emit_stxi(jit_state_t *_jit, jit_word_t i0, jit_gpr_t r0, jit_gpr_t r1)
1594 {
1595     stxi(i0, rn(r0), rn(r1));
1596 }
1597
1598 void
1599 _emit_ldxi_d(jit_state_t *_jit, jit_fpr_t r0, jit_gpr_t r1, jit_word_t i0)
1600 {
1601     ldxi_d(rn(r0), rn(r1), i0);
1602 }
1603
1604 void
1605 _emit_stxi_d(jit_state_t *_jit, jit_word_t i0, jit_gpr_t r0, jit_fpr_t r1)
1606 {
1607     stxi_d(i0, rn(r0), rn(r1));
1608 }
1609
1610 static jit_int32_t
1611 _jit_get_reg_pair(jit_state_t *_jit)
1612 {
1613     jit_int32_t         r1, r2;
1614     /* Try to find a register pair for use with operations that
1615      * require a odd based register pair. Search for the best
1616      * match to avoid spills or at least a valid operation.
1617      */
1618
1619     /* Try non callee save first */
1620     if (jit_reg_free_p(_R0) && jit_reg_free_p(_R1))
1621         r1 = _R0, r2 = _R1;
1622     else if (jit_reg_free_p(_R2) && jit_reg_free_p(_R3))
1623         r1 = _R2, r2 = _R3;
1624     else if (jit_reg_free_p(_R4) && jit_reg_free_p(_R5))
1625         r1 = _R4, r2 = _R5;
1626     /* Try callee save registers */
1627     else if (jit_reg_free_p(_R10) && jit_reg_free_p(_R11))
1628         r1 = _R10, r2 = _R11;
1629     else if (jit_reg_free_p(_R8) && jit_reg_free_p(_R9))
1630         r1 = _R8, r2 = _R9;
1631     else if (jit_reg_free_p(_R6) && jit_reg_free_p(_R7))
1632         r1 = _R6, r2 = _R7;
1633
1634     /* We *must* find a register pair */
1635     else if (jit_reg_free_if_spill_p(_R0) && jit_reg_free_if_spill_p(_R1))
1636         r1 = _R0, r2 = _R1;
1637     else if (jit_reg_free_if_spill_p(_R2) && jit_reg_free_if_spill_p(_R3))
1638         r1 = _R2, r2 = _R3;
1639     else if (jit_reg_free_if_spill_p(_R4) && jit_reg_free_if_spill_p(_R5))
1640         r1 = _R4, r2 = _R5;
1641     else if (jit_reg_free_if_spill_p(_R10) && jit_reg_free_if_spill_p(_R11))
1642         r1 = _R10, r2 = _R11;
1643     else if (jit_reg_free_if_spill_p(_R8) && jit_reg_free_if_spill_p(_R9))
1644         r1 = _R8, r2 = _R9;
1645     else if (jit_reg_free_if_spill_p(_R6) && jit_reg_free_if_spill_p(_R7))
1646         r1 = _R6, r2 = _R7;
1647     else
1648         /* Do not jit_get_reg() all registers to avoid it */
1649         abort();
1650
1651     (void)jit_get_reg(jit_class_gpr|jit_class_named|r1);
1652     (void)jit_get_reg(jit_class_gpr|jit_class_named|r2);
1653
1654     return (r1);
1655 }
1656
1657 static void
1658 _jit_unget_reg_pair(jit_state_t *_jit, jit_int32_t reg)
1659 {
1660     jit_int32_t         r1, r2;
1661     r1 = reg;
1662     switch (r1) {
1663         case _R0:       r2 = _R1;       break;
1664         case _R2:       r2 = _R3;       break;
1665         case _R4:       r2 = _R5;       break;
1666         case _R6:       r2 = _R7;       break;
1667         case _R8:       r2 = _R9;       break;
1668         case _R10:      r2 = _R11;      break;
1669         default:        abort();
1670     }
1671     jit_unget_reg(r1);
1672     jit_unget_reg(r2);
1673 }
1674
1675 static jit_int32_t
1676 _jit_get_reg_but_zero(jit_state_t *_jit, jit_int32_t flags)
1677 {
1678     jit_int32_t         reg;
1679     reg = jit_get_reg(jit_class_gpr);
1680     if (reg == _R0) {
1681         reg = jit_get_reg(jit_class_gpr|flags);
1682         jit_unget_reg(_R0);
1683     }
1684     return (reg);
1685 }
1686
1687 static void
1688 _patch(jit_state_t *_jit, jit_word_t instr, jit_node_t *node)
1689 {
1690     jit_int32_t         flag;
1691
1692     assert(node->flag & jit_flag_node);
1693     if (node->code == jit_code_movi)
1694         flag = node->v.n->flag;
1695     else
1696         flag = node->u.n->flag;
1697     assert(!(flag & jit_flag_patch));
1698     if (_jitc->patches.offset >= _jitc->patches.length) {
1699         jit_realloc((jit_pointer_t *)&_jitc->patches.ptr,
1700                     _jitc->patches.length * sizeof(jit_patch_t),
1701                     (_jitc->patches.length + 1024) * sizeof(jit_patch_t));
1702         _jitc->patches.length += 1024;
1703     }
1704     _jitc->patches.ptr[_jitc->patches.offset].inst = instr;
1705     _jitc->patches.ptr[_jitc->patches.offset].node = node;
1706     ++_jitc->patches.offset;
1707 }