deps/lightning/lib/jit_s390.c

   1 /*
   2  * Copyright (C) 2013-2023  Free Software Foundation, Inc.
   3  *
   4  * This file is part of GNU lightning.
   5  *
   6  * GNU lightning is free software; you can redistribute it and/or modify it
   7  * under the terms of the GNU Lesser General Public License as published
   8  * by the Free Software Foundation; either version 3, or (at your option)
   9  * any later version.
  10  *
  11  * GNU lightning is distributed in the hope that it will be useful, but
  12  * WITHOUT ANY WARRANTY; without even the implied warranty of MERCHANTABILITY
  13  * or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU Lesser General Public
  14  * License for more details.
  15  *
  16  * Authors:
  17  *      Paulo Cesar Pereira de Andrade
  18  */
  19 #define CHECK_FLOGR     0
  20
  21 #if CHECK_FLOGR
  22 #include <signal.h>
  23 #include <setjmp.h>
  24 #endif
  25
  26 #include <lightning.h>
  27 #include <lightning/jit_private.h>
  28
  29 #if __WORDSIZE == 32
  30 #  define NUM_FLOAT_REG_ARGS            2
  31 #else
  32 #  define NUM_FLOAT_REG_ARGS            4
  33 #endif
  34 #define jit_arg_reg_p(i)                ((i) >= 0 && (i) < 5)
  35 #define jit_arg_f_reg_p(i)              ((i) >= 0 && (i) < NUM_FLOAT_REG_ARGS)
  36
  37 /*
  38  * Types
  39  */
  40 typedef struct jit_va_list {
  41     /* The offsets are "1" based, as addresses are fixed in the
  42      * standard stack frame format. */
  43     jit_word_t          gpoff;
  44     jit_word_t          fpoff;
  45
  46     /* Easier when there is an explicitly defined type...
  47 (gdb) ptype ap
  48 type = struct __va_list_tag {
  49     long __gpr;
  50     long __fpr;
  51     void *__overflow_arg_area;
  52     void *__reg_save_area;
  53
  54     Note that gopff (__gpr) and fpoff (__fpr) are jit_word_t equivalent
  55     and, again, "1" (unit) based, so must be adjusted at va_arg time.
  56  */
  57     jit_pointer_t       over;
  58     jit_pointer_t       save;
  59
  60     /* For variadic functions, always allocate space to save callee
  61      * save fpr registers.
  62      * Note that s390 has a standard stack frame format that lightning
  63      * does not fully comply with, but for variadic functions it must,
  64      * for those (variadic) do not use the "empty" spaces for any
  65      * callee save fpr register, but save them after the va_list
  66      * space; and use the standard stack frame format, as required
  67      * by variadic functions (and have a compatible va_list pointer). */
  68     jit_float64_t       f8;
  69     jit_float64_t       f9;
  70     jit_float64_t       f10;
  71     jit_float64_t       f11;
  72     jit_float64_t       f12;
  73     jit_float64_t       f13;
  74     jit_float64_t       f14;
  75     jit_float64_t       f15;
  76 } jit_va_list_t;
  77
  78 /*
  79  * Prototypes
  80  */
  81 #define jit_get_reg_pair()              _jit_get_reg_pair(_jit)
  82 static jit_int32_t _jit_get_reg_pair(jit_state_t*);
  83 #define jit_unget_reg_pair(regno)       _jit_unget_reg_pair(_jit,regno)
  84 static void _jit_unget_reg_pair(jit_state_t*,jit_int32_t);
  85 #define jit_get_reg_but_zero(flags)     _jit_get_reg_but_zero(_jit,flags)
  86 static jit_int32_t _jit_get_reg_but_zero(jit_state_t*,jit_int32_t);
  87 #define jit_unget_reg_but_zero(reg)     jit_unget_reg(reg)
  88 #define patch(instr, node)              _patch(_jit, instr, node)
  89 static void _patch(jit_state_t*,jit_word_t,jit_node_t*);
  90
  91 /* libgcc */
  92 extern void __clear_cache(void *, void *);
  93
  94 #define PROTO                           1
  95 #  include "jit_s390-cpu.c"
  96 #  include "jit_s390-fpu.c"
  97 #  if CHECK_FLOGR
  98 #    include "jit_fallback.c"
  99 #  endif
 100 #undef PROTO
 101
 102 /*
 103  * Initialization
 104  */
 105 jit_cpu_t               jit_cpu;
 106 jit_register_t          _rvs[] = {
 107     { rc(gpr) | 0x0,                    "%r0" },
 108     { rc(gpr) | 0x1,                    "%r1" },
 109     { rc(gpr) | rc(sav) | 0xc,          "%r12" },
 110     { rc(gpr) | rc(sav) | 0xb,          "%r11" },
 111     { rc(gpr) | rc(sav) | 0xa,          "%r10" },
 112     { rc(gpr) | rc(sav) | 0x9,          "%r9" },
 113     { rc(gpr) | rc(sav) | 0x8,          "%r8" },
 114     { rc(gpr) | rc(sav) | 0x7,          "%r7" },
 115     { rc(gpr) | rc(arg) | rc(sav) | 0x6,"%r6" },
 116     { rc(gpr) | rc(arg) | 0x5,          "%r5" },
 117     { rc(gpr) | rc(arg) | 0x4,          "%r4" },
 118     { rc(gpr) | rc(arg) | 0x3,          "%r3" },
 119     { rc(gpr) | rc(arg) | 0x2,          "%r2" },
 120     { rc(sav) | 0xd,                    "%r13" },       /* used as JIT_FP */
 121     { 0xe,                              "%r14" },
 122     { rc(sav) | 0xf,                    "%r15" },
 123     { rc(fpr) | 0x1,                    "%f1" },
 124     { rc(fpr) | 0x3,                    "%f3" },
 125     { rc(fpr) | 0x5,                    "%f5" },
 126     { rc(fpr) | 0x7,                    "%f7" },
 127     { rc(fpr) | rc(sav) | 0xe,          "%f14" },
 128     /* Do not use as temporary to simplify stack layout */
 129     { 0xf,                              "%f15" },
 130     { rc(fpr) | rc(sav) | 0x8,          "%f8" },
 131     { rc(fpr) | rc(sav) | 0x9,          "%f9" },
 132     { rc(fpr) | rc(sav) | 0xa,          "%f10" },
 133     { rc(fpr) | rc(sav) | 0xb,          "%f11" },
 134     { rc(fpr) | rc(sav) | 0xc,          "%f12" },
 135     { rc(fpr) | rc(sav) | 0xd,          "%f13" },
 136     { rc(fpr) | rc(arg) | 0x6,          "%f6" },
 137     { rc(fpr) | rc(arg) | 0x4,          "%f4" },
 138     { rc(fpr) | rc(arg) | 0x2,          "%f2" },
 139     { rc(fpr) | rc(arg) | 0x0,          "%f0" },
 140     { _NOREG,                           "<none>" },
 141 };
 142 #if CHECK_FLOGR
 143 static sigjmp_buf       jit_env;
 144 #endif
 145
 146 /*
 147  * Implementation
 148  */
 149 #if CHECK_FLOGR
 150 static void
 151 sigill_handler(int signum)
 152 {
 153     jit_cpu.flogr = 0;
 154     siglongjmp(jit_env, 1);
 155 }
 156 #endif
 157
 158 void
 159 jit_get_cpu(void)
 160 {
 161 #if CHECK_FLOGR
 162     int                 r12, r13;
 163     struct              sigaction new_action, old_action;
 164     new_action.sa_handler = sigill_handler;
 165     sigemptyset(&new_action.sa_mask);
 166     new_action.sa_flags = 0;
 167     sigaction(SIGILL, NULL, &old_action);
 168     if (old_action.sa_handler != SIG_IGN) {
 169         sigaction(SIGILL, &new_action, NULL);
 170         if (!sigsetjmp(jit_env, 1)) {
 171             jit_cpu.flogr = 1;
 172             /* flogr %r12, %r12 */
 173             __asm__ volatile("lgr %%r12, %0; lgr %%r13, %1;"
 174                              "flogr %%r12, %%r12;"
 175                              "lgr %1, %%r13; lgr %0, %%r12;"
 176                              : "=r" (r12), "=r" (r13));
 177             sigaction(SIGILL, &old_action, NULL);
 178         }
 179     }
 180 #else
 181     /* By default, assume it is available */
 182     jit_cpu.flogr = 1;
 183 #endif
 184 }
 185
 186 void
 187 _jit_init(jit_state_t *_jit)
 188 {
 189     _jitc->reglen = jit_size(_rvs) - 1;
 190 }
 191
 192 void
 193 _jit_prolog(jit_state_t *_jit)
 194 {
 195     jit_int32_t         offset;
 196
 197     if (_jitc->function)
 198         jit_epilog();
 199     assert(jit_regset_cmp_ui(&_jitc->regarg, 0) == 0);
 200     jit_regset_set_ui(&_jitc->regsav, 0);
 201     offset = _jitc->functions.offset;
 202     if (offset >= _jitc->functions.length) {
 203         jit_realloc((jit_pointer_t *)&_jitc->functions.ptr,
 204                     _jitc->functions.length * sizeof(jit_function_t),
 205                     (_jitc->functions.length + 16) * sizeof(jit_function_t));
 206         _jitc->functions.length += 16;
 207     }
 208     _jitc->function = _jitc->functions.ptr + _jitc->functions.offset++;
 209     _jitc->function->self.size = stack_framesize;
 210     _jitc->function->self.argi = _jitc->function->self.argf =
 211         _jitc->function->self.aoff = _jitc->function->self.alen = 0;
 212     /* preallocate 8 bytes if not using a constant data buffer */
 213     if (_jitc->no_data)
 214         _jitc->function->self.aoff = -8;
 215     _jitc->function->self.call = jit_call_default;
 216     jit_alloc((jit_pointer_t *)&_jitc->function->regoff,
 217               _jitc->reglen * sizeof(jit_int32_t));
 218
 219     /* _no_link here does not mean the jit_link() call can be removed
 220      * by rewriting as:
 221      * _jitc->function->prolog = jit_new_node(jit_code_prolog);
 222      */
 223     _jitc->function->prolog = jit_new_node_no_link(jit_code_prolog);
 224     jit_link(_jitc->function->prolog);
 225     _jitc->function->prolog->w.w = offset;
 226     _jitc->function->epilog = jit_new_node_no_link(jit_code_epilog);
 227     /*  u:      label value
 228      *  v:      offset in blocks vector
 229      *  w:      offset in functions vector
 230      */
 231     _jitc->function->epilog->w.w = offset;
 232
 233     jit_regset_new(&_jitc->function->regset);
 234 }
 235
 236 jit_int32_t
 237 _jit_allocai(jit_state_t *_jit, jit_int32_t length)
 238 {
 239     assert(_jitc->function);
 240     switch (length) {
 241         case 0: case 1:                                         break;
 242         case 2:         _jitc->function->self.aoff &= -2;       break;
 243         case 3: case 4: _jitc->function->self.aoff &= -4;       break;
 244         default:        _jitc->function->self.aoff &= -8;       break;
 245     }
 246     _jitc->function->self.aoff -= length;
 247     if (!_jitc->realize) {
 248         jit_inc_synth_ww(allocai, _jitc->function->self.aoff, length);
 249         jit_dec_synth();
 250     }
 251     return (_jitc->function->self.aoff);
 252 }
 253
 254 void
 255 _jit_allocar(jit_state_t *_jit, jit_int32_t u, jit_int32_t v)
 256 {
 257     jit_int32_t          reg;
 258     assert(_jitc->function);
 259     jit_inc_synth_ww(allocar, u, v);
 260     if (!_jitc->function->allocar) {
 261         _jitc->function->aoffoff = jit_allocai(sizeof(jit_int32_t));
 262         _jitc->function->allocar = 1;
 263     }
 264     reg = jit_get_reg(jit_class_gpr);
 265     jit_negr(reg, v);
 266     jit_andi(reg, reg, -8);
 267     jit_ldxi_i(u, JIT_FP, _jitc->function->aoffoff);
 268     jit_addr(u, u, reg);
 269     jit_addr(JIT_SP, JIT_SP, reg);
 270     jit_stxi_i(_jitc->function->aoffoff, JIT_FP, u);
 271     jit_unget_reg(reg);
 272     jit_dec_synth();
 273 }
 274
 275 void
 276 _jit_ret(jit_state_t *_jit)
 277 {
 278     jit_node_t          *instr;
 279     assert(_jitc->function);
 280     jit_inc_synth(ret);
 281     /* jump to epilog */
 282     instr = jit_jmpi();
 283     jit_patch_at(instr, _jitc->function->epilog);
 284     jit_dec_synth();
 285 }
 286
 287 void
 288 _jit_retr(jit_state_t *_jit, jit_int32_t u, jit_code_t code)
 289 {
 290     jit_code_inc_synth_w(code, u);
 291     jit_movr(JIT_RET, u);
 292     jit_ret();
 293     jit_dec_synth();
 294 }
 295
 296 void
 297 _jit_reti(jit_state_t *_jit, jit_word_t u, jit_code_t code)
 298 {
 299     jit_code_inc_synth_w(code, u);
 300     jit_movi(JIT_RET, u);
 301     jit_ret();
 302     jit_dec_synth();
 303 }
 304
 305 void
 306 _jit_retr_f(jit_state_t *_jit, jit_int32_t u)
 307 {
 308     jit_inc_synth_w(retr_f, u);
 309     jit_movr_f(JIT_FRET, u);
 310     jit_ret();
 311     jit_dec_synth();
 312 }
 313
 314 void
 315 _jit_reti_f(jit_state_t *_jit, jit_float32_t u)
 316 {
 317     jit_inc_synth_f(reti_f, u);
 318     jit_movi_f(JIT_FRET, u);
 319     jit_ret();
 320     jit_dec_synth();
 321 }
 322
 323 void
 324 _jit_retr_d(jit_state_t *_jit, jit_int32_t u)
 325 {
 326     jit_inc_synth_w(retr_d, u);
 327     jit_movr_d(JIT_FRET, u);
 328     jit_ret();
 329     jit_dec_synth();
 330 }
 331
 332 void
 333 _jit_reti_d(jit_state_t *_jit, jit_float64_t u)
 334 {
 335     jit_inc_synth_d(reti_d, u);
 336     jit_movi_d(JIT_FRET, u);
 337     jit_ret();
 338     jit_dec_synth();
 339 }
 340
 341 void
 342 _jit_epilog(jit_state_t *_jit)
 343 {
 344     assert(_jitc->function);
 345     assert(_jitc->function->epilog->next == NULL);
 346     jit_link(_jitc->function->epilog);
 347     _jitc->function = NULL;
 348 }
 349
 350 jit_bool_t
 351 _jit_arg_register_p(jit_state_t *_jit, jit_node_t *u)
 352 {
 353     if (u->code >= jit_code_arg_c && u->code <= jit_code_arg)
 354         return (jit_arg_reg_p(u->u.w));
 355     assert(u->code == jit_code_arg_f || u->code == jit_code_arg_d);
 356     return (jit_arg_f_reg_p(u->u.w));
 357 }
 358
 359 void
 360 _jit_ellipsis(jit_state_t *_jit)
 361 {
 362     jit_inc_synth(ellipsis);
 363     if (_jitc->prepare) {
 364         jit_link_prepare();
 365         assert(!(_jitc->function->call.call & jit_call_varargs));
 366         _jitc->function->call.call |= jit_call_varargs;
 367     }
 368     else {
 369         jit_link_prolog();
 370         assert(!(_jitc->function->self.call & jit_call_varargs));
 371         _jitc->function->self.call |= jit_call_varargs;
 372
 373         /* Allocate va_list like object in the stack. */
 374         _jitc->function->vaoff = jit_allocai(sizeof(jit_va_list_t));
 375
 376         /* Initialize gp offset in save area. */
 377         if (jit_arg_reg_p(_jitc->function->self.argi))
 378             _jitc->function->vagp = _jitc->function->self.argi;
 379         else
 380             _jitc->function->vagp = 5;
 381
 382         /* Initialize fp offset in save area. */
 383         if (jit_arg_f_reg_p(_jitc->function->self.argf))
 384             _jitc->function->vafp = _jitc->function->self.argf;
 385         else
 386             _jitc->function->vafp = NUM_FLOAT_REG_ARGS;
 387     }
 388     jit_dec_synth();
 389 }
 390
 391 void
 392 _jit_va_push(jit_state_t *_jit, jit_int32_t u)
 393 {
 394     jit_inc_synth_w(va_push, u);
 395     jit_pushargr(u);
 396     jit_dec_synth();
 397 }
 398
 399 jit_node_t *
 400 _jit_arg(jit_state_t *_jit, jit_code_t code)
 401 {
 402     jit_node_t          *node;
 403     jit_int32_t          offset;
 404     assert(_jitc->function);
 405     assert(!(_jitc->function->self.call & jit_call_varargs));
 406 #if STRONG_TYPE_CHECKING
 407     assert(code >= jit_code_arg_c && code <= jit_code_arg);
 408 #endif
 409     if (jit_arg_reg_p(_jitc->function->self.argi))
 410         offset = _jitc->function->self.argi++;
 411     else {
 412         offset = _jitc->function->self.size;
 413         _jitc->function->self.size += sizeof(jit_word_t);
 414     }
 415     node = jit_new_node_ww(code, offset,
 416                            ++_jitc->function->self.argn);
 417     jit_link_prolog();
 418     return (node);
 419 }
 420
 421 jit_node_t *
 422 _jit_arg_f(jit_state_t *_jit)
 423 {
 424     jit_node_t          *node;
 425     jit_int32_t          offset;
 426     assert(_jitc->function);
 427     if (jit_arg_f_reg_p(_jitc->function->self.argf))
 428         offset = _jitc->function->self.argf++;
 429     else {
 430         offset = _jitc->function->self.size;
 431         _jitc->function->self.size += sizeof(jit_word_t);
 432     }
 433     node = jit_new_node_ww(jit_code_arg_f, offset,
 434                            ++_jitc->function->self.argn);
 435     jit_link_prolog();
 436     return (node);
 437 }
 438
 439 jit_node_t *
 440 _jit_arg_d(jit_state_t *_jit)
 441 {
 442     jit_node_t          *node;
 443     jit_int32_t          offset;
 444     assert(_jitc->function);
 445     if (jit_arg_f_reg_p(_jitc->function->self.argf))
 446         offset = _jitc->function->self.argf++;
 447     else {
 448         offset = _jitc->function->self.size;
 449         _jitc->function->self.size += sizeof(jit_float64_t);
 450     }
 451     node = jit_new_node_ww(jit_code_arg_d, offset,
 452                            ++_jitc->function->self.argn);
 453     jit_link_prolog();
 454     return (node);
 455 }
 456
 457 void
 458 _jit_getarg_c(jit_state_t *_jit, jit_int32_t u, jit_node_t *v)
 459 {
 460     assert_arg_type(v->code, jit_code_arg_c);
 461     jit_inc_synth_wp(getarg_c, u, v);
 462     if (jit_arg_reg_p(v->u.w))
 463         jit_extr_c(u, _R2 - v->u.w);
 464     else
 465         jit_ldxi_c(u, JIT_FP,
 466                    v->u.w + (__WORDSIZE >> 3) - sizeof(jit_int8_t));
 467     jit_dec_synth();
 468 }
 469
 470 void
 471 _jit_getarg_uc(jit_state_t *_jit, jit_int32_t u, jit_node_t *v)
 472 {
 473     assert_arg_type(v->code, jit_code_arg_c);
 474     jit_inc_synth_wp(getarg_uc, u, v);
 475     if (jit_arg_reg_p(v->u.w))
 476         jit_extr_uc(u, _R2 - v->u.w);
 477     else
 478         jit_ldxi_uc(u, JIT_FP,
 479                     v->u.w + (__WORDSIZE >> 3) - sizeof(jit_uint8_t));
 480     jit_dec_synth();
 481 }
 482
 483 void
 484 _jit_getarg_s(jit_state_t *_jit, jit_int32_t u, jit_node_t *v)
 485 {
 486     assert_arg_type(v->code, jit_code_arg_s);
 487     jit_inc_synth_wp(getarg_s, u, v);
 488     if (jit_arg_reg_p(v->u.w))
 489         jit_extr_s(u, _R2 - v->u.w);
 490     else
 491         jit_ldxi_s(u, JIT_FP,
 492                    v->u.w + (__WORDSIZE >> 3) - sizeof(jit_int16_t));
 493     jit_dec_synth();
 494 }
 495
 496 void
 497 _jit_getarg_us(jit_state_t *_jit, jit_int32_t u, jit_node_t *v)
 498 {
 499     assert_arg_type(v->code, jit_code_arg_s);
 500     jit_inc_synth_wp(getarg_us, u, v);
 501     if (jit_arg_reg_p(v->u.w))
 502         jit_extr_us(u, _R2 - v->u.w);
 503     else
 504         jit_ldxi_us(u, JIT_FP,
 505                     v->u.w + (__WORDSIZE >> 3) - sizeof(jit_uint16_t));
 506     jit_dec_synth();
 507 }
 508
 509 void
 510 _jit_getarg_i(jit_state_t *_jit, jit_int32_t u, jit_node_t *v)
 511 {
 512     assert_arg_type(v->code, jit_code_arg_i);
 513     jit_inc_synth_wp(getarg_i, u, v);
 514     if (jit_arg_reg_p(v->u.w)) {
 515 #if __WORDSIZE == 32
 516         jit_movr(u, _R2 - v->u.w);
 517 #else
 518         jit_extr_i(u, _R2 - v->u.w);
 519 #endif
 520     }
 521     else
 522         jit_ldxi_i(u, JIT_FP,
 523                    v->u.w + (__WORDSIZE >> 3) - sizeof(jit_int32_t));
 524     jit_dec_synth();
 525 }
 526
 527 #if __WORDSIZE == 64
 528 void
 529 _jit_getarg_ui(jit_state_t *_jit, jit_int32_t u, jit_node_t *v)
 530 {
 531     assert_arg_type(v->code, jit_code_arg_i);
 532     jit_inc_synth_wp(getarg_ui, u, v);
 533     if (jit_arg_reg_p(v->u.w))
 534         jit_extr_ui(u, _R2 - v->u.w);
 535     else
 536         jit_ldxi_ui(u, JIT_FP,
 537                     v->u.w + (__WORDSIZE >> 3) - sizeof(jit_uint32_t));
 538     jit_dec_synth();
 539 }
 540
 541 void
 542 _jit_getarg_l(jit_state_t *_jit, jit_int32_t u, jit_node_t *v)
 543 {
 544     assert_arg_type(v->code, jit_code_arg_l);
 545     jit_inc_synth_wp(getarg_l, u, v);
 546     if (jit_arg_reg_p(v->u.w))
 547         jit_movr(u, _R2 - v->u.w);
 548     else
 549         jit_ldxi_l(u, JIT_FP, v->u.w);
 550     jit_dec_synth();
 551 }
 552 #endif
 553
 554 void
 555 _jit_putargr(jit_state_t *_jit, jit_int32_t u, jit_node_t *v, jit_code_t code)
 556 {
 557     assert_putarg_type(code, v->code);
 558     jit_code_inc_synth_wp(code, u, v);
 559     if (jit_arg_reg_p(v->u.w))
 560         jit_movr(_R2 - v->u.w, u);
 561     else
 562         jit_stxi(v->u.w, JIT_FP, u);
 563     jit_dec_synth();
 564 }
 565
 566 void
 567 _jit_putargi(jit_state_t *_jit, jit_word_t u, jit_node_t *v, jit_code_t code)
 568 {
 569     jit_int32_t         regno;
 570     assert_putarg_type(code, v->code);
 571     jit_code_inc_synth_wp(code, u, v);
 572     if (jit_arg_reg_p(v->u.w))
 573         jit_movi(_R2 - v->u.w, u);
 574     else {
 575         regno = jit_get_reg(jit_class_gpr);
 576         jit_movi(regno, u);
 577         jit_stxi(v->u.w, JIT_FP, regno);
 578         jit_unget_reg(regno);
 579     }
 580     jit_dec_synth();
 581 }
 582
 583 void
 584 _jit_getarg_f(jit_state_t *_jit, jit_int32_t u, jit_node_t *v)
 585 {
 586     assert(v->code == jit_code_arg_f);
 587     jit_inc_synth_wp(getarg_f, u, v);
 588     if (jit_arg_f_reg_p(v->u.w))
 589         jit_movr_f(u, _F0 - v->u.w);
 590     else
 591         jit_ldxi_f(u, JIT_FP,
 592                    v->u.w
 593 #if __WORDSIZE == 64
 594                    + (__WORDSIZE >> 3) - sizeof(jit_float32_t)
 595 #endif
 596                    );
 597     jit_dec_synth();
 598 }
 599
 600 void
 601 _jit_putargr_f(jit_state_t *_jit, jit_int32_t u, jit_node_t *v)
 602 {
 603     assert(v->code == jit_code_arg_f);
 604     jit_inc_synth_wp(putargr_f, u, v);
 605     if (jit_arg_f_reg_p(v->u.w))
 606         jit_movr_f(_F0 - v->u.w, u);
 607     else
 608         jit_stxi_f(v->u.w
 609 #if __WORDSIZE == 64
 610                    + (__WORDSIZE >> 3) - sizeof(jit_float32_t)
 611 #endif
 612                    , JIT_FP, u);
 613     jit_dec_synth();
 614 }
 615
 616 void
 617 _jit_putargi_f(jit_state_t *_jit, jit_float32_t u, jit_node_t *v)
 618 {
 619     jit_int32_t         regno;
 620     assert(v->code == jit_code_arg_f);
 621     jit_inc_synth_fp(putargi_f, u, v);
 622     if (jit_arg_f_reg_p(v->u.w))
 623         jit_movi_f(_F0 - v->u.w, u);
 624     else {
 625         regno = jit_get_reg(jit_class_fpr);
 626         jit_movi_f(regno, u);
 627         jit_stxi_f(v->u.w
 628 #if __WORDSIZE == 64
 629                    + (__WORDSIZE >> 3) - sizeof(jit_float32_t)
 630 #endif
 631                    , JIT_FP, regno);
 632         jit_unget_reg(regno);
 633     }
 634     jit_dec_synth();
 635 }
 636
 637 void
 638 _jit_getarg_d(jit_state_t *_jit, jit_int32_t u, jit_node_t *v)
 639 {
 640     assert(v->code == jit_code_arg_d);
 641     jit_inc_synth_wp(getarg_d, u, v);
 642     if (jit_arg_f_reg_p(v->u.w))
 643         jit_movr_d(u, _F0 - v->u.w);
 644     else
 645         jit_ldxi_d(u, JIT_FP, v->u.w);
 646     jit_dec_synth();
 647 }
 648
 649 void
 650 _jit_putargr_d(jit_state_t *_jit, jit_int32_t u, jit_node_t *v)
 651 {
 652     assert(v->code == jit_code_arg_d);
 653     jit_inc_synth_wp(putargr_d, u, v);
 654     if (jit_arg_f_reg_p(v->u.w))
 655         jit_movr_d(_F0 - v->u.w, u);
 656     else
 657         jit_stxi_d(v->u.w, JIT_FP, u);
 658     jit_dec_synth();
 659 }
 660
 661 void
 662 _jit_putargi_d(jit_state_t *_jit, jit_float64_t u, jit_node_t *v)
 663 {
 664     jit_int32_t         regno;
 665     assert(v->code == jit_code_arg_d);
 666     jit_inc_synth_dp(putargi_d, u, v);
 667     if (jit_arg_f_reg_p(v->u.w))
 668         jit_movi_d(_F0 - v->u.w, u);
 669     else {
 670         regno = jit_get_reg(jit_class_fpr);
 671         jit_movi_d(regno, u);
 672         jit_stxi_d(v->u.w, JIT_FP, regno);
 673         jit_unget_reg(regno);
 674     }
 675     jit_dec_synth();
 676 }
 677
 678 void
 679 _jit_pushargr(jit_state_t *_jit, jit_int32_t u, jit_code_t code)
 680 {
 681     assert(_jitc->function);
 682     jit_code_inc_synth_w(code, u);
 683     jit_link_prepare();
 684     if (jit_arg_reg_p(_jitc->function->call.argi)) {
 685         jit_movr(_R2 - _jitc->function->call.argi, u);
 686         ++_jitc->function->call.argi;
 687     }
 688     else {
 689         jit_stxi(_jitc->function->call.size + stack_framesize, JIT_SP, u);
 690         _jitc->function->call.size += sizeof(jit_word_t);
 691     }
 692     jit_dec_synth();
 693 }
 694
 695 void
 696 _jit_pushargi(jit_state_t *_jit, jit_word_t u, jit_code_t code)
 697 {
 698     jit_int32_t          regno;
 699     assert(_jitc->function);
 700     jit_code_inc_synth_w(code, u);
 701     jit_link_prepare();
 702     if (jit_arg_reg_p(_jitc->function->call.argi)) {
 703         jit_movi(_R2 - _jitc->function->call.argi, u);
 704         ++_jitc->function->call.argi;
 705     }
 706     else {
 707         regno = jit_get_reg(jit_class_gpr);
 708         jit_movi(regno, u);
 709         jit_stxi(_jitc->function->call.size + stack_framesize, JIT_SP, regno);
 710         jit_unget_reg(regno);
 711         _jitc->function->call.size += sizeof(jit_word_t);
 712     }
 713     jit_dec_synth();
 714 }
 715
 716 void
 717 _jit_pushargr_f(jit_state_t *_jit, jit_int32_t u)
 718 {
 719     assert(_jitc->function);
 720     jit_inc_synth_w(pushargr_f, u);
 721     jit_link_prepare();
 722     if (jit_arg_f_reg_p(_jitc->function->call.argf)) {
 723         jit_movr_f(_F0 - _jitc->function->call.argf, u);
 724         ++_jitc->function->call.argf;
 725     }
 726     else {
 727         jit_stxi_f(_jitc->function->call.size + stack_framesize
 728 #if __WORDSIZE == 64
 729                    + (__WORDSIZE >> 3) - sizeof(jit_float32_t)
 730 #endif
 731                    , JIT_SP, u);
 732         _jitc->function->call.size += sizeof(jit_word_t);
 733     }
 734     jit_dec_synth();
 735 }
 736
 737 void
 738 _jit_pushargi_f(jit_state_t *_jit, jit_float32_t u)
 739 {
 740     jit_int32_t         regno;
 741     assert(_jitc->function);
 742     jit_inc_synth_f(pushargi_f, u);
 743     jit_link_prepare();
 744     if (jit_arg_f_reg_p(_jitc->function->call.argf)) {
 745         jit_movi_f(_F0 - _jitc->function->call.argf, u);
 746         ++_jitc->function->call.argf;
 747     }
 748     else {
 749         regno = jit_get_reg(jit_class_fpr);
 750         jit_movi_f(regno, u);
 751         jit_stxi_f(_jitc->function->call.size + stack_framesize
 752 #if __WORDSIZE == 64
 753                    + (__WORDSIZE >> 3) - sizeof(jit_float32_t)
 754 #endif
 755                    , JIT_SP, regno);
 756         jit_unget_reg(regno);
 757         _jitc->function->call.size += sizeof(jit_word_t);
 758     }
 759     jit_dec_synth();
 760 }
 761
 762 void
 763 _jit_pushargr_d(jit_state_t *_jit, jit_int32_t u)
 764 {
 765     assert(_jitc->function);
 766     jit_inc_synth_w(pushargr_d, u);
 767     jit_link_prepare();
 768     if (jit_arg_f_reg_p(_jitc->function->call.argf)) {
 769         jit_movr_d(_F0 - _jitc->function->call.argf, u);
 770         ++_jitc->function->call.argf;
 771     }
 772     else {
 773         jit_stxi_d(_jitc->function->call.size + stack_framesize, JIT_SP, u);
 774         _jitc->function->call.size += sizeof(jit_float64_t);
 775     }
 776     jit_dec_synth();
 777 }
 778
 779 void
 780 _jit_pushargi_d(jit_state_t *_jit, jit_float64_t u)
 781 {
 782     jit_int32_t         regno;
 783     assert(_jitc->function);
 784     jit_inc_synth_d(pushargi_d, u);
 785     jit_link_prepare();
 786     if (jit_arg_f_reg_p(_jitc->function->call.argf)) {
 787         jit_movi_d(_F0 - _jitc->function->call.argf, u);
 788         ++_jitc->function->call.argf;
 789     }
 790     else {
 791         regno = jit_get_reg(jit_class_fpr);
 792         jit_movi_d(regno, u);
 793         jit_stxi_d(_jitc->function->call.size + stack_framesize, JIT_SP, regno);
 794         jit_unget_reg(regno);
 795         _jitc->function->call.size += sizeof(jit_float64_t);
 796     }
 797     jit_dec_synth();
 798 }
 799
 800 jit_bool_t
 801 _jit_regarg_p(jit_state_t *_jit, jit_node_t *node, jit_int32_t regno)
 802 {
 803     jit_int32_t         spec;
 804     spec = jit_class(_rvs[regno].spec);
 805     if (spec & jit_class_arg) {
 806         regno = _R2 - regno;
 807         if (regno >= 0 && regno < node->v.w)
 808             return (1);
 809         if (spec & jit_class_fpr) {
 810             regno = _F0 - regno;
 811             if (regno >= 0 && regno < node->w.w)
 812                 return (1);
 813         }
 814     }
 815     return (0);
 816 }
 817
 818 void
 819 _jit_finishr(jit_state_t *_jit, jit_int32_t r0)
 820 {
 821     jit_node_t          *call;
 822     assert(_jitc->function);
 823     jit_inc_synth_w(finishr, r0);
 824     if (_jitc->function->self.alen < _jitc->function->call.size)
 825         _jitc->function->self.alen = _jitc->function->call.size;
 826     call = jit_callr(r0);
 827     call->v.w = _jitc->function->call.argi;
 828     call->w.w = _jitc->function->call.argf;
 829     _jitc->function->call.argi = _jitc->function->call.argf =
 830         _jitc->function->call.size = 0;
 831     _jitc->prepare = 0;
 832     jit_dec_synth();
 833 }
 834
 835 jit_node_t *
 836 _jit_finishi(jit_state_t *_jit, jit_pointer_t i0)
 837 {
 838     jit_node_t          *node;
 839     assert(_jitc->function);
 840     jit_inc_synth_w(finishi, (jit_word_t)i0);
 841     if (_jitc->function->self.alen < _jitc->function->call.size)
 842         _jitc->function->self.alen = _jitc->function->call.size;
 843     node = jit_calli(i0);
 844     node->v.w = _jitc->function->call.argi;
 845     node->w.w = _jitc->function->call.argf;
 846     _jitc->function->call.argi = _jitc->function->call.argf =
 847         _jitc->function->call.size = 0;
 848     _jitc->prepare = 0;
 849     jit_dec_synth();
 850     return (node);
 851 }
 852
 853 void
 854 _jit_retval_c(jit_state_t *_jit, jit_int32_t r0)
 855 {
 856     jit_inc_synth_w(retval_c, r0);
 857     jit_extr_c(r0, JIT_RET);
 858     jit_dec_synth();
 859 }
 860
 861 void
 862 _jit_retval_uc(jit_state_t *_jit, jit_int32_t r0)
 863 {
 864     jit_inc_synth_w(retval_uc, r0);
 865     jit_extr_uc(r0, JIT_RET);
 866     jit_dec_synth();
 867 }
 868
 869 void
 870 _jit_retval_s(jit_state_t *_jit, jit_int32_t r0)
 871 {
 872     jit_inc_synth_w(retval_s, r0);
 873     jit_extr_s(r0, JIT_RET);
 874     jit_dec_synth();
 875 }
 876
 877 void
 878 _jit_retval_us(jit_state_t *_jit, jit_int32_t r0)
 879 {
 880     jit_inc_synth_w(retval_us, r0);
 881     jit_extr_us(r0, JIT_RET);
 882     jit_dec_synth();
 883 }
 884
 885 void
 886 _jit_retval_i(jit_state_t *_jit, jit_int32_t r0)
 887 {
 888     jit_inc_synth_w(retval_i, r0);
 889 #if __WORDSIZE == 64
 890     jit_extr_i(r0, JIT_RET);
 891 #else
 892     jit_movr(r0, JIT_RET);
 893 #endif
 894     jit_dec_synth();
 895 }
 896
 897 #if __WORDSIZE == 64
 898 void
 899 _jit_retval_ui(jit_state_t *_jit, jit_int32_t r0)
 900 {
 901     jit_inc_synth_w(retval_ui, r0);
 902     jit_extr_ui(r0, JIT_RET);
 903     jit_dec_synth();
 904 }
 905
 906 void
 907 _jit_retval_l(jit_state_t *_jit, jit_int32_t r0)
 908 {
 909     jit_inc_synth_w(retval_l, r0);
 910     jit_movr(r0, JIT_RET);
 911     jit_dec_synth();
 912 }
 913 #endif
 914
 915 void
 916 _jit_retval_f(jit_state_t *_jit, jit_int32_t r0)
 917 {
 918     jit_inc_synth_w(retval_f, r0);
 919     jit_movr_f(r0, JIT_FRET);
 920     jit_dec_synth();
 921 }
 922
 923 void
 924 _jit_retval_d(jit_state_t *_jit, jit_int32_t r0)
 925 {
 926     jit_inc_synth_w(retval_d, r0);
 927     jit_movr_d(r0, JIT_FRET);
 928     jit_dec_synth();
 929 }
 930
 931 jit_pointer_t
 932 _emit_code(jit_state_t *_jit)
 933 {
 934     jit_node_t          *node;
 935     jit_node_t          *temp;
 936     jit_word_t           word;
 937     jit_int32_t          value;
 938     jit_int32_t          offset;
 939     struct {
 940         jit_node_t      *node;
 941         jit_word_t       word;
 942         jit_function_t   func;
 943 #if DEVEL_DISASSEMBLER
 944         jit_word_t       prevw;
 945 #endif
 946         jit_int32_t      patch_offset;
 947     } undo;
 948 #if DEVEL_DISASSEMBLER
 949     jit_word_t           prevw;
 950 #endif
 951
 952     _jitc->function = NULL;
 953
 954     jit_reglive_setup();
 955
 956     undo.word = 0;
 957     undo.node = NULL;
 958     undo.patch_offset = 0;
 959
 960 #define assert_data(node)               /**/
 961 #define case_rr(name, type)                                             \
 962             case jit_code_##name##r##type:                              \
 963                 name##r##type(rn(node->u.w), rn(node->v.w));            \
 964                 break
 965 #define case_rw(name, type)                                             \
 966             case jit_code_##name##i##type:                              \
 967                 name##i##type(rn(node->u.w), node->v.w);                \
 968                 break
 969 #define case_wr(name, type)                                             \
 970             case jit_code_##name##i##type:                              \
 971                 name##i##type(node->u.w, rn(node->v.w));                \
 972                 break
 973 #define case_rrr(name, type)                                            \
 974             case jit_code_##name##r##type:                              \
 975                 name##r##type(rn(node->u.w),                            \
 976                               rn(node->v.w), rn(node->w.w));            \
 977                 break
 978 #define case_rrrr(name, type)                                           \
 979             case jit_code_##name##r##type:                              \
 980                 name##r##type(rn(node->u.q.l), rn(node->u.q.h),         \
 981                               rn(node->v.w), rn(node->w.w));            \
 982                 break
 983 #define case_rrw(name, type)                                            \
 984             case jit_code_##name##i##type:                              \
 985                 name##i##type(rn(node->u.w), rn(node->v.w), node->w.w); \
 986                 break
 987 #define case_rrrw(name, type)                                           \
 988             case jit_code_##name##i##type:                              \
 989                 name##i##type(rn(node->u.q.l), rn(node->u.q.h),         \
 990                               rn(node->v.w), node->w.w);                \
 991                 break
 992 #define case_rrf(name)                                                  \
 993             case jit_code_##name##i_f:                                  \
 994                 assert_data(node);                                      \
 995                 name##i_f(rn(node->u.w), rn(node->v.w),                 \
 996                           (jit_float32_t *)node->w.n->u.w);             \
 997                 break
 998 #define case_rrd(name)                                                  \
 999             case jit_code_##name##i_d:                                  \
1000                 assert_data(node);                                      \
1001                 name##i_d(rn(node->u.w), rn(node->v.w),                 \
1002                           (jit_float64_t *)node->w.n->u.w);             \
1003                 break
1004 #define case_wrr(name, type)                                            \
1005             case jit_code_##name##i##type:                              \
1006                 name##i##type(node->u.w, rn(node->v.w), rn(node->w.w)); \
1007                 break
1008 #define case_brr(name, type)                                            \
1009             case jit_code_##name##r##type:                              \
1010                 temp = node->u.n;                                       \
1011                 assert(temp->code == jit_code_label ||                  \
1012                        temp->code == jit_code_epilog);                  \
1013                 if (temp->flag & jit_flag_patch)                        \
1014                     name##r##type(temp->u.w, rn(node->v.w),             \
1015                                   rn(node->w.w));                       \
1016                 else {                                                  \
1017                     word = name##r##type##_p(_jit->pc.w,                \
1018                                              rn(node->v.w),             \
1019                                              rn(node->w.w));            \
1020                     patch(word, node);                                  \
1021                 }                                                       \
1022                 break
1023 #define case_brw(name, type)                                            \
1024             case jit_code_##name##i##type:                              \
1025                 temp = node->u.n;                                       \
1026                 assert(temp->code == jit_code_label ||                  \
1027                        temp->code == jit_code_epilog);                  \
1028                 if (temp->flag & jit_flag_patch)                        \
1029                     name##i##type(temp->u.w,                            \
1030                                   rn(node->v.w), node->w.w);            \
1031                 else {                                                  \
1032                     word = name##i##type##_p(_jit->pc.w,                \
1033                                              rn(node->v.w), node->w.w); \
1034                     patch(word, node);                                  \
1035                 }                                                       \
1036                 break;
1037 #define case_brf(name)                                                  \
1038             case jit_code_##name##i_f:                                  \
1039                 temp = node->u.n;                                       \
1040                 assert(temp->code == jit_code_label ||                  \
1041                        temp->code == jit_code_epilog);                  \
1042                 if (temp->flag & jit_flag_patch)                        \
1043                     name##i_f(temp->u.w, rn(node->v.w),                 \
1044                               (jit_float32_t *)node->w.n->u.w);         \
1045                 else {                                                  \
1046                     word = name##i_f_p(_jit->pc.w, rn(node->v.w),       \
1047                                        (jit_float32_t *)node->w.n->u.w);\
1048                     patch(word, node);                                  \
1049                 }                                                       \
1050                 break
1051 #define case_brd(name)                                                  \
1052             case jit_code_##name##i_d:                                  \
1053                 temp = node->u.n;                                       \
1054                 assert(temp->code == jit_code_label ||                  \
1055                        temp->code == jit_code_epilog);                  \
1056                 if (temp->flag & jit_flag_patch)                        \
1057                     name##i_d(temp->u.w, rn(node->v.w),                 \
1058                               (jit_float64_t *)node->w.n->u.w);         \
1059                 else {                                                  \
1060                     word = name##i_d_p(_jit->pc.w, rn(node->v.w),       \
1061                                        (jit_float64_t *)node->w.n->u.w);\
1062                     patch(word, node);                                  \
1063                 }                                                       \
1064                 break
1065 #if DEVEL_DISASSEMBLER
1066     prevw = _jit->pc.w;
1067 #endif
1068     for (node = _jitc->head; node; node = node->next) {
1069         if (_jit->pc.uc >= _jitc->code.end)
1070             return (NULL);
1071
1072 #if DEVEL_DISASSEMBLER
1073         node->offset = (jit_uword_t)_jit->pc.w - (jit_uword_t)prevw;
1074         prevw = _jit->pc.w;
1075 #endif
1076         value = jit_classify(node->code);
1077         jit_regarg_set(node, value);
1078         switch (node->code) {
1079             case jit_code_align:
1080                 /* Must align to a power of two */
1081                 assert(!(node->u.w & (node->u.w - 1)));
1082                 if ((word = _jit->pc.w & (node->u.w - 1)))
1083                     nop(node->u.w - word);
1084                 break;
1085             case jit_code_skip:
1086                 nop((node->u.w + 1) & ~1);
1087                 break;
1088             case jit_code_note:         case jit_code_name:
1089                 node->u.w = _jit->pc.w;
1090                 break;
1091             case jit_code_label:
1092                 if ((node->link || (node->flag & jit_flag_use)) &&
1093                     (word = _jit->pc.w & 3))
1094                     nop(4 - word);
1095                 /* remember label is defined */
1096                 node->flag |= jit_flag_patch;
1097                 node->u.w = _jit->pc.w;
1098                 break;
1099                 case_rrr(add,);
1100                 case_rrw(add,);
1101                 case_rrr(addc,);
1102                 case_rrw(addc,);
1103                 case_rrr(addx,);
1104                 case_rrw(addx,);
1105                 case_rrr(sub,);
1106                 case_rrw(sub,);
1107                 case_rrr(subc,);
1108                 case_rrw(subc,);
1109                 case_rrr(subx,);
1110                 case_rrw(subx,);
1111                 case_rrw(rsb,);
1112                 case_rrr(mul,);
1113                 case_rrw(mul,);
1114                 case_rrrr(qmul,);
1115                 case_rrrw(qmul,);
1116                 case_rrrr(qmul, _u);
1117                 case_rrrw(qmul, _u);
1118                 case_rrr(div,);
1119                 case_rrw(div,);
1120                 case_rrr(div, _u);
1121                 case_rrw(div, _u);
1122                 case_rrr(rem,);
1123                 case_rrw(rem,);
1124                 case_rrr(rem, _u);
1125                 case_rrw(rem, _u);
1126                 case_rrrr(qdiv,);
1127                 case_rrrw(qdiv,);
1128                 case_rrrr(qdiv, _u);
1129                 case_rrrw(qdiv, _u);
1130                 case_rrr(lsh,);
1131                 case_rrw(lsh,);
1132                 case_rrr(rsh,);
1133                 case_rrw(rsh,);
1134                 case_rrr(rsh, _u);
1135                 case_rrw(rsh, _u);
1136                 case_rr(neg,);
1137                 case_rr(com,);
1138                 case_rr(clo,);
1139                 case_rr(clz,);
1140                 case_rr(cto,);
1141                 case_rr(ctz,);
1142                 case_rrr(and,);
1143                 case_rrw(and,);
1144                 case_rrr(or,);
1145                 case_rrw(or,);
1146                 case_rrr(xor,);
1147                 case_rrw(xor,);
1148                 case_rr(trunc, _f_i);
1149                 case_rr(trunc, _d_i);
1150 #if __WORDSIZE == 64
1151                 case_rr(trunc, _f_l);
1152                 case_rr(trunc, _d_l);
1153 #endif
1154                 case_rr(ld, _c);
1155                 case_rw(ld, _c);
1156                 case_rr(ld, _uc);
1157                 case_rw(ld, _uc);
1158                 case_rr(ld, _s);
1159                 case_rw(ld, _s);
1160                 case_rr(ld, _us);
1161                 case_rw(ld, _us);
1162                 case_rr(ld, _i);
1163                 case_rw(ld, _i);
1164 #if __WORDSIZE == 64
1165                 case_rr(ld, _ui);
1166                 case_rw(ld, _ui);
1167                 case_rr(ld, _l);
1168                 case_rw(ld, _l);
1169 #endif
1170                 case_rrr(ldx, _c);
1171                 case_rrw(ldx, _c);
1172                 case_rrr(ldx, _uc);
1173                 case_rrw(ldx, _uc);
1174                 case_rrr(ldx, _s);
1175                 case_rrw(ldx, _s);
1176                 case_rrr(ldx, _us);
1177                 case_rrw(ldx, _us);
1178                 case_rrr(ldx, _i);
1179                 case_rrw(ldx, _i);
1180 #if __WORDSIZE == 64
1181                 case_rrr(ldx, _ui);
1182                 case_rrw(ldx, _ui);
1183                 case_rrr(ldx, _l);
1184                 case_rrw(ldx, _l);
1185 #endif
1186                 case_rr(st, _c);
1187                 case_wr(st, _c);
1188                 case_rr(st, _s);
1189                 case_wr(st, _s);
1190                 case_rr(st, _i);
1191                 case_wr(st, _i);
1192 #if __WORDSIZE == 64
1193                 case_rr(st, _l);
1194                 case_wr(st, _l);
1195 #endif
1196                 case_rrr(stx, _c);
1197                 case_wrr(stx, _c);
1198                 case_rrr(stx, _s);
1199                 case_wrr(stx, _s);
1200                 case_rrr(stx, _i);
1201                 case_wrr(stx, _i);
1202 #if __WORDSIZE == 64
1203                 case_rrr(stx, _l);
1204                 case_wrr(stx, _l);
1205 #endif
1206                 case_rr(hton, _us);
1207                 case_rr(hton, _ui);
1208 #if __WORDSIZE == 64
1209                 case_rr(hton, _ul);
1210 #endif
1211                 case_rr(bswap, _us);
1212                 case_rr(bswap, _ui);
1213 #if __WORDSIZE == 64
1214                 case_rr(bswap, _ul);
1215 #endif
1216                 case_rr(ext, _c);
1217                 case_rr(ext, _uc);
1218                 case_rr(ext, _s);
1219                 case_rr(ext, _us);
1220 #if __WORDSIZE == 64
1221                 case_rr(ext, _i);
1222                 case_rr(ext, _ui);
1223 #endif
1224             case jit_code_casr:
1225                 casr(rn(node->u.w), rn(node->v.w),
1226                      rn(node->w.q.l), rn(node->w.q.h));
1227                 break;
1228             case jit_code_casi:
1229                 casi(rn(node->u.w), node->v.w,
1230                      rn(node->w.q.l), rn(node->w.q.h));
1231                 break;
1232                 case_rrr(movn,);
1233                 case_rrr(movz,);
1234                 case_rr(mov,);
1235             case jit_code_movi:
1236                 if (node->flag & jit_flag_node) {
1237                     temp = node->v.n;
1238                     if (temp->code == jit_code_data ||
1239                         (temp->code == jit_code_label &&
1240                          (temp->flag & jit_flag_patch)))
1241                         movi(rn(node->u.w), temp->u.w);
1242                     else {
1243                         assert(temp->code == jit_code_label ||
1244                                temp->code == jit_code_epilog);
1245                         word = movi_p(rn(node->u.w), temp->u.w);
1246                         patch(word, node);
1247                     }
1248                 }
1249                 else
1250                     movi(rn(node->u.w), node->v.w);
1251                 break;
1252                 case_rrr(lt,);
1253                 case_rrw(lt,);
1254                 case_rrr(lt, _u);
1255                 case_rrw(lt, _u);
1256                 case_rrr(le,);
1257                 case_rrw(le,);
1258                 case_rrr(le, _u);
1259                 case_rrw(le, _u);
1260                 case_rrr(eq,);
1261                 case_rrw(eq,);
1262                 case_rrr(ge,);
1263                 case_rrw(ge,);
1264                 case_rrr(ge, _u);
1265                 case_rrw(ge, _u);
1266                 case_rrr(gt,);
1267                 case_rrw(gt,);
1268                 case_rrr(gt, _u);
1269                 case_rrw(gt, _u);
1270                 case_rrr(ne,);
1271                 case_rrw(ne,);
1272                 case_brr(blt,);
1273                 case_brw(blt,);
1274                 case_brr(blt, _u);
1275                 case_brw(blt, _u);
1276                 case_brr(ble,);
1277                 case_brw(ble,);
1278                 case_brr(ble, _u);
1279                 case_brw(ble, _u);
1280                 case_brr(beq,);
1281                 case_brw(beq,);
1282                 case_brr(bge,);
1283                 case_brw(bge,);
1284                 case_brr(bge, _u);
1285                 case_brw(bge, _u);
1286                 case_brr(bgt,);
1287                 case_brw(bgt,);
1288                 case_brr(bgt, _u);
1289                 case_brw(bgt, _u);
1290                 case_brr(bne,);
1291                 case_brw(bne,);
1292                 case_brr(boadd,);
1293                 case_brw(boadd,);
1294                 case_brr(boadd, _u);
1295                 case_brw(boadd, _u);
1296                 case_brr(bxadd,);
1297                 case_brw(bxadd,);
1298                 case_brr(bxadd, _u);
1299                 case_brw(bxadd, _u);
1300                 case_brr(bosub,);
1301                 case_brw(bosub,);
1302                 case_brr(bosub, _u);
1303                 case_brw(bosub, _u);
1304                 case_brr(bxsub,);
1305                 case_brw(bxsub,);
1306                 case_brr(bxsub, _u);
1307                 case_brw(bxsub, _u);
1308                 case_brr(bms,);
1309                 case_brw(bms,);
1310                 case_brr(bmc,);
1311                 case_brw(bmc,);
1312                 case_rrr(add, _f);
1313                 case_rrf(add);
1314                 case_rrr(sub, _f);
1315                 case_rrf(sub);
1316                 case_rrf(rsb);
1317                 case_rrr(mul, _f);
1318                 case_rrf(mul);
1319                 case_rrr(div, _f);
1320                 case_rrf(div);
1321                 case_rr(abs, _f);
1322                 case_rr(neg, _f);
1323                 case_rr(sqrt, _f);
1324                 case_rr(ext, _f);
1325                 case_rr(ld, _f);
1326                 case_rw(ld, _f);
1327                 case_rrr(ldx, _f);
1328                 case_rrw(ldx, _f);
1329                 case_rr(st, _f);
1330                 case_wr(st, _f);
1331                 case_rrr(stx, _f);
1332                 case_wrr(stx, _f);
1333                 case_rr(mov, _f);
1334             case jit_code_movi_f:
1335                 assert_data(node);
1336                 movi_f(rn(node->u.w), (jit_float32_t *)node->v.n->u.w);
1337                 break;
1338                 case_rr(ext, _d_f);
1339                 case_rrr(lt, _f);
1340                 case_rrf(lt);
1341                 case_rrr(le, _f);
1342                 case_rrf(le);
1343                 case_rrr(eq, _f);
1344                 case_rrf(eq);
1345                 case_rrr(ge, _f);
1346                 case_rrf(ge);
1347                 case_rrr(gt, _f);
1348                 case_rrf(gt);
1349                 case_rrr(ne, _f);
1350                 case_rrf(ne);
1351                 case_rrr(unlt, _f);
1352                 case_rrf(unlt);
1353                 case_rrr(unle, _f);
1354                 case_rrf(unle);
1355                 case_rrr(uneq, _f);
1356                 case_rrf(uneq);
1357                 case_rrr(unge, _f);
1358                 case_rrf(unge);
1359                 case_rrr(ungt, _f);
1360                 case_rrf(ungt);
1361                 case_rrr(ltgt, _f);
1362                 case_rrf(ltgt);
1363                 case_rrr(ord, _f);
1364                 case_rrf(ord);
1365                 case_rrr(unord, _f);
1366                 case_rrf(unord);
1367                 case_brr(blt, _f);
1368                 case_brf(blt);
1369                 case_brr(ble, _f);
1370                 case_brf(ble);
1371                 case_brr(beq, _f);
1372                 case_brf(beq);
1373                 case_brr(bge, _f);
1374                 case_brf(bge);
1375                 case_brr(bgt, _f);
1376                 case_brf(bgt);
1377                 case_brr(bne, _f);
1378                 case_brf(bne);
1379                 case_brr(bunlt, _f);
1380                 case_brf(bunlt);
1381                 case_brr(bunle, _f);
1382                 case_brf(bunle);
1383                 case_brr(buneq, _f);
1384                 case_brf(buneq);
1385                 case_brr(bunge, _f);
1386                 case_brf(bunge);
1387                 case_brr(bungt, _f);
1388                 case_brf(bungt);
1389                 case_brr(bltgt, _f);
1390                 case_brf(bltgt);
1391                 case_brr(bord, _f);
1392                 case_brf(bord);
1393                 case_brr(bunord, _f);
1394                 case_brf(bunord);
1395                 case_rrr(add, _d);
1396                 case_rrd(add);
1397                 case_rrr(sub, _d);
1398                 case_rrd(sub);
1399                 case_rrd(rsb);
1400                 case_rrr(mul, _d);
1401                 case_rrd(mul);
1402                 case_rrr(div, _d);
1403                 case_rrd(div);
1404                 case_rr(abs, _d);
1405                 case_rr(neg, _d);
1406                 case_rr(sqrt, _d);
1407                 case_rr(ext, _d);
1408                 case_rr(ld, _d);
1409                 case_rw(ld, _d);
1410                 case_rrr(ldx, _d);
1411                 case_rrw(ldx, _d);
1412                 case_rr(st, _d);
1413                 case_wr(st, _d);
1414                 case_rrr(stx, _d);
1415                 case_wrr(stx, _d);
1416                 case_rr(mov, _d);
1417             case jit_code_movi_d:
1418                 assert_data(node);
1419                 movi_d(rn(node->u.w), (jit_float64_t *)node->v.n->u.w);
1420                 break;
1421                 case_rr(ext, _f_d);
1422                 case_rrr(lt, _d);
1423                 case_rrd(lt);
1424                 case_rrr(le, _d);
1425                 case_rrd(le);
1426                 case_rrr(eq, _d);
1427                 case_rrd(eq);
1428                 case_rrr(ge, _d);
1429                 case_rrd(ge);
1430                 case_rrr(gt, _d);
1431                 case_rrd(gt);
1432                 case_rrr(ne, _d);
1433                 case_rrd(ne);
1434                 case_rrr(unlt, _d);
1435                 case_rrd(unlt);
1436                 case_rrr(unle, _d);
1437                 case_rrd(unle);
1438                 case_rrr(uneq, _d);
1439                 case_rrd(uneq);
1440                 case_rrr(unge, _d);
1441                 case_rrd(unge);
1442                 case_rrr(ungt, _d);
1443                 case_rrd(ungt);
1444                 case_rrr(ltgt, _d);
1445                 case_rrd(ltgt);
1446                 case_rrr(ord, _d);
1447                 case_rrd(ord);
1448                 case_rrr(unord, _d);
1449                 case_rrd(unord);
1450                 case_brr(blt, _d);
1451                 case_brd(blt);
1452                 case_brr(ble, _d);
1453                 case_brd(ble);
1454                 case_brr(beq, _d);
1455                 case_brd(beq);
1456                 case_brr(bge, _d);
1457                 case_brd(bge);
1458                 case_brr(bgt, _d);
1459                 case_brd(bgt);
1460                 case_brr(bne, _d);
1461                 case_brd(bne);
1462                 case_brr(bunlt, _d);
1463                 case_brd(bunlt);
1464                 case_brr(bunle, _d);
1465                 case_brd(bunle);
1466                 case_brr(buneq, _d);
1467                 case_brd(buneq);
1468                 case_brr(bunge, _d);
1469                 case_brd(bunge);
1470                 case_brr(bungt, _d);
1471                 case_brd(bungt);
1472                 case_brr(bltgt, _d);
1473                 case_brd(bltgt);
1474                 case_brr(bord, _d);
1475                 case_brd(bord);
1476                 case_brr(bunord, _d);
1477                 case_brd(bunord);
1478             case jit_code_jmpr:
1479                 jmpr(rn(node->u.w));
1480                 break;
1481             case jit_code_jmpi:
1482                 if (node->flag & jit_flag_node) {
1483                     temp = node->u.n;
1484                     assert(temp->code == jit_code_label ||
1485                            temp->code == jit_code_epilog);
1486                     if (temp->flag & jit_flag_patch)
1487                         jmpi(temp->u.w, 1);
1488                     else {
1489                         word = _jit->code.length -
1490                             (_jit->pc.uc - _jit->code.ptr);
1491                         if (s32_p(word)) {
1492                             offset = s16_p(word);
1493                             word = jmpi(_jit->pc.w, offset);
1494                         }
1495                         else
1496                             word = jmpi_p(_jit->pc.w);
1497                         patch(word, node);
1498                     }
1499                 }
1500                 else
1501                     jmpi(node->u.w, 1);
1502                 break;
1503             case jit_code_callr:
1504                 callr(rn(node->u.w));
1505                 break;
1506             case jit_code_calli:
1507                 if (node->flag & jit_flag_node) {
1508                     temp = node->u.n;
1509                     assert(temp->code == jit_code_label ||
1510                            temp->code == jit_code_epilog);
1511                     if (temp->flag & jit_flag_patch)
1512                         calli(temp->u.w, 1);
1513                     else {
1514                         word = _jit->code.length -
1515                             (_jit->pc.uc - _jit->code.ptr);
1516                         if (s32_p(word)) {
1517                             offset =s16_p(word);
1518                             word = calli(_jit->pc.w, offset);
1519                         }
1520                         else
1521                             word = calli_p(_jit->pc.w);
1522                         patch(word, node);
1523                     }
1524                 }
1525                 else
1526                     calli(node->u.w, 1);
1527                 break;
1528             case jit_code_prolog:
1529                 _jitc->function = _jitc->functions.ptr + node->w.w;
1530                 undo.node = node;
1531                 undo.word = _jit->pc.w;
1532                 memcpy(&undo.func, _jitc->function, sizeof(undo.func));
1533 #if DEVEL_DISASSEMBLER
1534                 undo.prevw = prevw;
1535 #endif
1536                 undo.patch_offset = _jitc->patches.offset;
1537             restart_function:
1538                 _jitc->again = 0;
1539                 prolog(node);
1540                 break;
1541             case jit_code_epilog:
1542                 assert(_jitc->function == _jitc->functions.ptr + node->w.w);
1543                 if (_jitc->again) {
1544                     for (temp = undo.node->next;
1545                          temp != node; temp = temp->next) {
1546                         if (temp->code == jit_code_label ||
1547                             temp->code == jit_code_epilog)
1548                             temp->flag &= ~jit_flag_patch;
1549                     }
1550                     temp->flag &= ~jit_flag_patch;
1551                     node = undo.node;
1552                     _jit->pc.w = undo.word;
1553                     /* undo.func.self.aoff and undo.func.regset should not
1554                      * be undone, as they will be further updated, and are
1555                      * the reason of the undo. */
1556                     undo.func.self.aoff = _jitc->function->frame +
1557                         _jitc->function->self.aoff;
1558                     jit_regset_set(&undo.func.regset, &_jitc->function->regset);
1559                     /* allocar information also does not need to be undone */
1560                     undo.func.aoffoff = _jitc->function->aoffoff;
1561                     undo.func.allocar = _jitc->function->allocar;
1562                     memcpy(_jitc->function, &undo.func, sizeof(undo.func));
1563 #if DEVEL_DISASSEMBLER
1564                     prevw = undo.prevw;
1565 #endif
1566                     _jitc->patches.offset = undo.patch_offset;
1567                     goto restart_function;
1568                 }
1569                 if (node->link && (word = _jit->pc.w & 3))
1570                     nop(4 - word);
1571                 /* remember label is defined */
1572                 node->flag |= jit_flag_patch;
1573                 node->u.w = _jit->pc.w;
1574                 epilog(node);
1575                 _jitc->function = NULL;
1576                 break;
1577             case jit_code_va_start:
1578                 vastart(rn(node->u.w));
1579                 break;
1580             case jit_code_va_arg:
1581                 vaarg(rn(node->u.w), rn(node->v.w));
1582                 break;
1583             case jit_code_va_arg_d:
1584                 vaarg_d(rn(node->u.w), rn(node->v.w));
1585                 break;
1586             case jit_code_live:                 case jit_code_ellipsis:
1587             case jit_code_va_push:
1588             case jit_code_allocai:              case jit_code_allocar:
1589             case jit_code_arg_c:                case jit_code_arg_s:
1590             case jit_code_arg_i:
1591 #  if __WORDSIZE == 64
1592             case jit_code_arg_l:
1593 #  endif
1594             case jit_code_arg_f:                case jit_code_arg_d:
1595             case jit_code_va_end:
1596             case jit_code_ret:
1597             case jit_code_retr_c:               case jit_code_reti_c:
1598             case jit_code_retr_uc:              case jit_code_reti_uc:
1599             case jit_code_retr_s:               case jit_code_reti_s:
1600             case jit_code_retr_us:              case jit_code_reti_us:
1601             case jit_code_retr_i:               case jit_code_reti_i:
1602 #if __WORDSIZE == 64
1603             case jit_code_retr_ui:              case jit_code_reti_ui:
1604             case jit_code_retr_l:               case jit_code_reti_l:
1605 #endif
1606             case jit_code_retr_f:               case jit_code_reti_f:
1607             case jit_code_retr_d:               case jit_code_reti_d:
1608             case jit_code_getarg_c:             case jit_code_getarg_uc:
1609             case jit_code_getarg_s:             case jit_code_getarg_us:
1610             case jit_code_getarg_i:
1611 #if __WORDSIZE == 64
1612             case jit_code_getarg_ui:            case jit_code_getarg_l:
1613 #endif
1614             case jit_code_getarg_f:             case jit_code_getarg_d:
1615             case jit_code_putargr_c:            case jit_code_putargi_c:
1616             case jit_code_putargr_uc:           case jit_code_putargi_uc:
1617             case jit_code_putargr_s:            case jit_code_putargi_s:
1618             case jit_code_putargr_us:           case jit_code_putargi_us:
1619             case jit_code_putargr_i:            case jit_code_putargi_i:
1620 #if __WORDSIZE == 64
1621             case jit_code_putargr_ui:           case jit_code_putargi_ui:
1622             case jit_code_putargr_l:            case jit_code_putargi_l:
1623 #endif
1624             case jit_code_putargr_f:            case jit_code_putargi_f:
1625             case jit_code_putargr_d:            case jit_code_putargi_d:
1626             case jit_code_pushargr_c:           case jit_code_pushargi_c:
1627             case jit_code_pushargr_uc:          case jit_code_pushargi_uc:
1628             case jit_code_pushargr_s:           case jit_code_pushargi_s:
1629             case jit_code_pushargr_us:          case jit_code_pushargi_us:
1630             case jit_code_pushargr_i:           case jit_code_pushargi_i:
1631 #if __WORDSIZE == 64
1632             case jit_code_pushargr_ui:          case jit_code_pushargi_ui:
1633             case jit_code_pushargr_l:           case jit_code_pushargi_l:
1634 #endif
1635             case jit_code_pushargr_f:           case jit_code_pushargi_f:
1636             case jit_code_pushargr_d:           case jit_code_pushargi_d:
1637             case jit_code_retval_c:             case jit_code_retval_uc:
1638             case jit_code_retval_s:             case jit_code_retval_us:
1639             case jit_code_retval_i:
1640 #if __WORDSIZE == 64
1641             case jit_code_retval_ui:            case jit_code_retval_l:
1642 #endif
1643             case jit_code_retval_f:             case jit_code_retval_d:
1644             case jit_code_prepare:
1645             case jit_code_finishr:              case jit_code_finishi:
1646                 break;
1647             default:
1648                 abort();
1649         }
1650         jit_regarg_clr(node, value);
1651         assert(_jitc->regarg == 0 && _jitc->synth == 0);
1652         /* update register live state */
1653         jit_reglive(node);
1654     }
1655 #undef case_brw
1656 #undef case_brr
1657 #undef case_wrr
1658 #undef case_rrw
1659 #undef case_rrr
1660 #undef case_wr
1661 #undef case_rw
1662 #undef case_rr
1663
1664     for (offset = 0; offset < _jitc->patches.offset; offset++) {
1665         node = _jitc->patches.ptr[offset].node;
1666         word = node->code == jit_code_movi ? node->v.n->u.w : node->u.n->u.w;
1667         patch_at(_jitc->patches.ptr[offset].inst, word);
1668     }
1669
1670     jit_flush(_jit->code.ptr, _jit->pc.uc);
1671
1672     return (_jit->code.ptr);
1673 }
1674
1675 #define CODE                            1
1676 #  include "jit_s390-cpu.c"
1677 #  include "jit_s390-fpu.c"
1678 #  if CHECK_FLOGR
1679 #    include "jit_fallback.c"
1680 #  endif
1681 #undef CODE
1682
1683 void
1684 jit_flush(void *fptr, void *tptr)
1685 {
1686 #if defined(__GNUC__)
1687     jit_word_t          f, t, s;
1688
1689     s = sysconf(_SC_PAGE_SIZE);
1690     f = (jit_word_t)fptr & -s;
1691     t = (((jit_word_t)tptr) + s - 1) & -s;
1692     __clear_cache((void *)f, (void *)t);
1693 #endif
1694 }
1695
1696 void
1697 _emit_ldxi(jit_state_t *_jit, jit_gpr_t r0, jit_gpr_t r1, jit_word_t i0)
1698 {
1699     ldxi(rn(r0), rn(r1), i0);
1700 }
1701
1702 void
1703 _emit_stxi(jit_state_t *_jit, jit_word_t i0, jit_gpr_t r0, jit_gpr_t r1)
1704 {
1705     stxi(i0, rn(r0), rn(r1));
1706 }
1707
1708 void
1709 _emit_ldxi_d(jit_state_t *_jit, jit_fpr_t r0, jit_gpr_t r1, jit_word_t i0)
1710 {
1711     ldxi_d(rn(r0), rn(r1), i0);
1712 }
1713
1714 void
1715 _emit_stxi_d(jit_state_t *_jit, jit_word_t i0, jit_gpr_t r0, jit_fpr_t r1)
1716 {
1717     stxi_d(i0, rn(r0), rn(r1));
1718 }
1719
1720 static jit_int32_t
1721 _jit_get_reg_pair(jit_state_t *_jit)
1722 {
1723     jit_int32_t         r1, r2;
1724     /* Try to find a register pair for use with operations that
1725      * require a odd based register pair. Search for the best
1726      * match to avoid spills or at least a valid operation.
1727      */
1728
1729     /* Try non callee save first */
1730     if (jit_reg_free_p(_R0) && jit_reg_free_p(_R1))
1731         r1 = _R0, r2 = _R1;
1732     else if (jit_reg_free_p(_R2) && jit_reg_free_p(_R3))
1733         r1 = _R2, r2 = _R3;
1734     else if (jit_reg_free_p(_R4) && jit_reg_free_p(_R5))
1735         r1 = _R4, r2 = _R5;
1736     /* Try callee save registers */
1737     else if (jit_reg_free_p(_R10) && jit_reg_free_p(_R11))
1738         r1 = _R10, r2 = _R11;
1739     else if (jit_reg_free_p(_R8) && jit_reg_free_p(_R9))
1740         r1 = _R8, r2 = _R9;
1741     else if (jit_reg_free_p(_R6) && jit_reg_free_p(_R7))
1742         r1 = _R6, r2 = _R7;
1743
1744     /* We *must* find a register pair */
1745     else if (jit_reg_free_if_spill_p(_R0) && jit_reg_free_if_spill_p(_R1))
1746         r1 = _R0, r2 = _R1;
1747     else if (jit_reg_free_if_spill_p(_R2) && jit_reg_free_if_spill_p(_R3))
1748         r1 = _R2, r2 = _R3;
1749     else if (jit_reg_free_if_spill_p(_R4) && jit_reg_free_if_spill_p(_R5))
1750         r1 = _R4, r2 = _R5;
1751     else if (jit_reg_free_if_spill_p(_R10) && jit_reg_free_if_spill_p(_R11))
1752         r1 = _R10, r2 = _R11;
1753     else if (jit_reg_free_if_spill_p(_R8) && jit_reg_free_if_spill_p(_R9))
1754         r1 = _R8, r2 = _R9;
1755     else if (jit_reg_free_if_spill_p(_R6) && jit_reg_free_if_spill_p(_R7))
1756         r1 = _R6, r2 = _R7;
1757     else
1758         /* Do not jit_get_reg() all registers to avoid it */
1759         abort();
1760
1761     (void)jit_get_reg(jit_class_gpr|jit_class_named|r1);
1762     (void)jit_get_reg(jit_class_gpr|jit_class_named|r2);
1763
1764     return (r1);
1765 }
1766
1767 static void
1768 _jit_unget_reg_pair(jit_state_t *_jit, jit_int32_t reg)
1769 {
1770     jit_int32_t         r1, r2;
1771     r1 = reg;
1772     switch (r1) {
1773         case _R0:       r2 = _R1;       break;
1774         case _R2:       r2 = _R3;       break;
1775         case _R4:       r2 = _R5;       break;
1776         case _R6:       r2 = _R7;       break;
1777         case _R8:       r2 = _R9;       break;
1778         case _R10:      r2 = _R11;      break;
1779         default:        abort();
1780     }
1781     jit_unget_reg(r1);
1782     jit_unget_reg(r2);
1783 }
1784
1785 static jit_int32_t
1786 _jit_get_reg_but_zero(jit_state_t *_jit, jit_int32_t flags)
1787 {
1788     jit_int32_t         reg;
1789     reg = jit_get_reg(jit_class_gpr);
1790     if (reg == _R0) {
1791         reg = jit_get_reg(jit_class_gpr|flags);
1792         jit_unget_reg(_R0);
1793     }
1794     return (reg);
1795 }
1796
1797 static void
1798 _patch(jit_state_t *_jit, jit_word_t instr, jit_node_t *node)
1799 {
1800     jit_int32_t         flag;
1801
1802     assert(node->flag & jit_flag_node);
1803     if (node->code == jit_code_movi)
1804         flag = node->v.n->flag;
1805     else
1806         flag = node->u.n->flag;
1807     assert(!(flag & jit_flag_patch));
1808     if (_jitc->patches.offset >= _jitc->patches.length) {
1809         jit_realloc((jit_pointer_t *)&_jitc->patches.ptr,
1810                     _jitc->patches.length * sizeof(jit_patch_t),
1811                     (_jitc->patches.length + 1024) * sizeof(jit_patch_t));
1812         _jitc->patches.length += 1024;
1813     }
1814     _jitc->patches.ptr[_jitc->patches.offset].inst = instr;
1815     _jitc->patches.ptr[_jitc->patches.offset].node = node;
1816     ++_jitc->patches.offset;
1817 }