deps/lightning/lib/jit_s390.c

   1 /*
   2  * Copyright (C) 2013-2023  Free Software Foundation, Inc.
   3  *
   4  * This file is part of GNU lightning.
   5  *
   6  * GNU lightning is free software; you can redistribute it and/or modify it
   7  * under the terms of the GNU Lesser General Public License as published
   8  * by the Free Software Foundation; either version 3, or (at your option)
   9  * any later version.
  10  *
  11  * GNU lightning is distributed in the hope that it will be useful, but
  12  * WITHOUT ANY WARRANTY; without even the implied warranty of MERCHANTABILITY
  13  * or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU Lesser General Public
  14  * License for more details.
  15  *
  16  * Authors:
  17  *      Paulo Cesar Pereira de Andrade
  18  */
  19 #define CHECK_FLOGR     0
  20
  21 #if CHECK_FLOGR
  22 #include <signal.h>
  23 #include <setjmp.h>
  24 #endif
  25
  26 #include <lightning.h>
  27 #include <lightning/jit_private.h>
  28
  29 #if __WORDSIZE == 32
  30 #  define NUM_FLOAT_REG_ARGS            2
  31 #else
  32 #  define NUM_FLOAT_REG_ARGS            4
  33 #endif
  34 #define jit_arg_reg_p(i)                ((i) >= 0 && (i) < 5)
  35 #define jit_arg_f_reg_p(i)              ((i) >= 0 && (i) < NUM_FLOAT_REG_ARGS)
  36
  37 /*
  38  * Types
  39  */
  40 typedef struct jit_va_list {
  41     /* The offsets are "1" based, as addresses are fixed in the
  42      * standard stack frame format. */
  43     jit_word_t          gpoff;
  44     jit_word_t          fpoff;
  45
  46     /* Easier when there is an explicitly defined type...
  47 (gdb) ptype ap
  48 type = struct __va_list_tag {
  49     long __gpr;
  50     long __fpr;
  51     void *__overflow_arg_area;
  52     void *__reg_save_area;
  53
  54     Note that gopff (__gpr) and fpoff (__fpr) are jit_word_t equivalent
  55     and, again, "1" (unit) based, so must be adjusted at va_arg time.
  56  */
  57     jit_pointer_t       over;
  58     jit_pointer_t       save;
  59
  60     /* For variadic functions, always allocate space to save callee
  61      * save fpr registers.
  62      * Note that s390 has a standard stack frame format that lightning
  63      * does not fully comply with, but for variadic functions it must,
  64      * for those (variadic) do not use the "empty" spaces for any
  65      * callee save fpr register, but save them after the va_list
  66      * space; and use the standard stack frame format, as required
  67      * by variadic functions (and have a compatible va_list pointer). */
  68     jit_float64_t       f8;
  69     jit_float64_t       f9;
  70     jit_float64_t       f10;
  71     jit_float64_t       f11;
  72     jit_float64_t       f12;
  73     jit_float64_t       f13;
  74     jit_float64_t       f14;
  75     jit_float64_t       f15;
  76 } jit_va_list_t;
  77
  78 /*
  79  * Prototypes
  80  */
  81 #define jit_get_reg_pair()              _jit_get_reg_pair(_jit)
  82 static jit_int32_t _jit_get_reg_pair(jit_state_t*);
  83 #define jit_unget_reg_pair(regno)       _jit_unget_reg_pair(_jit,regno)
  84 static void _jit_unget_reg_pair(jit_state_t*,jit_int32_t);
  85 #define jit_get_reg_but_zero(flags)     _jit_get_reg_but_zero(_jit,flags)
  86 static jit_int32_t _jit_get_reg_but_zero(jit_state_t*,jit_int32_t);
  87 #define jit_unget_reg_but_zero(reg)     jit_unget_reg(reg)
  88 #define patch(instr, node)              _patch(_jit, instr, node)
  89 static void _patch(jit_state_t*,jit_word_t,jit_node_t*);
  90
  91 /* libgcc */
  92 extern void __clear_cache(void *, void *);
  93
  94 #define PROTO                           1
  95 #  include "jit_s390-cpu.c"
  96 #  include "jit_s390-fpu.c"
  97 #  include "jit_fallback.c"
  98 #undef PROTO
  99
 100 /*
 101  * Initialization
 102  */
 103 jit_cpu_t               jit_cpu;
 104 jit_register_t          _rvs[] = {
 105     { rc(gpr) | 0x0,                    "%r0" },
 106     { rc(gpr) | 0x1,                    "%r1" },
 107     { rc(gpr) | rc(sav) | 0xc,          "%r12" },
 108     { rc(gpr) | rc(sav) | 0xb,          "%r11" },
 109     { rc(gpr) | rc(sav) | 0xa,          "%r10" },
 110     { rc(gpr) | rc(sav) | 0x9,          "%r9" },
 111     { rc(gpr) | rc(sav) | 0x8,          "%r8" },
 112     { rc(gpr) | rc(sav) | 0x7,          "%r7" },
 113     { rc(gpr) | rc(arg) | rc(sav) | 0x6,"%r6" },
 114     { rc(gpr) | rc(arg) | 0x5,          "%r5" },
 115     { rc(gpr) | rc(arg) | 0x4,          "%r4" },
 116     { rc(gpr) | rc(arg) | 0x3,          "%r3" },
 117     { rc(gpr) | rc(arg) | 0x2,          "%r2" },
 118     { rc(sav) | 0xd,                    "%r13" },       /* used as JIT_FP */
 119     { 0xe,                              "%r14" },
 120     { rc(sav) | 0xf,                    "%r15" },
 121     { rc(fpr) | 0x1,                    "%f1" },
 122     { rc(fpr) | 0x3,                    "%f3" },
 123     { rc(fpr) | 0x5,                    "%f5" },
 124     { rc(fpr) | 0x7,                    "%f7" },
 125     { rc(fpr) | rc(sav) | 0xe,          "%f14" },
 126     /* Do not use as temporary to simplify stack layout */
 127     { 0xf,                              "%f15" },
 128     { rc(fpr) | rc(sav) | 0x8,          "%f8" },
 129     { rc(fpr) | rc(sav) | 0x9,          "%f9" },
 130     { rc(fpr) | rc(sav) | 0xa,          "%f10" },
 131     { rc(fpr) | rc(sav) | 0xb,          "%f11" },
 132     { rc(fpr) | rc(sav) | 0xc,          "%f12" },
 133     { rc(fpr) | rc(sav) | 0xd,          "%f13" },
 134     { rc(fpr) | rc(arg) | 0x6,          "%f6" },
 135     { rc(fpr) | rc(arg) | 0x4,          "%f4" },
 136     { rc(fpr) | rc(arg) | 0x2,          "%f2" },
 137     { rc(fpr) | rc(arg) | 0x0,          "%f0" },
 138     { _NOREG,                           "<none>" },
 139 };
 140 #if CHECK_FLOGR
 141 static sigjmp_buf       jit_env;
 142 #endif
 143
 144 /*
 145  * Implementation
 146  */
 147 #if CHECK_FLOGR
 148 static void
 149 sigill_handler(int signum)
 150 {
 151     jit_cpu.flogr = 0;
 152     siglongjmp(jit_env, 1);
 153 }
 154 #endif
 155
 156 void
 157 jit_get_cpu(void)
 158 {
 159 #if CHECK_FLOGR
 160     int                 r12, r13;
 161     struct              sigaction new_action, old_action;
 162     new_action.sa_handler = sigill_handler;
 163     sigemptyset(&new_action.sa_mask);
 164     new_action.sa_flags = 0;
 165     sigaction(SIGILL, NULL, &old_action);
 166     if (old_action.sa_handler != SIG_IGN) {
 167         sigaction(SIGILL, &new_action, NULL);
 168         if (!sigsetjmp(jit_env, 1)) {
 169             jit_cpu.flogr = 1;
 170             /* flogr %r12, %r12 */
 171             __asm__ volatile("lgr %%r12, %0; lgr %%r13, %1;"
 172                              "flogr %%r12, %%r12;"
 173                              "lgr %1, %%r13; lgr %0, %%r12;"
 174                              : "=r" (r12), "=r" (r13));
 175             sigaction(SIGILL, &old_action, NULL);
 176         }
 177     }
 178 #else
 179     /* By default, assume it is available */
 180     jit_cpu.flogr = 1;
 181 #endif
 182 }
 183
 184 void
 185 _jit_init(jit_state_t *_jit)
 186 {
 187     _jitc->reglen = jit_size(_rvs) - 1;
 188 }
 189
 190 void
 191 _jit_prolog(jit_state_t *_jit)
 192 {
 193     jit_int32_t         offset;
 194
 195     if (_jitc->function)
 196         jit_epilog();
 197     assert(jit_regset_cmp_ui(&_jitc->regarg, 0) == 0);
 198     jit_regset_set_ui(&_jitc->regsav, 0);
 199     offset = _jitc->functions.offset;
 200     if (offset >= _jitc->functions.length) {
 201         jit_realloc((jit_pointer_t *)&_jitc->functions.ptr,
 202                     _jitc->functions.length * sizeof(jit_function_t),
 203                     (_jitc->functions.length + 16) * sizeof(jit_function_t));
 204         _jitc->functions.length += 16;
 205     }
 206     _jitc->function = _jitc->functions.ptr + _jitc->functions.offset++;
 207     _jitc->function->self.size = stack_framesize;
 208     _jitc->function->self.argi = _jitc->function->self.argf =
 209         _jitc->function->self.aoff = _jitc->function->self.alen = 0;
 210     /* preallocate 8 bytes if not using a constant data buffer */
 211     if (_jitc->no_data)
 212         _jitc->function->self.aoff = -8;
 213     _jitc->function->self.call = jit_call_default;
 214     jit_alloc((jit_pointer_t *)&_jitc->function->regoff,
 215               _jitc->reglen * sizeof(jit_int32_t));
 216
 217     /* _no_link here does not mean the jit_link() call can be removed
 218      * by rewriting as:
 219      * _jitc->function->prolog = jit_new_node(jit_code_prolog);
 220      */
 221     _jitc->function->prolog = jit_new_node_no_link(jit_code_prolog);
 222     jit_link(_jitc->function->prolog);
 223     _jitc->function->prolog->w.w = offset;
 224     _jitc->function->epilog = jit_new_node_no_link(jit_code_epilog);
 225     /*  u:      label value
 226      *  v:      offset in blocks vector
 227      *  w:      offset in functions vector
 228      */
 229     _jitc->function->epilog->w.w = offset;
 230
 231     jit_regset_new(&_jitc->function->regset);
 232 }
 233
 234 jit_int32_t
 235 _jit_allocai(jit_state_t *_jit, jit_int32_t length)
 236 {
 237     assert(_jitc->function);
 238     switch (length) {
 239         case 0: case 1:                                         break;
 240         case 2:         _jitc->function->self.aoff &= -2;       break;
 241         case 3: case 4: _jitc->function->self.aoff &= -4;       break;
 242         default:        _jitc->function->self.aoff &= -8;       break;
 243     }
 244     _jitc->function->self.aoff -= length;
 245     if (!_jitc->realize) {
 246         jit_inc_synth_ww(allocai, _jitc->function->self.aoff, length);
 247         jit_dec_synth();
 248     }
 249     return (_jitc->function->self.aoff);
 250 }
 251
 252 void
 253 _jit_allocar(jit_state_t *_jit, jit_int32_t u, jit_int32_t v)
 254 {
 255     jit_int32_t          reg;
 256     assert(_jitc->function);
 257     jit_inc_synth_ww(allocar, u, v);
 258     if (!_jitc->function->allocar) {
 259         _jitc->function->aoffoff = jit_allocai(sizeof(jit_int32_t));
 260         _jitc->function->allocar = 1;
 261     }
 262     reg = jit_get_reg(jit_class_gpr);
 263     jit_negr(reg, v);
 264     jit_andi(reg, reg, -8);
 265     jit_ldxi_i(u, JIT_FP, _jitc->function->aoffoff);
 266     jit_addr(u, u, reg);
 267     jit_addr(JIT_SP, JIT_SP, reg);
 268     jit_stxi_i(_jitc->function->aoffoff, JIT_FP, u);
 269     jit_unget_reg(reg);
 270     jit_dec_synth();
 271 }
 272
 273 void
 274 _jit_ret(jit_state_t *_jit)
 275 {
 276     jit_node_t          *instr;
 277     assert(_jitc->function);
 278     jit_inc_synth(ret);
 279     /* jump to epilog */
 280     instr = jit_jmpi();
 281     jit_patch_at(instr, _jitc->function->epilog);
 282     jit_dec_synth();
 283 }
 284
 285 void
 286 _jit_retr(jit_state_t *_jit, jit_int32_t u, jit_code_t code)
 287 {
 288     jit_code_inc_synth_w(code, u);
 289     jit_movr(JIT_RET, u);
 290     jit_ret();
 291     jit_dec_synth();
 292 }
 293
 294 void
 295 _jit_reti(jit_state_t *_jit, jit_word_t u, jit_code_t code)
 296 {
 297     jit_code_inc_synth_w(code, u);
 298     jit_movi(JIT_RET, u);
 299     jit_ret();
 300     jit_dec_synth();
 301 }
 302
 303 void
 304 _jit_retr_f(jit_state_t *_jit, jit_int32_t u)
 305 {
 306     jit_inc_synth_w(retr_f, u);
 307     jit_movr_f(JIT_FRET, u);
 308     jit_ret();
 309     jit_dec_synth();
 310 }
 311
 312 void
 313 _jit_reti_f(jit_state_t *_jit, jit_float32_t u)
 314 {
 315     jit_inc_synth_f(reti_f, u);
 316     jit_movi_f(JIT_FRET, u);
 317     jit_ret();
 318     jit_dec_synth();
 319 }
 320
 321 void
 322 _jit_retr_d(jit_state_t *_jit, jit_int32_t u)
 323 {
 324     jit_inc_synth_w(retr_d, u);
 325     jit_movr_d(JIT_FRET, u);
 326     jit_ret();
 327     jit_dec_synth();
 328 }
 329
 330 void
 331 _jit_reti_d(jit_state_t *_jit, jit_float64_t u)
 332 {
 333     jit_inc_synth_d(reti_d, u);
 334     jit_movi_d(JIT_FRET, u);
 335     jit_ret();
 336     jit_dec_synth();
 337 }
 338
 339 void
 340 _jit_epilog(jit_state_t *_jit)
 341 {
 342     assert(_jitc->function);
 343     assert(_jitc->function->epilog->next == NULL);
 344     jit_link(_jitc->function->epilog);
 345     _jitc->function = NULL;
 346 }
 347
 348 jit_bool_t
 349 _jit_arg_register_p(jit_state_t *_jit, jit_node_t *u)
 350 {
 351     if (u->code >= jit_code_arg_c && u->code <= jit_code_arg)
 352         return (jit_arg_reg_p(u->u.w));
 353     assert(u->code == jit_code_arg_f || u->code == jit_code_arg_d);
 354     return (jit_arg_f_reg_p(u->u.w));
 355 }
 356
 357 void
 358 _jit_ellipsis(jit_state_t *_jit)
 359 {
 360     jit_inc_synth(ellipsis);
 361     if (_jitc->prepare) {
 362         jit_link_prepare();
 363         assert(!(_jitc->function->call.call & jit_call_varargs));
 364         _jitc->function->call.call |= jit_call_varargs;
 365     }
 366     else {
 367         jit_link_prolog();
 368         assert(!(_jitc->function->self.call & jit_call_varargs));
 369         _jitc->function->self.call |= jit_call_varargs;
 370
 371         /* Allocate va_list like object in the stack. */
 372         _jitc->function->vaoff = jit_allocai(sizeof(jit_va_list_t));
 373
 374         /* Initialize gp offset in save area. */
 375         if (jit_arg_reg_p(_jitc->function->self.argi))
 376             _jitc->function->vagp = _jitc->function->self.argi;
 377         else
 378             _jitc->function->vagp = 5;
 379
 380         /* Initialize fp offset in save area. */
 381         if (jit_arg_f_reg_p(_jitc->function->self.argf))
 382             _jitc->function->vafp = _jitc->function->self.argf;
 383         else
 384             _jitc->function->vafp = NUM_FLOAT_REG_ARGS;
 385     }
 386     jit_dec_synth();
 387 }
 388
 389 void
 390 _jit_va_push(jit_state_t *_jit, jit_int32_t u)
 391 {
 392     jit_inc_synth_w(va_push, u);
 393     jit_pushargr(u);
 394     jit_dec_synth();
 395 }
 396
 397 jit_node_t *
 398 _jit_arg(jit_state_t *_jit, jit_code_t code)
 399 {
 400     jit_node_t          *node;
 401     jit_int32_t          offset;
 402     assert(_jitc->function);
 403     assert(!(_jitc->function->self.call & jit_call_varargs));
 404 #if STRONG_TYPE_CHECKING
 405     assert(code >= jit_code_arg_c && code <= jit_code_arg);
 406 #endif
 407     if (jit_arg_reg_p(_jitc->function->self.argi))
 408         offset = _jitc->function->self.argi++;
 409     else {
 410         offset = _jitc->function->self.size;
 411         _jitc->function->self.size += sizeof(jit_word_t);
 412     }
 413     node = jit_new_node_ww(code, offset,
 414                            ++_jitc->function->self.argn);
 415     jit_link_prolog();
 416     return (node);
 417 }
 418
 419 jit_node_t *
 420 _jit_arg_f(jit_state_t *_jit)
 421 {
 422     jit_node_t          *node;
 423     jit_int32_t          offset;
 424     assert(_jitc->function);
 425     if (jit_arg_f_reg_p(_jitc->function->self.argf))
 426         offset = _jitc->function->self.argf++;
 427     else {
 428         offset = _jitc->function->self.size;
 429         _jitc->function->self.size += sizeof(jit_word_t);
 430     }
 431     node = jit_new_node_ww(jit_code_arg_f, offset,
 432                            ++_jitc->function->self.argn);
 433     jit_link_prolog();
 434     return (node);
 435 }
 436
 437 jit_node_t *
 438 _jit_arg_d(jit_state_t *_jit)
 439 {
 440     jit_node_t          *node;
 441     jit_int32_t          offset;
 442     assert(_jitc->function);
 443     if (jit_arg_f_reg_p(_jitc->function->self.argf))
 444         offset = _jitc->function->self.argf++;
 445     else {
 446         offset = _jitc->function->self.size;
 447         _jitc->function->self.size += sizeof(jit_float64_t);
 448     }
 449     node = jit_new_node_ww(jit_code_arg_d, offset,
 450                            ++_jitc->function->self.argn);
 451     jit_link_prolog();
 452     return (node);
 453 }
 454
 455 void
 456 _jit_getarg_c(jit_state_t *_jit, jit_int32_t u, jit_node_t *v)
 457 {
 458     assert_arg_type(v->code, jit_code_arg_c);
 459     jit_inc_synth_wp(getarg_c, u, v);
 460     if (jit_arg_reg_p(v->u.w))
 461         jit_extr_c(u, _R2 - v->u.w);
 462     else
 463         jit_ldxi_c(u, JIT_FP,
 464                    v->u.w + (__WORDSIZE >> 3) - sizeof(jit_int8_t));
 465     jit_dec_synth();
 466 }
 467
 468 void
 469 _jit_getarg_uc(jit_state_t *_jit, jit_int32_t u, jit_node_t *v)
 470 {
 471     assert_arg_type(v->code, jit_code_arg_c);
 472     jit_inc_synth_wp(getarg_uc, u, v);
 473     if (jit_arg_reg_p(v->u.w))
 474         jit_extr_uc(u, _R2 - v->u.w);
 475     else
 476         jit_ldxi_uc(u, JIT_FP,
 477                     v->u.w + (__WORDSIZE >> 3) - sizeof(jit_uint8_t));
 478     jit_dec_synth();
 479 }
 480
 481 void
 482 _jit_getarg_s(jit_state_t *_jit, jit_int32_t u, jit_node_t *v)
 483 {
 484     assert_arg_type(v->code, jit_code_arg_s);
 485     jit_inc_synth_wp(getarg_s, u, v);
 486     if (jit_arg_reg_p(v->u.w))
 487         jit_extr_s(u, _R2 - v->u.w);
 488     else
 489         jit_ldxi_s(u, JIT_FP,
 490                    v->u.w + (__WORDSIZE >> 3) - sizeof(jit_int16_t));
 491     jit_dec_synth();
 492 }
 493
 494 void
 495 _jit_getarg_us(jit_state_t *_jit, jit_int32_t u, jit_node_t *v)
 496 {
 497     assert_arg_type(v->code, jit_code_arg_s);
 498     jit_inc_synth_wp(getarg_us, u, v);
 499     if (jit_arg_reg_p(v->u.w))
 500         jit_extr_us(u, _R2 - v->u.w);
 501     else
 502         jit_ldxi_us(u, JIT_FP,
 503                     v->u.w + (__WORDSIZE >> 3) - sizeof(jit_uint16_t));
 504     jit_dec_synth();
 505 }
 506
 507 void
 508 _jit_getarg_i(jit_state_t *_jit, jit_int32_t u, jit_node_t *v)
 509 {
 510     assert_arg_type(v->code, jit_code_arg_i);
 511     jit_inc_synth_wp(getarg_i, u, v);
 512     if (jit_arg_reg_p(v->u.w)) {
 513 #if __WORDSIZE == 32
 514         jit_movr(u, _R2 - v->u.w);
 515 #else
 516         jit_extr_i(u, _R2 - v->u.w);
 517 #endif
 518     }
 519     else
 520         jit_ldxi_i(u, JIT_FP,
 521                    v->u.w + (__WORDSIZE >> 3) - sizeof(jit_int32_t));
 522     jit_dec_synth();
 523 }
 524
 525 #if __WORDSIZE == 64
 526 void
 527 _jit_getarg_ui(jit_state_t *_jit, jit_int32_t u, jit_node_t *v)
 528 {
 529     assert_arg_type(v->code, jit_code_arg_i);
 530     jit_inc_synth_wp(getarg_ui, u, v);
 531     if (jit_arg_reg_p(v->u.w))
 532         jit_extr_ui(u, _R2 - v->u.w);
 533     else
 534         jit_ldxi_ui(u, JIT_FP,
 535                     v->u.w + (__WORDSIZE >> 3) - sizeof(jit_uint32_t));
 536     jit_dec_synth();
 537 }
 538
 539 void
 540 _jit_getarg_l(jit_state_t *_jit, jit_int32_t u, jit_node_t *v)
 541 {
 542     assert_arg_type(v->code, jit_code_arg_l);
 543     jit_inc_synth_wp(getarg_l, u, v);
 544     if (jit_arg_reg_p(v->u.w))
 545         jit_movr(u, _R2 - v->u.w);
 546     else
 547         jit_ldxi_l(u, JIT_FP, v->u.w);
 548     jit_dec_synth();
 549 }
 550 #endif
 551
 552 void
 553 _jit_putargr(jit_state_t *_jit, jit_int32_t u, jit_node_t *v, jit_code_t code)
 554 {
 555     assert_putarg_type(code, v->code);
 556     jit_code_inc_synth_wp(code, u, v);
 557     if (jit_arg_reg_p(v->u.w))
 558         jit_movr(_R2 - v->u.w, u);
 559     else
 560         jit_stxi(v->u.w, JIT_FP, u);
 561     jit_dec_synth();
 562 }
 563
 564 void
 565 _jit_putargi(jit_state_t *_jit, jit_word_t u, jit_node_t *v, jit_code_t code)
 566 {
 567     jit_int32_t         regno;
 568     assert_putarg_type(code, v->code);
 569     jit_code_inc_synth_wp(code, u, v);
 570     if (jit_arg_reg_p(v->u.w))
 571         jit_movi(_R2 - v->u.w, u);
 572     else {
 573         regno = jit_get_reg(jit_class_gpr);
 574         jit_movi(regno, u);
 575         jit_stxi(v->u.w, JIT_FP, regno);
 576         jit_unget_reg(regno);
 577     }
 578     jit_dec_synth();
 579 }
 580
 581 void
 582 _jit_getarg_f(jit_state_t *_jit, jit_int32_t u, jit_node_t *v)
 583 {
 584     assert(v->code == jit_code_arg_f);
 585     jit_inc_synth_wp(getarg_f, u, v);
 586     if (jit_arg_f_reg_p(v->u.w))
 587         jit_movr_f(u, _F0 - v->u.w);
 588     else
 589         jit_ldxi_f(u, JIT_FP,
 590                    v->u.w
 591 #if __WORDSIZE == 64
 592                    + (__WORDSIZE >> 3) - sizeof(jit_float32_t)
 593 #endif
 594                    );
 595     jit_dec_synth();
 596 }
 597
 598 void
 599 _jit_putargr_f(jit_state_t *_jit, jit_int32_t u, jit_node_t *v)
 600 {
 601     assert(v->code == jit_code_arg_f);
 602     jit_inc_synth_wp(putargr_f, u, v);
 603     if (jit_arg_f_reg_p(v->u.w))
 604         jit_movr_f(_F0 - v->u.w, u);
 605     else
 606         jit_stxi_f(v->u.w
 607 #if __WORDSIZE == 64
 608                    + (__WORDSIZE >> 3) - sizeof(jit_float32_t)
 609 #endif
 610                    , JIT_FP, u);
 611     jit_dec_synth();
 612 }
 613
 614 void
 615 _jit_putargi_f(jit_state_t *_jit, jit_float32_t u, jit_node_t *v)
 616 {
 617     jit_int32_t         regno;
 618     assert(v->code == jit_code_arg_f);
 619     jit_inc_synth_fp(putargi_f, u, v);
 620     if (jit_arg_f_reg_p(v->u.w))
 621         jit_movi_f(_F0 - v->u.w, u);
 622     else {
 623         regno = jit_get_reg(jit_class_fpr);
 624         jit_movi_f(regno, u);
 625         jit_stxi_f(v->u.w
 626 #if __WORDSIZE == 64
 627                    + (__WORDSIZE >> 3) - sizeof(jit_float32_t)
 628 #endif
 629                    , JIT_FP, regno);
 630         jit_unget_reg(regno);
 631     }
 632     jit_dec_synth();
 633 }
 634
 635 void
 636 _jit_getarg_d(jit_state_t *_jit, jit_int32_t u, jit_node_t *v)
 637 {
 638     assert(v->code == jit_code_arg_d);
 639     jit_inc_synth_wp(getarg_d, u, v);
 640     if (jit_arg_f_reg_p(v->u.w))
 641         jit_movr_d(u, _F0 - v->u.w);
 642     else
 643         jit_ldxi_d(u, JIT_FP, v->u.w);
 644     jit_dec_synth();
 645 }
 646
 647 void
 648 _jit_putargr_d(jit_state_t *_jit, jit_int32_t u, jit_node_t *v)
 649 {
 650     assert(v->code == jit_code_arg_d);
 651     jit_inc_synth_wp(putargr_d, u, v);
 652     if (jit_arg_f_reg_p(v->u.w))
 653         jit_movr_d(_F0 - v->u.w, u);
 654     else
 655         jit_stxi_d(v->u.w, JIT_FP, u);
 656     jit_dec_synth();
 657 }
 658
 659 void
 660 _jit_putargi_d(jit_state_t *_jit, jit_float64_t u, jit_node_t *v)
 661 {
 662     jit_int32_t         regno;
 663     assert(v->code == jit_code_arg_d);
 664     jit_inc_synth_dp(putargi_d, u, v);
 665     if (jit_arg_f_reg_p(v->u.w))
 666         jit_movi_d(_F0 - v->u.w, u);
 667     else {
 668         regno = jit_get_reg(jit_class_fpr);
 669         jit_movi_d(regno, u);
 670         jit_stxi_d(v->u.w, JIT_FP, regno);
 671         jit_unget_reg(regno);
 672     }
 673     jit_dec_synth();
 674 }
 675
 676 void
 677 _jit_pushargr(jit_state_t *_jit, jit_int32_t u, jit_code_t code)
 678 {
 679     assert(_jitc->function);
 680     jit_code_inc_synth_w(code, u);
 681     jit_link_prepare();
 682     if (jit_arg_reg_p(_jitc->function->call.argi)) {
 683         jit_movr(_R2 - _jitc->function->call.argi, u);
 684         ++_jitc->function->call.argi;
 685     }
 686     else {
 687         jit_stxi(_jitc->function->call.size + stack_framesize, JIT_SP, u);
 688         _jitc->function->call.size += sizeof(jit_word_t);
 689     }
 690     jit_dec_synth();
 691 }
 692
 693 void
 694 _jit_pushargi(jit_state_t *_jit, jit_word_t u, jit_code_t code)
 695 {
 696     jit_int32_t          regno;
 697     assert(_jitc->function);
 698     jit_code_inc_synth_w(code, u);
 699     jit_link_prepare();
 700     if (jit_arg_reg_p(_jitc->function->call.argi)) {
 701         jit_movi(_R2 - _jitc->function->call.argi, u);
 702         ++_jitc->function->call.argi;
 703     }
 704     else {
 705         regno = jit_get_reg(jit_class_gpr);
 706         jit_movi(regno, u);
 707         jit_stxi(_jitc->function->call.size + stack_framesize, JIT_SP, regno);
 708         jit_unget_reg(regno);
 709         _jitc->function->call.size += sizeof(jit_word_t);
 710     }
 711     jit_dec_synth();
 712 }
 713
 714 void
 715 _jit_pushargr_f(jit_state_t *_jit, jit_int32_t u)
 716 {
 717     assert(_jitc->function);
 718     jit_inc_synth_w(pushargr_f, u);
 719     jit_link_prepare();
 720     if (jit_arg_f_reg_p(_jitc->function->call.argf)) {
 721         jit_movr_f(_F0 - _jitc->function->call.argf, u);
 722         ++_jitc->function->call.argf;
 723     }
 724     else {
 725         jit_stxi_f(_jitc->function->call.size + stack_framesize
 726 #if __WORDSIZE == 64
 727                    + (__WORDSIZE >> 3) - sizeof(jit_float32_t)
 728 #endif
 729                    , JIT_SP, u);
 730         _jitc->function->call.size += sizeof(jit_word_t);
 731     }
 732     jit_dec_synth();
 733 }
 734
 735 void
 736 _jit_pushargi_f(jit_state_t *_jit, jit_float32_t u)
 737 {
 738     jit_int32_t         regno;
 739     assert(_jitc->function);
 740     jit_inc_synth_f(pushargi_f, u);
 741     jit_link_prepare();
 742     if (jit_arg_f_reg_p(_jitc->function->call.argf)) {
 743         jit_movi_f(_F0 - _jitc->function->call.argf, u);
 744         ++_jitc->function->call.argf;
 745     }
 746     else {
 747         regno = jit_get_reg(jit_class_fpr);
 748         jit_movi_f(regno, u);
 749         jit_stxi_f(_jitc->function->call.size + stack_framesize
 750 #if __WORDSIZE == 64
 751                    + (__WORDSIZE >> 3) - sizeof(jit_float32_t)
 752 #endif
 753                    , JIT_SP, regno);
 754         jit_unget_reg(regno);
 755         _jitc->function->call.size += sizeof(jit_word_t);
 756     }
 757     jit_dec_synth();
 758 }
 759
 760 void
 761 _jit_pushargr_d(jit_state_t *_jit, jit_int32_t u)
 762 {
 763     assert(_jitc->function);
 764     jit_inc_synth_w(pushargr_d, u);
 765     jit_link_prepare();
 766     if (jit_arg_f_reg_p(_jitc->function->call.argf)) {
 767         jit_movr_d(_F0 - _jitc->function->call.argf, u);
 768         ++_jitc->function->call.argf;
 769     }
 770     else {
 771         jit_stxi_d(_jitc->function->call.size + stack_framesize, JIT_SP, u);
 772         _jitc->function->call.size += sizeof(jit_float64_t);
 773     }
 774     jit_dec_synth();
 775 }
 776
 777 void
 778 _jit_pushargi_d(jit_state_t *_jit, jit_float64_t u)
 779 {
 780     jit_int32_t         regno;
 781     assert(_jitc->function);
 782     jit_inc_synth_d(pushargi_d, u);
 783     jit_link_prepare();
 784     if (jit_arg_f_reg_p(_jitc->function->call.argf)) {
 785         jit_movi_d(_F0 - _jitc->function->call.argf, u);
 786         ++_jitc->function->call.argf;
 787     }
 788     else {
 789         regno = jit_get_reg(jit_class_fpr);
 790         jit_movi_d(regno, u);
 791         jit_stxi_d(_jitc->function->call.size + stack_framesize, JIT_SP, regno);
 792         jit_unget_reg(regno);
 793         _jitc->function->call.size += sizeof(jit_float64_t);
 794     }
 795     jit_dec_synth();
 796 }
 797
 798 jit_bool_t
 799 _jit_regarg_p(jit_state_t *_jit, jit_node_t *node, jit_int32_t regno)
 800 {
 801     jit_int32_t         spec;
 802     spec = jit_class(_rvs[regno].spec);
 803     if (spec & jit_class_arg) {
 804         regno = _R2 - regno;
 805         if (regno >= 0 && regno < node->v.w)
 806             return (1);
 807         if (spec & jit_class_fpr) {
 808             regno = _F0 - regno;
 809             if (regno >= 0 && regno < node->w.w)
 810                 return (1);
 811         }
 812     }
 813     return (0);
 814 }
 815
 816 void
 817 _jit_finishr(jit_state_t *_jit, jit_int32_t r0)
 818 {
 819     jit_node_t          *call;
 820     assert(_jitc->function);
 821     jit_inc_synth_w(finishr, r0);
 822     if (_jitc->function->self.alen < _jitc->function->call.size)
 823         _jitc->function->self.alen = _jitc->function->call.size;
 824     call = jit_callr(r0);
 825     call->v.w = _jitc->function->call.argi;
 826     call->w.w = _jitc->function->call.argf;
 827     _jitc->function->call.argi = _jitc->function->call.argf =
 828         _jitc->function->call.size = 0;
 829     _jitc->prepare = 0;
 830     jit_dec_synth();
 831 }
 832
 833 jit_node_t *
 834 _jit_finishi(jit_state_t *_jit, jit_pointer_t i0)
 835 {
 836     jit_node_t          *node;
 837     assert(_jitc->function);
 838     jit_inc_synth_w(finishi, (jit_word_t)i0);
 839     if (_jitc->function->self.alen < _jitc->function->call.size)
 840         _jitc->function->self.alen = _jitc->function->call.size;
 841     node = jit_calli(i0);
 842     node->v.w = _jitc->function->call.argi;
 843     node->w.w = _jitc->function->call.argf;
 844     _jitc->function->call.argi = _jitc->function->call.argf =
 845         _jitc->function->call.size = 0;
 846     _jitc->prepare = 0;
 847     jit_dec_synth();
 848     return (node);
 849 }
 850
 851 void
 852 _jit_retval_c(jit_state_t *_jit, jit_int32_t r0)
 853 {
 854     jit_inc_synth_w(retval_c, r0);
 855     jit_extr_c(r0, JIT_RET);
 856     jit_dec_synth();
 857 }
 858
 859 void
 860 _jit_retval_uc(jit_state_t *_jit, jit_int32_t r0)
 861 {
 862     jit_inc_synth_w(retval_uc, r0);
 863     jit_extr_uc(r0, JIT_RET);
 864     jit_dec_synth();
 865 }
 866
 867 void
 868 _jit_retval_s(jit_state_t *_jit, jit_int32_t r0)
 869 {
 870     jit_inc_synth_w(retval_s, r0);
 871     jit_extr_s(r0, JIT_RET);
 872     jit_dec_synth();
 873 }
 874
 875 void
 876 _jit_retval_us(jit_state_t *_jit, jit_int32_t r0)
 877 {
 878     jit_inc_synth_w(retval_us, r0);
 879     jit_extr_us(r0, JIT_RET);
 880     jit_dec_synth();
 881 }
 882
 883 void
 884 _jit_retval_i(jit_state_t *_jit, jit_int32_t r0)
 885 {
 886     jit_inc_synth_w(retval_i, r0);
 887 #if __WORDSIZE == 64
 888     jit_extr_i(r0, JIT_RET);
 889 #else
 890     jit_movr(r0, JIT_RET);
 891 #endif
 892     jit_dec_synth();
 893 }
 894
 895 #if __WORDSIZE == 64
 896 void
 897 _jit_retval_ui(jit_state_t *_jit, jit_int32_t r0)
 898 {
 899     jit_inc_synth_w(retval_ui, r0);
 900     jit_extr_ui(r0, JIT_RET);
 901     jit_dec_synth();
 902 }
 903
 904 void
 905 _jit_retval_l(jit_state_t *_jit, jit_int32_t r0)
 906 {
 907     jit_inc_synth_w(retval_l, r0);
 908     jit_movr(r0, JIT_RET);
 909     jit_dec_synth();
 910 }
 911 #endif
 912
 913 void
 914 _jit_retval_f(jit_state_t *_jit, jit_int32_t r0)
 915 {
 916     jit_inc_synth_w(retval_f, r0);
 917     jit_movr_f(r0, JIT_FRET);
 918     jit_dec_synth();
 919 }
 920
 921 void
 922 _jit_retval_d(jit_state_t *_jit, jit_int32_t r0)
 923 {
 924     jit_inc_synth_w(retval_d, r0);
 925     jit_movr_d(r0, JIT_FRET);
 926     jit_dec_synth();
 927 }
 928
 929 jit_pointer_t
 930 _emit_code(jit_state_t *_jit)
 931 {
 932     jit_node_t          *node;
 933     jit_node_t          *temp;
 934     jit_word_t           word;
 935     jit_int32_t          value;
 936     jit_int32_t          offset;
 937     struct {
 938         jit_node_t      *node;
 939         jit_word_t       word;
 940         jit_function_t   func;
 941 #if DEVEL_DISASSEMBLER
 942         jit_word_t       prevw;
 943 #endif
 944         jit_int32_t      patch_offset;
 945     } undo;
 946 #if DEVEL_DISASSEMBLER
 947     jit_word_t           prevw;
 948 #endif
 949
 950     _jitc->function = NULL;
 951
 952     jit_reglive_setup();
 953
 954     undo.word = 0;
 955     undo.node = NULL;
 956     undo.patch_offset = 0;
 957
 958 #define assert_data(node)               /**/
 959 #define case_rr(name, type)                                             \
 960             case jit_code_##name##r##type:                              \
 961                 name##r##type(rn(node->u.w), rn(node->v.w));            \
 962                 break
 963 #define case_rw(name, type)                                             \
 964             case jit_code_##name##i##type:                              \
 965                 name##i##type(rn(node->u.w), node->v.w);                \
 966                 break
 967 #define case_wr(name, type)                                             \
 968             case jit_code_##name##i##type:                              \
 969                 name##i##type(node->u.w, rn(node->v.w));                \
 970                 break
 971 #define case_rrr(name, type)                                            \
 972             case jit_code_##name##r##type:                              \
 973                 name##r##type(rn(node->u.w),                            \
 974                               rn(node->v.w), rn(node->w.w));            \
 975                 break
 976 #define case_rrrr(name, type)                                           \
 977             case jit_code_##name##r##type:                              \
 978                 name##r##type(rn(node->u.q.l), rn(node->u.q.h),         \
 979                               rn(node->v.w), rn(node->w.w));            \
 980                 break
 981 #define case_rqr(name, type)                                            \
 982             case jit_code_##name##r##type:                              \
 983                 name##r##type(rn(node->u.w), rn(node->v.q.l),           \
 984                               rn(node->v.q.h), rn(node->w.w));          \
 985             case jit_code_##name##i##type:                              \
 986                 break;
 987 #define case_rrw(name, type)                                            \
 988             case jit_code_##name##i##type:                              \
 989                 name##i##type(rn(node->u.w), rn(node->v.w), node->w.w); \
 990                 break
 991 #define case_rrrw(name, type)                                           \
 992             case jit_code_##name##i##type:                              \
 993                 name##i##type(rn(node->u.q.l), rn(node->u.q.h),         \
 994                               rn(node->v.w), node->w.w);                \
 995                 break
 996 #define case_rrf(name)                                                  \
 997             case jit_code_##name##i_f:                                  \
 998                 assert_data(node);                                      \
 999                 name##i_f(rn(node->u.w), rn(node->v.w),                 \
1000                           (jit_float32_t *)node->w.n->u.w);             \
1001                 break
1002 #define case_rrd(name)                                                  \
1003             case jit_code_##name##i_d:                                  \
1004                 assert_data(node);                                      \
1005                 name##i_d(rn(node->u.w), rn(node->v.w),                 \
1006                           (jit_float64_t *)node->w.n->u.w);             \
1007                 break
1008 #define case_wrr(name, type)                                            \
1009             case jit_code_##name##i##type:                              \
1010                 name##i##type(node->u.w, rn(node->v.w), rn(node->w.w)); \
1011                 break
1012 #define case_brr(name, type)                                            \
1013             case jit_code_##name##r##type:                              \
1014                 temp = node->u.n;                                       \
1015                 assert(temp->code == jit_code_label ||                  \
1016                        temp->code == jit_code_epilog);                  \
1017                 if (temp->flag & jit_flag_patch)                        \
1018                     name##r##type(temp->u.w, rn(node->v.w),             \
1019                                   rn(node->w.w));                       \
1020                 else {                                                  \
1021                     word = name##r##type##_p(_jit->pc.w,                \
1022                                              rn(node->v.w),             \
1023                                              rn(node->w.w));            \
1024                     patch(word, node);                                  \
1025                 }                                                       \
1026                 break
1027 #define case_brw(name, type)                                            \
1028             case jit_code_##name##i##type:                              \
1029                 temp = node->u.n;                                       \
1030                 assert(temp->code == jit_code_label ||                  \
1031                        temp->code == jit_code_epilog);                  \
1032                 if (temp->flag & jit_flag_patch)                        \
1033                     name##i##type(temp->u.w,                            \
1034                                   rn(node->v.w), node->w.w);            \
1035                 else {                                                  \
1036                     word = name##i##type##_p(_jit->pc.w,                \
1037                                              rn(node->v.w), node->w.w); \
1038                     patch(word, node);                                  \
1039                 }                                                       \
1040                 break;
1041 #define case_brf(name)                                                  \
1042             case jit_code_##name##i_f:                                  \
1043                 temp = node->u.n;                                       \
1044                 assert(temp->code == jit_code_label ||                  \
1045                        temp->code == jit_code_epilog);                  \
1046                 if (temp->flag & jit_flag_patch)                        \
1047                     name##i_f(temp->u.w, rn(node->v.w),                 \
1048                               (jit_float32_t *)node->w.n->u.w);         \
1049                 else {                                                  \
1050                     word = name##i_f_p(_jit->pc.w, rn(node->v.w),       \
1051                                        (jit_float32_t *)node->w.n->u.w);\
1052                     patch(word, node);                                  \
1053                 }                                                       \
1054                 break
1055 #define case_brd(name)                                                  \
1056             case jit_code_##name##i_d:                                  \
1057                 temp = node->u.n;                                       \
1058                 assert(temp->code == jit_code_label ||                  \
1059                        temp->code == jit_code_epilog);                  \
1060                 if (temp->flag & jit_flag_patch)                        \
1061                     name##i_d(temp->u.w, rn(node->v.w),                 \
1062                               (jit_float64_t *)node->w.n->u.w);         \
1063                 else {                                                  \
1064                     word = name##i_d_p(_jit->pc.w, rn(node->v.w),       \
1065                                        (jit_float64_t *)node->w.n->u.w);\
1066                     patch(word, node);                                  \
1067                 }                                                       \
1068                 break
1069 #if DEVEL_DISASSEMBLER
1070     prevw = _jit->pc.w;
1071 #endif
1072     for (node = _jitc->head; node; node = node->next) {
1073         if (_jit->pc.uc >= _jitc->code.end)
1074             return (NULL);
1075
1076 #if DEVEL_DISASSEMBLER
1077         node->offset = (jit_uword_t)_jit->pc.w - (jit_uword_t)prevw;
1078         prevw = _jit->pc.w;
1079 #endif
1080         value = jit_classify(node->code);
1081         jit_regarg_set(node, value);
1082         switch (node->code) {
1083             case jit_code_align:
1084                 /* Must align to a power of two */
1085                 assert(!(node->u.w & (node->u.w - 1)));
1086                 if ((word = _jit->pc.w & (node->u.w - 1)))
1087                     nop(node->u.w - word);
1088                 break;
1089             case jit_code_skip:
1090                 nop((node->u.w + 1) & ~1);
1091                 break;
1092             case jit_code_note:         case jit_code_name:
1093                 node->u.w = _jit->pc.w;
1094                 break;
1095             case jit_code_label:
1096                 if ((node->link || (node->flag & jit_flag_use)) &&
1097                     (word = _jit->pc.w & 3))
1098                     nop(4 - word);
1099                 /* remember label is defined */
1100                 node->flag |= jit_flag_patch;
1101                 node->u.w = _jit->pc.w;
1102                 break;
1103                 case_rrr(add,);
1104                 case_rrw(add,);
1105                 case_rrr(addc,);
1106                 case_rrw(addc,);
1107                 case_rrr(addx,);
1108                 case_rrw(addx,);
1109                 case_rrr(sub,);
1110                 case_rrw(sub,);
1111                 case_rrr(subc,);
1112                 case_rrw(subc,);
1113                 case_rrr(subx,);
1114                 case_rrw(subx,);
1115                 case_rrw(rsb,);
1116                 case_rrr(mul,);
1117                 case_rrw(mul,);
1118                 case_rrr(hmul,);
1119                 case_rrw(hmul,);
1120                 case_rrr(hmul, _u);
1121                 case_rrw(hmul, _u);
1122                 case_rrrr(qmul,);
1123                 case_rrrw(qmul,);
1124                 case_rrrr(qmul, _u);
1125                 case_rrrw(qmul, _u);
1126                 case_rrr(div,);
1127                 case_rrw(div,);
1128                 case_rrr(div, _u);
1129                 case_rrw(div, _u);
1130                 case_rrr(rem,);
1131                 case_rrw(rem,);
1132                 case_rrr(rem, _u);
1133                 case_rrw(rem, _u);
1134                 case_rrrr(qdiv,);
1135                 case_rrrw(qdiv,);
1136                 case_rrrr(qdiv, _u);
1137                 case_rrrw(qdiv, _u);
1138                 case_rrr(lsh,);
1139                 case_rrw(lsh,);
1140 #define qlshr(r0, r1, r2, r3)   fallback_qlshr(r0, r1, r2, r3)
1141 #define qlshi(r0, r1, r2, i0)   fallback_qlshi(r0, r1, r2, i0)
1142 #define qlshr_u(r0, r1, r2, r3) fallback_qlshr_u(r0, r1, r2, r3)
1143 #define qlshi_u(r0, r1, r2, i0) fallback_qlshi_u(r0, r1, r2, i0)
1144                 case_rrrr(qlsh,);
1145                 case_rrrw(qlsh,);
1146                 case_rrrr(qlsh, _u);
1147                 case_rrrw(qlsh, _u);
1148                 case_rrr(rsh,);
1149                 case_rrw(rsh,);
1150                 case_rrr(rsh, _u);
1151                 case_rrw(rsh, _u);
1152 #define qrshr(r0, r1, r2, r3)   fallback_qrshr(r0, r1, r2, r3)
1153 #define qrshi(r0, r1, r2, i0)   fallback_qrshi(r0, r1, r2, i0)
1154 #define qrshr_u(r0, r1, r2, r3) fallback_qrshr_u(r0, r1, r2, r3)
1155 #define qrshi_u(r0, r1, r2, i0) fallback_qrshi_u(r0, r1, r2, i0)
1156                 case_rrrr(qrsh,);
1157                 case_rrrw(qrsh,);
1158                 case_rrrr(qrsh, _u);
1159                 case_rrrw(qrsh, _u);
1160                 case_rrr(lrot,);
1161                 case_rrw(lrot,);
1162                 case_rrr(rrot,);
1163                 case_rrw(rrot,);
1164                 case_rr(neg,);
1165                 case_rr(com,);
1166                 case_rr(clo,);
1167                 case_rr(clz,);
1168                 case_rr(cto,);
1169                 case_rr(ctz,);
1170 #define rbitr(r0, r1)   fallback_rbit(r0, r1)
1171 #define popcntr(r0, r1) fallback_popcnt(r0, r1)
1172                 case_rr(rbit,);
1173                 case_rr(popcnt,);
1174                 case_rrr(and,);
1175                 case_rrw(and,);
1176                 case_rrr(or,);
1177                 case_rrw(or,);
1178                 case_rrr(xor,);
1179                 case_rrw(xor,);
1180                 case_rr(trunc, _f_i);
1181                 case_rr(trunc, _d_i);
1182 #if __WORDSIZE == 64
1183                 case_rr(trunc, _f_l);
1184                 case_rr(trunc, _d_l);
1185 #endif
1186                 case_rr(ld, _c);
1187                 case_rw(ld, _c);
1188                 case_rr(ld, _uc);
1189                 case_rw(ld, _uc);
1190                 case_rr(ld, _s);
1191                 case_rw(ld, _s);
1192                 case_rr(ld, _us);
1193                 case_rw(ld, _us);
1194                 case_rr(ld, _i);
1195                 case_rw(ld, _i);
1196 #if __WORDSIZE == 64
1197                 case_rr(ld, _ui);
1198                 case_rw(ld, _ui);
1199                 case_rr(ld, _l);
1200                 case_rw(ld, _l);
1201 #endif
1202                 case_rrr(ldx, _c);
1203                 case_rrw(ldx, _c);
1204                 case_rrr(ldx, _uc);
1205                 case_rrw(ldx, _uc);
1206                 case_rrr(ldx, _s);
1207                 case_rrw(ldx, _s);
1208                 case_rrr(ldx, _us);
1209                 case_rrw(ldx, _us);
1210                 case_rrr(ldx, _i);
1211                 case_rrw(ldx, _i);
1212 #if __WORDSIZE == 64
1213                 case_rrr(ldx, _ui);
1214                 case_rrw(ldx, _ui);
1215                 case_rrr(ldx, _l);
1216                 case_rrw(ldx, _l);
1217 #endif
1218 #define unldr(r0, r1, i0)       fallback_unldr(r0, r1, i0)
1219             case jit_code_unldr:
1220                 unldr(rn(node->u.w), rn(node->v.w), node->w.w);
1221                 break;
1222 #define unldi(r0, i0, i1)       fallback_unldi(r0, i0, i1)
1223             case jit_code_unldi:
1224                 unldi(rn(node->u.w), node->v.w, node->w.w);
1225                 break;
1226 #define unldr_u(r0, r1, i0)     fallback_unldr_u(r0, r1, i0)
1227             case jit_code_unldr_u:
1228                 unldr_u(rn(node->u.w), rn(node->v.w), node->w.w);
1229                 break;
1230 #define unldi_u(r0, i0, i1)     fallback_unldi_u(r0, i0, i1)
1231             case jit_code_unldi_u:
1232                 unldi_u(rn(node->u.w), node->v.w, node->w.w);
1233                 break;
1234                 case_rr(st, _c);
1235                 case_wr(st, _c);
1236                 case_rr(st, _s);
1237                 case_wr(st, _s);
1238                 case_rr(st, _i);
1239                 case_wr(st, _i);
1240 #if __WORDSIZE == 64
1241                 case_rr(st, _l);
1242                 case_wr(st, _l);
1243 #endif
1244                 case_rrr(stx, _c);
1245                 case_wrr(stx, _c);
1246                 case_rrr(stx, _s);
1247                 case_wrr(stx, _s);
1248                 case_rrr(stx, _i);
1249                 case_wrr(stx, _i);
1250 #if __WORDSIZE == 64
1251                 case_rrr(stx, _l);
1252                 case_wrr(stx, _l);
1253 #endif
1254 #define unstr(r0, r1, i0)       fallback_unstr(r0, r1, i0)
1255             case jit_code_unstr:
1256                 unstr(rn(node->u.w), rn(node->v.w), node->w.w);
1257                 break;
1258 #define unsti(i0, r0, i1)       fallback_unsti(i0, r0, i1)
1259             case jit_code_unsti:
1260                 unsti(node->u.w, rn(node->v.w), node->w.w);
1261                 break;
1262                 case_rr(hton, _us);
1263                 case_rr(hton, _ui);
1264 #if __WORDSIZE == 64
1265                 case_rr(hton, _ul);
1266 #endif
1267                 case_rr(bswap, _us);
1268                 case_rr(bswap, _ui);
1269 #if __WORDSIZE == 64
1270                 case_rr(bswap, _ul);
1271 #endif
1272             case jit_code_extr:
1273                 extr(rn(node->u.w), rn(node->v.w), node->w.q.l, node->w.q.h);
1274                 break;
1275             case jit_code_extr_u:
1276                 extr_u(rn(node->u.w), rn(node->v.w), node->w.q.l, node->w.q.h);
1277                 break;
1278             case jit_code_depr:
1279                 depr(rn(node->u.w), rn(node->v.w), node->w.q.l, node->w.q.h);
1280                 break;
1281             case jit_code_depi:
1282                 depi(rn(node->u.w), node->v.w, node->w.q.l, node->w.q.h);
1283                 break;
1284                 case_rr(ext, _c);
1285                 case_rr(ext, _uc);
1286                 case_rr(ext, _s);
1287                 case_rr(ext, _us);
1288 #if __WORDSIZE == 64
1289                 case_rr(ext, _i);
1290                 case_rr(ext, _ui);
1291 #endif
1292             case jit_code_casr:
1293                 casr(rn(node->u.w), rn(node->v.w),
1294                      rn(node->w.q.l), rn(node->w.q.h));
1295                 break;
1296             case jit_code_casi:
1297                 casi(rn(node->u.w), node->v.w,
1298                      rn(node->w.q.l), rn(node->w.q.h));
1299                 break;
1300                 case_rrr(movn,);
1301                 case_rrr(movz,);
1302                 case_rr(mov,);
1303             case jit_code_movi:
1304                 if (node->flag & jit_flag_node) {
1305                     temp = node->v.n;
1306                     if (temp->code == jit_code_data ||
1307                         (temp->code == jit_code_label &&
1308                          (temp->flag & jit_flag_patch)))
1309                         movi(rn(node->u.w), temp->u.w);
1310                     else {
1311                         assert(temp->code == jit_code_label ||
1312                                temp->code == jit_code_epilog);
1313                         word = movi_p(rn(node->u.w), temp->u.w);
1314                         patch(word, node);
1315                     }
1316                 }
1317                 else
1318                     movi(rn(node->u.w), node->v.w);
1319                 break;
1320                 case_rrr(lt,);
1321                 case_rrw(lt,);
1322                 case_rrr(lt, _u);
1323                 case_rrw(lt, _u);
1324                 case_rrr(le,);
1325                 case_rrw(le,);
1326                 case_rrr(le, _u);
1327                 case_rrw(le, _u);
1328                 case_rrr(eq,);
1329                 case_rrw(eq,);
1330                 case_rrr(ge,);
1331                 case_rrw(ge,);
1332                 case_rrr(ge, _u);
1333                 case_rrw(ge, _u);
1334                 case_rrr(gt,);
1335                 case_rrw(gt,);
1336                 case_rrr(gt, _u);
1337                 case_rrw(gt, _u);
1338                 case_rrr(ne,);
1339                 case_rrw(ne,);
1340                 case_brr(blt,);
1341                 case_brw(blt,);
1342                 case_brr(blt, _u);
1343                 case_brw(blt, _u);
1344                 case_brr(ble,);
1345                 case_brw(ble,);
1346                 case_brr(ble, _u);
1347                 case_brw(ble, _u);
1348                 case_brr(beq,);
1349                 case_brw(beq,);
1350                 case_brr(bge,);
1351                 case_brw(bge,);
1352                 case_brr(bge, _u);
1353                 case_brw(bge, _u);
1354                 case_brr(bgt,);
1355                 case_brw(bgt,);
1356                 case_brr(bgt, _u);
1357                 case_brw(bgt, _u);
1358                 case_brr(bne,);
1359                 case_brw(bne,);
1360                 case_brr(boadd,);
1361                 case_brw(boadd,);
1362                 case_brr(boadd, _u);
1363                 case_brw(boadd, _u);
1364                 case_brr(bxadd,);
1365                 case_brw(bxadd,);
1366                 case_brr(bxadd, _u);
1367                 case_brw(bxadd, _u);
1368                 case_brr(bosub,);
1369                 case_brw(bosub,);
1370                 case_brr(bosub, _u);
1371                 case_brw(bosub, _u);
1372                 case_brr(bxsub,);
1373                 case_brw(bxsub,);
1374                 case_brr(bxsub, _u);
1375                 case_brw(bxsub, _u);
1376                 case_brr(bms,);
1377                 case_brw(bms,);
1378                 case_brr(bmc,);
1379                 case_brw(bmc,);
1380                 case_rrr(add, _f);
1381                 case_rrf(add);
1382                 case_rrr(sub, _f);
1383                 case_rrf(sub);
1384                 case_rrf(rsb);
1385                 case_rrr(mul, _f);
1386                 case_rrf(mul);
1387                 case_rrr(div, _f);
1388                 case_rrf(div);
1389                 case_rr(abs, _f);
1390                 case_rr(neg, _f);
1391                 case_rr(sqrt, _f);
1392                 case_rqr(fma, _f);
1393                 case_rqr(fms, _f);
1394                 case_rqr(fnma, _f);
1395                 case_rqr(fnms, _f);
1396                 case_rr(ext, _f);
1397                 case_rr(ld, _f);
1398                 case_rw(ld, _f);
1399                 case_rrr(ldx, _f);
1400                 case_rrw(ldx, _f);
1401 #define unldr_x(r0, r1, i0)     fallback_unldr_x(r0, r1, i0)
1402             case jit_code_unldr_x:
1403                 unldr_x(rn(node->u.w), rn(node->v.w), node->w.w);
1404                 break;
1405 #define unldi_x(r0, i0, i1)     fallback_unldi_x(r0, i0, i1)
1406             case jit_code_unldi_x:
1407                 unldi_x(rn(node->u.w), node->v.w, node->w.w);
1408                 break;
1409                 case_rr(st, _f);
1410                 case_wr(st, _f);
1411                 case_rrr(stx, _f);
1412                 case_wrr(stx, _f);
1413 #define unstr_x(r0, r1, i0)     fallback_unstr_x(r0, r1, i0)
1414             case jit_code_unstr_x:
1415                 unstr_x(rn(node->u.w), rn(node->v.w), node->w.w);
1416                 break;
1417 #define unsti_x(i0, r0, i1)     fallback_unsti_x(i0, r0, i1)
1418             case jit_code_unsti_x:
1419                 unsti_x(node->u.w, rn(node->v.w), node->w.w);
1420                 break;
1421                 case_rr(mov, _f);
1422             case jit_code_movi_f:
1423                 assert_data(node);
1424                 movi_f(rn(node->u.w), (jit_float32_t *)node->v.n->u.w);
1425                 break;
1426                 case_rr(ext, _d_f);
1427                 case_rrr(lt, _f);
1428                 case_rrf(lt);
1429                 case_rrr(le, _f);
1430                 case_rrf(le);
1431                 case_rrr(eq, _f);
1432                 case_rrf(eq);
1433                 case_rrr(ge, _f);
1434                 case_rrf(ge);
1435                 case_rrr(gt, _f);
1436                 case_rrf(gt);
1437                 case_rrr(ne, _f);
1438                 case_rrf(ne);
1439                 case_rrr(unlt, _f);
1440                 case_rrf(unlt);
1441                 case_rrr(unle, _f);
1442                 case_rrf(unle);
1443                 case_rrr(uneq, _f);
1444                 case_rrf(uneq);
1445                 case_rrr(unge, _f);
1446                 case_rrf(unge);
1447                 case_rrr(ungt, _f);
1448                 case_rrf(ungt);
1449                 case_rrr(ltgt, _f);
1450                 case_rrf(ltgt);
1451                 case_rrr(ord, _f);
1452                 case_rrf(ord);
1453                 case_rrr(unord, _f);
1454                 case_rrf(unord);
1455                 case_brr(blt, _f);
1456                 case_brf(blt);
1457                 case_brr(ble, _f);
1458                 case_brf(ble);
1459                 case_brr(beq, _f);
1460                 case_brf(beq);
1461                 case_brr(bge, _f);
1462                 case_brf(bge);
1463                 case_brr(bgt, _f);
1464                 case_brf(bgt);
1465                 case_brr(bne, _f);
1466                 case_brf(bne);
1467                 case_brr(bunlt, _f);
1468                 case_brf(bunlt);
1469                 case_brr(bunle, _f);
1470                 case_brf(bunle);
1471                 case_brr(buneq, _f);
1472                 case_brf(buneq);
1473                 case_brr(bunge, _f);
1474                 case_brf(bunge);
1475                 case_brr(bungt, _f);
1476                 case_brf(bungt);
1477                 case_brr(bltgt, _f);
1478                 case_brf(bltgt);
1479                 case_brr(bord, _f);
1480                 case_brf(bord);
1481                 case_brr(bunord, _f);
1482                 case_brf(bunord);
1483                 case_rrr(add, _d);
1484                 case_rrd(add);
1485                 case_rrr(sub, _d);
1486                 case_rrd(sub);
1487                 case_rrd(rsb);
1488                 case_rrr(mul, _d);
1489                 case_rrd(mul);
1490                 case_rrr(div, _d);
1491                 case_rrd(div);
1492                 case_rr(abs, _d);
1493                 case_rr(neg, _d);
1494                 case_rr(sqrt, _d);
1495                 case_rqr(fma, _d);
1496                 case_rqr(fms, _d);
1497                 case_rqr(fnma, _d);
1498                 case_rqr(fnms, _d);
1499                 case_rr(ext, _d);
1500                 case_rr(ld, _d);
1501                 case_rw(ld, _d);
1502                 case_rrr(ldx, _d);
1503                 case_rrw(ldx, _d);
1504                 case_rr(st, _d);
1505                 case_wr(st, _d);
1506                 case_rrr(stx, _d);
1507                 case_wrr(stx, _d);
1508                 case_rr(mov, _d);
1509             case jit_code_movi_d:
1510                 assert_data(node);
1511                 movi_d(rn(node->u.w), (jit_float64_t *)node->v.n->u.w);
1512                 break;
1513                 case_rr(ext, _f_d);
1514                 case_rrr(lt, _d);
1515                 case_rrd(lt);
1516                 case_rrr(le, _d);
1517                 case_rrd(le);
1518                 case_rrr(eq, _d);
1519                 case_rrd(eq);
1520                 case_rrr(ge, _d);
1521                 case_rrd(ge);
1522                 case_rrr(gt, _d);
1523                 case_rrd(gt);
1524                 case_rrr(ne, _d);
1525                 case_rrd(ne);
1526                 case_rrr(unlt, _d);
1527                 case_rrd(unlt);
1528                 case_rrr(unle, _d);
1529                 case_rrd(unle);
1530                 case_rrr(uneq, _d);
1531                 case_rrd(uneq);
1532                 case_rrr(unge, _d);
1533                 case_rrd(unge);
1534                 case_rrr(ungt, _d);
1535                 case_rrd(ungt);
1536                 case_rrr(ltgt, _d);
1537                 case_rrd(ltgt);
1538                 case_rrr(ord, _d);
1539                 case_rrd(ord);
1540                 case_rrr(unord, _d);
1541                 case_rrd(unord);
1542                 case_brr(blt, _d);
1543                 case_brd(blt);
1544                 case_brr(ble, _d);
1545                 case_brd(ble);
1546                 case_brr(beq, _d);
1547                 case_brd(beq);
1548                 case_brr(bge, _d);
1549                 case_brd(bge);
1550                 case_brr(bgt, _d);
1551                 case_brd(bgt);
1552                 case_brr(bne, _d);
1553                 case_brd(bne);
1554                 case_brr(bunlt, _d);
1555                 case_brd(bunlt);
1556                 case_brr(bunle, _d);
1557                 case_brd(bunle);
1558                 case_brr(buneq, _d);
1559                 case_brd(buneq);
1560                 case_brr(bunge, _d);
1561                 case_brd(bunge);
1562                 case_brr(bungt, _d);
1563                 case_brd(bungt);
1564                 case_brr(bltgt, _d);
1565                 case_brd(bltgt);
1566                 case_brr(bord, _d);
1567                 case_brd(bord);
1568                 case_brr(bunord, _d);
1569                 case_brd(bunord);
1570             case jit_code_jmpr:
1571                 jmpr(rn(node->u.w));
1572                 break;
1573             case jit_code_jmpi:
1574                 if (node->flag & jit_flag_node) {
1575                     temp = node->u.n;
1576                     assert(temp->code == jit_code_label ||
1577                            temp->code == jit_code_epilog);
1578                     if (temp->flag & jit_flag_patch)
1579                         jmpi(temp->u.w, 1);
1580                     else {
1581                         word = _jit->code.length -
1582                             (_jit->pc.uc - _jit->code.ptr);
1583                         if (s32_p(word)) {
1584                             offset = s16_p(word);
1585                             word = jmpi(_jit->pc.w, offset);
1586                         }
1587                         else
1588                             word = jmpi_p(_jit->pc.w);
1589                         patch(word, node);
1590                     }
1591                 }
1592                 else
1593                     jmpi(node->u.w, 1);
1594                 break;
1595             case jit_code_callr:
1596                 callr(rn(node->u.w));
1597                 break;
1598             case jit_code_calli:
1599                 if (node->flag & jit_flag_node) {
1600                     temp = node->u.n;
1601                     assert(temp->code == jit_code_label ||
1602                            temp->code == jit_code_epilog);
1603                     if (temp->flag & jit_flag_patch)
1604                         calli(temp->u.w, 1);
1605                     else {
1606                         word = _jit->code.length -
1607                             (_jit->pc.uc - _jit->code.ptr);
1608                         if (s32_p(word)) {
1609                             offset =s16_p(word);
1610                             word = calli(_jit->pc.w, offset);
1611                         }
1612                         else
1613                             word = calli_p(_jit->pc.w);
1614                         patch(word, node);
1615                     }
1616                 }
1617                 else
1618                     calli(node->u.w, 1);
1619                 break;
1620             case jit_code_prolog:
1621                 _jitc->function = _jitc->functions.ptr + node->w.w;
1622                 undo.node = node;
1623                 undo.word = _jit->pc.w;
1624                 memcpy(&undo.func, _jitc->function, sizeof(undo.func));
1625 #if DEVEL_DISASSEMBLER
1626                 undo.prevw = prevw;
1627 #endif
1628                 undo.patch_offset = _jitc->patches.offset;
1629             restart_function:
1630                 _jitc->again = 0;
1631                 prolog(node);
1632                 break;
1633             case jit_code_epilog:
1634                 assert(_jitc->function == _jitc->functions.ptr + node->w.w);
1635                 if (_jitc->again) {
1636                     for (temp = undo.node->next;
1637                          temp != node; temp = temp->next) {
1638                         if (temp->code == jit_code_label ||
1639                             temp->code == jit_code_epilog)
1640                             temp->flag &= ~jit_flag_patch;
1641                     }
1642                     temp->flag &= ~jit_flag_patch;
1643                     node = undo.node;
1644                     _jit->pc.w = undo.word;
1645                     /* undo.func.self.aoff and undo.func.regset should not
1646                      * be undone, as they will be further updated, and are
1647                      * the reason of the undo. */
1648                     undo.func.self.aoff = _jitc->function->frame +
1649                         _jitc->function->self.aoff;
1650                     jit_regset_set(&undo.func.regset, &_jitc->function->regset);
1651                     /* allocar information also does not need to be undone */
1652                     undo.func.aoffoff = _jitc->function->aoffoff;
1653                     undo.func.allocar = _jitc->function->allocar;
1654                     /* cvt_offset must also not be undone */
1655                     undo.func.cvt_offset = _jitc->function->cvt_offset;
1656                     memcpy(_jitc->function, &undo.func, sizeof(undo.func));
1657 #if DEVEL_DISASSEMBLER
1658                     prevw = undo.prevw;
1659 #endif
1660                     _jitc->patches.offset = undo.patch_offset;
1661                     goto restart_function;
1662                 }
1663                 if (node->link && (word = _jit->pc.w & 3))
1664                     nop(4 - word);
1665                 /* remember label is defined */
1666                 node->flag |= jit_flag_patch;
1667                 node->u.w = _jit->pc.w;
1668                 epilog(node);
1669                 _jitc->function = NULL;
1670                 break;
1671             case jit_code_movr_w_f:
1672                 movr_w_f(rn(node->u.w), rn(node->v.w));
1673                 break;
1674             case jit_code_movr_f_w:
1675                 movr_f_w(rn(node->u.w), rn(node->v.w));
1676                 break;
1677             case jit_code_movi_f_w:
1678                 assert(node->flag & jit_flag_data);
1679                 movi_f_w(rn(node->u.w), *(jit_float32_t *)node->v.n->u.w);
1680                 break;
1681             case jit_code_movi_w_f:
1682                 movi_w_f(rn(node->u.w), node->v.w);
1683                 break;
1684 #if __WORDSIZE == 32
1685             case jit_code_movr_ww_d:
1686                 movr_ww_d(rn(node->u.w), rn(node->v.w), rn(node->w.w));
1687                 break;
1688             case jit_code_movr_d_ww:
1689                 movr_d_ww(rn(node->u.w), rn(node->v.w), rn(node->w.w));
1690                 break;
1691             case jit_code_movi_d_ww:
1692                 assert(node->flag & jit_flag_data);
1693                 movi_d_ww(rn(node->u.w), rn(node->v.w),
1694                           *(jit_float64_t *)node->w.n->u.w);
1695                 break;
1696             case jit_code_movi_ww_d:
1697                 movi_ww_d(rn(node->u.w), node->v.w, node->w.w);
1698                 break;
1699 #else
1700             case jit_code_movr_w_d:
1701                 movr_w_d(rn(node->u.w), rn(node->v.w));
1702                 break;
1703             case jit_code_movr_d_w:
1704                 movr_d_w(rn(node->u.w), rn(node->v.w));
1705                 break;
1706             case jit_code_movi_d_w:
1707                 assert(node->flag & jit_flag_data);
1708                 movi_d_w(rn(node->u.w), *(jit_float64_t *)node->v.n->u.w);
1709                 break;
1710             case jit_code_movi_w_d:
1711                 movi_w_d(rn(node->u.w), node->v.w);
1712                 break;
1713 #endif
1714             case jit_code_va_start:
1715                 vastart(rn(node->u.w));
1716                 break;
1717             case jit_code_va_arg:
1718                 vaarg(rn(node->u.w), rn(node->v.w));
1719                 break;
1720             case jit_code_va_arg_d:
1721                 vaarg_d(rn(node->u.w), rn(node->v.w));
1722                 break;
1723             case jit_code_live:                 case jit_code_ellipsis:
1724             case jit_code_va_push:
1725             case jit_code_allocai:              case jit_code_allocar:
1726             case jit_code_arg_c:                case jit_code_arg_s:
1727             case jit_code_arg_i:
1728 #  if __WORDSIZE == 64
1729             case jit_code_arg_l:
1730 #  endif
1731             case jit_code_arg_f:                case jit_code_arg_d:
1732             case jit_code_va_end:
1733             case jit_code_ret:
1734             case jit_code_retr_c:               case jit_code_reti_c:
1735             case jit_code_retr_uc:              case jit_code_reti_uc:
1736             case jit_code_retr_s:               case jit_code_reti_s:
1737             case jit_code_retr_us:              case jit_code_reti_us:
1738             case jit_code_retr_i:               case jit_code_reti_i:
1739 #if __WORDSIZE == 64
1740             case jit_code_retr_ui:              case jit_code_reti_ui:
1741             case jit_code_retr_l:               case jit_code_reti_l:
1742 #endif
1743             case jit_code_retr_f:               case jit_code_reti_f:
1744             case jit_code_retr_d:               case jit_code_reti_d:
1745             case jit_code_getarg_c:             case jit_code_getarg_uc:
1746             case jit_code_getarg_s:             case jit_code_getarg_us:
1747             case jit_code_getarg_i:
1748 #if __WORDSIZE == 64
1749             case jit_code_getarg_ui:            case jit_code_getarg_l:
1750 #endif
1751             case jit_code_getarg_f:             case jit_code_getarg_d:
1752             case jit_code_putargr_c:            case jit_code_putargi_c:
1753             case jit_code_putargr_uc:           case jit_code_putargi_uc:
1754             case jit_code_putargr_s:            case jit_code_putargi_s:
1755             case jit_code_putargr_us:           case jit_code_putargi_us:
1756             case jit_code_putargr_i:            case jit_code_putargi_i:
1757 #if __WORDSIZE == 64
1758             case jit_code_putargr_ui:           case jit_code_putargi_ui:
1759             case jit_code_putargr_l:            case jit_code_putargi_l:
1760 #endif
1761             case jit_code_putargr_f:            case jit_code_putargi_f:
1762             case jit_code_putargr_d:            case jit_code_putargi_d:
1763             case jit_code_pushargr_c:           case jit_code_pushargi_c:
1764             case jit_code_pushargr_uc:          case jit_code_pushargi_uc:
1765             case jit_code_pushargr_s:           case jit_code_pushargi_s:
1766             case jit_code_pushargr_us:          case jit_code_pushargi_us:
1767             case jit_code_pushargr_i:           case jit_code_pushargi_i:
1768 #if __WORDSIZE == 64
1769             case jit_code_pushargr_ui:          case jit_code_pushargi_ui:
1770             case jit_code_pushargr_l:           case jit_code_pushargi_l:
1771 #endif
1772             case jit_code_pushargr_f:           case jit_code_pushargi_f:
1773             case jit_code_pushargr_d:           case jit_code_pushargi_d:
1774             case jit_code_retval_c:             case jit_code_retval_uc:
1775             case jit_code_retval_s:             case jit_code_retval_us:
1776             case jit_code_retval_i:
1777 #if __WORDSIZE == 64
1778             case jit_code_retval_ui:            case jit_code_retval_l:
1779 #endif
1780             case jit_code_retval_f:             case jit_code_retval_d:
1781             case jit_code_prepare:
1782             case jit_code_finishr:              case jit_code_finishi:
1783             case jit_code_negi_f:               case jit_code_absi_f:
1784             case jit_code_sqrti_f:              case jit_code_negi_d:
1785             case jit_code_absi_d:               case jit_code_sqrti_d:
1786                 break;
1787             case jit_code_negi:
1788                 negi(rn(node->u.w), node->v.w);
1789                 break;
1790             case jit_code_comi:
1791                 comi(rn(node->u.w), node->v.w);
1792                 break;
1793             case jit_code_exti_c:
1794                 exti_c(rn(node->u.w), node->v.w);
1795                 break;
1796             case jit_code_exti_uc:
1797                 exti_uc(rn(node->u.w), node->v.w);
1798                 break;
1799             case jit_code_exti_s:
1800                 exti_s(rn(node->u.w), node->v.w);
1801                 break;
1802             case jit_code_exti_us:
1803                 exti_us(rn(node->u.w), node->v.w);
1804                 break;
1805             case jit_code_bswapi_us:
1806                 bswapi_us(rn(node->u.w), node->v.w);
1807                 break;
1808             case jit_code_bswapi_ui:
1809                 bswapi_ui(rn(node->u.w), node->v.w);
1810                 break;
1811             case jit_code_htoni_us:
1812                 htoni_us(rn(node->u.w), node->v.w);
1813                 break;
1814             case jit_code_htoni_ui:
1815                 htoni_ui(rn(node->u.w), node->v.w);
1816                 break;
1817 #if __WORDSIZE == 64
1818             case jit_code_exti_i:
1819                 exti_i(rn(node->u.w), node->v.w);
1820                 break;
1821             case jit_code_exti_ui:
1822                 exti_ui(rn(node->u.w), node->v.w);
1823                 break;
1824             case jit_code_bswapi_ul:
1825                 bswapi_ul(rn(node->u.w), node->v.w);
1826                 break;
1827             case jit_code_htoni_ul:
1828                 htoni_ul(rn(node->u.w), node->v.w);
1829                 break;
1830 #endif
1831             case jit_code_cloi:
1832                 cloi(rn(node->u.w), node->v.w);
1833                 break;
1834             case jit_code_clzi:
1835                 clzi(rn(node->u.w), node->v.w);
1836                 break;
1837             case jit_code_ctoi:
1838                 ctoi(rn(node->u.w), node->v.w);
1839                 break;
1840             case jit_code_ctzi:
1841                 ctzi(rn(node->u.w), node->v.w);
1842                 break;
1843             case jit_code_rbiti:
1844                 rbiti(rn(node->u.w), node->v.w);
1845                 break;
1846             case jit_code_popcnti:
1847                 popcnti(rn(node->u.w), node->v.w);
1848                 break;
1849             case jit_code_exti:
1850                 exti(rn(node->u.w), node->v.w, node->w.q.l, node->w.q.h);
1851                 break;
1852             case jit_code_exti_u:
1853                 exti_u(rn(node->u.w), node->v.w, node->w.q.l, node->w.q.h);
1854                 break;
1855             default:
1856                 abort();
1857         }
1858         jit_regarg_clr(node, value);
1859         assert(_jitc->regarg == 0 && _jitc->synth == 0);
1860         /* update register live state */
1861         jit_reglive(node);
1862     }
1863 #undef case_brw
1864 #undef case_brr
1865 #undef case_wrr
1866 #undef case_rrw
1867 #undef case_rrr
1868 #undef case_wr
1869 #undef case_rw
1870 #undef case_rr
1871
1872     for (offset = 0; offset < _jitc->patches.offset; offset++) {
1873         node = _jitc->patches.ptr[offset].node;
1874         word = node->code == jit_code_movi ? node->v.n->u.w : node->u.n->u.w;
1875         patch_at(_jitc->patches.ptr[offset].inst, word);
1876     }
1877
1878     jit_flush(_jit->code.ptr, _jit->pc.uc);
1879
1880     return (_jit->code.ptr);
1881 }
1882
1883 #define CODE                            1
1884 #  include "jit_s390-cpu.c"
1885 #  include "jit_s390-fpu.c"
1886 #  include "jit_fallback.c"
1887 #undef CODE
1888
1889 void
1890 jit_flush(void *fptr, void *tptr)
1891 {
1892 #if defined(__GNUC__)
1893     jit_word_t          f, t, s;
1894
1895     s = sysconf(_SC_PAGE_SIZE);
1896     f = (jit_word_t)fptr & -s;
1897     t = (((jit_word_t)tptr) + s - 1) & -s;
1898     __clear_cache((void *)f, (void *)t);
1899 #endif
1900 }
1901
1902 void
1903 _emit_ldxi(jit_state_t *_jit, jit_gpr_t r0, jit_gpr_t r1, jit_word_t i0)
1904 {
1905     ldxi(rn(r0), rn(r1), i0);
1906 }
1907
1908 void
1909 _emit_stxi(jit_state_t *_jit, jit_word_t i0, jit_gpr_t r0, jit_gpr_t r1)
1910 {
1911     stxi(i0, rn(r0), rn(r1));
1912 }
1913
1914 void
1915 _emit_ldxi_d(jit_state_t *_jit, jit_fpr_t r0, jit_gpr_t r1, jit_word_t i0)
1916 {
1917     ldxi_d(rn(r0), rn(r1), i0);
1918 }
1919
1920 void
1921 _emit_stxi_d(jit_state_t *_jit, jit_word_t i0, jit_gpr_t r0, jit_fpr_t r1)
1922 {
1923     stxi_d(i0, rn(r0), rn(r1));
1924 }
1925
1926 static jit_int32_t
1927 _jit_get_reg_pair(jit_state_t *_jit)
1928 {
1929     jit_int32_t         r1, r2;
1930     /* Try to find a register pair for use with operations that
1931      * require a odd based register pair. Search for the best
1932      * match to avoid spills or at least a valid operation.
1933      */
1934
1935     /* Try non callee save first */
1936     if (jit_reg_free_p(_R0) && jit_reg_free_p(_R1))
1937         r1 = _R0, r2 = _R1;
1938     else if (jit_reg_free_p(_R2) && jit_reg_free_p(_R3))
1939         r1 = _R2, r2 = _R3;
1940     else if (jit_reg_free_p(_R4) && jit_reg_free_p(_R5))
1941         r1 = _R4, r2 = _R5;
1942     /* Try callee save registers */
1943     else if (jit_reg_free_p(_R10) && jit_reg_free_p(_R11))
1944         r1 = _R10, r2 = _R11;
1945     else if (jit_reg_free_p(_R8) && jit_reg_free_p(_R9))
1946         r1 = _R8, r2 = _R9;
1947     else if (jit_reg_free_p(_R6) && jit_reg_free_p(_R7))
1948         r1 = _R6, r2 = _R7;
1949
1950     /* We *must* find a register pair */
1951     else if (jit_reg_free_if_spill_p(_R0) && jit_reg_free_if_spill_p(_R1))
1952         r1 = _R0, r2 = _R1;
1953     else if (jit_reg_free_if_spill_p(_R2) && jit_reg_free_if_spill_p(_R3))
1954         r1 = _R2, r2 = _R3;
1955     else if (jit_reg_free_if_spill_p(_R4) && jit_reg_free_if_spill_p(_R5))
1956         r1 = _R4, r2 = _R5;
1957     else if (jit_reg_free_if_spill_p(_R10) && jit_reg_free_if_spill_p(_R11))
1958         r1 = _R10, r2 = _R11;
1959     else if (jit_reg_free_if_spill_p(_R8) && jit_reg_free_if_spill_p(_R9))
1960         r1 = _R8, r2 = _R9;
1961     else if (jit_reg_free_if_spill_p(_R6) && jit_reg_free_if_spill_p(_R7))
1962         r1 = _R6, r2 = _R7;
1963     else
1964         /* Do not jit_get_reg() all registers to avoid it */
1965         abort();
1966
1967     (void)jit_get_reg(jit_class_gpr|jit_class_named|r1);
1968     (void)jit_get_reg(jit_class_gpr|jit_class_named|r2);
1969
1970     return (r1);
1971 }
1972
1973 static void
1974 _jit_unget_reg_pair(jit_state_t *_jit, jit_int32_t reg)
1975 {
1976     jit_int32_t         r1, r2;
1977     r1 = reg;
1978     switch (r1) {
1979         case _R0:       r2 = _R1;       break;
1980         case _R2:       r2 = _R3;       break;
1981         case _R4:       r2 = _R5;       break;
1982         case _R6:       r2 = _R7;       break;
1983         case _R8:       r2 = _R9;       break;
1984         case _R10:      r2 = _R11;      break;
1985         default:        abort();
1986     }
1987     jit_unget_reg(r1);
1988     jit_unget_reg(r2);
1989 }
1990
1991 static jit_int32_t
1992 _jit_get_reg_but_zero(jit_state_t *_jit, jit_int32_t flags)
1993 {
1994     jit_int32_t         reg;
1995     reg = jit_get_reg(jit_class_gpr);
1996     if (reg == _R0) {
1997         reg = jit_get_reg(jit_class_gpr|flags);
1998         jit_unget_reg(_R0);
1999     }
2000     return (reg);
2001 }
2002
2003 static void
2004 _patch(jit_state_t *_jit, jit_word_t instr, jit_node_t *node)
2005 {
2006     jit_int32_t         flag;
2007
2008     assert(node->flag & jit_flag_node);
2009     if (node->code == jit_code_movi)
2010         flag = node->v.n->flag;
2011     else
2012         flag = node->u.n->flag;
2013     assert(!(flag & jit_flag_patch));
2014     if (_jitc->patches.offset >= _jitc->patches.length) {
2015         jit_realloc((jit_pointer_t *)&_jitc->patches.ptr,
2016                     _jitc->patches.length * sizeof(jit_patch_t),
2017                     (_jitc->patches.length + 1024) * sizeof(jit_patch_t));
2018         _jitc->patches.length += 1024;
2019     }
2020     _jitc->patches.ptr[_jitc->patches.offset].inst = instr;
2021     _jitc->patches.ptr[_jitc->patches.offset].node = node;
2022     ++_jitc->patches.offset;
2023 }