deps/lightning/lib/jit_alpha.c

   1 /*
   2  * Copyright (C) 2014-2023  Free Software Foundation, Inc.
   3  *
   4  * This file is part of GNU lightning.
   5  *
   6  * GNU lightning is free software; you can redistribute it and/or modify it
   7  * under the terms of the GNU Lesser General Public License as published
   8  * by the Free Software Foundation; either version 3, or (at your option)
   9  * any later version.
  10  *
  11  * GNU lightning is distributed in the hope that it will be useful, but
  12  * WITHOUT ANY WARRANTY; without even the implied warranty of MERCHANTABILITY
  13  * or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU Lesser General Public
  14  * License for more details.
  15  *
  16  * Authors:
  17  *      Paulo Cesar Pereira de Andrade
  18  */
  19
  20 #define jit_arg_reg_p(i)                ((i) >= 0 && (i) < 6)
  21 #define jit_arg_f_reg_p(i)              ((i) >= 0 && (i) < 6)
  22 #if __BYTE_ORDER == __LITTLE_ENDIAN
  23 #  define C_DISP                        0
  24 #  define S_DISP                        0
  25 #  define I_DISP                        0
  26 #  define F_DISP                        0
  27 #else
  28 #  define C_DISP                        8 - sizeof(jit_int8_t)
  29 #  define S_DISP                        8 - sizeof(jit_int16_t)
  30 #  define I_DISP                        8 - sizeof(jit_int32_t)
  31 #  define F_DISP                        8 - sizeof(jit_float32_t)
  32 #endif
  33
  34 /*
  35  * Types
  36  */
  37 /*
  38  * What I could understand from gcc/config/alpha/alpha.c:alpha_build_builtin_va_list()
  39  * and other helpers, as well as objdump of simple test programs; could not
  40  * get gdb working on the test system I had access...
  41  *
  42  * base-48 to base is where up to 6 float registers are saved.
  43  * base to base+48 is where up to 6 integer registers are saved.
  44  * base+48... is where varargs arguments are stored.
  45  *
  46  *      if (offset < 48) {
  47  *              if (type == double)
  48  *                      offset -= 48;
  49  *      }
  50  *      load(reg, base, offset);
  51  *      offset += 8;
  52  */
  53 typedef struct jit_va_list {
  54     jit_pointer_t       base;
  55     jit_word_t          offset;
  56 } jit_va_list_t;
  57
  58 /*
  59  * Prototypes
  60  */
  61 #define patch(instr, node)              _patch(_jit, instr, node)
  62 static void _patch(jit_state_t*,jit_word_t,jit_node_t*);
  63
  64 #define PROTO                           1
  65 #  include "jit_alpha-cpu.c"
  66 #  include "jit_alpha-fpu.c"
  67 #undef PROTO
  68
  69 /*
  70  * Initialization
  71  */
  72 jit_register_t          _rvs[] = {
  73     { rc(gpr) | 0x1c,                   "at" },
  74     { rc(gpr) | 0x00,                   "v0" },
  75     { rc(gpr) | 0x01,                   "t0" },
  76     { rc(gpr) | 0x02,                   "t1" },
  77     { rc(gpr) | 0x03,                   "t2" },
  78     { rc(gpr) | 0x04,                   "t3" },
  79     { rc(gpr) | 0x05,                   "t4" },
  80     { rc(gpr) | 0x06,                   "t5" },
  81     { rc(gpr) | 0x07,                   "t6" },
  82     { rc(gpr) | 0x08,                   "t7" },
  83     { rc(gpr) | 0x16,                   "t8" },
  84     { rc(gpr) | 0x17,                   "t9" },
  85     { rc(gpr) | 0x18,                   "t10" },
  86     { rc(gpr) | 0x19,                   "t11" },
  87     { rc(sav) | rc(gpr) | 0x09,         "s0" },
  88     { rc(sav) | rc(gpr) | 0x0a,         "s1" },
  89     { rc(sav) | rc(gpr) | 0x0b,         "s2" },
  90     { rc(sav) | rc(gpr) | 0x0c,         "s3" },
  91     { rc(sav) | rc(gpr) | 0x0d,         "s4" },
  92     { rc(sav) | rc(gpr) | 0x0e,         "s5" },
  93     { 0x0f,                             "fp" },
  94     { rc(arg) | rc(gpr) | 0x15,         "a5" },
  95     { rc(arg) | rc(gpr) | 0x14,         "a4" },
  96     { rc(arg) | rc(gpr) | 0x13,         "a3" },
  97     { rc(arg) | rc(gpr) | 0x12,         "a2" },
  98     { rc(arg) | rc(gpr) | 0x11,         "a1" },
  99     { rc(arg) | rc(gpr) | 0x10,         "a0" },
 100     { 0x1a,                             "ra" },
 101     { 0x1b,                             "pv" },
 102     { 0x1d,                             "gp" },
 103     { 0x1e,                             "sp" },
 104     { 0x1f,                             "zero" },
 105     { rc(fpr) | 0x00,                   "$f0" },
 106     { rc(fpr) | 0x01,                   "$f1" },
 107     { rc(sav) | rc(fpr) | 0x02,         "$f2" },
 108     { rc(sav) | rc(fpr) | 0x03,         "$f3" },
 109     { rc(sav) | rc(fpr) | 0x04,         "$f4" },
 110     { rc(sav) | rc(fpr) | 0x05,         "$f5" },
 111     { rc(sav) | rc(fpr) | 0x06,         "$f6" },
 112     { rc(sav) | rc(fpr) | 0x07,         "$f7" },
 113     { rc(sav) | rc(fpr) | 0x08,         "$f8" },
 114     { rc(sav) | rc(fpr) | 0x09,         "$f9" },
 115     { rc(fpr) | 0x0a,                   "$f10" },
 116     { rc(fpr) | 0x0b,                   "$f11" },
 117     { rc(fpr) | 0x0c,                   "$f12" },
 118     { rc(fpr) | 0x0d,                   "$f13" },
 119     { rc(fpr) | 0x0e,                   "$f14" },
 120     { rc(fpr) | 0x0f,                   "$f15" },
 121     { rc(arg) | rc(fpr) | 0x15,         "$f21" },
 122     { rc(arg) | rc(fpr) | 0x14,         "$f20" },
 123     { rc(arg) | rc(fpr) | 0x13,         "$f19" },
 124     { rc(arg) | rc(fpr) | 0x12,         "$f18" },
 125     { rc(arg) | rc(fpr) | 0x11,         "$f17" },
 126     { rc(arg) | rc(fpr) | 0x10,         "$f16" },
 127     { rc(fpr) | 0x16,                   "$f22" },
 128     { rc(fpr) | 0x17,                   "$f23" },
 129     { rc(fpr) | 0x18,                   "$f24" },
 130     { rc(fpr) | 0x19,                   "$f25" },
 131     { rc(fpr) | 0x1a,                   "$f26" },
 132     { rc(fpr) | 0x1b,                   "$f27" },
 133     { rc(fpr) | 0x1c,                   "$f28" },
 134     { rc(fpr) | 0x1d,                   "$f29" },
 135     { rc(fpr) | 0x1e,                   "$f30" },
 136     { 0x1f,                             "$f31" },
 137     { _NOREG,                           "<none>" },
 138 };
 139
 140 /*
 141  * Implementation
 142  */
 143 void
 144 jit_get_cpu(void)
 145 {
 146 }
 147
 148 void
 149 _jit_init(jit_state_t *_jit)
 150 {
 151     _jitc->reglen = jit_size(_rvs) - 1;
 152     jit_carry = _NOREG;
 153 }
 154
 155 void
 156 _jit_prolog(jit_state_t *_jit)
 157 {
 158     jit_int32_t          offset;
 159
 160     if (_jitc->function)
 161         jit_epilog();
 162     assert(jit_regset_cmp_ui(&_jitc->regarg, 0) == 0);
 163     jit_regset_set_ui(&_jitc->regsav, 0);
 164     offset = _jitc->functions.offset;
 165     if (offset >= _jitc->functions.length) {
 166         jit_realloc((jit_pointer_t *)&_jitc->functions.ptr,
 167                     _jitc->functions.length * sizeof(jit_function_t),
 168                     (_jitc->functions.length + 16) * sizeof(jit_function_t));
 169         _jitc->functions.length += 16;
 170     }
 171     _jitc->function = _jitc->functions.ptr + _jitc->functions.offset++;
 172     _jitc->function->self.size = stack_framesize;
 173     _jitc->function->self.argi = _jitc->function->self.alen = 0;
 174     /* float conversion */
 175     _jitc->function->self.aoff = -8;
 176     _jitc->function->self.call = jit_call_default;
 177     jit_alloc((jit_pointer_t *)&_jitc->function->regoff,
 178               _jitc->reglen * sizeof(jit_int32_t));
 179
 180     /* _no_link here does not mean the jit_link() call can be removed
 181      * by rewriting as:
 182      * _jitc->function->prolog = jit_new_node(jit_code_prolog);
 183      */
 184     _jitc->function->prolog = jit_new_node_no_link(jit_code_prolog);
 185     jit_link(_jitc->function->prolog);
 186     _jitc->function->prolog->w.w = offset;
 187     _jitc->function->epilog = jit_new_node_no_link(jit_code_epilog);
 188     /*  u:      label value
 189      *  v:      offset in blocks vector
 190      *  w:      offset in functions vector
 191      */
 192     _jitc->function->epilog->w.w = offset;
 193
 194     jit_regset_new(&_jitc->function->regset);
 195 }
 196
 197 jit_int32_t
 198 _jit_allocai(jit_state_t *_jit, jit_int32_t length)
 199 {
 200     assert(_jitc->function != NULL);
 201     switch (length) {
 202         case 0: case 1:                                         break;
 203         case 2:         _jitc->function->self.aoff &= -2;       break;
 204         case 3: case 4: _jitc->function->self.aoff &= -4;       break;
 205         default:        _jitc->function->self.aoff &= -8;       break;
 206     }
 207     _jitc->function->self.aoff -= length;
 208     if (!_jitc->realize) {
 209         jit_inc_synth_ww(allocai, _jitc->function->self.aoff, length);
 210         jit_dec_synth();
 211     }
 212     return (_jitc->function->self.aoff);
 213 }
 214
 215 void
 216 _jit_allocar(jit_state_t *_jit, jit_int32_t u, jit_int32_t v)
 217 {
 218     jit_int32_t          reg;
 219     assert(_jitc->function != NULL);
 220     jit_inc_synth_ww(allocar, u, v);
 221     if (!_jitc->function->allocar) {
 222         _jitc->function->aoffoff = jit_allocai(sizeof(jit_int32_t));
 223         _jitc->function->allocar = 1;
 224     }
 225     reg = jit_get_reg(jit_class_gpr);
 226     jit_negr(reg, v);
 227     jit_andi(reg, reg, -8);
 228     jit_ldxi_i(u, JIT_FP, _jitc->function->aoffoff);
 229     jit_addr(u, u, reg);
 230     jit_addr(JIT_SP, JIT_SP, reg);
 231     jit_stxi_i(_jitc->function->aoffoff, JIT_FP, u);
 232     jit_unget_reg(reg);
 233     jit_dec_synth();
 234 }
 235
 236 void
 237 _jit_ret(jit_state_t *_jit)
 238 {
 239     jit_node_t          *instr;
 240     assert(_jitc->function != NULL);
 241     jit_inc_synth(ret);
 242     /* jump to epilog */
 243     instr = jit_jmpi();
 244     jit_patch_at(instr, _jitc->function->epilog);
 245     jit_dec_synth();
 246 }
 247
 248 void
 249 _jit_retr(jit_state_t *_jit, jit_int32_t u, jit_code_t code)
 250 {
 251     jit_code_inc_synth_w(code, u);
 252     jit_movr(JIT_RET, u);
 253     jit_ret();
 254     jit_dec_synth();
 255 }
 256
 257 void
 258 _jit_reti(jit_state_t *_jit, jit_word_t u, jit_code_t code)
 259 {
 260     jit_code_inc_synth_w(code, u);
 261     jit_movi(JIT_RET, u);
 262     jit_ret();
 263     jit_dec_synth();
 264 }
 265
 266 void
 267 _jit_retr_f(jit_state_t *_jit, jit_int32_t u)
 268 {
 269     jit_inc_synth_w(retr_f, u);
 270     if (u != JIT_FRET)
 271         jit_movr_f(JIT_FRET, u);
 272     else
 273         jit_live(JIT_FRET);
 274     jit_ret();
 275     jit_dec_synth();
 276 }
 277
 278 void
 279 _jit_reti_f(jit_state_t *_jit, jit_float32_t u)
 280 {
 281     jit_inc_synth_f(reti_f, u);
 282     jit_movi_f(JIT_FRET, u);
 283     jit_ret();
 284     jit_dec_synth();
 285 }
 286
 287 void
 288 _jit_retr_d(jit_state_t *_jit, jit_int32_t u)
 289 {
 290     jit_inc_synth_w(retr_d, u);
 291     if (u != JIT_FRET)
 292         jit_movr_d(JIT_FRET, u);
 293     else
 294         jit_live(JIT_FRET);
 295     jit_ret();
 296     jit_dec_synth();
 297 }
 298
 299 void
 300 _jit_reti_d(jit_state_t *_jit, jit_float64_t u)
 301 {
 302     jit_inc_synth_d(reti_d, u);
 303     jit_movi_d(JIT_FRET, u);
 304     jit_ret();
 305     jit_dec_synth();
 306 }
 307
 308 void
 309 _jit_epilog(jit_state_t *_jit)
 310 {
 311     assert(_jitc->function != NULL);
 312     assert(_jitc->function->epilog->next == NULL);
 313     jit_link(_jitc->function->epilog);
 314     _jitc->function = NULL;
 315 }
 316
 317 jit_bool_t
 318 _jit_arg_register_p(jit_state_t *_jit, jit_node_t *u)
 319 {
 320     if (u->code >= jit_code_arg_c && u->code <= jit_code_arg)
 321         return (jit_arg_reg_p(u->u.w));
 322     assert(u->code == jit_code_arg_f || u->code == jit_code_arg_d);
 323     return (jit_arg_f_reg_p(u->u.w));
 324 }
 325
 326 void
 327 _jit_ellipsis(jit_state_t *_jit)
 328 {
 329     jit_inc_synth(ellipsis);
 330     if (_jitc->prepare) {
 331         jit_link_prepare();
 332         assert(!(_jitc->function->call.call & jit_call_varargs));
 333         _jitc->function->call.call |= jit_call_varargs;
 334     }
 335     else {
 336         jit_link_prolog();
 337         assert(!(_jitc->function->self.call & jit_call_varargs));
 338         _jitc->function->self.call |= jit_call_varargs;
 339
 340         /* Allocate va_list like object in the stack */
 341         _jitc->function->vaoff = jit_allocai(sizeof(jit_va_list_t));
 342         _jitc->function->vagp = _jitc->function->self.argi;
 343     }
 344     jit_dec_synth();
 345 }
 346
 347 void
 348 _jit_va_push(jit_state_t *_jit, jit_int32_t u)
 349 {
 350     jit_int32_t         reg;
 351     jit_inc_synth_w(va_push, u);
 352     reg = jit_get_reg(jit_class_gpr);
 353     jit_ldxi(reg, u, offsetof(jit_va_list_t, base));
 354     jit_pushargr(reg);
 355     jit_ldxi(reg, u, offsetof(jit_va_list_t, offset));
 356     jit_pushargr(reg);
 357     jit_unget_reg(reg);
 358     jit_dec_synth();
 359 }
 360
 361 jit_node_t *
 362 _jit_arg(jit_state_t *_jit, jit_code_t code)
 363 {
 364     jit_node_t          *node;
 365     jit_int32_t          offset;
 366     assert(_jitc->function != NULL);
 367     assert(!(_jitc->function->self.call & jit_call_varargs));
 368 #if STRONG_TYPE_CHECKING
 369     assert(code >= jit_code_arg_c && code <= jit_code_arg);
 370 #endif
 371     if (jit_arg_reg_p(_jitc->function->self.argi))
 372         offset = _jitc->function->self.argi++;
 373     else {
 374         offset = _jitc->function->self.size;
 375         _jitc->function->self.size += 8;
 376     }
 377     node = jit_new_node_ww(code, offset,
 378                            ++_jitc->function->self.argn);
 379     jit_link_prolog();
 380     return (node);
 381 }
 382
 383 jit_node_t *
 384 _jit_arg_f(jit_state_t *_jit)
 385 {
 386     jit_node_t          *node;
 387     jit_int32_t          offset;
 388     assert(_jitc->function != NULL);
 389     if (jit_arg_f_reg_p(_jitc->function->self.argi))
 390         offset = _jitc->function->self.argi++;
 391     else {
 392         offset = _jitc->function->self.size;
 393         _jitc->function->self.size += 8;
 394     }
 395     node = jit_new_node_ww(jit_code_arg_f, offset,
 396                            ++_jitc->function->self.argn);
 397     jit_link_prolog();
 398     return (node);
 399 }
 400
 401 jit_node_t *
 402 _jit_arg_d(jit_state_t *_jit)
 403 {
 404     jit_node_t          *node;
 405     jit_int32_t          offset;
 406     assert(_jitc->function != NULL);
 407     if (jit_arg_f_reg_p(_jitc->function->self.argi))
 408         offset = _jitc->function->self.argi++;
 409     else {
 410         offset = _jitc->function->self.size;
 411         _jitc->function->self.size += 8;
 412     }
 413     node = jit_new_node_ww(jit_code_arg_d, offset,
 414                            ++_jitc->function->self.argn);
 415     jit_link_prolog();
 416     return (node);
 417 }
 418
 419 void
 420 _jit_getarg_c(jit_state_t *_jit, jit_int32_t u, jit_node_t *v)
 421 {
 422     assert_arg_type(v->code, jit_code_arg_c);
 423     jit_inc_synth_wp(getarg_c, u, v);
 424     if (jit_arg_reg_p(v->u.w))
 425         jit_extr_c(u, _A0 - v->u.w);
 426     else
 427         jit_ldxi_c(u, _FP, v->u.w + C_DISP);
 428     jit_dec_synth();
 429 }
 430
 431 void
 432 _jit_getarg_uc(jit_state_t *_jit, jit_int32_t u, jit_node_t *v)
 433 {
 434     assert_arg_type(v->code, jit_code_arg_c);
 435     jit_inc_synth_wp(getarg_uc, u, v);
 436     if (jit_arg_reg_p(v->u.w))
 437         jit_extr_uc(u, _A0 - v->u.w);
 438     else
 439         jit_ldxi_uc(u, _FP, v->u.w + C_DISP);
 440     jit_dec_synth();
 441 }
 442
 443 void
 444 _jit_getarg_s(jit_state_t *_jit, jit_int32_t u, jit_node_t *v)
 445 {
 446     assert_arg_type(v->code, jit_code_arg_s);
 447     jit_inc_synth_wp(getarg_s, u, v);
 448     if (jit_arg_reg_p(v->u.w))
 449         jit_extr_s(u, _A0 - v->u.w);
 450     else
 451         jit_ldxi_s(u, _FP, v->u.w + S_DISP);
 452     jit_dec_synth();
 453 }
 454
 455 void
 456 _jit_getarg_us(jit_state_t *_jit, jit_int32_t u, jit_node_t *v)
 457 {
 458     assert_arg_type(v->code, jit_code_arg_s);
 459     jit_inc_synth_wp(getarg_us, u, v);
 460     if (jit_arg_reg_p(v->u.w))
 461         jit_extr_us(u, _A0 - v->u.w);
 462     else
 463         jit_ldxi_us(u, _FP, v->u.w + S_DISP);
 464     jit_dec_synth();
 465 }
 466
 467 void
 468 _jit_getarg_i(jit_state_t *_jit, jit_int32_t u, jit_node_t *v)
 469 {
 470     assert_arg_type(v->code, jit_code_arg_i);
 471     jit_inc_synth_wp(getarg_i, u, v);
 472     if (jit_arg_reg_p(v->u.w))
 473         jit_extr_i(u, _A0 - v->u.w);
 474     else
 475         jit_ldxi_i(u, _FP, v->u.w + I_DISP);
 476     jit_dec_synth();
 477 }
 478
 479 void
 480 _jit_getarg_ui(jit_state_t *_jit, jit_int32_t u, jit_node_t *v)
 481 {
 482     assert_arg_type(v->code, jit_code_arg_i);
 483     jit_inc_synth_wp(getarg_ui, u, v);
 484     if (jit_arg_reg_p(v->u.w))
 485         jit_extr_ui(u, _A0 - v->u.w);
 486     else
 487         jit_ldxi_ui(u, _FP, v->u.w + I_DISP);
 488     jit_dec_synth();
 489 }
 490
 491 void
 492 _jit_getarg_l(jit_state_t *_jit, jit_int32_t u, jit_node_t *v)
 493 {
 494     assert_arg_type(v->code, jit_code_arg_l);
 495     jit_inc_synth_wp(getarg_l, u, v);
 496     if (jit_arg_reg_p(v->u.w))
 497         jit_movr(u, _A0 - v->u.w);
 498     else
 499         jit_ldxi_l(u, _FP, v->u.w);
 500     jit_dec_synth();
 501 }
 502
 503 void
 504 _jit_putargr(jit_state_t *_jit, jit_int32_t u, jit_node_t *v, jit_code_t code)
 505 {
 506     assert_putarg_type(code, v->code);
 507     jit_code_inc_synth_wp(code, u, v);
 508     if (jit_arg_reg_p(v->u.w))
 509         jit_movr(_A0 - v->u.w, u);
 510     else
 511         jit_stxi(v->u.w, _FP, u);
 512     jit_dec_synth();
 513 }
 514
 515 void
 516 _jit_putargi(jit_state_t *_jit, jit_word_t u, jit_node_t *v, jit_code_t code)
 517 {
 518     jit_int32_t         regno;
 519     assert_putarg_type(code, v->code);
 520     jit_code_inc_synth_wp(code, u, v);
 521     if (jit_arg_reg_p(v->u.w))
 522         jit_movi(_A0 - v->u.w, u);
 523     else {
 524         regno = jit_get_reg(jit_class_gpr);
 525         jit_movi(regno, u);
 526         jit_stxi(v->u.w, _FP, regno);
 527         jit_unget_reg(regno);
 528     }
 529     jit_dec_synth();
 530 }
 531
 532 void
 533 _jit_getarg_f(jit_state_t *_jit, jit_int32_t u, jit_node_t *v)
 534 {
 535     assert(v->code == jit_code_arg_f);
 536     jit_inc_synth_wp(getarg_f, u, v);
 537     if (jit_arg_f_reg_p(v->u.w))
 538         jit_movr_f(u, _F16 - v->u.w);
 539     else
 540         jit_ldxi_f(u, _FP, v->u.w + F_DISP);
 541     jit_dec_synth();
 542 }
 543
 544 void
 545 _jit_putargr_f(jit_state_t *_jit, jit_int32_t u, jit_node_t *v)
 546 {
 547     assert(v->code == jit_code_arg_f);
 548     jit_inc_synth_wp(putargr_f, u, v);
 549     if (jit_arg_f_reg_p(v->u.w))
 550         jit_movr_f(_F16 - v->u.w, u);
 551     else
 552         jit_stxi_f(v->u.w, _FP, u + F_DISP);
 553     jit_dec_synth();
 554 }
 555
 556 void
 557 _jit_putargi_f(jit_state_t *_jit, jit_float32_t u, jit_node_t *v)
 558 {
 559     jit_int32_t         regno;
 560     assert(v->code == jit_code_arg_f);
 561     jit_inc_synth_fp(putargi_f, u, v);
 562     if (jit_arg_f_reg_p(v->u.w))
 563         jit_movi_f(_F16 - v->u.w, u);
 564     else {
 565         regno = jit_get_reg(jit_class_fpr);
 566         jit_movi_f(regno, u);
 567         jit_stxi_f(v->u.w, _FP, regno + F_DISP);
 568         jit_unget_reg(regno);
 569     }
 570     jit_dec_synth();
 571 }
 572
 573 void
 574 _jit_getarg_d(jit_state_t *_jit, jit_int32_t u, jit_node_t *v)
 575 {
 576     assert(v->code == jit_code_arg_d);
 577     jit_inc_synth_wp(getarg_d, u, v);
 578     if (jit_arg_f_reg_p(v->u.w))
 579         jit_movr_d(u, _F16 - v->u.w);
 580     else
 581         jit_ldxi_d(u, _FP, v->u.w);
 582     jit_dec_synth();
 583 }
 584
 585 void
 586 _jit_putargr_d(jit_state_t *_jit, jit_int32_t u, jit_node_t *v)
 587 {
 588     assert(v->code == jit_code_arg_d);
 589     jit_inc_synth_wp(putargr_d, u, v);
 590     if (jit_arg_f_reg_p(v->u.w))
 591         jit_movr_d(_F16 - v->u.w, u);
 592     else
 593         jit_stxi_d(v->u.w, _FP, u);
 594     jit_dec_synth();
 595 }
 596
 597 void
 598 _jit_putargi_d(jit_state_t *_jit, jit_float64_t u, jit_node_t *v)
 599 {
 600     jit_int32_t         regno;
 601     assert(v->code == jit_code_arg_d);
 602     jit_inc_synth_dp(putargi_d, u, v);
 603     if (jit_arg_f_reg_p(v->u.w))
 604         jit_movi_d(_F16 - v->u.w, u);
 605     else {
 606         regno = jit_get_reg(jit_class_fpr);
 607         jit_movi_d(regno, u);
 608         jit_stxi_d(v->u.w, _FP, regno);
 609         jit_unget_reg(regno);
 610     }
 611     jit_dec_synth();
 612 }
 613
 614 void
 615 _jit_pushargr(jit_state_t *_jit, jit_int32_t u, jit_code_t code)
 616 {
 617     assert(_jitc->function != NULL);
 618     jit_code_inc_synth_w(code, u);
 619     jit_link_prepare();
 620     if (jit_arg_reg_p(_jitc->function->call.argi)) {
 621         jit_movr(_A0 - _jitc->function->call.argi, u);
 622         ++_jitc->function->call.argi;
 623     }
 624     else {
 625         jit_stxi(_jitc->function->call.size, JIT_SP, u);
 626         _jitc->function->call.size += 8;
 627     }
 628     jit_dec_synth();
 629 }
 630
 631 void
 632 _jit_pushargi(jit_state_t *_jit, jit_int64_t u, jit_code_t code)
 633 {
 634     jit_int32_t         regno;
 635     assert(_jitc->function != NULL);
 636     jit_code_inc_synth_w(code, u);
 637     jit_link_prepare();
 638     if (jit_arg_reg_p(_jitc->function->call.argi)) {
 639         jit_movi(_A0 - _jitc->function->call.argi, u);
 640         ++_jitc->function->call.argi;
 641     }
 642     else {
 643         regno = jit_get_reg(jit_class_gpr);
 644         jit_movi(regno, u);
 645         jit_stxi(_jitc->function->call.size, JIT_SP, regno);
 646         _jitc->function->call.size += 8;
 647         jit_unget_reg(regno);
 648     }
 649     jit_dec_synth();
 650 }
 651
 652 void
 653 _jit_pushargr_f(jit_state_t *_jit, jit_int32_t u)
 654 {
 655     assert(_jitc->function != NULL);
 656     jit_inc_synth_w(pushargr_f, u);
 657     jit_link_prepare();
 658     if (jit_arg_f_reg_p(_jitc->function->call.argi)) {
 659         jit_movr_f(_F16 - _jitc->function->call.argi, u);
 660         ++_jitc->function->call.argi;
 661     }
 662     else {
 663         jit_stxi_f(_jitc->function->call.size + F_DISP, JIT_SP, u);
 664         _jitc->function->call.size += 8;
 665     }
 666     jit_dec_synth();
 667 }
 668
 669 void
 670 _jit_pushargi_f(jit_state_t *_jit, jit_float32_t u)
 671 {
 672     jit_int32_t         regno;
 673     assert(_jitc->function != NULL);
 674     jit_inc_synth_f(pushargi_f, u);
 675     jit_link_prepare();
 676     if (jit_arg_f_reg_p(_jitc->function->call.argi)) {
 677         jit_movi_f(_F16 - _jitc->function->call.argi, u);
 678         ++_jitc->function->call.argi;
 679     }
 680     else {
 681         regno = jit_get_reg(jit_class_fpr);
 682         jit_movi_f(regno, u);
 683         jit_stxi_f(_jitc->function->call.size + F_DISP, JIT_SP, regno);
 684         _jitc->function->call.size += 8;
 685         jit_unget_reg(regno);
 686     }
 687     jit_dec_synth();
 688 }
 689
 690 void
 691 _jit_pushargr_d(jit_state_t *_jit, jit_int32_t u)
 692 {
 693     assert(_jitc->function != NULL);
 694     jit_inc_synth_w(pushargr_d, u);
 695     jit_link_prepare();
 696     if (jit_arg_f_reg_p(_jitc->function->call.argi)) {
 697         jit_movr_d(_F16 - _jitc->function->call.argi, u);
 698         ++_jitc->function->call.argi;
 699     }
 700     else {
 701         jit_stxi_d(_jitc->function->call.size, JIT_SP, u);
 702         _jitc->function->call.size += 8;
 703     }
 704     jit_dec_synth();
 705 }
 706
 707 void
 708 _jit_pushargi_d(jit_state_t *_jit, jit_float64_t u)
 709 {
 710     jit_int32_t         regno;
 711     assert(_jitc->function != NULL);
 712     jit_inc_synth_d(pushargi_d, u);
 713     jit_link_prepare();
 714     if (jit_arg_f_reg_p(_jitc->function->call.argi)) {
 715         jit_movi_d(_F16 - _jitc->function->call.argi, u);
 716         ++_jitc->function->call.argi;
 717     }
 718     else {
 719         regno = jit_get_reg(jit_class_fpr);
 720         jit_movi_d(regno, u);
 721         jit_stxi_d(_jitc->function->call.size, JIT_SP, regno);
 722         _jitc->function->call.size += 8;
 723         jit_unget_reg(regno);
 724     }
 725     jit_dec_synth();
 726 }
 727
 728 jit_bool_t
 729 _jit_regarg_p(jit_state_t *_jit, jit_node_t *node, jit_int32_t regno)
 730 {
 731     jit_int32_t         spec;
 732
 733     spec = jit_class(_rvs[regno].spec);
 734     if (spec & jit_class_arg) {
 735         if (spec & jit_class_gpr) {
 736             regno = _A0 - regno;
 737             if (regno >= 0 && regno < node->v.w)
 738                 return (1);
 739         }
 740         else if (spec & jit_class_fpr) {
 741             regno = _F16 - regno;
 742             if (regno >= 0 && regno < node->w.w)
 743                 return (1);
 744         }
 745     }
 746
 747     return (0);
 748 }
 749
 750 void
 751 _jit_finishr(jit_state_t *_jit, jit_int32_t r0)
 752 {
 753     jit_node_t          *call;
 754     assert(_jitc->function != NULL);
 755     jit_inc_synth_w(finishr, r0);
 756     if (_jitc->function->self.alen < _jitc->function->call.size)
 757         _jitc->function->self.alen = _jitc->function->call.size;
 758     call = jit_callr(r0);
 759     call->v.w = call->w.w = _jitc->function->self.argi;
 760     _jitc->function->call.argi = _jitc->function->call.size = 0;
 761     _jitc->prepare = 0;
 762     jit_dec_synth();
 763 }
 764
 765 jit_node_t *
 766 _jit_finishi(jit_state_t *_jit, jit_pointer_t i0)
 767 {
 768     jit_node_t          *call;
 769     assert(_jitc->function != NULL);
 770     jit_inc_synth_w(finishi, (jit_word_t)i0);
 771     if (_jitc->function->self.alen < _jitc->function->call.size)
 772         _jitc->function->self.alen = _jitc->function->call.size;
 773     call = jit_calli(i0);
 774     call->v.w = call->w.w = _jitc->function->self.argf;
 775     _jitc->function->call.argi = _jitc->function->call.size = 0;
 776     _jitc->prepare = 0;
 777     jit_dec_synth();
 778     return (call);
 779 }
 780
 781 void
 782 _jit_retval_c(jit_state_t *_jit, jit_int32_t r0)
 783 {
 784     jit_inc_synth_w(retval_c, r0);
 785     jit_extr_c(r0, JIT_RET);
 786     jit_dec_synth();
 787 }
 788
 789 void
 790 _jit_retval_uc(jit_state_t *_jit, jit_int32_t r0)
 791 {
 792     jit_inc_synth_w(retval_uc, r0);
 793     jit_extr_uc(r0, JIT_RET);
 794     jit_dec_synth();
 795 }
 796
 797 void
 798 _jit_retval_s(jit_state_t *_jit, jit_int32_t r0)
 799 {
 800     jit_inc_synth_w(retval_s, r0);
 801     jit_extr_s(r0, JIT_RET);
 802     jit_dec_synth();
 803 }
 804
 805 void
 806 _jit_retval_us(jit_state_t *_jit, jit_int32_t r0)
 807 {
 808     jit_inc_synth_w(retval_us, r0);
 809     jit_extr_us(r0, JIT_RET);
 810     jit_dec_synth();
 811 }
 812
 813 void
 814 _jit_retval_i(jit_state_t *_jit, jit_int32_t r0)
 815 {
 816     jit_inc_synth_w(retval_i, r0);
 817     jit_extr_i(r0, JIT_RET);
 818     jit_dec_synth();
 819 }
 820
 821 void
 822 _jit_retval_ui(jit_state_t *_jit, jit_int32_t r0)
 823 {
 824     jit_inc_synth_w(retval_ui, r0);
 825     jit_extr_ui(r0, JIT_RET);
 826     jit_dec_synth();
 827 }
 828
 829 void
 830 _jit_retval_l(jit_state_t *_jit, jit_int32_t r0)
 831 {
 832     jit_inc_synth_w(retval_l, r0);
 833     if (r0 != JIT_RET)
 834         jit_movr(r0, JIT_RET);
 835     jit_dec_synth();
 836 }
 837
 838 void
 839 _jit_retval_f(jit_state_t *_jit, jit_int32_t r0)
 840 {
 841     jit_inc_synth_w(retval_f, r0);
 842     if (r0 != JIT_FRET)
 843         jit_movr_f(r0, JIT_FRET);
 844     jit_dec_synth();
 845 }
 846
 847 void
 848 _jit_retval_d(jit_state_t *_jit, jit_int32_t r0)
 849 {
 850     jit_inc_synth_w(retval_d, r0);
 851     if (r0 != JIT_FRET)
 852         jit_movr_d(r0, JIT_FRET);
 853     jit_dec_synth();
 854 }
 855
 856 jit_pointer_t
 857 _emit_code(jit_state_t *_jit)
 858 {
 859     jit_node_t          *node;
 860     jit_node_t          *temp;
 861     jit_word_t           word;
 862     jit_word_t           value;
 863     jit_int32_t          offset;
 864     struct {
 865         jit_node_t      *node;
 866         jit_uint8_t     *data;
 867         jit_word_t       word;
 868         jit_function_t   func;
 869 #if DEVEL_DISASSEMBLER
 870         jit_word_t       prevw;
 871 #endif
 872         jit_int32_t      const_offset;
 873         jit_int32_t      patch_offset;
 874     } undo;
 875 #if DEVEL_DISASSEMBLER
 876     jit_word_t           prevw;
 877 #endif
 878
 879     _jitc->function = NULL;
 880
 881     jit_reglive_setup();
 882
 883     undo.word = 0;
 884     undo.node = NULL;
 885     undo.const_offset = undo.patch_offset = 0;
 886 #define case_rr(name, type)                                             \
 887             case jit_code_##name##r##type:                              \
 888                 name##r##type(rn(node->u.w), rn(node->v.w));            \
 889                 break
 890 #define case_rw(name, type)                                             \
 891             case jit_code_##name##i##type:                              \
 892                 name##i##type(rn(node->u.w), node->v.w);                \
 893                 break
 894 #define case_wr(name, type)                                             \
 895             case jit_code_##name##i##type:                              \
 896                 name##i##type(node->u.w, rn(node->v.w));                \
 897                 break
 898 #define case_rrr(name, type)                                            \
 899             case jit_code_##name##r##type:                              \
 900                 name##r##type(rn(node->u.w),                            \
 901                               rn(node->v.w), rn(node->w.w));            \
 902                 break
 903 #define case_rrw(name, type)                                            \
 904             case jit_code_##name##i##type:                              \
 905                 name##i##type(rn(node->u.w), rn(node->v.w), node->w.w); \
 906                 break
 907 #define case_rrf(name, type, size)                                      \
 908             case jit_code_##name##i##type:                              \
 909                 assert(node->flag & jit_flag_data);                     \
 910                 name##i##type(rn(node->u.w), rn(node->v.w),             \
 911                               (jit_float##size##_t *)node->w.n->u.w);   \
 912                 break
 913 #define case_rrrr(name, type)                                           \
 914             case jit_code_##name##r##type:                              \
 915                 name##r##type(rn(node->u.q.l), rn(node->u.q.h),         \
 916                               rn(node->v.w), rn(node->w.w));            \
 917                 break
 918 #define case_rrrw(name, type)                                           \
 919             case jit_code_##name##i##type:                              \
 920                 name##i##type(rn(node->u.q.l), rn(node->u.q.h),         \
 921                               rn(node->v.w), node->w.w);                \
 922                 break
 923 #define case_wrr(name, type)                                            \
 924             case jit_code_##name##i##type:                              \
 925                 name##i##type(node->u.w, rn(node->v.w), rn(node->w.w)); \
 926                 break
 927 #define case_brr(name, type)                                            \
 928             case jit_code_##name##r##type:                              \
 929                 temp = node->u.n;                                       \
 930                 assert(temp->code == jit_code_label ||                  \
 931                        temp->code == jit_code_epilog);                  \
 932                 if (temp->flag & jit_flag_patch)                        \
 933                     name##r##type(temp->u.w, rn(node->v.w),             \
 934                                   rn(node->w.w));                       \
 935                 else {                                                  \
 936                     word = name##r##type(_jit->pc.w,                    \
 937                                          rn(node->v.w), rn(node->w.w)); \
 938                     patch(word, node);                                  \
 939                 }                                                       \
 940                 break
 941 #define case_brw(name, type)                                            \
 942             case jit_code_##name##i##type:                              \
 943                 temp = node->u.n;                                       \
 944                 assert(temp->code == jit_code_label ||                  \
 945                        temp->code == jit_code_epilog);                  \
 946                 if (temp->flag & jit_flag_patch)                        \
 947                     name##i##type(temp->u.w,                            \
 948                                   rn(node->v.w), node->w.w);            \
 949                 else {                                                  \
 950                     word = name##i##type(_jit->pc.w,                    \
 951                                          rn(node->v.w), node->w.w);     \
 952                     patch(word, node);                                  \
 953                 }                                                       \
 954                 break
 955 #define case_brf(name, type, size)                                      \
 956             case jit_code_##name##i##type:                              \
 957                 temp = node->u.n;                                       \
 958                 assert(temp->code == jit_code_label ||                  \
 959                        temp->code == jit_code_epilog);                  \
 960                 if (temp->flag & jit_flag_patch)                        \
 961                     name##i##type(temp->u.w, rn(node->v.w),             \
 962                                 (jit_float##size##_t *)node->w.n->u.w); \
 963                 else {                                                  \
 964                     word = name##i##type(_jit->pc.w, rn(node->v.w),     \
 965                                 (jit_float##size##_t *)node->w.n->u.w); \
 966                     patch(word, node);                                  \
 967                 }                                                       \
 968                 break
 969 #if DEVEL_DISASSEMBLER
 970     prevw = _jit->pc.w;
 971 #endif
 972     for (node = _jitc->head; node; node = node->next) {
 973         if (_jit->pc.uc >= _jitc->code.end)
 974             return (NULL);
 975
 976 #if DEVEL_DISASSEMBLER
 977         node->offset = (jit_uword_t)_jit->pc.w - (jit_uword_t)prevw;
 978         prevw = _jit->pc.w;
 979 #endif
 980         value = jit_classify(node->code);
 981         jit_regarg_set(node, value);
 982         switch (node->code) {
 983             case jit_code_align:
 984                 /* Must align to a power of two */
 985                 assert(!(node->u.w & (node->u.w - 1)));
 986                 if ((word = _jit->pc.w & (node->u.w - 1)))
 987                     nop(node->u.w - word);
 988                 break;
 989             case jit_code_skip:
 990                 nop((node->u.w + 3) & ~3);
 991                 break;
 992             case jit_code_note:         case jit_code_name:
 993                 node->u.w = _jit->pc.w;
 994                 break;
 995             case jit_code_label:
 996                 /* remember label is defined */
 997                 node->flag |= jit_flag_patch;
 998                 node->u.w = _jit->pc.w;
 999                 break;
1000                 case_rrr(add,);
1001                 case_rrw(add,);
1002                 case_rrr(addc,);
1003                 case_rrw(addc,);
1004                 case_rrr(addx,);
1005                 case_rrw(addx,);
1006                 case_rrr(sub,);
1007                 case_rrw(sub,);
1008                 case_rrr(subc,);
1009                 case_rrw(subc,);
1010                 case_rrr(subx,);
1011                 case_rrw(subx,);
1012                 case_rrw(rsb,);
1013                 case_rrr(mul,);
1014                 case_rrw(mul,);
1015                 case_rrrr(qmul,);
1016                 case_rrrw(qmul,);
1017                 case_rrrr(qmul, _u);
1018                 case_rrrw(qmul, _u);
1019                 case_rrr(div,);
1020                 case_rrw(div,);
1021                 case_rrr(div, _u);
1022                 case_rrw(div, _u);
1023                 case_rrrr(qdiv,);
1024                 case_rrrw(qdiv,);
1025                 case_rrrr(qdiv, _u);
1026                 case_rrrw(qdiv, _u);
1027                 case_rrr(rem,);
1028                 case_rrw(rem,);
1029                 case_rrr(rem, _u);
1030                 case_rrw(rem, _u);
1031                 case_rrr(lsh,);
1032                 case_rrw(lsh,);
1033                 case_rrr(rsh,);
1034                 case_rrw(rsh,);
1035                 case_rrr(rsh, _u);
1036                 case_rrw(rsh, _u);
1037                 case_rrr(and,);
1038                 case_rrw(and,);
1039                 case_rrr(or,);
1040                 case_rrw(or,);
1041                 case_rrr(xor,);
1042                 case_rrw(xor,);
1043                 case_rr(trunc, _f_i);
1044                 case_rr(trunc, _d_i);
1045                 case_rr(trunc, _f_l);
1046                 case_rr(trunc, _d_l);
1047                 case_rr(ld, _c);
1048                 case_rw(ld, _c);
1049                 case_rr(ld, _uc);
1050                 case_rw(ld, _uc);
1051                 case_rr(ld, _s);
1052                 case_rw(ld, _s);
1053                 case_rr(ld, _us);
1054                 case_rw(ld, _us);
1055                 case_rr(ld, _i);
1056                 case_rw(ld, _i);
1057                 case_rr(ld, _ui);
1058                 case_rw(ld, _ui);
1059                 case_rr(ld, _l);
1060                 case_rw(ld, _l);
1061                 case_rrr(ldx, _c);
1062                 case_rrw(ldx, _c);
1063                 case_rrr(ldx, _uc);
1064                 case_rrw(ldx, _uc);
1065                 case_rrr(ldx, _s);
1066                 case_rrw(ldx, _s);
1067                 case_rrr(ldx, _us);
1068                 case_rrw(ldx, _us);
1069                 case_rrr(ldx, _i);
1070                 case_rrw(ldx, _i);
1071                 case_rrr(ldx, _ui);
1072                 case_rrw(ldx, _ui);
1073                 case_rrr(ldx, _l);
1074                 case_rrw(ldx, _l);
1075                 case_rr(st, _c);
1076                 case_wr(st, _c);
1077                 case_rr(st, _s);
1078                 case_wr(st, _s);
1079                 case_rr(st, _i);
1080                 case_wr(st, _i);
1081                 case_rr(st, _l);
1082                 case_wr(st, _l);
1083                 case_rrr(stx, _c);
1084                 case_wrr(stx, _c);
1085                 case_rrr(stx, _s);
1086                 case_wrr(stx, _s);
1087                 case_rrr(stx, _i);
1088                 case_wrr(stx, _i);
1089                 case_rrr(stx, _l);
1090                 case_wrr(stx, _l);
1091                 case_rr(hton, _us);
1092                 case_rr(hton, _ui);
1093                 case_rr(hton, _ul);
1094                 case_rr(bswap, _us);
1095                 case_rr(bswap, _ui);
1096                 case_rr(bswap, _ul);
1097                 case_rr(ext, _c);
1098                 case_rr(ext, _uc);
1099                 case_rr(ext, _s);
1100                 case_rr(ext, _us);
1101                 case_rr(ext, _i);
1102                 case_rr(ext, _ui);
1103             case jit_code_casr:
1104                 casr(rn(node->u.w), rn(node->v.w),
1105                      rn(node->w.q.l), rn(node->w.q.h));
1106                 break;
1107             case jit_code_casi:
1108                 casi(rn(node->u.w), node->v.w,
1109                      rn(node->w.q.l), rn(node->w.q.h));
1110                 break;
1111                 case_rrr(movn,);
1112                 case_rrr(movz,);
1113                 case_rr(mov,);
1114             case jit_code_movi:
1115                 if (node->flag & jit_flag_node) {
1116                     temp = node->v.n;
1117                     if (temp->code == jit_code_data ||
1118                         (temp->code == jit_code_label &&
1119                          (temp->flag & jit_flag_patch)))
1120                         movi(rn(node->u.w), temp->u.w);
1121                     else {
1122                         assert(temp->code == jit_code_label ||
1123                                temp->code == jit_code_epilog);
1124                         word = movi_p(rn(node->u.w), node->v.w);
1125                         patch(word, node);
1126                     }
1127                 }
1128                 else
1129                     movi(rn(node->u.w), node->v.w);
1130                 break;
1131                 case_rr(neg,);
1132                 case_rr(com,);
1133                 case_rr(clo,);
1134                 case_rr(clz,);
1135                 case_rr(cto,);
1136                 case_rr(ctz,);
1137                 case_rrr(lt,);
1138                 case_rrw(lt,);
1139                 case_rrr(lt, _u);
1140                 case_rrw(lt, _u);
1141                 case_rrr(le,);
1142                 case_rrw(le,);
1143                 case_rrr(le, _u);
1144                 case_rrw(le, _u);
1145                 case_rrr(eq,);
1146                 case_rrw(eq,);
1147                 case_rrr(ge,);
1148                 case_rrw(ge,);
1149                 case_rrr(ge, _u);
1150                 case_rrw(ge, _u);
1151                 case_rrr(gt,);
1152                 case_rrw(gt,);
1153                 case_rrr(gt, _u);
1154                 case_rrw(gt, _u);
1155                 case_rrr(ne,);
1156                 case_rrw(ne,);
1157                 case_brr(blt,);
1158                 case_brw(blt,);
1159                 case_brr(blt, _u);
1160                 case_brw(blt, _u);
1161                 case_brr(ble,);
1162                 case_brw(ble,);
1163                 case_brr(ble, _u);
1164                 case_brw(ble, _u);
1165                 case_brr(beq,);
1166                 case_brw(beq,);
1167                 case_brr(bge,);
1168                 case_brw(bge,);
1169                 case_brr(bge, _u);
1170                 case_brw(bge, _u);
1171                 case_brr(bgt,);
1172                 case_brw(bgt,);
1173                 case_brr(bgt, _u);
1174                 case_brw(bgt, _u);
1175                 case_brr(bne,);
1176                 case_brw(bne,);
1177                 case_brr(boadd,);
1178                 case_brw(boadd,);
1179                 case_brr(boadd, _u);
1180                 case_brw(boadd, _u);
1181                 case_brr(bxadd,);
1182                 case_brw(bxadd,);
1183                 case_brr(bxadd, _u);
1184                 case_brw(bxadd, _u);
1185                 case_brr(bosub,);
1186                 case_brw(bosub,);
1187                 case_brr(bosub, _u);
1188                 case_brw(bosub, _u);
1189                 case_brr(bxsub,);
1190                 case_brw(bxsub,);
1191                 case_brr(bxsub, _u);
1192                 case_brw(bxsub, _u);
1193                 case_brr(bms,);
1194                 case_brw(bms,);
1195                 case_brr(bmc,);
1196                 case_brw(bmc,);
1197                 case_rrr(add, _f);
1198                 case_rrf(add, _f, 32);
1199                 case_rrr(sub, _f);
1200                 case_rrf(sub, _f, 32);
1201                 case_rrf(rsb, _f, 32);
1202                 case_rrr(mul, _f);
1203                 case_rrf(mul, _f, 32);
1204                 case_rrr(div, _f);
1205                 case_rrf(div, _f, 32);
1206                 case_rr(abs, _f);
1207                 case_rr(neg, _f);
1208                 case_rr(sqrt, _f);
1209                 case_rr(ext, _f);
1210                 case_rr(ld, _f);
1211                 case_rw(ld, _f);
1212                 case_rrr(ldx, _f);
1213                 case_rrw(ldx, _f);
1214                 case_rr(st, _f);
1215                 case_wr(st, _f);
1216                 case_rrr(stx, _f);
1217                 case_wrr(stx, _f);
1218                 case_rr(mov, _f);
1219             case jit_code_movi_f:
1220                 assert(node->flag & jit_flag_data);
1221                 movi_f(rn(node->u.w), (jit_float32_t *)node->v.n->u.w);
1222                 break;
1223                 case_rr(ext, _d_f);
1224                 case_rrr(lt, _f);
1225                 case_rrf(lt, _f, 32);
1226                 case_rrr(le, _f);
1227                 case_rrf(le, _f, 32);
1228                 case_rrr(eq, _f);
1229                 case_rrf(eq, _f, 32);
1230                 case_rrr(ge, _f);
1231                 case_rrf(ge, _f, 32);
1232                 case_rrr(gt, _f);
1233                 case_rrf(gt, _f, 32);
1234                 case_rrr(ne, _f);
1235                 case_rrf(ne, _f, 32);
1236                 case_rrr(unlt, _f);
1237                 case_rrf(unlt, _f, 32);
1238                 case_rrr(unle, _f);
1239                 case_rrf(unle, _f, 32);
1240                 case_rrr(uneq, _f);
1241                 case_rrf(uneq, _f, 32);
1242                 case_rrr(unge, _f);
1243                 case_rrf(unge, _f, 32);
1244                 case_rrr(ungt, _f);
1245                 case_rrf(ungt, _f, 32);
1246                 case_rrr(ltgt, _f);
1247                 case_rrf(ltgt, _f, 32);
1248                 case_rrr(ord, _f);
1249                 case_rrf(ord, _f, 32);
1250                 case_rrr(unord, _f);
1251                 case_rrf(unord, _f, 32);
1252                 case_brr(blt, _f);
1253                 case_brf(blt, _f, 32);
1254                 case_brr(ble, _f);
1255                 case_brf(ble, _f, 32);
1256                 case_brr(beq, _f);
1257                 case_brf(beq, _f, 32);
1258                 case_brr(bge, _f);
1259                 case_brf(bge, _f, 32);
1260                 case_brr(bgt, _f);
1261                 case_brf(bgt, _f, 32);
1262                 case_brr(bne, _f);
1263                 case_brf(bne, _f, 32);
1264                 case_brr(bunlt, _f);
1265                 case_brf(bunlt, _f, 32);
1266                 case_brr(bunle, _f);
1267                 case_brf(bunle, _f, 32);
1268                 case_brr(buneq, _f);
1269                 case_brf(buneq, _f, 32);
1270                 case_brr(bunge, _f);
1271                 case_brf(bunge, _f, 32);
1272                 case_brr(bungt, _f);
1273                 case_brf(bungt, _f, 32);
1274                 case_brr(bltgt, _f);
1275                 case_brf(bltgt, _f, 32);
1276                 case_brr(bord, _f);
1277                 case_brf(bord, _f, 32);
1278                 case_brr(bunord, _f);
1279                 case_brf(bunord, _f, 32);
1280                 case_rrr(add, _d);
1281                 case_rrf(add, _d, 64);
1282                 case_rrr(sub, _d);
1283                 case_rrf(sub, _d, 64);
1284                 case_rrf(rsb, _d, 64);
1285                 case_rrr(mul, _d);
1286                 case_rrf(mul, _d, 64);
1287                 case_rrr(div, _d);
1288                 case_rrf(div, _d, 64);
1289                 case_rr(abs, _d);
1290                 case_rr(neg, _d);
1291                 case_rr(sqrt, _d);
1292                 case_rr(ext, _d);
1293                 case_rr(ld, _d);
1294                 case_rw(ld, _d);
1295                 case_rrr(ldx, _d);
1296                 case_rrw(ldx, _d);
1297                 case_rr(st, _d);
1298                 case_wr(st, _d);
1299                 case_rrr(stx, _d);
1300                 case_wrr(stx, _d);
1301                 case_rr(mov, _d);
1302             case jit_code_movi_d:
1303                 assert(node->flag & jit_flag_data);
1304                 movi_d(rn(node->u.w), (jit_float64_t *)node->v.n->u.w);
1305                 break;
1306                 case_rr(ext, _f_d);
1307                 case_rrr(lt, _d);
1308                 case_rrf(lt, _d, 64);
1309                 case_rrr(le, _d);
1310                 case_rrf(le, _d, 64);
1311                 case_rrr(eq, _d);
1312                 case_rrf(eq, _d, 64);
1313                 case_rrr(ge, _d);
1314                 case_rrf(ge, _d, 64);
1315                 case_rrr(gt, _d);
1316                 case_rrf(gt, _d, 64);
1317                 case_rrr(ne, _d);
1318                 case_rrf(ne, _d, 64);
1319                 case_rrr(unlt, _d);
1320                 case_rrf(unlt, _d, 64);
1321                 case_rrr(unle, _d);
1322                 case_rrf(unle, _d, 64);
1323                 case_rrr(uneq, _d);
1324                 case_rrf(uneq, _d, 64);
1325                 case_rrr(unge, _d);
1326                 case_rrf(unge, _d, 64);
1327                 case_rrr(ungt, _d);
1328                 case_rrf(ungt, _d, 64);
1329                 case_rrr(ltgt, _d);
1330                 case_rrf(ltgt, _d, 64);
1331                 case_rrr(ord, _d);
1332                 case_rrf(ord, _d, 64);
1333                 case_rrr(unord, _d);
1334                 case_rrf(unord, _d, 64);
1335                 case_brr(blt, _d);
1336                 case_brf(blt, _d, 64);
1337                 case_brr(ble, _d);
1338                 case_brf(ble, _d, 64);
1339                 case_brr(beq, _d);
1340                 case_brf(beq, _d, 64);
1341                 case_brr(bge, _d);
1342                 case_brf(bge, _d, 64);
1343                 case_brr(bgt, _d);
1344                 case_brf(bgt, _d, 64);
1345                 case_brr(bne, _d);
1346                 case_brf(bne, _d, 64);
1347                 case_brr(bunlt, _d);
1348                 case_brf(bunlt, _d, 64);
1349                 case_brr(bunle, _d);
1350                 case_brf(bunle, _d, 64);
1351                 case_brr(buneq, _d);
1352                 case_brf(buneq, _d, 64);
1353                 case_brr(bunge, _d);
1354                 case_brf(bunge, _d, 64);
1355                 case_brr(bungt, _d);
1356                 case_brf(bungt, _d, 64);
1357                 case_brr(bltgt, _d);
1358                 case_brf(bltgt, _d, 64);
1359                 case_brr(bord, _d);
1360                 case_brf(bord, _d, 64);
1361                 case_brr(bunord, _d);
1362                 case_brf(bunord, _d, 64);
1363             case jit_code_jmpr:
1364                 jmpr(rn(node->u.w));
1365                 break;
1366             case jit_code_jmpi:
1367                 if (node->flag & jit_flag_node) {
1368                     temp = node->u.n;
1369                     assert(temp->code == jit_code_label ||
1370                            temp->code == jit_code_epilog);
1371                     if (temp->flag & jit_flag_patch)
1372                         jmpi(temp->u.w);
1373                     else {
1374                         word = _jit->code.length -
1375                             (_jit->pc.uc - _jit->code.ptr);
1376                         if (_s21_p(word))
1377                             word = jmpi(_jit->pc.w);
1378                         else
1379                             word = jmpi_p(_jit->pc.w);
1380                         patch(word, node);
1381                     }
1382                 }
1383                 else
1384                     jmpi(node->u.w);
1385                 break;
1386             case jit_code_callr:
1387                 callr(rn(node->u.w));
1388                 break;
1389             case jit_code_calli:
1390                 if (node->flag & jit_flag_node) {
1391                     temp = node->u.n;
1392                     assert(temp->code == jit_code_label ||
1393                            temp->code == jit_code_epilog);
1394                     if (!(temp->flag & jit_flag_patch)) {
1395                         word = calli_p(temp->u.w);
1396                         patch(word, node);
1397                     }
1398                     else
1399                         calli(temp->u.w);
1400                 }
1401                 else
1402                     calli(node->u.w);
1403                 break;
1404             case jit_code_prolog:
1405                 _jitc->function = _jitc->functions.ptr + node->w.w;
1406                 undo.node = node;
1407                 undo.word = _jit->pc.w;
1408                 memcpy(&undo.func, _jitc->function, sizeof(undo.func));
1409 #if DEVEL_DISASSEMBLER
1410                 undo.prevw = prevw;
1411 #endif
1412                 undo.patch_offset = _jitc->patches.offset;
1413             restart_function:
1414                 _jitc->again = 0;
1415                 prolog(node);
1416                 break;
1417             case jit_code_epilog:
1418                 assert(_jitc->function == _jitc->functions.ptr + node->w.w);
1419                 if (_jitc->again) {
1420                     for (temp = undo.node->next;
1421                          temp != node; temp = temp->next) {
1422                         if (temp->code == jit_code_label ||
1423                             temp->code == jit_code_epilog)
1424                             temp->flag &= ~jit_flag_patch;
1425                     }
1426                     temp->flag &= ~jit_flag_patch;
1427                     node = undo.node;
1428                     _jit->pc.w = undo.word;
1429                     /* undo.func.self.aoff and undo.func.regset should not
1430                      * be undone, as they will be further updated, and are
1431                      * the reason of the undo. */
1432                     undo.func.self.aoff = _jitc->function->frame +
1433                         _jitc->function->self.aoff;
1434                     /* allocar information also does not need to be undone */
1435                     undo.func.aoffoff = _jitc->function->aoffoff;
1436                     undo.func.allocar = _jitc->function->allocar;
1437                     jit_regset_set(&undo.func.regset, &_jitc->function->regset);
1438                     memcpy(_jitc->function, &undo.func, sizeof(undo.func));
1439 #if DEVEL_DISASSEMBLER
1440                     prevw = undo.prevw;
1441 #endif
1442                     _jitc->patches.offset = undo.patch_offset;
1443                     goto restart_function;
1444                 }
1445                 /* remember label is defined */
1446                 node->flag |= jit_flag_patch;
1447                 node->u.w = _jit->pc.w;
1448                 epilog(node);
1449                 _jitc->function = NULL;
1450                 break;
1451             case jit_code_va_start:
1452                 vastart(rn(node->u.w));
1453                 break;
1454             case jit_code_va_arg:
1455                 vaarg(rn(node->u.w), rn(node->v.w));
1456                 break;
1457             case jit_code_va_arg_d:
1458                 vaarg_d(rn(node->u.w), rn(node->v.w));
1459                 break;
1460             case jit_code_live:                 case jit_code_ellipsis:
1461             case jit_code_va_push:
1462             case jit_code_allocai:              case jit_code_allocar:
1463             case jit_code_arg_c:                case jit_code_arg_s:
1464             case jit_code_arg_i:                case jit_code_arg_l:
1465             case jit_code_arg_f:                case jit_code_arg_d:
1466             case jit_code_va_end:
1467             case jit_code_ret:
1468             case jit_code_retr_c:               case jit_code_reti_c:
1469             case jit_code_retr_uc:              case jit_code_reti_uc:
1470             case jit_code_retr_s:               case jit_code_reti_s:
1471             case jit_code_retr_us:              case jit_code_reti_us:
1472             case jit_code_retr_i:               case jit_code_reti_i:
1473             case jit_code_retr_ui:              case jit_code_reti_ui:
1474             case jit_code_retr_l:               case jit_code_reti_l:
1475             case jit_code_retr_f:               case jit_code_reti_f:
1476             case jit_code_retr_d:               case jit_code_reti_d:
1477             case jit_code_getarg_c:             case jit_code_getarg_uc:
1478             case jit_code_getarg_s:             case jit_code_getarg_us:
1479             case jit_code_getarg_i:             case jit_code_getarg_ui:
1480             case jit_code_getarg_l:
1481             case jit_code_getarg_f:             case jit_code_getarg_d:
1482             case jit_code_putargr_c:            case jit_code_putargi_c:
1483             case jit_code_putargr_uc:           case jit_code_putargi_uc:
1484             case jit_code_putargr_s:            case jit_code_putargi_s:
1485             case jit_code_putargr_us:           case jit_code_putargi_us:
1486             case jit_code_putargr_i:            case jit_code_putargi_i:
1487             case jit_code_putargr_ui:           case jit_code_putargi_ui:
1488             case jit_code_putargr_l:            case jit_code_putargi_l:
1489             case jit_code_putargr_f:            case jit_code_putargi_f:
1490             case jit_code_putargr_d:            case jit_code_putargi_d:
1491             case jit_code_pushargr_c:           case jit_code_pushargi_c:
1492             case jit_code_pushargr_uc:          case jit_code_pushargi_uc:
1493             case jit_code_pushargr_s:           case jit_code_pushargi_s:
1494             case jit_code_pushargr_us:          case jit_code_pushargi_us:
1495             case jit_code_pushargr_i:           case jit_code_pushargi_i:
1496             case jit_code_pushargr_ui:          case jit_code_pushargi_ui:
1497             case jit_code_pushargr_l:           case jit_code_pushargi_l:
1498             case jit_code_pushargr_f:           case jit_code_pushargi_f:
1499             case jit_code_pushargr_d:           case jit_code_pushargi_d:
1500             case jit_code_retval_c:             case jit_code_retval_uc:
1501             case jit_code_retval_s:             case jit_code_retval_us:
1502             case jit_code_retval_i:
1503             case jit_code_retval_ui:            case jit_code_retval_l:
1504             case jit_code_retval_f:             case jit_code_retval_d:
1505             case jit_code_prepare:
1506             case jit_code_finishr:              case jit_code_finishi:
1507                 break;
1508             default:
1509                 abort();
1510         }
1511         if (jit_carry != _NOREG) {
1512             switch (node->code) {
1513                 case jit_code_note:
1514                 case jit_code_addcr:            case jit_code_addci:
1515                 case jit_code_addxr:            case jit_code_addxi:
1516                 case jit_code_subcr:            case jit_code_subci:
1517                 case jit_code_subxr:            case jit_code_subxi:
1518                     break;
1519                 default:
1520                     jit_unget_reg(jit_carry);
1521                     jit_carry = _NOREG;
1522                     break;
1523             }
1524         }
1525         jit_regarg_clr(node, value);
1526         assert(_jitc->regarg == 0 ||
1527                (jit_carry != _NOREG && _jitc->regarg == (1 << jit_carry)));
1528         assert(_jitc->synth == 0);
1529         /* update register live state */
1530         jit_reglive(node);
1531     }
1532 #undef case_brf
1533 #undef case_brw
1534 #undef case_brr
1535 #undef case_wrr
1536 #undef case_rrrw
1537 #undef case_rrrr
1538 #undef case_rrf
1539 #undef case_rrw
1540 #undef case_rrr
1541 #undef case_wr
1542 #undef case_rw
1543 #undef case_rr
1544     for (offset = 0; offset < _jitc->patches.offset; offset++) {
1545         node = _jitc->patches.ptr[offset].node;
1546         word = node->code == jit_code_movi ? node->v.n->u.w : node->u.n->u.w;
1547         patch_at(_jitc->patches.ptr[offset].inst, word);
1548     }
1549
1550     jit_flush(_jit->code.ptr, _jit->pc.uc);
1551
1552     return (_jit->code.ptr);
1553 }
1554
1555 #define CODE                            1
1556 #  include "jit_alpha-cpu.c"
1557 #  include "jit_alpha-fpu.c"
1558 #undef CODE
1559
1560 void
1561 jit_flush(void *fptr, void *tptr)
1562 {
1563 }
1564
1565 void
1566 _emit_ldxi(jit_state_t *_jit, jit_int32_t r0, jit_int32_t r1, jit_word_t i0)
1567 {
1568     ldxi(rn(r0), rn(r1), i0);
1569 }
1570
1571 void
1572 _emit_stxi(jit_state_t *_jit, jit_word_t i0, jit_int32_t r0, jit_int32_t r1)
1573 {
1574     stxi(i0, rn(r0), rn(r1));
1575 }
1576
1577 void
1578 _emit_ldxi_d(jit_state_t *_jit, jit_int32_t r0, jit_int32_t r1, jit_word_t i0)
1579 {
1580     ldxi_d(rn(r0), rn(r1), i0);
1581 }
1582
1583 void
1584 _emit_stxi_d(jit_state_t *_jit, jit_word_t i0, jit_int32_t r0, jit_int32_t r1)
1585 {
1586     stxi_d(i0, rn(r0), rn(r1));
1587 }
1588
1589 static void
1590 _patch(jit_state_t *_jit, jit_word_t instr, jit_node_t *node)
1591 {
1592     jit_int32_t          flag;
1593
1594     assert(node->flag & jit_flag_node);
1595     if (node->code == jit_code_movi)
1596         flag = node->v.n->flag;
1597     else
1598         flag = node->u.n->flag;
1599      assert(!(flag & jit_flag_patch));
1600     if (_jitc->patches.offset >= _jitc->patches.length) {
1601         jit_realloc((jit_pointer_t *)&_jitc->patches.ptr,
1602                     _jitc->patches.length * sizeof(jit_patch_t),
1603                     (_jitc->patches.length + 1024) * sizeof(jit_patch_t));
1604         _jitc->patches.length += 1024;
1605     }
1606     _jitc->patches.ptr[_jitc->patches.offset].inst = instr;
1607     _jitc->patches.ptr[_jitc->patches.offset].node = node;
1608     ++_jitc->patches.offset;
1609 }