deps/lightning/lib/jit_aarch64.c

   1 /*
   2  * Copyright (C) 2013-2019  Free Software Foundation, Inc.
   3  *
   4  * This file is part of GNU lightning.
   5  *
   6  * GNU lightning is free software; you can redistribute it and/or modify it
   7  * under the terms of the GNU Lesser General Public License as published
   8  * by the Free Software Foundation; either version 3, or (at your option)
   9  * any later version.
  10  *
  11  * GNU lightning is distributed in the hope that it will be useful, but
  12  * WITHOUT ANY WARRANTY; without even the implied warranty of MERCHANTABILITY
  13  * or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU Lesser General Public
  14  * License for more details.
  15  *
  16  * Authors:
  17  *      Paulo Cesar Pereira de Andrade
  18  */
  19
  20 #define jit_arg_reg_p(i)                ((i) >= 0 && (i) < 8)
  21 #define jit_arg_f_reg_p(i)              ((i) >= 0 && (i) < 8)
  22
  23 typedef struct jit_qreg {
  24     jit_float64_t       l;
  25     jit_float64_t       h;
  26 } jit_qreg_t;
  27
  28 #define va_gp_top_offset                offsetof(jit_va_list_t, q0)
  29 #define va_fp_top_offset                sizeof(jit_va_list_t)
  30 typedef struct jit_va_list {
  31     jit_pointer_t       stack;
  32     jit_pointer_t       gptop;
  33     jit_pointer_t       fptop;
  34     jit_int32_t         gpoff;
  35     jit_int32_t         fpoff;
  36
  37     jit_int64_t         x0;
  38     jit_int64_t         x1;
  39     jit_int64_t         x2;
  40     jit_int64_t         x3;
  41     jit_int64_t         x4;
  42     jit_int64_t         x5;
  43     jit_int64_t         x6;
  44     jit_int64_t         x7;
  45
  46     jit_qreg_t          q0;
  47     jit_qreg_t          q1;
  48     jit_qreg_t          q2;
  49     jit_qreg_t          q3;
  50     jit_qreg_t          q4;
  51     jit_qreg_t          q5;
  52     jit_qreg_t          q6;
  53     jit_qreg_t          q7;
  54 } jit_va_list_t;
  55
  56 /*
  57  * Prototypes
  58  */
  59 #define patch(instr, node)              _patch(_jit, instr, node)
  60 static void _patch(jit_state_t*,jit_word_t,jit_node_t*);
  61
  62 /* libgcc */
  63 extern void __clear_cache(void *, void *);
  64
  65 #define PROTO                           1
  66 #  include "jit_aarch64-cpu.c"
  67 #  include "jit_aarch64-fpu.c"
  68 #undef PROTO
  69
  70 /*
  71  * Initialization
  72  */
  73 jit_register_t          _rvs[] = {
  74     { rc(gpr) | 0x08,                   "x8" },
  75     { rc(gpr) | 0x12,                   "x18" },
  76     { rc(gpr) | 0x11,                   "x17" },
  77     { rc(gpr) | 0x10,                   "x16" },
  78     { rc(gpr) | 0x09,                   "x9" },
  79     { rc(gpr) | 0x0a,                   "x10" },
  80     { rc(gpr) | 0x0b,                   "x11" },
  81     { rc(gpr) | 0x0c,                   "x12" },
  82     { rc(gpr) | 0x0d,                   "x13" },
  83     { rc(gpr) | 0x0e,                   "x14" },
  84     { rc(gpr) | 0x0f,                   "x15" },
  85     { rc(sav) | rc(gpr) | 0x13,         "x19" },
  86     { rc(sav) | rc(gpr) | 0x14,         "x20" },
  87     { rc(sav) | rc(gpr) | 0x15,         "x21" },
  88     { rc(sav) | rc(gpr) | 0x16,         "x22" },
  89     { rc(sav) | rc(gpr) | 0x17,         "x23" },
  90     { rc(sav) | rc(gpr) | 0x18,         "x24" },
  91     { rc(sav) | rc(gpr) | 0x19,         "x25" },
  92     { rc(sav) | rc(gpr) | 0x1a,         "x26" },
  93     { rc(sav) | rc(gpr) | 0x1b,         "x27" },
  94     { rc(sav) | rc(gpr) | 0x1c,         "x28" },
  95     { 0x1f,                             "sp" },
  96     { 0x1e,                             "lr" },
  97     { 0x1d,                             "fp" },
  98     { rc(arg) | rc(gpr) | 0x07,         "x7" },
  99     { rc(arg) | rc(gpr) | 0x06,         "x6" },
 100     { rc(arg) | rc(gpr) | 0x05,         "x5" },
 101     { rc(arg) | rc(gpr) | 0x04,         "x4" },
 102     { rc(arg) | rc(gpr) | 0x03,         "x3" },
 103     { rc(arg) | rc(gpr) | 0x02,         "x2" },
 104     { rc(arg) | rc(gpr) | 0x01,         "x1" },
 105     { rc(arg) | rc(gpr) | 0x00,         "x0" },
 106     { rc(fpr) | 0x1f,                   "v31" },
 107     { rc(fpr) | 0x1e,                   "v30" },
 108     { rc(fpr) | 0x1d,                   "v29" },
 109     { rc(fpr) | 0x1c,                   "v28" },
 110     { rc(fpr) | 0x1b,                   "v27" },
 111     { rc(fpr) | 0x1a,                   "v26" },
 112     { rc(fpr) | 0x19,                   "v25" },
 113     { rc(fpr) | 0x18,                   "v24" },
 114     { rc(fpr) | 0x17,                   "v23" },
 115     { rc(fpr) | 0x16,                   "v22" },
 116     { rc(fpr) | 0x15,                   "v21" },
 117     { rc(fpr) | 0x14,                   "v20" },
 118     { rc(fpr) | 0x13,                   "v19" },
 119     { rc(fpr) | 0x12,                   "v18" },
 120     { rc(fpr) | 0x11,                   "v17" },
 121     { rc(fpr) | 0x10,                   "v16" },
 122     { rc(sav) | rc(fpr) | 0x08,         "v8" },
 123     { rc(sav) | rc(fpr) | 0x09,         "v9" },
 124     { rc(sav) | rc(fpr) | 0x0a,         "v10" },
 125     { rc(sav) | rc(fpr) | 0x0b,         "v11" },
 126     { rc(sav) | rc(fpr) | 0x0c,         "v12" },
 127     { rc(sav) | rc(fpr) | 0x0d,         "v13" },
 128     { rc(sav) | rc(fpr) | 0x0e,         "v14" },
 129     { rc(sav) | rc(fpr) | 0x0f,         "v15" },
 130     { rc(arg) | rc(fpr) | 0x07,         "v7" },
 131     { rc(arg) | rc(fpr) | 0x06,         "v6" },
 132     { rc(arg) | rc(fpr) | 0x05,         "v5" },
 133     { rc(arg) | rc(fpr) | 0x04,         "v4" },
 134     { rc(arg) | rc(fpr) | 0x03,         "v3" },
 135     { rc(arg) | rc(fpr) | 0x02,         "v2" },
 136     { rc(arg) | rc(fpr) | 0x01,         "v1" },
 137     { rc(arg) | rc(fpr) | 0x00,         "v0" },
 138     { _NOREG,                           "<none>" },
 139 };
 140
 141 /*
 142  * Implementation
 143  */
 144 void
 145 jit_get_cpu(void)
 146 {
 147 }
 148
 149 void
 150 _jit_init(jit_state_t *_jit)
 151 {
 152     _jitc->reglen = jit_size(_rvs) - 1;
 153 }
 154
 155 void
 156 _jit_prolog(jit_state_t *_jit)
 157 {
 158     jit_int32_t          offset;
 159
 160     if (_jitc->function)
 161         jit_epilog();
 162     assert(jit_regset_cmp_ui(&_jitc->regarg, 0) == 0);
 163     jit_regset_set_ui(&_jitc->regsav, 0);
 164     offset = _jitc->functions.offset;
 165     if (offset >= _jitc->functions.length) {
 166         jit_realloc((jit_pointer_t *)&_jitc->functions.ptr,
 167                     _jitc->functions.length * sizeof(jit_function_t),
 168                     (_jitc->functions.length + 16) * sizeof(jit_function_t));
 169         _jitc->functions.length += 16;
 170     }
 171     _jitc->function = _jitc->functions.ptr + _jitc->functions.offset++;
 172     _jitc->function->self.size = stack_framesize;
 173     _jitc->function->self.argi = _jitc->function->self.argf =
 174         _jitc->function->self.alen = 0;
 175     _jitc->function->self.aoff = 0;
 176     _jitc->function->self.call = jit_call_default;
 177     jit_alloc((jit_pointer_t *)&_jitc->function->regoff,
 178               _jitc->reglen * sizeof(jit_int32_t));
 179
 180     /* _no_link here does not mean the jit_link() call can be removed
 181      * by rewriting as:
 182      * _jitc->function->prolog = jit_new_node(jit_code_prolog);
 183      */
 184     _jitc->function->prolog = jit_new_node_no_link(jit_code_prolog);
 185     jit_link(_jitc->function->prolog);
 186     _jitc->function->prolog->w.w = offset;
 187     _jitc->function->epilog = jit_new_node_no_link(jit_code_epilog);
 188     /*  u:      label value
 189      *  v:      offset in blocks vector
 190      *  w:      offset in functions vector
 191      */
 192     _jitc->function->epilog->w.w = offset;
 193
 194     jit_regset_new(&_jitc->function->regset);
 195 }
 196
 197 jit_int32_t
 198 _jit_allocai(jit_state_t *_jit, jit_int32_t length)
 199 {
 200     assert(_jitc->function);
 201     switch (length) {
 202         case 0: case 1:                                         break;
 203         case 2:         _jitc->function->self.aoff &= -2;       break;
 204         case 3: case 4: _jitc->function->self.aoff &= -4;       break;
 205         default:        _jitc->function->self.aoff &= -8;       break;
 206     }
 207     _jitc->function->self.aoff -= length;
 208     if (!_jitc->realize) {
 209         jit_inc_synth_ww(allocai, _jitc->function->self.aoff, length);
 210         jit_dec_synth();
 211     }
 212     return (_jitc->function->self.aoff);
 213 }
 214
 215 void
 216 _jit_allocar(jit_state_t *_jit, jit_int32_t u, jit_int32_t v)
 217 {
 218     jit_int32_t          r0, r1;
 219     assert(_jitc->function);
 220     jit_inc_synth_ww(allocar, u, v);
 221     if (!_jitc->function->allocar) {
 222         _jitc->function->aoffoff = jit_allocai(sizeof(jit_int32_t));
 223         _jitc->function->allocar = 1;
 224     }
 225     r0 = jit_get_reg(jit_class_gpr);
 226     jit_negr(r0, v);
 227     jit_andi(r0, r0, -16);
 228     jit_ldxi_i(u, JIT_FP, _jitc->function->aoffoff);
 229     jit_addr(u, u, r0);
 230     /* Cannot "addr sp, sp, reg" because in this context "sp" is "[w|x]zr",
 231      * the zero register */
 232 #if 0
 233     jit_addr(JIT_SP, JIT_SP, r0);
 234 #else
 235     r1 = jit_get_reg(jit_class_gpr);
 236     /* note that "mov r1, sp" does not work, but the proper encoding
 237      * can be triggered before actually emiting with "add r1, sp, 0" */
 238     jit_addi(r1, JIT_SP, 0);
 239     jit_addr(r1, r1, r0);
 240     jit_addi(JIT_SP, r1, 0);
 241     jit_unget_reg(r1);
 242 #endif
 243     jit_stxi_i(_jitc->function->aoffoff, JIT_FP, u);
 244     jit_unget_reg(r0);
 245     jit_dec_synth();
 246 }
 247
 248 void
 249 _jit_ret(jit_state_t *_jit)
 250 {
 251     jit_node_t          *instr;
 252     assert(_jitc->function);
 253     jit_inc_synth(ret);
 254     /* jump to epilog */
 255     instr = jit_jmpi();
 256     jit_patch_at(instr, _jitc->function->epilog);
 257     jit_dec_synth();
 258 }
 259
 260 void
 261 _jit_retr(jit_state_t *_jit, jit_int32_t u)
 262 {
 263     jit_inc_synth_w(retr, u);
 264     if (JIT_RET != u)
 265         jit_movr(JIT_RET, u);
 266     jit_live(JIT_RET);
 267     jit_ret();
 268     jit_dec_synth();
 269 }
 270
 271 void
 272 _jit_reti(jit_state_t *_jit, jit_word_t u)
 273 {
 274     jit_inc_synth_w(reti, u);
 275     jit_movi(JIT_RET, u);
 276     jit_ret();
 277     jit_dec_synth();
 278 }
 279
 280 void
 281 _jit_retr_f(jit_state_t *_jit, jit_int32_t u)
 282 {
 283     jit_inc_synth_w(retr_f, u);
 284     if (u != JIT_FRET)
 285         jit_movr_f(JIT_FRET, u);
 286     else
 287         jit_live(JIT_FRET);
 288     jit_ret();
 289     jit_dec_synth();
 290 }
 291
 292 void
 293 _jit_reti_f(jit_state_t *_jit, jit_float32_t u)
 294 {
 295     jit_inc_synth_f(reti_f, u);
 296     jit_movi_f(JIT_FRET, u);
 297     jit_ret();
 298     jit_dec_synth();
 299 }
 300
 301 void
 302 _jit_retr_d(jit_state_t *_jit, jit_int32_t u)
 303 {
 304     jit_inc_synth_w(retr_d, u);
 305     if (u != JIT_FRET)
 306         jit_movr_d(JIT_FRET, u);
 307     else
 308         jit_live(JIT_FRET);
 309     jit_ret();
 310     jit_dec_synth();
 311 }
 312
 313 void
 314 _jit_reti_d(jit_state_t *_jit, jit_float64_t u)
 315 {
 316     jit_inc_synth_d(reti_d, u);
 317     jit_movi_d(JIT_FRET, u);
 318     jit_ret();
 319     jit_dec_synth();
 320 }
 321
 322 void
 323 _jit_epilog(jit_state_t *_jit)
 324 {
 325     assert(_jitc->function);
 326     assert(_jitc->function->epilog->next == NULL);
 327     jit_link(_jitc->function->epilog);
 328     _jitc->function = NULL;
 329 }
 330
 331 jit_bool_t
 332 _jit_arg_register_p(jit_state_t *_jit, jit_node_t *u)
 333 {
 334     if (u->code == jit_code_arg)
 335         return (jit_arg_reg_p(u->u.w));
 336     assert(u->code == jit_code_arg_f || u->code == jit_code_arg_d);
 337     return (jit_arg_f_reg_p(u->u.w));
 338 }
 339
 340 void
 341 _jit_ellipsis(jit_state_t *_jit)
 342 {
 343     jit_inc_synth(ellipsis);
 344     if (_jitc->prepare) {
 345         jit_link_prepare();
 346         assert(!(_jitc->function->call.call & jit_call_varargs));
 347         _jitc->function->call.call |= jit_call_varargs;
 348     }
 349     else {
 350         jit_link_prolog();
 351         assert(!(_jitc->function->self.call & jit_call_varargs));
 352         _jitc->function->self.call |= jit_call_varargs;
 353
 354         /* Allocate va_list like object in the stack,
 355          * with enough space to save all argument
 356          * registers, and use fixed offsets for them. */
 357         _jitc->function->vaoff = jit_allocai(sizeof(jit_va_list_t));
 358
 359         /* Initialize gp offset in save area. */
 360         if (jit_arg_reg_p(_jitc->function->self.argi))
 361             _jitc->function->vagp = (8 - _jitc->function->self.argi) * -8;
 362         else
 363             _jitc->function->vagp = 0;
 364
 365         /* Initialize fp offset in save area. */
 366         if (jit_arg_f_reg_p(_jitc->function->self.argf))
 367             _jitc->function->vafp = (8 - _jitc->function->self.argf) * -16;
 368         else
 369             _jitc->function->vafp = 0;
 370     }
 371     jit_dec_synth();
 372 }
 373
 374 void
 375 _jit_va_push(jit_state_t *_jit, jit_int32_t u)
 376 {
 377     jit_inc_synth_w(va_push, u);
 378     jit_pushargr(u);
 379     jit_dec_synth();
 380 }
 381
 382 jit_node_t *
 383 _jit_arg(jit_state_t *_jit)
 384 {
 385     jit_node_t          *node;
 386     jit_int32_t          offset;
 387     assert(_jitc->function);
 388     assert(!(_jitc->function->self.call & jit_call_varargs));
 389     if (jit_arg_reg_p(_jitc->function->self.argi))
 390         offset = _jitc->function->self.argi++;
 391     else {
 392         offset = _jitc->function->self.size;
 393         _jitc->function->self.size += sizeof(jit_word_t);
 394     }
 395     node = jit_new_node_ww(jit_code_arg, offset,
 396                            ++_jitc->function->self.argn);
 397     jit_link_prolog();
 398     return (node);
 399 }
 400
 401 jit_node_t *
 402 _jit_arg_f(jit_state_t *_jit)
 403 {
 404     jit_node_t          *node;
 405     jit_int32_t          offset;
 406     assert(_jitc->function);
 407     assert(!(_jitc->function->self.call & jit_call_varargs));
 408     if (jit_arg_f_reg_p(_jitc->function->self.argf))
 409         offset = _jitc->function->self.argf++;
 410     else {
 411         offset = _jitc->function->self.size;
 412         _jitc->function->self.size += sizeof(jit_word_t);
 413     }
 414     node = jit_new_node_ww(jit_code_arg_f, offset,
 415                            ++_jitc->function->self.argn);
 416     jit_link_prolog();
 417     return (node);
 418 }
 419
 420 jit_node_t *
 421 _jit_arg_d(jit_state_t *_jit)
 422 {
 423     jit_node_t          *node;
 424     jit_int32_t          offset;
 425     assert(_jitc->function);
 426     assert(!(_jitc->function->self.call & jit_call_varargs));
 427     if (jit_arg_f_reg_p(_jitc->function->self.argf))
 428         offset = _jitc->function->self.argf++;
 429     else {
 430         offset = _jitc->function->self.size;
 431         _jitc->function->self.size += sizeof(jit_word_t);
 432     }
 433     node = jit_new_node_ww(jit_code_arg_d, offset,
 434                            ++_jitc->function->self.argn);
 435     jit_link_prolog();
 436     return (node);
 437 }
 438
 439 void
 440 _jit_getarg_c(jit_state_t *_jit, jit_int32_t u, jit_node_t *v)
 441 {
 442     assert(v->code == jit_code_arg);
 443     jit_inc_synth_wp(getarg_c, u, v);
 444     if (jit_arg_reg_p(v->u.w))
 445         jit_extr_c(u, JIT_RA0 - v->u.w);
 446     else
 447         jit_ldxi_c(u, JIT_FP, v->u.w);
 448     jit_dec_synth();
 449 }
 450
 451 void
 452 _jit_getarg_uc(jit_state_t *_jit, jit_int32_t u, jit_node_t *v)
 453 {
 454     assert(v->code == jit_code_arg);
 455     jit_inc_synth_wp(getarg_uc, u, v);
 456     if (jit_arg_reg_p(v->u.w))
 457         jit_extr_uc(u, JIT_RA0 - v->u.w);
 458     else
 459         jit_ldxi_uc(u, JIT_FP, v->u.w);
 460     jit_dec_synth();
 461 }
 462
 463 void
 464 _jit_getarg_s(jit_state_t *_jit, jit_int32_t u, jit_node_t *v)
 465 {
 466     assert(v->code == jit_code_arg);
 467     jit_inc_synth_wp(getarg_s, u, v);
 468     if (jit_arg_reg_p(v->u.w))
 469         jit_extr_s(u, JIT_RA0 - v->u.w);
 470     else
 471         jit_ldxi_s(u, JIT_FP, v->u.w);
 472     jit_dec_synth();
 473 }
 474
 475 void
 476 _jit_getarg_us(jit_state_t *_jit, jit_int32_t u, jit_node_t *v)
 477 {
 478     assert(v->code == jit_code_arg);
 479     jit_inc_synth_wp(getarg_us, u, v);
 480     if (jit_arg_reg_p(v->u.w))
 481         jit_extr_us(u, JIT_RA0 - v->u.w);
 482     else
 483         jit_ldxi_us(u, JIT_FP, v->u.w);
 484     jit_dec_synth();
 485 }
 486
 487 void
 488 _jit_getarg_i(jit_state_t *_jit, jit_int32_t u, jit_node_t *v)
 489 {
 490     assert(v->code == jit_code_arg);
 491     jit_inc_synth_wp(getarg_i, u, v);
 492     if (jit_arg_reg_p(v->u.w))
 493         jit_extr_i(u, JIT_RA0 - v->u.w);
 494     else
 495         jit_ldxi_i(u, JIT_FP, v->u.w);
 496     jit_dec_synth();
 497 }
 498
 499 void
 500 _jit_getarg_ui(jit_state_t *_jit, jit_int32_t u, jit_node_t *v)
 501 {
 502     assert(v->code == jit_code_arg);
 503     jit_inc_synth_wp(getarg_ui, u, v);
 504     if (jit_arg_reg_p(v->u.w))
 505         jit_extr_ui(u, JIT_RA0 - v->u.w);
 506     else
 507         jit_ldxi_ui(u, JIT_FP, v->u.w);
 508     jit_dec_synth();
 509 }
 510
 511 void
 512 _jit_getarg_l(jit_state_t *_jit, jit_int32_t u, jit_node_t *v)
 513 {
 514     assert(v->code == jit_code_arg);
 515     jit_inc_synth_wp(getarg_l, u, v);
 516     if (jit_arg_reg_p(v->u.w))
 517         jit_movr(u, JIT_RA0 - v->u.w);
 518     else
 519         jit_ldxi_l(u, JIT_FP, v->u.w);
 520     jit_dec_synth();
 521 }
 522
 523 void
 524 _jit_putargr(jit_state_t *_jit, jit_int32_t u, jit_node_t *v)
 525 {
 526     assert(v->code == jit_code_arg);
 527     jit_inc_synth_wp(putargr, u, v);
 528     if (jit_arg_reg_p(v->u.w))
 529         jit_movr(JIT_RA0 - v->u.w, u);
 530     else
 531         jit_stxi(v->u.w, JIT_FP, u);
 532     jit_dec_synth();
 533 }
 534
 535 void
 536 _jit_putargi(jit_state_t *_jit, jit_word_t u, jit_node_t *v)
 537 {
 538     jit_int32_t         regno;
 539     assert(v->code == jit_code_arg);
 540     jit_inc_synth_wp(putargi, u, v);
 541     if (jit_arg_reg_p(v->u.w))
 542         jit_movi(JIT_RA0 - v->u.w, u);
 543     else {
 544         regno = jit_get_reg(jit_class_gpr);
 545         jit_movi(regno, u);
 546         jit_stxi(v->u.w, JIT_FP, regno);
 547         jit_unget_reg(regno);
 548     }
 549     jit_dec_synth();
 550 }
 551
 552 void
 553 _jit_getarg_f(jit_state_t *_jit, jit_int32_t u, jit_node_t *v)
 554 {
 555     assert(v->code == jit_code_arg_f);
 556     jit_inc_synth_wp(getarg_f, u, v);
 557     if (jit_arg_reg_p(v->u.w))
 558         jit_movr_f(u, JIT_FA0 - v->u.w);
 559     else
 560         jit_ldxi_f(u, JIT_FP, v->u.w);
 561     jit_dec_synth();
 562 }
 563
 564 void
 565 _jit_putargr_f(jit_state_t *_jit, jit_int32_t u, jit_node_t *v)
 566 {
 567     assert(v->code == jit_code_arg_f);
 568     jit_inc_synth_wp(putargr_f, u, v);
 569     if (jit_arg_f_reg_p(v->u.w))
 570         jit_movr_f(JIT_FA0 - v->u.w, u);
 571     else
 572         jit_stxi_f(v->u.w, JIT_FP, u);
 573     jit_dec_synth();
 574 }
 575
 576 void
 577 _jit_putargi_f(jit_state_t *_jit, jit_float32_t u, jit_node_t *v)
 578 {
 579     jit_int32_t         regno;
 580     assert(v->code == jit_code_arg_f);
 581     jit_inc_synth_fp(putargi_f, u, v);
 582     if (jit_arg_f_reg_p(v->u.w))
 583         jit_movi_f(JIT_FA0 - v->u.w, u);
 584     else {
 585         regno = jit_get_reg(jit_class_fpr);
 586         jit_movi_f(regno, u);
 587         jit_stxi_f(v->u.w, JIT_FP, regno);
 588         jit_unget_reg(regno);
 589     }
 590     jit_dec_synth();
 591 }
 592
 593 void
 594 _jit_getarg_d(jit_state_t *_jit, jit_int32_t u, jit_node_t *v)
 595 {
 596     assert(v->code == jit_code_arg_d);
 597     jit_inc_synth_wp(getarg_d, u, v);
 598     if (jit_arg_f_reg_p(v->u.w))
 599         jit_movr_d(u, JIT_FA0 - v->u.w);
 600     else
 601         jit_ldxi_d(u, JIT_FP, v->u.w);
 602     jit_dec_synth();
 603 }
 604
 605 void
 606 _jit_putargr_d(jit_state_t *_jit, jit_int32_t u, jit_node_t *v)
 607 {
 608     assert(v->code == jit_code_arg_d);
 609     jit_inc_synth_wp(putargr_d, u, v);
 610     if (jit_arg_reg_p(v->u.w))
 611         jit_movr_d(JIT_FA0 - v->u.w, u);
 612     else
 613         jit_stxi_d(v->u.w, JIT_FP, u);
 614     jit_dec_synth();
 615 }
 616
 617 void
 618 _jit_putargi_d(jit_state_t *_jit, jit_float64_t u, jit_node_t *v)
 619 {
 620     jit_int32_t         regno;
 621     assert(v->code == jit_code_arg_d);
 622     jit_inc_synth_dp(putargi_d, u, v);
 623     if (jit_arg_reg_p(v->u.w))
 624         jit_movi_d(JIT_FA0 - v->u.w, u);
 625     else {
 626         regno = jit_get_reg(jit_class_fpr);
 627         jit_movi_d(regno, u);
 628         jit_stxi_d(v->u.w, JIT_FP, regno);
 629         jit_unget_reg(regno);
 630     }
 631     jit_dec_synth();
 632 }
 633
 634 void
 635 _jit_pushargr(jit_state_t *_jit, jit_int32_t u)
 636 {
 637     assert(_jitc->function);
 638     jit_inc_synth_w(pushargr, u);
 639     jit_link_prepare();
 640     if (jit_arg_reg_p(_jitc->function->call.argi)) {
 641         jit_movr(JIT_RA0 - _jitc->function->call.argi, u);
 642         ++_jitc->function->call.argi;
 643     }
 644     else {
 645         jit_stxi(_jitc->function->call.size, JIT_SP, u);
 646         _jitc->function->call.size += sizeof(jit_word_t);
 647     }
 648     jit_dec_synth();
 649 }
 650
 651 void
 652 _jit_pushargi(jit_state_t *_jit, jit_word_t u)
 653 {
 654     jit_int32_t          regno;
 655     assert(_jitc->function);
 656     jit_inc_synth_w(pushargi, u);
 657     jit_link_prepare();
 658     if (jit_arg_reg_p(_jitc->function->call.argi)) {
 659         jit_movi(JIT_RA0 - _jitc->function->call.argi, u);
 660         ++_jitc->function->call.argi;
 661     }
 662     else {
 663         regno = jit_get_reg(jit_class_gpr);
 664         jit_movi(regno, u);
 665         jit_stxi(_jitc->function->call.size, JIT_SP, regno);
 666         jit_unget_reg(regno);
 667         _jitc->function->call.size += sizeof(jit_word_t);
 668     }
 669     jit_dec_synth();
 670 }
 671
 672 void
 673 _jit_pushargr_f(jit_state_t *_jit, jit_int32_t u)
 674 {
 675     assert(_jitc->function);
 676     jit_inc_synth_w(pushargr_f, u);
 677     jit_link_prepare();
 678     if (jit_arg_f_reg_p(_jitc->function->call.argf)) {
 679         jit_movr_f(JIT_FA0 - _jitc->function->call.argf, u);
 680         ++_jitc->function->call.argf;
 681     }
 682     else {
 683         jit_stxi_f(_jitc->function->call.size, JIT_SP, u);
 684         _jitc->function->call.size += sizeof(jit_word_t);
 685     }
 686     jit_dec_synth();
 687 }
 688
 689 void
 690 _jit_pushargi_f(jit_state_t *_jit, jit_float32_t u)
 691 {
 692     jit_int32_t         regno;
 693     assert(_jitc->function);
 694     jit_inc_synth_f(pushargi_f, u);
 695     jit_link_prepare();
 696     if (jit_arg_f_reg_p(_jitc->function->call.argf)) {
 697         jit_movi_f(JIT_FA0 - _jitc->function->call.argf, u);
 698         ++_jitc->function->call.argf;
 699     }
 700     else {
 701         regno = jit_get_reg(jit_class_fpr);
 702         jit_movi_f(regno, u);
 703         jit_stxi_f(_jitc->function->call.size, JIT_SP, regno);
 704         jit_unget_reg(regno);
 705         _jitc->function->call.size += sizeof(jit_word_t);
 706     }
 707     jit_dec_synth();
 708 }
 709
 710 void
 711 _jit_pushargr_d(jit_state_t *_jit, jit_int32_t u)
 712 {
 713     assert(_jitc->function);
 714     jit_inc_synth_w(pushargr_d, u);
 715     jit_link_prepare();
 716     if (jit_arg_f_reg_p(_jitc->function->call.argf)) {
 717         jit_movr_d(JIT_FA0 - _jitc->function->call.argf, u);
 718         ++_jitc->function->call.argf;
 719     }
 720     else {
 721         jit_stxi_d(_jitc->function->call.size, JIT_SP, u);
 722         _jitc->function->call.size += sizeof(jit_word_t);
 723     }
 724     jit_dec_synth();
 725 }
 726
 727 void
 728 _jit_pushargi_d(jit_state_t *_jit, jit_float64_t u)
 729 {
 730     jit_int32_t         regno;
 731     assert(_jitc->function);
 732     jit_inc_synth_d(pushargi_d, u);
 733     jit_link_prepare();
 734     if (jit_arg_f_reg_p(_jitc->function->call.argf)) {
 735         jit_movi_d(JIT_FA0 - _jitc->function->call.argf, u);
 736         ++_jitc->function->call.argf;
 737     }
 738     else {
 739         regno = jit_get_reg(jit_class_fpr);
 740         jit_movi_d(regno, u);
 741         jit_stxi_d(_jitc->function->call.size, JIT_SP, regno);
 742         jit_unget_reg(regno);
 743         _jitc->function->call.size += sizeof(jit_word_t);
 744     }
 745     jit_dec_synth();
 746 }
 747
 748 jit_bool_t
 749 _jit_regarg_p(jit_state_t *_jit, jit_node_t *node, jit_int32_t regno)
 750 {
 751     jit_int32_t         spec;
 752     spec = jit_class(_rvs[regno].spec);
 753     if (spec & jit_class_arg) {
 754         regno = JIT_RA0 - regno;
 755         if (regno >= 0 && regno < node->v.w)
 756             return (1);
 757         if (spec & jit_class_fpr) {
 758             regno = JIT_FA0 - regno;
 759             if (regno >= 0 && regno < node->w.w)
 760                 return (1);
 761         }
 762     }
 763
 764     return (0);
 765 }
 766
 767 void
 768 _jit_finishr(jit_state_t *_jit, jit_int32_t r0)
 769 {
 770     jit_node_t          *node;
 771     assert(_jitc->function);
 772     jit_inc_synth_w(finishr, r0);
 773     if (_jitc->function->self.alen < _jitc->function->call.size)
 774         _jitc->function->self.alen = _jitc->function->call.size;
 775     node = jit_callr(r0);
 776     node->v.w = _jitc->function->self.argi;
 777     node->w.w = _jitc->function->call.argf;
 778     _jitc->function->call.argi = _jitc->function->call.argf =
 779         _jitc->function->call.size = 0;
 780     _jitc->prepare = 0;
 781     jit_dec_synth();
 782 }
 783
 784 jit_node_t *
 785 _jit_finishi(jit_state_t *_jit, jit_pointer_t i0)
 786 {
 787     jit_node_t          *node;
 788     assert(_jitc->function);
 789     jit_inc_synth_w(finishi, (jit_word_t)i0);
 790     if (_jitc->function->self.alen < _jitc->function->call.size)
 791         _jitc->function->self.alen = _jitc->function->call.size;
 792     node = jit_calli(i0);
 793     node->v.w = _jitc->function->call.argi;
 794     node->w.w = _jitc->function->call.argf;
 795     _jitc->function->call.argi = _jitc->function->call.argf =
 796         _jitc->function->call.size = 0;
 797     _jitc->prepare = 0;
 798     jit_dec_synth();
 799     return (node);
 800 }
 801
 802 void
 803 _jit_retval_c(jit_state_t *_jit, jit_int32_t r0)
 804 {
 805     jit_inc_synth_w(retval_c, r0);
 806     jit_extr_c(r0, JIT_RET);
 807     jit_dec_synth();
 808 }
 809
 810 void
 811 _jit_retval_uc(jit_state_t *_jit, jit_int32_t r0)
 812 {
 813     jit_inc_synth_w(retval_uc, r0);
 814     jit_extr_uc(r0, JIT_RET);
 815     jit_dec_synth();
 816 }
 817
 818 void
 819 _jit_retval_s(jit_state_t *_jit, jit_int32_t r0)
 820 {
 821     jit_inc_synth_w(retval_s, r0);
 822     jit_extr_s(r0, JIT_RET);
 823     jit_dec_synth();
 824 }
 825
 826 void
 827 _jit_retval_us(jit_state_t *_jit, jit_int32_t r0)
 828 {
 829     jit_inc_synth_w(retval_us, r0);
 830     jit_extr_us(r0, JIT_RET);
 831     jit_dec_synth();
 832 }
 833
 834 void
 835 _jit_retval_i(jit_state_t *_jit, jit_int32_t r0)
 836 {
 837     jit_inc_synth_w(retval_i, r0);
 838     jit_extr_i(r0, JIT_RET);
 839     jit_dec_synth();
 840 }
 841
 842 void
 843 _jit_retval_ui(jit_state_t *_jit, jit_int32_t r0)
 844 {
 845     jit_inc_synth_w(retval_ui, r0);
 846     jit_extr_ui(r0, JIT_RET);
 847     jit_dec_synth();
 848 }
 849
 850 void
 851 _jit_retval_l(jit_state_t *_jit, jit_int32_t r0)
 852 {
 853     jit_inc_synth_w(retval_l, r0);
 854     if (r0 != JIT_RET)
 855         jit_movr(r0, JIT_RET);
 856     jit_dec_synth();
 857 }
 858
 859 void
 860 _jit_retval_f(jit_state_t *_jit, jit_int32_t r0)
 861 {
 862     jit_inc_synth_w(retval_f, r0);
 863     if (r0 != JIT_FRET)
 864         jit_movr_f(r0, JIT_FRET);
 865     jit_dec_synth();
 866 }
 867
 868 void
 869 _jit_retval_d(jit_state_t *_jit, jit_int32_t r0)
 870 {
 871     jit_inc_synth_w(retval_d, r0);
 872     if (r0 != JIT_FRET)
 873         jit_movr_d(r0, JIT_FRET);
 874     jit_dec_synth();
 875 }
 876
 877 jit_pointer_t
 878 _emit_code(jit_state_t *_jit)
 879 {
 880     jit_node_t          *node;
 881     jit_node_t          *temp;
 882     jit_word_t           word;
 883     jit_word_t           value;
 884     jit_int32_t          offset;
 885     struct {
 886         jit_node_t      *node;
 887         jit_uint8_t     *data;
 888         jit_word_t       word;
 889 #if DEVEL_DISASSEMBLER
 890         jit_word_t       prevw;
 891 #endif
 892         jit_int32_t      const_offset;
 893         jit_int32_t      patch_offset;
 894     } undo;
 895 #if DEVEL_DISASSEMBLER
 896     jit_word_t           prevw;
 897 #endif
 898
 899     _jitc->function = NULL;
 900
 901     jit_reglive_setup();
 902
 903     undo.word = 0;
 904     undo.node = NULL;
 905     undo.const_offset = undo.patch_offset = 0;
 906 #  define assert_data(node)             /**/
 907 #define case_rr(name, type)                                             \
 908             case jit_code_##name##r##type:                              \
 909                 name##r##type(rn(node->u.w), rn(node->v.w));            \
 910                 break
 911 #define case_rw(name, type)                                             \
 912             case jit_code_##name##i##type:                              \
 913                 name##i##type(rn(node->u.w), node->v.w);                \
 914                 break
 915 #define case_wr(name, type)                                             \
 916             case jit_code_##name##i##type:                              \
 917                 name##i##type(node->u.w, rn(node->v.w));                \
 918                 break
 919 #define case_rrr(name, type)                                            \
 920             case jit_code_##name##r##type:                              \
 921                 name##r##type(rn(node->u.w),                            \
 922                               rn(node->v.w), rn(node->w.w));            \
 923                 break
 924 #define case_rrrr(name, type)                                           \
 925             case jit_code_##name##r##type:                              \
 926                 name##r##type(rn(node->u.q.l), rn(node->u.q.h),         \
 927                               rn(node->v.w), rn(node->w.w));            \
 928                 break
 929 #define case_rrw(name, type)                                            \
 930             case jit_code_##name##i##type:                              \
 931                 name##i##type(rn(node->u.w), rn(node->v.w), node->w.w); \
 932                 break
 933 #define case_rrrw(name, type)                                           \
 934             case jit_code_##name##i##type:                              \
 935                 name##i##type(rn(node->u.q.l), rn(node->u.q.h),         \
 936                               rn(node->v.w), node->w.w);                \
 937                 break
 938 #define case_rrf(name)                                                  \
 939             case jit_code_##name##i_f:                                  \
 940                 assert_data(node);                                      \
 941                 name##i_f(rn(node->u.w), rn(node->v.w), node->w.f);     \
 942                 break
 943 #define case_rrd(name)                                                  \
 944             case jit_code_##name##i_d:                                  \
 945                 assert_data(node);                                      \
 946                 name##i_d(rn(node->u.w), rn(node->v.w), node->w.d);     \
 947                 break
 948 #define case_wrr(name, type)                                            \
 949             case jit_code_##name##i##type:                              \
 950                 name##i##type(node->u.w, rn(node->v.w), rn(node->w.w)); \
 951                 break
 952 #define case_brr(name, type)                                            \
 953             case jit_code_##name##r##type:                              \
 954                 temp = node->u.n;                                       \
 955                 assert(temp->code == jit_code_label ||                  \
 956                        temp->code == jit_code_epilog);                  \
 957                 if (temp->flag & jit_flag_patch)                        \
 958                     name##r##type(temp->u.w, rn(node->v.w),             \
 959                                   rn(node->w.w));                       \
 960                 else {                                                  \
 961                     word = name##r##type(_jit->pc.w,                    \
 962                                          rn(node->v.w), rn(node->w.w)); \
 963                     patch(word, node);                                  \
 964                 }                                                       \
 965                 break
 966 #define case_brw(name, type)                                            \
 967             case jit_code_##name##i##type:                              \
 968                 temp = node->u.n;                                       \
 969                 assert(temp->code == jit_code_label ||                  \
 970                        temp->code == jit_code_epilog);                  \
 971                 if (temp->flag & jit_flag_patch)                        \
 972                     name##i##type(temp->u.w,                            \
 973                                   rn(node->v.w), node->w.w);            \
 974                 else {                                                  \
 975                     word = name##i##type(_jit->pc.w,                    \
 976                                          rn(node->v.w), node->w.w);     \
 977                     patch(word, node);                                  \
 978                 }                                                       \
 979                 break;
 980 #define case_brf(name)                                                  \
 981             case jit_code_##name##i_f:                                  \
 982                 temp = node->u.n;                                       \
 983                 assert(temp->code == jit_code_label ||                  \
 984                        temp->code == jit_code_epilog);                  \
 985                 if (temp->flag & jit_flag_patch)                        \
 986                     name##i_f(temp->u.w, rn(node->v.w), node->w.f);     \
 987                 else {                                                  \
 988                     word = name##i_f(_jit->pc.w, rn(node->v.w),         \
 989                                      node->w.f);                        \
 990                     patch(word, node);                                  \
 991                 }                                                       \
 992                 break
 993 #define case_brd(name)                                                  \
 994             case jit_code_##name##i_d:                                  \
 995                 temp = node->u.n;                                       \
 996                 assert(temp->code == jit_code_label ||                  \
 997                        temp->code == jit_code_epilog);                  \
 998                 if (temp->flag & jit_flag_patch)                        \
 999                     name##i_d(temp->u.w, rn(node->v.w), node->w.d);     \
1000                 else {                                                  \
1001                     word = name##i_d(_jit->pc.w, rn(node->v.w),         \
1002                                      node->w.d);                        \
1003                     patch(word, node);                                  \
1004                 }                                                       \
1005                 break
1006 #if DEVEL_DISASSEMBLER
1007     prevw = _jit->pc.w;
1008 #endif
1009     for (node = _jitc->head; node; node = node->next) {
1010         if (_jit->pc.uc >= _jitc->code.end)
1011             return (NULL);
1012
1013 #if DEVEL_DISASSEMBLER
1014         node->offset = (jit_uword_t)_jit->pc.w - (jit_uword_t)prevw;
1015         prevw = _jit->pc.w;
1016 #endif
1017         value = jit_classify(node->code);
1018         jit_regarg_set(node, value);
1019         switch (node->code) {
1020             case jit_code_align:
1021                 assert(!(node->u.w & (node->u.w - 1)) &&
1022                        node->u.w <= sizeof(jit_word_t));
1023                 if (node->u.w == sizeof(jit_word_t) &&
1024                     (word = _jit->pc.w & (sizeof(jit_word_t) - 1)))
1025                     nop(sizeof(jit_word_t) - word);
1026                 break;
1027             case jit_code_note:         case jit_code_name:
1028                 node->u.w = _jit->pc.w;
1029                 break;
1030             case jit_code_label:
1031                 /* remember label is defined */
1032                 node->flag |= jit_flag_patch;
1033                 node->u.w = _jit->pc.w;
1034                 break;
1035                 case_rrr(add,);
1036                 case_rrw(add,);
1037                 case_rrr(addc,);
1038                 case_rrw(addc,);
1039                 case_rrr(addx,);
1040                 case_rrw(addx,);
1041                 case_rrr(sub,);
1042                 case_rrw(sub,);
1043                 case_rrr(subc,);
1044                 case_rrw(subc,);
1045                 case_rrr(subx,);
1046                 case_rrw(subx,);
1047                 case_rrw(rsb,);
1048                 case_rrr(mul,);
1049                 case_rrw(mul,);
1050                 case_rrrr(qmul,);
1051                 case_rrrw(qmul,);
1052                 case_rrrr(qmul, _u);
1053                 case_rrrw(qmul, _u);
1054                 case_rrr(div,);
1055                 case_rrw(div,);
1056                 case_rrr(div, _u);
1057                 case_rrw(div, _u);
1058                 case_rrrr(qdiv,);
1059                 case_rrrw(qdiv,);
1060                 case_rrrr(qdiv, _u);
1061                 case_rrrw(qdiv, _u);
1062                 case_rrr(rem,);
1063                 case_rrw(rem,);
1064                 case_rrr(rem, _u);
1065                 case_rrw(rem, _u);
1066                 case_rrr(lsh,);
1067                 case_rrw(lsh,);
1068                 case_rrr(rsh,);
1069                 case_rrw(rsh,);
1070                 case_rrr(rsh, _u);
1071                 case_rrw(rsh, _u);
1072                 case_rr(neg,);
1073                 case_rr(com,);
1074                 case_rrr(and,);
1075                 case_rrw(and,);
1076                 case_rrr(or,);
1077                 case_rrw(or,);
1078                 case_rrr(xor,);
1079                 case_rrw(xor,);
1080                 case_rr(trunc, _f_i);
1081                 case_rr(trunc, _d_i);
1082                 case_rr(trunc, _f_l);
1083                 case_rr(trunc, _d_l);
1084                 case_rr(ld, _c);
1085                 case_rw(ld, _c);
1086                 case_rr(ld, _uc);
1087                 case_rw(ld, _uc);
1088                 case_rr(ld, _s);
1089                 case_rw(ld, _s);
1090                 case_rr(ld, _us);
1091                 case_rw(ld, _us);
1092                 case_rr(ld, _i);
1093                 case_rw(ld, _i);
1094                 case_rr(ld, _ui);
1095                 case_rw(ld, _ui);
1096                 case_rr(ld, _l);
1097                 case_rw(ld, _l);
1098                 case_rrr(ldx, _c);
1099                 case_rrw(ldx, _c);
1100                 case_rrr(ldx, _uc);
1101                 case_rrw(ldx, _uc);
1102                 case_rrr(ldx, _s);
1103                 case_rrw(ldx, _s);
1104                 case_rrr(ldx, _us);
1105                 case_rrw(ldx, _us);
1106                 case_rrr(ldx, _i);
1107                 case_rrw(ldx, _i);
1108                 case_rrr(ldx, _ui);
1109                 case_rrw(ldx, _ui);
1110                 case_rrr(ldx, _l);
1111                 case_rrw(ldx, _l);
1112                 case_rr(st, _c);
1113                 case_wr(st, _c);
1114                 case_rr(st, _s);
1115                 case_wr(st, _s);
1116                 case_rr(st, _i);
1117                 case_wr(st, _i);
1118                 case_rr(st, _l);
1119                 case_wr(st, _l);
1120                 case_rrr(stx, _c);
1121                 case_wrr(stx, _c);
1122                 case_rrr(stx, _s);
1123                 case_wrr(stx, _s);
1124                 case_rrr(stx, _i);
1125                 case_wrr(stx, _i);
1126                 case_rrr(stx, _l);
1127                 case_wrr(stx, _l);
1128                 case_rr(hton, _us);
1129                 case_rr(hton, _ui);
1130                 case_rr(hton, _ul);
1131                 case_rr(bswap, _us);
1132                 case_rr(bswap, _ui);
1133                 case_rr(bswap, _ul);
1134                 case_rr(ext, _c);
1135                 case_rr(ext, _uc);
1136                 case_rr(ext, _s);
1137                 case_rr(ext, _us);
1138                 case_rr(ext, _i);
1139                 case_rr(ext, _ui);
1140                 case_rr(mov,);
1141                 case_rrr(movn,);
1142                 case_rrr(movz,);
1143             case jit_code_movi:
1144                 if (node->flag & jit_flag_node) {
1145                     temp = node->v.n;
1146                     if (temp->code == jit_code_data ||
1147                         (temp->code == jit_code_label &&
1148                          (temp->flag & jit_flag_patch)))
1149                         movi(rn(node->u.w), temp->u.w);
1150                     else {
1151                         assert(temp->code == jit_code_label ||
1152                                temp->code == jit_code_epilog);
1153                         word = movi_p(rn(node->u.w), temp->u.w);
1154                         patch(word, node);
1155                     }
1156                 }
1157                 else
1158                     movi(rn(node->u.w), node->v.w);
1159                 break;
1160                 case_rrr(lt,);
1161                 case_rrw(lt,);
1162                 case_rrr(lt, _u);
1163                 case_rrw(lt, _u);
1164                 case_rrr(le,);
1165                 case_rrw(le,);
1166                 case_rrr(le, _u);
1167                 case_rrw(le, _u);
1168                 case_rrr(eq,);
1169                 case_rrw(eq,);
1170                 case_rrr(ge,);
1171                 case_rrw(ge,);
1172                 case_rrr(ge, _u);
1173                 case_rrw(ge, _u);
1174                 case_rrr(gt,);
1175                 case_rrw(gt,);
1176                 case_rrr(gt, _u);
1177                 case_rrw(gt, _u);
1178                 case_rrr(ne,);
1179                 case_rrw(ne,);
1180                 case_brr(blt,);
1181                 case_brw(blt,);
1182                 case_brr(blt, _u);
1183                 case_brw(blt, _u);
1184                 case_brr(ble,);
1185                 case_brw(ble,);
1186                 case_brr(ble, _u);
1187                 case_brw(ble, _u);
1188                 case_brr(beq,);
1189                 case_brw(beq,);
1190                 case_brr(bge,);
1191                 case_brw(bge,);
1192                 case_brr(bge, _u);
1193                 case_brw(bge, _u);
1194                 case_brr(bgt,);
1195                 case_brw(bgt,);
1196                 case_brr(bgt, _u);
1197                 case_brw(bgt, _u);
1198                 case_brr(bne,);
1199                 case_brw(bne,);
1200                 case_brr(boadd,);
1201                 case_brw(boadd,);
1202                 case_brr(boadd, _u);
1203                 case_brw(boadd, _u);
1204                 case_brr(bxadd,);
1205                 case_brw(bxadd,);
1206                 case_brr(bxadd, _u);
1207                 case_brw(bxadd, _u);
1208                 case_brr(bosub,);
1209                 case_brw(bosub,);
1210                 case_brr(bosub, _u);
1211                 case_brw(bosub, _u);
1212                 case_brr(bxsub,);
1213                 case_brw(bxsub,);
1214                 case_brr(bxsub, _u);
1215                 case_brw(bxsub, _u);
1216                 case_brr(bms,);
1217                 case_brw(bms,);
1218                 case_brr(bmc,);
1219                 case_brw(bmc,);
1220                 case_rrr(add, _f);
1221                 case_rrf(add);
1222                 case_rrr(sub, _f);
1223                 case_rrf(sub);
1224                 case_rrf(rsb);
1225                 case_rrr(mul, _f);
1226                 case_rrf(mul);
1227                 case_rrr(div, _f);
1228                 case_rrf(div);
1229                 case_rr(abs, _f);
1230                 case_rr(neg, _f);
1231                 case_rr(sqrt, _f);
1232                 case_rr(ext, _f);
1233                 case_rr(ld, _f);
1234                 case_rw(ld, _f);
1235                 case_rrr(ldx, _f);
1236                 case_rrw(ldx, _f);
1237                 case_rr(st, _f);
1238                 case_wr(st, _f);
1239                 case_rrr(stx, _f);
1240                 case_wrr(stx, _f);
1241                 case_rr(mov, _f);
1242             case jit_code_movi_f:
1243                 assert_data(node);
1244                 movi_f(rn(node->u.w), node->v.f);
1245                 break;
1246                 case_rr(ext, _d_f);
1247                 case_rrr(lt, _f);
1248                 case_rrf(lt);
1249                 case_rrr(le, _f);
1250                 case_rrf(le);
1251                 case_rrr(eq, _f);
1252                 case_rrf(eq);
1253                 case_rrr(ge, _f);
1254                 case_rrf(ge);
1255                 case_rrr(gt, _f);
1256                 case_rrf(gt);
1257                 case_rrr(ne, _f);
1258                 case_rrf(ne);
1259                 case_rrr(unlt, _f);
1260                 case_rrf(unlt);
1261                 case_rrr(unle, _f);
1262                 case_rrf(unle);
1263                 case_rrr(uneq, _f);
1264                 case_rrf(uneq);
1265                 case_rrr(unge, _f);
1266                 case_rrf(unge);
1267                 case_rrr(ungt, _f);
1268                 case_rrf(ungt);
1269                 case_rrr(ltgt, _f);
1270                 case_rrf(ltgt);
1271                 case_rrr(ord, _f);
1272                 case_rrf(ord);
1273                 case_rrr(unord, _f);
1274                 case_rrf(unord);
1275                 case_brr(blt, _f);
1276                 case_brf(blt);
1277                 case_brr(ble, _f);
1278                 case_brf(ble);
1279                 case_brr(beq, _f);
1280                 case_brf(beq);
1281                 case_brr(bge, _f);
1282                 case_brf(bge);
1283                 case_brr(bgt, _f);
1284                 case_brf(bgt);
1285                 case_brr(bne, _f);
1286                 case_brf(bne);
1287                 case_brr(bunlt, _f);
1288                 case_brf(bunlt);
1289                 case_brr(bunle, _f);
1290                 case_brf(bunle);
1291                 case_brr(buneq, _f);
1292                 case_brf(buneq);
1293                 case_brr(bunge, _f);
1294                 case_brf(bunge);
1295                 case_brr(bungt, _f);
1296                 case_brf(bungt);
1297                 case_brr(bltgt, _f);
1298                 case_brf(bltgt);
1299                 case_brr(bord, _f);
1300                 case_brf(bord);
1301                 case_brr(bunord, _f);
1302                 case_brf(bunord);
1303                 case_rrr(add, _d);
1304                 case_rrd(add);
1305                 case_rrr(sub, _d);
1306                 case_rrd(sub);
1307                 case_rrd(rsb);
1308                 case_rrr(mul, _d);
1309                 case_rrd(mul);
1310                 case_rrr(div, _d);
1311                 case_rrd(div);
1312                 case_rr(abs, _d);
1313                 case_rr(neg, _d);
1314                 case_rr(sqrt, _d);
1315                 case_rr(ext, _d);
1316                 case_rr(ld, _d);
1317                 case_rw(ld, _d);
1318                 case_rrr(ldx, _d);
1319                 case_rrw(ldx, _d);
1320                 case_rr(st, _d);
1321                 case_wr(st, _d);
1322                 case_rrr(stx, _d);
1323                 case_wrr(stx, _d);
1324                 case_rr(mov, _d);
1325             case jit_code_movi_d:
1326                 assert_data(node);
1327                 movi_d(rn(node->u.w), node->v.d);
1328                 break;
1329                 case_rr(ext, _f_d);
1330                 case_rrr(lt, _d);
1331                 case_rrd(lt);
1332                 case_rrr(le, _d);
1333                 case_rrd(le);
1334                 case_rrr(eq, _d);
1335                 case_rrd(eq);
1336                 case_rrr(ge, _d);
1337                 case_rrd(ge);
1338                 case_rrr(gt, _d);
1339                 case_rrd(gt);
1340                 case_rrr(ne, _d);
1341                 case_rrd(ne);
1342                 case_rrr(unlt, _d);
1343                 case_rrd(unlt);
1344                 case_rrr(unle, _d);
1345                 case_rrd(unle);
1346                 case_rrr(uneq, _d);
1347                 case_rrd(uneq);
1348                 case_rrr(unge, _d);
1349                 case_rrd(unge);
1350                 case_rrr(ungt, _d);
1351                 case_rrd(ungt);
1352                 case_rrr(ltgt, _d);
1353                 case_rrd(ltgt);
1354                 case_rrr(ord, _d);
1355                 case_rrd(ord);
1356                 case_rrr(unord, _d);
1357                 case_rrd(unord);
1358                 case_brr(blt, _d);
1359                 case_brd(blt);
1360                 case_brr(ble, _d);
1361                 case_brd(ble);
1362                 case_brr(beq, _d);
1363                 case_brd(beq);
1364                 case_brr(bge, _d);
1365                 case_brd(bge);
1366                 case_brr(bgt, _d);
1367                 case_brd(bgt);
1368                 case_brr(bne, _d);
1369                 case_brd(bne);
1370                 case_brr(bunlt, _d);
1371                 case_brd(bunlt);
1372                 case_brr(bunle, _d);
1373                 case_brd(bunle);
1374                 case_brr(buneq, _d);
1375                 case_brd(buneq);
1376                 case_brr(bunge, _d);
1377                 case_brd(bunge);
1378                 case_brr(bungt, _d);
1379                 case_brd(bungt);
1380                 case_brr(bltgt, _d);
1381                 case_brd(bltgt);
1382                 case_brr(bord, _d);
1383                 case_brd(bord);
1384                 case_brr(bunord, _d);
1385                 case_brd(bunord);
1386             case jit_code_jmpr:
1387                 jmpr(rn(node->u.w));
1388                 break;
1389             case jit_code_jmpi:
1390                 if (node->flag & jit_flag_node) {
1391                     temp = node->u.n;
1392                     assert(temp->code == jit_code_label ||
1393                            temp->code == jit_code_epilog);
1394                     if (temp->flag & jit_flag_patch)
1395                         jmpi(temp->u.w);
1396                     else {
1397                         word = jmpi_p(_jit->pc.w);
1398                         patch(word, node);
1399                     }
1400                 }
1401                 else
1402                     jmpi(node->u.w);
1403                 break;
1404             case jit_code_callr:
1405                 callr(rn(node->u.w));
1406                 break;
1407             case jit_code_calli:
1408                 if (node->flag & jit_flag_node) {
1409                     temp = node->u.n;
1410                     assert(temp->code == jit_code_label ||
1411                            temp->code == jit_code_epilog);
1412                     if (temp->flag & jit_flag_patch)
1413                         calli(temp->u.w);
1414                     else {
1415                         word = calli_p(_jit->pc.w);
1416                         patch(word, node);
1417                     }
1418                 }
1419                 else
1420                     calli(node->u.w);
1421                 break;
1422             case jit_code_prolog:
1423                 _jitc->function = _jitc->functions.ptr + node->w.w;
1424                 undo.node = node;
1425                 undo.word = _jit->pc.w;
1426 #if DEVEL_DISASSEMBLER
1427                 undo.prevw = prevw;
1428 #endif
1429                 undo.patch_offset = _jitc->patches.offset;
1430             restart_function:
1431                 _jitc->again = 0;
1432                 prolog(node);
1433                 break;
1434             case jit_code_epilog:
1435                 assert(_jitc->function == _jitc->functions.ptr + node->w.w);
1436                 if (_jitc->again) {
1437                     for (temp = undo.node->next;
1438                          temp != node; temp = temp->next) {
1439                         if (temp->code == jit_code_label ||
1440                             temp->code == jit_code_epilog)
1441                             temp->flag &= ~jit_flag_patch;
1442                     }
1443                     temp->flag &= ~jit_flag_patch;
1444                     node = undo.node;
1445                     _jit->pc.w = undo.word;
1446 #if DEVEL_DISASSEMBLER
1447                     prevw = undo.prevw;
1448 #endif
1449                     _jitc->patches.offset = undo.patch_offset;
1450                     goto restart_function;
1451                 }
1452                 /* remember label is defined */
1453                 node->flag |= jit_flag_patch;
1454                 node->u.w = _jit->pc.w;
1455                 epilog(node);
1456                 _jitc->function = NULL;
1457                 break;
1458             case jit_code_va_start:
1459                 vastart(rn(node->u.w));
1460                 break;
1461             case jit_code_va_arg:
1462                 vaarg(rn(node->u.w), rn(node->v.w));
1463                 break;
1464             case jit_code_va_arg_d:
1465                 vaarg_d(rn(node->u.w), rn(node->v.w));
1466                 break;
1467             case jit_code_live:                 case jit_code_ellipsis:
1468             case jit_code_va_push:
1469             case jit_code_allocai:              case jit_code_allocar:
1470             case jit_code_arg:
1471             case jit_code_arg_f:                case jit_code_arg_d:
1472             case jit_code_va_end:
1473             case jit_code_ret:
1474             case jit_code_retr:                 case jit_code_reti:
1475             case jit_code_retr_f:               case jit_code_reti_f:
1476             case jit_code_retr_d:               case jit_code_reti_d:
1477             case jit_code_getarg_c:             case jit_code_getarg_uc:
1478             case jit_code_getarg_s:             case jit_code_getarg_us:
1479             case jit_code_getarg_i:             case jit_code_getarg_ui:
1480             case jit_code_getarg_l:
1481             case jit_code_getarg_f:             case jit_code_getarg_d:
1482             case jit_code_putargr:              case jit_code_putargi:
1483             case jit_code_putargr_f:            case jit_code_putargi_f:
1484             case jit_code_putargr_d:            case jit_code_putargi_d:
1485             case jit_code_pushargr:             case jit_code_pushargi:
1486             case jit_code_pushargr_f:           case jit_code_pushargi_f:
1487             case jit_code_pushargr_d:           case jit_code_pushargi_d:
1488             case jit_code_retval_c:             case jit_code_retval_uc:
1489             case jit_code_retval_s:             case jit_code_retval_us:
1490             case jit_code_retval_i:
1491             case jit_code_retval_ui:            case jit_code_retval_l:
1492             case jit_code_retval_f:             case jit_code_retval_d:
1493             case jit_code_prepare:
1494             case jit_code_finishr:              case jit_code_finishi:
1495                 break;
1496             default:
1497                 abort();
1498         }
1499         jit_regarg_clr(node, value);
1500         assert(_jitc->regarg == 0 && _jitc->synth == 0);
1501         /* update register live state */
1502         jit_reglive(node);
1503     }
1504 #undef case_brw
1505 #undef case_brr
1506 #undef case_wrr
1507 #undef case_rrw
1508 #undef case_rrr
1509 #undef case_wr
1510 #undef case_rw
1511 #undef case_rr
1512
1513     for (offset = 0; offset < _jitc->patches.offset; offset++) {
1514         node = _jitc->patches.ptr[offset].node;
1515         word = _jitc->patches.ptr[offset].inst;
1516         value = node->code == jit_code_movi ? node->v.n->u.w : node->u.n->u.w;
1517         patch_at(word, value);
1518     }
1519
1520     jit_flush(_jit->code.ptr, _jit->pc.uc);
1521
1522     return (_jit->code.ptr);
1523 }
1524
1525 #define CODE                            1
1526 #  include "jit_aarch64-cpu.c"
1527 #  include "jit_aarch64-fpu.c"
1528 #undef CODE
1529
1530 void
1531 jit_flush(void *fptr, void *tptr)
1532 {
1533 #if defined(__GNUC__)
1534     jit_word_t          f, t, s;
1535
1536     s = sysconf(_SC_PAGE_SIZE);
1537     f = (jit_word_t)fptr & -s;
1538     t = (((jit_word_t)tptr) + s - 1) & -s;
1539     __clear_cache((void *)f, (void *)t);
1540 #endif
1541 }
1542
1543 void
1544 _emit_ldxi(jit_state_t *_jit, jit_int32_t r0, jit_int32_t r1, jit_word_t i0)
1545 {
1546     ldxi(rn(r0), rn(r1), i0);
1547 }
1548
1549 void
1550 _emit_stxi(jit_state_t *_jit, jit_word_t i0, jit_int32_t r0, jit_int32_t r1)
1551 {
1552     stxi(i0, rn(r0), rn(r1));
1553 }
1554
1555 void
1556 _emit_ldxi_d(jit_state_t *_jit, jit_int32_t r0, jit_int32_t r1, jit_word_t i0)
1557 {
1558     ldxi_d(rn(r0), rn(r1), i0);
1559 }
1560
1561 void
1562 _emit_stxi_d(jit_state_t *_jit, jit_word_t i0, jit_int32_t r0, jit_int32_t r1)
1563 {
1564     stxi_d(i0, rn(r0), rn(r1));
1565 }
1566
1567 static void
1568 _patch(jit_state_t *_jit, jit_word_t instr, jit_node_t *node)
1569 {
1570     jit_int32_t          flag;
1571
1572     assert(node->flag & jit_flag_node);
1573     if (node->code == jit_code_movi)
1574         flag = node->v.n->flag;
1575     else
1576         flag = node->u.n->flag;
1577     assert(!(flag & jit_flag_patch));
1578     if (_jitc->patches.offset >= _jitc->patches.length) {
1579         jit_realloc((jit_pointer_t *)&_jitc->patches.ptr,
1580                     _jitc->patches.length * sizeof(jit_patch_t),
1581                     (_jitc->patches.length + 1024) * sizeof(jit_patch_t));
1582         _jitc->patches.length += 1024;
1583     }
1584     _jitc->patches.ptr[_jitc->patches.offset].inst = instr;
1585     _jitc->patches.ptr[_jitc->patches.offset].node = node;
1586     ++_jitc->patches.offset;
1587 }