deps/lightning/lib/jit_ia64-cpu.c

   1 /*
   2  * Copyright (C) 2013-2023  Free Software Foundation, Inc.
   3  *
   4  * This file is part of GNU lightning.
   5  *
   6  * GNU lightning is free software; you can redistribute it and/or modify it
   7  * under the terms of the GNU Lesser General Public License as published
   8  * by the Free Software Foundation; either version 3, or (at your option)
   9  * any later version.
  10  *
  11  * GNU lightning is distributed in the hope that it will be useful, but
  12  * WITHOUT ANY WARRANTY; without even the implied warranty of MERCHANTABILITY
  13  * or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU Lesser General Public
  14  * License for more details.
  15  *
  16  * Authors:
  17  *      Paulo Cesar Pereira de Andrade
  18  */
  19
  20 #if PROTO
  21 #define jit_unaligned_p()               (jit_cpu.unaligned)
  22 #define stack_framesize                 144
  23 #define params_offset                   16
  24 #define INST_NONE                       0       /* should never be generated */
  25 #define INST_STOP                       1       /* or'ed if stop is required */
  26 #define INST_A                          2       /* M- or I- unit */
  27 #define INST_As                         3
  28 #define INST_I                          4
  29 #define INST_Is                         5
  30 #define INST_M                          6
  31 #define INST_Ms                         7
  32 #define INST_F                          8
  33 #define INST_Fs                         9
  34 #define INST_B                          10
  35 #define INST_Bs                         11
  36 #define INST_L                          12
  37 #define INST_Ls                         13
  38 #define INST_X                          14
  39 #define INST_Xs                         15
  40
  41 /* Data and instructions are referenced by 64-bit addresses. Instructions
  42  * are stored in memory in little endian byte order, in which the least
  43  * significant byte appears in the lowest addressed byte of a memory
  44  * location. For data, modes for both big and little endian byte order are
  45  * supported and can be controlled by a bit in the User Mask Register.
  46  */
  47 #define il(ii)                          *_jit->pc.ul++ = ii
  48 #define set_bundle(p, l, h, tm, s0, s1, s2)                             \
  49     do {                                                                \
  50         l = tm | ((s0 & 0x1ffffffffffL) << 5L) | (s1 << 46L);           \
  51         h = ((s1 >> 18L) & 0x7fffffLL) | (s2 << 23L);                   \
  52         p[0] = byte_swap_if_big_endian(l);                              \
  53         p[1] = byte_swap_if_big_endian(h);                              \
  54     } while (0)
  55 #define get_bundle(p, l, h, tm, s0, s1, s2)                             \
  56     do {                                                                \
  57         l = byte_swap_if_big_endian(p[0]);                              \
  58         h = byte_swap_if_big_endian(p[1]);                              \
  59         tm = l & 0x1f;                                                  \
  60         s0 = (l >> 5L) & 0x1ffffffffffL;                                \
  61         s1 = ((l >> 46L) & 0x3ffffL) | ((h & 0x7fffffL) << 18L);        \
  62         s2 = (h >> 23L) & 0x1ffffffffffL;                               \
  63     } while (0)
  64
  65 /*  Need to insert a stop if a modified register would (or could)
  66  *  be read in the same cycle.
  67  */
  68 #define TSTREG1(r0)                                                     \
  69     do {                                                                \
  70         if (jit_regset_tstbit(&_jitc->regs, r0))                        \
  71             stop();                                                     \
  72     } while (0)
  73 #define TSTREG2(r0, r1)                                                 \
  74     do {                                                                \
  75         if (jit_regset_tstbit(&_jitc->regs, r0) ||                      \
  76             jit_regset_tstbit(&_jitc->regs, r1))                        \
  77             stop();                                                     \
  78     } while (0)
  79 #define TSTPRED(p0)                                                     \
  80     do {                                                                \
  81         if (p0 && (_jitc->pred & (1 << p0)))                            \
  82             stop();                                                     \
  83     } while (0)
  84 /* Record register was modified */
  85 #define SETREG(r0)              jit_regset_setbit(&_jitc->regs, r0)
  86
  87 /* Avoid using constants in macros and code */
  88 typedef enum {
  89     GR_0,               GR_1,           GR_2,           GR_3,
  90     GR_4,               GR_5,           GR_6,           GR_7,
  91     GR_8,               GR_9,           GR_10,          GR_11,
  92     GR_12,              GR_13,          GR_14,          GR_15,
  93     GR_16,              GR_17,          GR_18,          GR_19,
  94     GR_20,              GR_21,          GR_22,          GR_23,
  95     GR_24,              GR_25,          GR_26,          GR_27,
  96     GR_28,              GR_29,          GR_30,          GR_31,
  97     GR_32,              GR_33,          GR_34,          GR_35,
  98     GR_36,              GR_37,          GR_38,          GR_39,
  99     GR_40,              GR_41,          GR_42,          GR_43,
 100     /* GR_44...GR_127 */
 101 } gr_reg_t;
 102
 103 typedef enum {
 104     PR_0,               /* constant - always 1 */
 105     /* p0-p5             -  preserved */
 106     PR_1,               PR_2,           PR_3,           PR_4,
 107     PR_5,
 108     /* p6-p15            - scratch */
 109     PR_6,               PR_7,           PR_8,           PR_9,
 110     PR_10,              PR_11,          PR_12,          PR_13,
 111     PR_14,              PR_15,
 112     /* p16-...           - preserved - rotating */
 113 } pr_reg_t;
 114
 115 typedef enum {
 116     BR_0,               /* scratch - Return link */
 117     /* b1-b5             - preserved */
 118     BR_1,               BR_2,           BR_3,           BR_4,
 119     BR_5,
 120     /* b6-b7            - scratch */
 121     BR_6,               BR_7
 122 } br_reg_t;
 123
 124 typedef enum {
 125     AR_KR0,             AR_KR1,         AR_KR2,         AR_KR3,
 126     AR_KR4,             AR_KR5,         AR_KR6,         AR_KR7,
 127     AR_8,               AR_9,           AR_10,          AR_11,
 128     AR_12,              AR_13,          AR_14,          AR_15,
 129     AR_RSC,             AR_BSP,         AR_BSPSTORE,    AR_RNAT,
 130     AR_20,              AR_FCR,         AR_22,          AR_23,
 131     AR_EFLAG,           AR_CSD,         AR_SSD,         AR_CFLG,
 132     AR_FSR,             AR_FIR,         AR_FDR,         AR_31,
 133     AR_CCV,             AR_33,          AR_34,          AR_35,
 134     AR_UNAT,            AR_37,          AR_38,          AR_39,
 135     AR_FPSR,            AR_41,          AR_42,          AR_43,
 136     AR_ITC,             AR_RUC,         AR_46,          AR_47,
 137     AR_48,              AR_49,          AR_50,          AR_51,
 138     AR_52,              AR_53,          AR_54,          AR_55,
 139     AR_56,              AR_57,          AR_58,          AR_59,
 140     AR_60,              AR_61,          AR_62,          AR_63,
 141     AR_PFS,             AR_LC,          AR_EC,
 142     /* AR_67 ... AR_127 */
 143 } ar_reg_t;
 144
 145 typedef enum {
 146     TM_M_I_I_,  TM_M_I_Is,      TM_M_IsI_,      TM_M_IsIs,
 147     TM_M_L_X_,  TM_M_L_Xs,      TM_ILL_06,      TM_ILL_07,
 148     TM_M_M_I_,  TM_M_M_Is,      TM_MsM_I_,      TM_MsM_Is,
 149     TM_M_F_I_,  TM_M_F_Is,      TM_M_M_F_,      TM_M_M_Fs,
 150     TM_M_I_B_,  TM_M_I_Bs,      TM_M_B_B_,      TM_M_B_Bs,
 151     TM_ILL_14,  TM_ILL_15,      TM_B_B_B_,      TM_B_B_Bs,
 152     TM_M_M_B_,  TM_M_M_Bs,      TM_ILL_1A,      TM_ILL_1B,
 153     TM_M_F_B_,  TM_M_F_Bs,      TM_ILL_1E,      TM_ILL_1F,
 154 } template_t;
 155
 156 #define MWH_SPTK                        0
 157 #define MWH_NONE                        1
 158 #define MWH_DPTK                        2
 159
 160 #define IH_NONE                         0
 161 #define IH_IMP                          1
 162
 163 #define LD_NONE                         0
 164 #define LD_NT1                          1
 165 #define LD_NTA                          3
 166
 167 #define ST_NONE                         0
 168 #define ST_NTA                          3
 169
 170 #define LF_NONE                         0
 171 #define LF_NT1                          1
 172 #define LF_NT2                          2
 173 #define LF_NTA                          3
 174
 175 #define BR_PH_FEW                       0
 176 #define BR_PH_MANY                      1
 177
 178 #define BR_BWH_SPTK                     0
 179 #define BR_BWH_SPNT                     1
 180 #define BR_BWH_DPTK                     2
 181 #define BR_BWH_DPNT                     3
 182
 183 #define BRI_BWH_SPTK                    1
 184 #define BRI_BWH_SPNT                    3
 185 #define BRI_BWH_DPTK                    5
 186 #define BRI_BWH_DPNT                    7
 187
 188 #define BR_DH_NONE                      0
 189 #define BR_DH_CLR                       1
 190
 191 #define BR_IH_NONE                      0
 192 #define BR_IH_IMP                       1
 193
 194 #define BR_IPWH_SPTK                    0
 195 #define BR_IPWH_LOOP                    1
 196 #define BR_IPWH_DPTK                    2
 197 #define BR_IPWH_EXIT                    3
 198
 199 #define BR_INDWH_SPTK                   0
 200 #define BR_INDWH_DPTK                   2
 201
 202 #define MUX_BRCST                       0
 203 #define MUX_REV                         11
 204 #define MUX_MIX                         8
 205 #define MUX_SHUF                        9
 206 #define MUX_ALT                         10
 207
 208 #define ldr(r0,r1)                      ldr_l(r0,r1)
 209 #define ldi(r0,i0)                      ldi_l(r0,i0)
 210 #define ldxr(r0,r1,r2)                  ldxr_l(r0,r1,r2)
 211 #define ldxi(r0,r1,i0)                  ldxi_l(r0,r1,i0)
 212 #define str(r0,r1)                      str_l(r0,r1)
 213 #define sti(i0,r0)                      sti_l(i0,r0)
 214 #define stxr(r0,r1,r2)                  stxr_l(r0,r1,r2)
 215 #define stxi(i0,r0,r1)                  stxi_l(i0,r0,r1)
 216
 217 #if !HAVE_FFSL
 218 #  define ffsl(l)                       __builtin_ffsl(l)
 219 #endif
 220
 221 /* libgcc */
 222 #if defined(__GNUC__)
 223 extern long __divdi3(long,long);
 224 extern unsigned long __udivdi3(unsigned long,unsigned long);
 225 extern long __moddi3(long,long);
 226 extern unsigned long __umoddi3(unsigned long,unsigned long);
 227 #else
 228 static long __divdi3(long,long);
 229 static unsigned long __udivdi3(unsigned long,unsigned long);
 230 static long __moddi3(long,long);
 231 static unsigned long __umoddi3(unsigned long,unsigned long);
 232 #endif
 233 #define out(n,tm,s0,s1,s2)              _out(_jit,n,tm,s0,s1,s2)
 234 static void _out(jit_state_t*,int,int,jit_word_t,jit_word_t,jit_word_t);
 235 #define stop()                          _stop(_jit)
 236 static void _stop(jit_state_t*);
 237 #define sync()                          _sync(_jit)
 238 static void _sync(jit_state_t*);
 239 #define flush()                         _flush(_jit)
 240 static void _flush(jit_state_t*);
 241 #define inst(w, t)                      _inst(_jit, w, t)
 242 static void _inst(jit_state_t*, jit_word_t, jit_uint8_t);
 243 #define A1(x4,x2,r3,r2,r1)              _A1(_jit,0,x4,x2,r3,r2,r1)
 244 static void _A1(jit_state_t*, jit_word_t,
 245                 jit_word_t,jit_word_t,jit_word_t,jit_word_t,jit_word_t);
 246 #define A2(x4,ct,r3,r2,r1)              A1(x4,ct,r3,r2,r1)
 247 #define A3(x4,x2,r3,im,r1)              _A3(_jit,0,x4,x2,r3,im,r1)
 248 static void _A3(jit_state_t*, jit_word_t,
 249                 jit_word_t,jit_word_t,jit_word_t,jit_word_t,jit_word_t);
 250 #define A4(x2a,r3,im,r1)                _A4(_jit,0,x2a,r3,im,r1)
 251 static void _A4(jit_state_t*,jit_word_t,
 252                 jit_word_t,jit_word_t,jit_word_t,jit_word_t);
 253 #define A5(r3,im,r1)                    _A5(_jit,0,r3,im,r1)
 254 static void _A5(jit_state_t*,jit_word_t,
 255                 jit_word_t,jit_word_t,jit_word_t);
 256 #define A6(o,x2,ta,p2,r3,r2,c,p1)       _A6(_jit,0,o,x2,ta,p2,r3,r2,c,p1)
 257 static void _A6(jit_state_t*,jit_word_t,
 258                 jit_word_t,jit_word_t,jit_word_t,jit_word_t,
 259                 jit_word_t,jit_word_t,jit_word_t,jit_word_t);
 260 #define A7(o,x2,ta,p2,r3,c,p1)          _A7(_jit,0,o,x2,ta,p2,r3,c,p1)
 261 static void _A7(jit_state_t*,jit_word_t,
 262                 jit_word_t,jit_word_t,jit_word_t,
 263                 jit_word_t,jit_word_t,jit_word_t,jit_word_t)
 264     maybe_unused;
 265 #define A8(o,x2,ta,p2,r3,im,c,p1)       _A8(_jit,0,o,x2,ta,p2,r3,im,c,p1)
 266 static void _A8(jit_state_t*,jit_word_t,
 267                 jit_word_t,jit_word_t,jit_word_t,jit_word_t,
 268                 jit_word_t,jit_word_t,jit_word_t,jit_word_t);
 269 #define A9(za,zb,x4,x2,r3,r2,r1)        _A9(_jit,0,za,zb,x4,x2,r3,r2,r1)
 270 static void _A9(jit_state_t*,jit_word_t,
 271                 jit_word_t,jit_word_t,jit_word_t,
 272                 jit_word_t,jit_word_t,jit_word_t,jit_word_t)
 273     maybe_unused;
 274 #define A10(x4,ct,r3,r2,r1)             A9(0,1,x4,ct,r3,r2,r1)
 275 #define I1(ct,x2,r3,r2,r1)              _I1(_jit,0,ct,x2,r3,r2,r1)
 276 static void _I1(jit_state_t*,jit_word_t,
 277                 jit_word_t,jit_word_t,jit_word_t,jit_word_t,jit_word_t)
 278     maybe_unused;
 279 #define I2(za,x2a,zb,x2c,x2b,r3,r2,r1)  _I2(_jit,0,za,x2a,zb,x2c,x2b,r3,r2,r1)
 280 static void _I2(jit_state_t*,jit_word_t,
 281                 jit_word_t,jit_word_t,jit_word_t,jit_word_t,
 282                 jit_word_t,jit_word_t,jit_word_t,jit_word_t)
 283     maybe_unused;
 284 #define I3(mbt,r2,r1)                   _I3(_jit,0,mbt,r2,r1)
 285 static void _I3(jit_state_t*,jit_word_t,
 286                 jit_word_t,jit_word_t,jit_word_t)
 287 #if __BYTE_ORDER == __BIG_ENDIAN
 288     maybe_unused
 289 #endif
 290     ;
 291 #define I4(mht,r2,r1)                   _I4(_jit,0,mht,r2,r1)
 292 static void _I4(jit_state_t*,jit_word_t,
 293                 jit_word_t,jit_word_t,jit_word_t)
 294     maybe_unused;
 295 #define I5(za,zb,x2b,r3,r2,r1)          _I5(_jit,0,za,zb,x2b,r3,r2,r1)
 296 static void _I5(jit_state_t*,jit_word_t,
 297                 jit_word_t,jit_word_t,jit_word_t,
 298                 jit_word_t,jit_word_t,jit_word_t);
 299 #define I6(za,zb,x2b,r3,ct,r1)          _I6(_jit,0,za,zb,x2b,r3,ct,r1)
 300 static void _I6(jit_state_t*,jit_word_t,
 301                 jit_word_t,jit_word_t,jit_word_t,
 302                 jit_word_t,jit_word_t,jit_word_t)
 303     maybe_unused;
 304 #define I7(za,zb,r3,r2,r1)              _I7(_jit,0,za,zb,r3,r2,r1)
 305 static void _I7(jit_state_t*,jit_word_t,
 306                 jit_word_t,jit_word_t,jit_word_t,jit_word_t,jit_word_t);
 307 #define I8(za,zb,ct,r2,r1)              _I8(_jit,0,za,zb,ct,r2,r1)
 308 static void _I8(jit_state_t*,jit_word_t,
 309                 jit_word_t,jit_word_t,jit_word_t,jit_word_t,jit_word_t)
 310     maybe_unused;
 311 #define I9(x2c,r3,r1)                   _I9(_jit,0,x2c,r3,r1)
 312 static void _I9(jit_state_t*,jit_word_t,
 313                 jit_word_t,jit_word_t,jit_word_t)
 314     maybe_unused;
 315 #define I10(ct,r3,r2,r1)                _I10(_jit,0,ct,r3,r2,r1)
 316 static void _I10(jit_state_t*,jit_word_t,
 317                  jit_word_t,jit_word_t,jit_word_t,jit_word_t)
 318     maybe_unused;
 319 #define I11(len,r3,pos,y,r1)            _I11(_jit,0,len,r3,pos,y,r1)
 320 static void _I11(jit_state_t*,jit_word_t,
 321                  jit_word_t,jit_word_t,jit_word_t,jit_word_t,jit_word_t)
 322     maybe_unused;
 323 #define I12(len,pos,r2,r1)              _I12(_jit,0,len,pos,r2,r1)
 324 static void _I12(jit_state_t*,jit_word_t,
 325                  jit_word_t,jit_word_t,jit_word_t,jit_word_t)
 326     maybe_unused;
 327 #define I13(len,pos,im,r1)              _I13(_jit,0,len,pos,im,r1)
 328 static void _I13(jit_state_t*,jit_word_t,
 329                  jit_word_t,jit_word_t,jit_word_t,jit_word_t)
 330     maybe_unused;
 331 #define I14(s,len,r3,pos,r1)            _I14(_jit,0,s,len,r3,pos,r1)
 332 static void _I14(jit_state_t*,jit_word_t,
 333                  jit_word_t,jit_word_t,jit_word_t,jit_word_t,jit_word_t)
 334     maybe_unused;
 335 #define I15(pos,len,r3,r2,r1)           _I15(_jit,0,pos,len,r3,r2,r1)
 336 static void _I15(jit_state_t*,jit_word_t,
 337                  jit_word_t,jit_word_t,jit_word_t,jit_word_t,jit_word_t)
 338     maybe_unused;
 339 #define I16(tb,ta,p2,r3,pos,c,p1)       _I16(_jit,0,tb,ta,p2,r3,pos,c,p1)
 340 static void _I16(jit_state_t*,jit_word_t,
 341                  jit_word_t,jit_word_t,jit_word_t,
 342                  jit_word_t,jit_word_t,jit_word_t,jit_word_t)
 343     maybe_unused;
 344 #define I17(tb,ta,p2,r3,c,p1)           _I17(_jit,0,tb,ta,p2,r3,c,p1)
 345 static void _I17(jit_state_t*,jit_word_t,
 346                  jit_word_t,jit_word_t,jit_word_t,
 347                  jit_word_t,jit_word_t,jit_word_t)
 348     maybe_unused;
 349 #define I18(im,y)                       _I18(_jit,0,im,y)
 350 static void _I18(jit_state_t*,jit_word_t,
 351                  jit_word_t,jit_word_t)
 352     maybe_unused;
 353 #define I19(im)                         _I19(_jit,0,im)
 354 static void _I19(jit_state_t*,jit_word_t,
 355                  jit_word_t)
 356     maybe_unused;
 357 #define I20(r2,im)                      _I20(_jit,0,r2,im)
 358 static void _I20(jit_state_t*,jit_word_t,
 359                  jit_word_t,jit_word_t)
 360     maybe_unused;
 361 #define I21(im,ih,x,wh,r2,b1)           _I21(_jit,0,im,ih,x,wh,r2,b1)
 362 static void _I21(jit_state_t*,jit_word_t,
 363                  jit_word_t,jit_word_t,
 364                  jit_word_t,jit_word_t,jit_word_t,jit_word_t);
 365 #define I22(b2,r1)                      _I22(_jit,0,b2,r1)
 366 static void _I22(jit_state_t*,jit_word_t,
 367                  jit_word_t,jit_word_t);
 368 #define I23(r2,im)                      _I23(_jit,0,r2,im)
 369 static void _I23(jit_state_t*,jit_word_t,
 370                  jit_word_t,jit_word_t)
 371     maybe_unused;
 372 #define I24(im)                         _I24(_jit,0,im)
 373 static void _I24(jit_state_t*,jit_word_t,
 374                  jit_word_t)
 375     maybe_unused;
 376 #define I25(x6,r1)                      _I25(_jit,0,x6,r1)
 377 static void _I25(jit_state_t*,jit_word_t,
 378                  jit_word_t,jit_word_t)
 379     maybe_unused;
 380 #define I26(ar,r2)                      _I26(_jit,0,ar,r2)
 381 static void _I26(jit_state_t*,jit_word_t,
 382                  jit_word_t,jit_word_t);
 383 #define I27(ar,im)                      _I27(_jit,0,ar,im)
 384 static void _I27(jit_state_t*,jit_word_t,
 385                  jit_word_t,jit_word_t)
 386     maybe_unused;
 387 #define I28(ar,r1)                      _I28(_jit,0,ar,r1)
 388 static void _I28(jit_state_t*,jit_word_t,
 389                  jit_word_t,jit_word_t)
 390     maybe_unused;
 391 #define I29(x6,r3,r1)                   _I29(_jit,0,x6,r3,r1)
 392 static void _I29(jit_state_t*,jit_word_t,
 393                  jit_word_t,jit_word_t,jit_word_t);
 394 #define I30(tb,ta,p2,im,c,p1)           _I30(_jit,0,ta,tb,p2,im,c,p1)
 395 static void _I30(jit_state_t*,jit_word_t,
 396                  jit_word_t,jit_word_t,jit_word_t,
 397                  jit_word_t,jit_word_t,jit_word_t)
 398     maybe_unused;
 399 #define M1(x6,ht,x,r3,r1)               _M1(_jit,0,x6,ht,x,r3,r1)
 400 static void _M1(jit_state_t*,jit_word_t,
 401                 jit_word_t,jit_word_t,jit_word_t,jit_word_t,jit_word_t);
 402 #define M2(x6,ht,r3,r2,r1)              _M2(_jit,0,x6,ht,r3,r2,r1)
 403 static void _M2(jit_state_t*,jit_word_t,
 404                 jit_word_t,jit_word_t,jit_word_t,jit_word_t,jit_word_t)
 405     maybe_unused;
 406 #define M3(x6,ht,r3,im,r1)              _M3(_jit,0,x6,ht,r3,im,r1)
 407 static void _M3(jit_state_t*,jit_word_t,
 408                 jit_word_t,jit_word_t,jit_word_t,jit_word_t,jit_word_t);
 409 #define M5(x6,ht,r3,r2,im)              _M5(_jit,0,x6,ht,r3,r2,im)
 410 static void _M5(jit_state_t*,jit_word_t,
 411                 jit_word_t,jit_word_t,jit_word_t,jit_word_t,jit_word_t)
 412     maybe_unused;
 413 #define M6(x6,ht,x,r3,r2)               _M6(_jit,0,x6,ht,x,r3,r2)
 414 static void _M6(jit_state_t*,jit_word_t,
 415                 jit_word_t,jit_word_t,jit_word_t,jit_word_t,jit_word_t);
 416 #define M13(x6,ht,r3,f2)                _M13(_jit,0,x6,ht,r3,f2)
 417 static void _M13(jit_state_t*,jit_word_t,
 418                  jit_word_t,jit_word_t,jit_word_t,jit_word_t);
 419 #define M14(x6,ht,r3,r2)                _M14(_jit,0,x6,ht,r3,r2)
 420 static void _M14(jit_state_t*,jit_word_t,
 421                  jit_word_t,jit_word_t,jit_word_t,jit_word_t)
 422     maybe_unused;
 423 #define M15(x6,ht,r3,im)                _M15(_jit,0,x6,ht,r3,im)
 424 static void _M15(jit_state_t*,jit_word_t,
 425                  jit_word_t,jit_word_t,jit_word_t,jit_word_t)
 426     maybe_unused;
 427 #define M16(x6,ht,r3,r2,r1)             _M16(_jit,0,x6,ht,r3,r2,r1)
 428 static void _M16(jit_state_t*,jit_word_t,
 429                  jit_word_t,jit_word_t,jit_word_t,jit_word_t,jit_word_t)
 430     maybe_unused;
 431 #define M17(x6,ht,r3,im,r1)             _M17(_jit,0,x6,ht,r3,im,r1)
 432 static void _M17(jit_state_t*,jit_word_t,
 433                  jit_word_t,jit_word_t,jit_word_t,jit_word_t,jit_word_t)
 434     maybe_unused;
 435 #define M20(r2,im)                      M20x(0x1,r2,im)
 436 #define M20x(x3,r2,im)                  _M20x(_jit,0,x3,r2,im)
 437 static void _M20x(jit_state_t*,jit_word_t,
 438                   jit_word_t,jit_word_t,jit_word_t)
 439     maybe_unused;
 440 #define M22(x3,im,r1)                   M22x(x3,im,r1)
 441 #define M22x(x3,im,r1)                  _M22x(_jit,0,x3,im,r1)
 442 static void _M22x(jit_state_t*,jit_word_t,
 443                   jit_word_t,jit_word_t,jit_word_t)
 444     maybe_unused;
 445 #define M24(x2,x4)                      _M24(_jit,0,x2,x4)
 446 #define M25(x4)                         M24(0,x4)
 447 static void _M24(jit_state_t*,jit_word_t,
 448                  jit_word_t,jit_word_t)
 449     maybe_unused;
 450 #define M26(r1)                         M26x(2,r1)
 451 #define M26x(x4,r1)                     _M26x(_jit,0,x4,r1)
 452 static void _M26x(jit_state_t*,jit_word_t,
 453                   jit_word_t,jit_word_t)
 454     maybe_unused;
 455 #define M28(x,r3)                       _M28(_jit,0,x,r3)
 456 static void _M28(jit_state_t*,jit_word_t,
 457                  jit_word_t,jit_word_t)
 458     maybe_unused;
 459 #define M29(ar,r2)                      _M29(_jit,0,ar,r2)
 460 static void _M29(jit_state_t*,jit_word_t,
 461                  jit_word_t,jit_word_t)
 462     maybe_unused;
 463 #define M30(ar,im)                      _M30(_jit,0,ar,im)
 464 static void _M30(jit_state_t*,jit_word_t,jit_word_t,jit_word_t)
 465     maybe_unused;
 466 #define M31(ar,r1)                      _M31(_jit,0,ar,r1)
 467 static void _M31(jit_state_t*,jit_word_t,
 468                  jit_word_t,jit_word_t)
 469     maybe_unused;
 470 #define M32(cr,r2)                      _M32(_jit,0,cr,r2)
 471 static void _M32(jit_state_t*,jit_word_t,
 472                  jit_word_t,jit_word_t)
 473     maybe_unused;
 474 #define M33(cr,r1)                      _M33(_jit,0,cr,r1)
 475 static void _M33(jit_state_t*,jit_word_t,
 476                  jit_word_t,jit_word_t)
 477     maybe_unused;
 478 #define M34(sor,sol,sof,r1)             _M34(_jit,0,sor,sol,sof,r1)
 479 static void _M34(jit_state_t*,jit_word_t,
 480                  jit_word_t,jit_word_t,jit_word_t,jit_word_t);
 481 #define M35(x6,r2)                      _M35(_jit,0,x6,r2)
 482 static void _M35(jit_state_t*,jit_word_t,
 483                  jit_word_t,jit_word_t)
 484     maybe_unused;
 485 #define M36(x6,r1)                      _M36(_jit,0,x6,r1)
 486 static void _M36(jit_state_t*,jit_word_t,
 487                  jit_word_t,jit_word_t)
 488     maybe_unused;
 489 #define M37(im)                         _M37(_jit,0,im)
 490 static void _M37(jit_state_t*,jit_word_t,
 491                  jit_word_t)
 492     maybe_unused;
 493 #define M38(x6,r3,r2,r1)                _M38(_jit,0,x6,r3,r2,r1)
 494 static void _M38(jit_state_t*,jit_word_t,
 495                  jit_word_t,jit_word_t,jit_word_t,jit_word_t)
 496     maybe_unused;
 497 #define M39(x6,r3,im,r1)                _M39(_jit,0,x6,r3,im,r1)
 498 static void _M39(jit_state_t*,jit_word_t,
 499                  jit_word_t,jit_word_t,jit_word_t,jit_word_t)
 500     maybe_unused;
 501 #define M40(x6,r3,im)                   _M40(_jit,0,x6,r3,im)
 502 static void _M40(jit_state_t*,jit_word_t,
 503                  jit_word_t,jit_word_t,jit_word_t)
 504     maybe_unused;
 505 #define M41(x6,r2)                      _M41(_jit,0,x6,r2)
 506 static void _M41(jit_state_t*,jit_word_t,
 507                  jit_word_t,jit_word_t)
 508     maybe_unused;
 509 #define M42(x6,r3,r2)                   _M42(_jit,0,x6,r3,r2)
 510 static void _M42(jit_state_t*,jit_word_t,
 511                  jit_word_t,jit_word_t,jit_word_t)
 512     maybe_unused;
 513 #define M43(x6,r3,r1)                   _M43(_jit,0,x6,r3,r1)
 514 static void _M43(jit_state_t*,jit_word_t,
 515                  jit_word_t,jit_word_t,jit_word_t)
 516     maybe_unused;
 517 #define M44(x4,im)                      _M44(_jit,0,x4,im)
 518 static void _M44(jit_state_t*,jit_word_t,
 519                  jit_word_t,jit_word_t)
 520     maybe_unused;
 521 #define M45(x6,r3,r2)                   _M45(_jit,0,x6,r3,r2)
 522 static void _M45(jit_state_t*,jit_word_t,
 523                  jit_word_t,jit_word_t,jit_word_t)
 524     maybe_unused;
 525 #define M46(x6,r3,r1)                   _M46(_jit,0,x6,r3,r1)
 526 #define M47(x6,r3)                      M46(x6,r3,0)
 527 static void _M46(jit_state_t*,jit_word_t,
 528                  jit_word_t,jit_word_t,jit_word_t)
 529     maybe_unused;
 530 #define M48(y,im)                       _M48(_jit,0,y,im)
 531 static void _M48(jit_state_t*,jit_word_t,
 532                  jit_word_t,jit_word_t)
 533     maybe_unused;
 534 #define B1(d,wh,im,p,tp)                _B1(_jit,0,d,wh,im,p,tp)
 535 #define B2(d,wh,im,p,tp)                B1(d,wh,im,p,tp)
 536 static void _B1(jit_state_t*,jit_word_t,
 537                 jit_word_t,jit_word_t,jit_word_t,jit_word_t,jit_word_t);
 538 #define B3(d,wh,im,p,b)                 _B3(_jit,0,d,wh,im,p,b)
 539 static void _B3(jit_state_t*,jit_word_t,
 540                 jit_word_t,jit_word_t,jit_word_t,jit_word_t,jit_word_t)
 541     maybe_unused;
 542 #define B4(d,wh,x6,b,p,tp)              _B4(_jit,0,d,wh,x6,b,p,tp)
 543 static void _B4(jit_state_t*,jit_word_t,
 544                 jit_word_t,jit_word_t,jit_word_t,
 545                 jit_word_t,jit_word_t,jit_word_t);
 546 #define B5(d,wh,b2,p,b1)                _B5(_jit,0,d,wh,b2,p,b1)
 547 static void _B5(jit_state_t*,jit_word_t,
 548                 jit_word_t,jit_word_t,jit_word_t,jit_word_t,jit_word_t);
 549 #define B6(ih,im,tag,wh)                _B6(_jit,0,ih,im,tag,wh)
 550 static void _B6(jit_state_t*,jit_word_t,
 551                 jit_word_t,jit_word_t,jit_word_t,jit_word_t)
 552     maybe_unused;
 553 #define B7(ih,x6,b2,tag,wh)             _B7(_jit,0,ih,x6,b2,tag,wh)
 554 static void _B7(jit_state_t*,jit_word_t,
 555                 jit_word_t,jit_word_t,jit_word_t,jit_word_t,jit_word_t)
 556     maybe_unused;
 557 #define B8(x6)                          _B8(_jit,0,x6)
 558 static void _B8(jit_state_t*,jit_word_t,
 559                 jit_word_t)
 560     maybe_unused;
 561 #define B9(op,x6,im)                    _B9(_jit,0,op,x6,im)
 562 static void _B9(jit_state_t*,jit_word_t,
 563                 jit_word_t,jit_word_t,jit_word_t)
 564     maybe_unused;
 565 #define X1(im)                          _X1(_jit,0,im)
 566 static void _X1(jit_state_t*,jit_word_t,
 567                 jit_word_t)
 568     maybe_unused;
 569 #define X2(r1,im)                       _X2(_jit,0,r1,im)
 570 static void _X2(jit_state_t*,jit_word_t,
 571                 jit_word_t,jit_word_t);
 572 #define X3x(o,d,wh,p,tp,im)             _X3x(_jit,0,o,d,wh,p,tp,im)
 573 #define X3(d,wh,p,tp,im)                X3x(0xc,d,wh,p,tp,im)
 574 #define X4(d,wh,p,tp,im)                X3x(0xd,d,wh,p,tp,im)
 575 static void _X3x(jit_state_t*,jit_word_t,
 576                  jit_word_t,jit_word_t,jit_word_t,
 577                  jit_word_t,jit_word_t,jit_word_t);
 578 #define X5(y,im)                        _X5(_jit,0,y,im)
 579 static void _X5(jit_state_t*,jit_word_t,
 580                 jit_word_t,jit_word_t)
 581     maybe_unused;
 582
 583 /* add */
 584 #define ADD(r1,r2,r3)                   A1(0,0,r3,r2,r1)
 585 #define ADD1(r1,r2,r3)                  A1(0,1,r3,r2,r1)
 586 #define ADDS(r1,im,r3)                  A4(2,r3,im,r1)
 587 #define ADDS_p(r1,im,r3,_p)             _A4(_jit,_p,2,r3,im,r1)
 588 #define ADDL(r1,im,r3)                  A5(r3,im,r1)
 589 #define ADDL_p(r1,im,r3,_p)             _A5(_jit,_p,r3,im,r1)
 590 /* addp4 */
 591 #define ADDP4(r1,r2,r3)                 A1(2,0,r3,r2,r1)
 592 #define ADDIP4(r1,im,r3)                A4(3,r3,im,r1)
 593 /* alloc */
 594 #define ALLOCR(r1,i,l,o,r)              M34((r)>>3,(i)+(l),(i)+(l)+(o),r1)
 595 #define ALLOC(r1,i,o)                   ALLOCR(r1,i,0,o,0)
 596 /* and */
 597 #define AND(r1,r2,r3)                   A1(3,0,r3,r2,r1)
 598 #define ANDI(r1,im,r3)                  A3(0xb,0,r3,im,r1)
 599 /* andcm */
 600 #define ANDCM(r1,r2,r3)                 A1(3,1,r3,r2,r1)
 601 #define ANDCMI(r1,im,r3)                A3(0xb,1,r3,im,r1)
 602 /* br */
 603 #define BRI(im)                         B1(BR_DH_NONE,BR_BWH_SPTK,im,BR_PH_FEW,0)
 604 #define BRI_COND(im,_p)                 _B1(_jit,_p,BR_DH_NONE,BR_BWH_SPTK,im,BR_PH_FEW,0)
 605 #define BRI_WEXIT(im)                   B1(BR_DH_NONE,BR_BWH_SPTK,im,BR_PH_MANY,2)
 606 #define BRI_WTOP(im)                    B1(BR_DH_NONE,BR_BWH_SPTK,im,BR_PH_MANY,3)
 607 #define BRI_CALL(b,im)                  B3(BR_DH_NONE,BR_BWH_SPTK,im,BR_PH_MANY,b)
 608 #define BRI_CLOOP(im)                   B2(BR_DH_NONE,BR_BWH_SPTK,im,BR_PH_MANY,5)
 609 #define BRI_CEXIT(im)                   B2(BR_DH_NONE,BR_BWH_SPTK,im,BR_PH_MANY,6)
 610 #define BRI_CTOP(im)                    B2(BR_DH_NONE,BR_BWH_SPTK,im,BR_PH_MANY,7)
 611 #define BR_COND(b,_p)                   _B4(_jit,_p,BR_DH_NONE,BR_BWH_SPTK,0x20,b,BR_PH_FEW,0)
 612 #define BR(b)                           B4(BR_DH_NONE,BR_BWH_SPTK,0x20,b,BR_PH_FEW,0)
 613 #define BR_IA(b)                        B4(BR_DH_NONE,BR_BWH_SPTK,0x20,b,BR_PH_MANY,1)
 614 #define BR_RET(b)                       B4(BR_DH_NONE,BR_BWH_SPTK,0x21,b,BR_PH_MANY,4)
 615 #define BR_CALL(b1,b2)                  B5(BR_DH_NONE,BRI_BWH_SPTK,b2,BR_PH_MANY,b1)
 616 /* break */
 617 #define BREAK_I(im)                     I19(im)
 618 #define BREAK_M(im)                     M37(im)
 619 #define BREAK_B(im)                     B9(0,0,im)
 620 #define BREAK_X(im)                     X1(im)
 621 /* brl */
 622 #define BRL(im)                         X3(BR_DH_NONE,BR_BWH_SPTK,BR_PH_MANY,0,im)
 623 #define BRL_COND(im,_p)                 _X3(_jit,_p,BR_DH_NONE,BR_BWH_SPTK,BR_PH_MANY,0,im)
 624 #define BRL_CALL(b1,im)                 X4(BR_DH_NONE,BR_BWH_SPTK,BR_PH_MANY,b1,im)
 625 /* brp */
 626 #define BRP(im,tag)                     B6(BR_IH_NONE,im,tag,BR_IPWH_SPTK)
 627 #define BRPI(b2,tag)                    B7(BR_IH_NONE,0x10,b2,tag,BR_INDWH_SPTK)
 628 #define BRPI_RET(b2,tag)                B7(BR_IH_NONE,0x11,b2,tag,BR_INDWH_SPTK)
 629 /* bsw */
 630 #define BSW_0()                         B8(0x0c)
 631 #define BSW_1()                         B8(0x0d)
 632 /* chk */
 633 #define CHK_S_I(r2,im)                  I20(r2,im)
 634 #define CHK_S_M(r2,im)                  M20(r2,im)
 635 #define CHK_A_NC(r1,im)                 M22(0x4,im,r1)
 636 #define CHK_A_CLR(r1,im)                M22(0x5,im,r1)
 637 /* clrrrb */
 638 #define CLRRRB()                        B8(0x04)
 639 #define CLRRRB_PR()                     B8(0x05)
 640 /* clz */
 641 #define CLZ(r1,r3)                      I9(3,r3,r1)
 642 /* cmp */
 643 #define CMP_LT(p1,p2,r2,r3)             A6(0xc,0,0,p2,r3,r2,0,p1)
 644 #define CMP_LT_p(p1,p2,r2,r3,_p)        A6(_jit,_p,0xc,0,0,p2,r3,r2,0,p1)
 645 #define CMP_LTU(p1,p2,r2,r3)            A6(0xd,0,0,p2,r3,r2,0,p1)
 646 #define CMP_EQ(p1,p2,r2,r3)             A6(0xe,0,0,p2,r3,r2,0,p1)
 647 #define CMP_LT_UNC(p1,p2,r2,r3)         A6(0xc,0,0,p2,r3,r2,1,p1)
 648 #define CMP_LTU_UNC(p1,p2,r2,r3)        A6(0xd,0,0,p2,r3,r2,1,p1)
 649 #define CMP_EQ_UNC(p1,p2,r2,r3)         A6(0xe,0,0,p2,r3,r2,1,p1)
 650 #define CMP_EQ_AND(p1,p2,r2,r3)         A6(0xc,0,1,p2,r3,r2,0,p1)
 651 #define CMP_EQ_OR(p1,p2,r2,r3)          A6(0xd,0,1,p2,r3,r2,0,p1)
 652 #define CMP_EQ_OR_ANDCM(p1,p2,r2,r3)    A6(0xe,0,1,p2,r3,r2,0,p1)
 653 #define CMP_NE_AND(p1,p2,r2,r3)         A6(0xc,0,1,p2,r3,r2,1,p1)
 654 #define CMP_NE_OR(p1,p2,r2,r3)          A6(0xd,0,1,p2,r3,r2,1,p1)
 655 #define CMP_NE_OR_ANDCM(p1,p2,r2,r3)    A6(0xe,0,1,p2,r3,r2,1,p1)
 656 #define CMPI_LT(p1,p2,im,r3)            A8(0xc,2,0,p2,r3,im,0,p1)
 657 #define CMPI_LTU(p1,p2,im,r3)           A8(0xd,2,0,p2,r3,im,0,p1)
 658 #define CMPI_EQ(p1,p2,im,r3)            A8(0xe,2,0,p2,r3,im,0,p1)
 659 #define CMPI_EQ_p(p1,p2,im,r3,_p)       _A8(_jit,_p,0xe,2,0,p2,r3,im,0,p1)
 660 #define CMPI_LT_UNC(p1,p2,im,r3)        A8(0xc,2,0,p2,r3,im,1,p1)
 661 #define CMPI_LTU_UNC(p1,p2,im,r3)       A8(0xd,2,0,p2,r3,im,1,p1)
 662 #define CMPI_EQ_UNC(p1,p2,im,r3)        A8(0xe,2,0,p2,r3,im,1,p1)
 663 #define CMPI_EQ_AND(p1,p2,im,r3)        A8(0xc,2,1,p2,r3,im,0,p1)
 664 #define CMPI_EQ_OR(p1,p2,im,r3)         A8(0xd,2,1,p2,r3,im,0,p1)
 665 #define CMPI_EQ_ANDCM(p1,p2,im,r3)      A8(0xe,2,1,p2,r3,im,0,p1)
 666 #define CMPI_NE_AND(p1,p2,im,r3)        A8(0xc,2,1,p2,r3,im,1,p1)
 667 #define CMPI_NE_OR(p1,p2,im,r3)         A8(0xd,2,1,p2,r3,im,1,p1)
 668 #define CMPI_NE_ANDCM(p1,p2,im,r3)      A8(0xe,2,1,p2,r3,im,1,p1)
 669 #define ZCMP_GT_AND(p1,p2,r3)           A7(0xc,0,0,p2,r3,0,p1)
 670 #define ZCMP_GT_OR(p1,p2,r3)            A7(0xd,0,0,p2,r3,0,p1)
 671 #define ZCMP_GT_ANDCM(p1,p2,r3)         A7(0xe,0,0,p2,r3,0,p1)
 672 #define ZCMP_LE_AND(p1,p2,r3)           A7(0xc,0,0,p2,r3,1,p1)
 673 #define ZCMP_LE_OR(p1,p2,r3)            A7(0xd,0,0,p2,r3,1,p1)
 674 #define ZCMP_LE_ANDCM(p1,p2,r3)         A7(0xe,0,0,p2,r3,1,p1)
 675 #define ZCMP_GE_AND(p1,p2,r3)           A7(0xc,0,1,p2,r3,0,p1)
 676 #define ZCMP_GE_OR(p1,p2,r3)            A7(0xd,0,1,p2,r3,0,p1)
 677 #define ZCMP_GE_ANDCM(p1,p2,r3)         A7(0xe,0,1,p2,r3,0,p1)
 678 #define ZCMP_LT_AND(p1,p2,r3)           A7(0xc,0,1,p2,r3,1,p1)
 679 #define ZCMP_LT_OR(p1,p2,r3)            A7(0xd,0,1,p2,r3,1,p1)
 680 #define ZCMP_LT_ANDCM(p1,p2,r3)         A7(0xe,0,1,p2,r3,1,p1)
 681 /* cmp4 */
 682 #define CMP4_LT(p1,p2,r2,r3)            A6(0xc,1,0,p2,r3,r2,0,p1)
 683 #define CMP4_LTU(p1,p2,r2,r3)           A6(0xd,1,0,p2,r3,r2,0,p1)
 684 #define CMP4_EQ(p1,p2,r2,r3)            A6(0xe,1,0,p2,r3,r2,0,p1)
 685 #define CMP4_LT_UNC(p1,p2,r2,r3)        A6(0xc,1,0,p2,r3,r2,1,p1)
 686 #define CMP4_LTU_UNC(p1,p2,r2,r3)       A6(0xd,1,0,p2,r3,r2,1,p1)
 687 #define CMP4_EQ_UNC(p1,p2,r2,r3)        A6(0xe,1,0,p2,r3,r2,1,p1)
 688 #define CMP4_EQ_AND(p1,p2,r2,r3)        A6(0xc,1,1,p2,r3,r2,0,p1)
 689 #define CMP4_EQ_OR(p1,p2,r2,r3)         A6(0xd,1,1,p2,r3,r2,0,p1)
 690 #define CMP4_EQ_XOR(p1,p2,r2,r3)        A6(0xe,1,1,p2,r3,r2,0,p1)
 691 #define CMP4_NE_AND(p1,p2,r2,r3)        A6(0xc,1,1,p2,r3,r2,1,p1)
 692 #define CMP4_NE_OR(p1,p2,r2,r3)         A6(0xd,1,1,p2,r3,r2,1,p1)
 693 #define CMP4_NE_XOR(p1,p2,r2,r3)        A6(0xe,1,1,p2,r3,r2,1,p1)
 694 #define CMP4I_LT(p1,p2,im,r3)           A8(0xc,3,0,p2,r3,im,0,p1)
 695 #define CMP4I_LTU(p1,p2,im,r3)          A8(0xd,3,0,p2,r3,im,0,p1)
 696 #define CMP4I_EQ(p1,p2,im,r3)           A8(0xe,3,0,p2,r3,im,0,p1)
 697 #define CMP4I_LT_UNC(p1,p2,im,r3)       A8(0xc,3,0,p2,r3,im,1,p1)
 698 #define CMP4I_LTU_UNC(p1,p2,im,r3)      A8(0xd,3,0,p2,r3,im,1,p1)
 699 #define CMP4I_EQ_UNC(p1,p2,im,r3)       A8(0xe,3,0,p2,r3,im,1,p1)
 700 #define CMP4I_EQ_AND(p1,p2,im,r3)       A8(0xc,3,1,p2,r3,im,0,p1)
 701 #define CMP4I_EQ_OR(p1,p2,im,r3)        A8(0xd,3,1,p2,r3,im,0,p1)
 702 #define CMP4I_EQ_ANDCM(p1,p2,im,r3)     A8(0xe,3,1,p2,r3,im,0,p1)
 703 #define CMP4I_NE_AND(p1,p2,im,r3)       A8(0xc,3,1,p2,r3,im,1,p1)
 704 #define CMP4I_NE_OR(p1,p2,im,r3)        A8(0xd,3,1,p2,r3,im,1,p1)
 705 #define CMP4I_NE_ANDCM(p1,p2,im,r3)     A8(0xe,3,1,p2,r3,im,1,p1)
 706 #define ZCMP4_GT_AND(p1,p2,r3)          A7(0xc,1,0,p2,r3,0,p1)
 707 #define ZCMP4_GT_OR(p1,p2,r3)           A7(0xd,1,0,p2,r3,0,p1)
 708 #define ZCMP4_GT_ANDCM(p1,p2,r3)        A7(0xe,1,0,p2,r3,0,p1)
 709 #define ZCMP4_LE_AND(p1,p2,r3)          A7(0xc,1,0,p2,r3,1,p1)
 710 #define ZCMP4_LE_OR(p1,p2,r3)           A7(0xd,1,0,p2,r3,1,p1)
 711 #define ZCMP4_LE_ANDCM(p1,p2,r3)        A7(0xe,1,0,p2,r3,1,p1)
 712 #define ZCMP4_GE_AND(p1,p2,r3)          A7(0xc,1,1,p2,r3,0,p1)
 713 #define ZCMP4_GE_OR(p1,p2,r3)           A7(0xd,1,1,p2,r3,0,p1)
 714 #define ZCMP4_GE_ANDCM(p1,p2,r3)        A7(0xe,1,1,p2,r3,0,p1)
 715 #define ZCMP4_LT_AND(p1,p2,r3)          A7(0xc,1,1,p2,r3,1,p1)
 716 #define ZCMP4_LT_OR(p1,p2,r3)           A7(0xd,1,1,p2,r3,1,p1)
 717 #define ZCMP4_LT_ANDCM(p1,p2,r3)        A7(0xe,1,1,p2,r3,1,p1)
 718 /* cmpxchg */
 719 #define CMPXCHG1_ACQ(r1,r3,r2)          M16(0x00,LD_NONE,r3,r2,r1)
 720 #define CMPXCHG2_ACQ(r1,r3,r2)          M16(0x01,LD_NONE,r3,r2,r1)
 721 #define CMPXCHG4_ACQ(r1,r3,r2)          M16(0x02,LD_NONE,r3,r2,r1)
 722 #define CMPXCHG8_ACQ(r1,r3,r2)          M16(0x03,LD_NONE,r3,r2,r1)
 723 #define CMPXCHG1_REL(r1,r3,r2)          M16(0x04,LD_NONE,r3,r2,r1)
 724 #define CMPXCHG2_REL(r1,r3,r2)          M16(0x05,LD_NONE,r3,r2,r1)
 725 #define CMPXCHG4_REL(r1,r3,r2)          M16(0x06,LD_NONE,r3,r2,r1)
 726 #define CMPXCHG8_REL(r1,r3,r2)          M16(0x07,LD_NONE,r3,r2,r1)
 727 #define CMP8XCHG16_ACQ(r1,r3,r2)        M16(0x20,LD_NONE,r3,r2,r1)
 728 #define CMP8XCHG16_REL(r1,r3,r2)        M16(0x24,LD_NONE,r3,r2,r1)
 729 /* cover */
 730 #define COVER()                         B8(0x02)
 731 /* cxz */
 732 #define CZX1_L(r1,r3)                   I29(0x18,r3,r1)
 733 #define CZX2_L(r1,r3)                   I29(0x19,r3,r1)
 734 #define CZX1_R(r1,r3)                   I29(0x1c,r3,r1)
 735 #define CZX2_R(r1,r3)                   I29(0x1d,r3,r1)
 736 /* dep */
 737 #define DEP_Z(r1,r2,pos,len)            I12(len,pos,r2,r1)
 738 #define DEPI_Z(r1,im,pos,len)           I13(len,pos,im,r1)
 739 #define DEPs(r1,r3,pos,len)             I14(1,len,r3,pos,r1)
 740 #define DEPu(r1,r3,pos,len)             I14(0,len,r3,pos,r1)
 741 #define DEP(r1,r2,r3,pos,len)           I15(pos,len,r3,r2,r1)
 742 /* epc */
 743 #define EPC()                           B8(0x10)
 744 /* extr */
 745 #define EXTR(r1,r3,pos,len)             I11(len,r3,pos,1,r1)
 746 #define EXTR_U(r1,r3,pos,len)           I11(len,r3,pos,0,r1)
 747 /* fc */
 748 #define FC(r3)                          M28(0,r3)
 749 #define FC_I(r3)                        M28(1,r3)
 750 /* fetchadd */
 751 #define FETCHADD4_ACQ(r1,r3,im)         M17(0x12,LD_NONE,r3,im,r1)
 752 #define FETCHADD8_ACQ(r1,r3,im)         M17(0x13,LD_NONE,r3,im,r1)
 753 #define FETCHADD4_REL(r1,r3,im)         M17(0x16,LD_NONE,r3,im,r1)
 754 #define FETCHADD8_REL(r1,r3,im)         M17(0x17,LD_NONE,r3,im,r1)
 755 /* flushrs */
 756 #define FLUSHRS()                       M25(0xc)
 757 /* fwb */
 758 #define FWB()                           M24(2,0)
 759 /* hint */
 760 #define HINT_I(im)                      I18(im,1)
 761 #define HINT_M(im)                      M48(1,im)
 762 #define HINT_B(im)                      B9(2,1,im)
 763 #define HINT_X(im)                      X5(1,im)
 764 /* invala */
 765 #define INVALA()                        M24(1,0)
 766 #define INVALA_E(r1)                    M26(r1)
 767 /* itc */
 768 #define ITC_I(r2)                       M41(0x2f,r2)
 769 #define ITC_D(r2)                       M41(0x2e,r2)
 770 /* itr */
 771 #define ITR_I(r3,r2)                    M42(0x0f,r3,r2)
 772 #define ITR_D(r3,r2)                    M42(0x0e,r3,r2)
 773 /* ld */
 774 #define LD1(r1,r3)                      M1(0x00,LD_NONE,0,r3,r1)
 775 #define LD2(r1,r3)                      M1(0x01,LD_NONE,0,r3,r1)
 776 #define LD4(r1,r3)                      M1(0x02,LD_NONE,0,r3,r1)
 777 #define LD8(r1,r3)                      M1(0x03,LD_NONE,0,r3,r1)
 778 #define LD1_S(r1,r3)                    M1(0x04,LD_NONE,0,r3,r1)
 779 #define LD2_S(r1,r3)                    M1(0x05,LD_NONE,0,r3,r1)
 780 #define LD4_S(r1,r3)                    M1(0x06,LD_NONE,0,r3,r1)
 781 #define LD8_S(r1,r3)                    M1(0x07,LD_NONE,0,r3,r1)
 782 #define LD1_A(r1,r3)                    M1(0x08,LD_NONE,0,r3,r1)
 783 #define LD2_A(r1,r3)                    M1(0x09,LD_NONE,0,r3,r1)
 784 #define LD4_A(r1,r3)                    M1(0x0a,LD_NONE,0,r3,r1)
 785 #define LD8_A(r1,r3)                    M1(0x0b,LD_NONE,0,r3,r1)
 786 #define LD1_SA(r1,r3)                   M1(0x0c,LD_NONE,0,r3,r1)
 787 #define LD2_SA(r1,r3)                   M1(0x0d,LD_NONE,0,r3,r1)
 788 #define LD4_SA(r1,r3)                   M1(0x0e,LD_NONE,0,r3,r1)
 789 #define LD8_SA(r1,r3)                   M1(0x0f,LD_NONE,0,r3,r1)
 790 #define LD1_BIAS(r1,r3)                 M1(0x10,LD_NONE,0,r3,r1)
 791 #define LD2_BIAS(r1,r3)                 M1(0x11,LD_NONE,0,r3,r1)
 792 #define LD4_BIAS(r1,r3)                 M1(0x12,LD_NONE,0,r3,r1)
 793 #define LD8_BIAS(r1,r3)                 M1(0x13,LD_NONE,0,r3,r1)
 794 #define LD1_ACQ(r1,r3)                  M1(0x14,LD_NONE,0,r3,r1)
 795 #define LD2_ACQ(r1,r3)                  M1(0x15,LD_NONE,0,r3,r1)
 796 #define LD4_ACQ(r1,r3)                  M1(0x16,LD_NONE,0,r3,r1)
 797 #define LD8_ACQ(r1,r3)                  M1(0x17,LD_NONE,0,r3,r1)
 798 #define LD8_FILL(r1,r3)                 M1(0x1b,LD_NONE,0,r3,r1)
 799 #define LD1_C_CLR(r1,r3)                M1(0x20,LD_NONE,0,r3,r1)
 800 #define LD2_C_CLR(r1,r3)                M1(0x21,LD_NONE,0,r3,r1)
 801 #define LD4_C_CLR(r1,r3)                M1(0x22,LD_NONE,0,r3,r1)
 802 #define LD8_C_CLR(r1,r3)                M1(0x23,LD_NONE,0,r3,r1)
 803 #define LD1_C_NC(r1,r3)                 M1(0x24,LD_NONE,0,r3,r1)
 804 #define LD2_C_NC(r1,r3)                 M1(0x25,LD_NONE,0,r3,r1)
 805 #define LD4_C_NC(r1,r3)                 M1(0x26,LD_NONE,0,r3,r1)
 806 #define LD8_C_NC(r1,r3)                 M1(0x27,LD_NONE,0,r3,r1)
 807 #define LD1_C_CLR_ACQ(r1,r3)            M1(0x28,LD_NONE,0,r3,r1)
 808 #define LD2_C_CLR_ACQ(r1,r3)            M1(0x29,LD_NONE,0,r3,r1)
 809 #define LD4_C_CLR_ACQ(r1,r3)            M1(0x2a,LD_NONE,0,r3,r1)
 810 #define LD8_C_CLR_ACQ(r1,r3)            M1(0x2b,LD_NONE,0,r3,r1)
 811 #define LD16(r1,r3)                     M1(0x28,LD_NONE,1,r3,r1)
 812 #define LD16_ACQ(r1,r3)                 M1(0x2c,LD_NONE,1,r3,r1)
 813 #define LD1_inc(r1,r3,im)               M3(0x00,LD_NONE,r3,im,r1)
 814 #define LD2_inc(r1,r3,im)               M3(0x01,LD_NONE,r3,im,r1)
 815 #define LD4_inc(r1,r3,im)               M3(0x02,LD_NONE,r3,im,r1)
 816 #define LD8_inc(r1,r3,im)               M3(0x03,LD_NONE,r3,im,r1)
 817 #define LD1_S_inc(r1,r3,im)             M3(0x04,LD_NONE,r3,im,r1)
 818 #define LD2_S_inc(r1,r3,im)             M3(0x05,LD_NONE,r3,im,r1)
 819 #define LD4_S_inc(r1,r3,im)             M3(0x06,LD_NONE,r3,im,r1)
 820 #define LD8_S_inc(r1,r3,im)             M3(0x07,LD_NONE,r3,im,r1)
 821 #define LD1_A_inc(r1,r3,im)             M3(0x08,LD_NONE,r3,im,r1)
 822 #define LD2_A_inc(r1,r3,im)             M3(0x09,LD_NONE,r3,im,r1)
 823 #define LD4_A_inc(r1,r3,im)             M3(0x0a,LD_NONE,r3,im,r1)
 824 #define LD8_A_inc(r1,r3,im)             M3(0x0b,LD_NONE,r3,im,r1)
 825 #define LD1_SA_inc(r1,r3,im)            M3(0x0c,LD_NONE,r3,im,r1)
 826 #define LD2_SA_inc(r1,r3,im)            M3(0x0d,LD_NONE,r3,im,r1)
 827 #define LD4_SA_inc(r1,r3,im)            M3(0x0e,LD_NONE,r3,im,r1)
 828 #define LD8_SA_inc(r1,r3,im)            M3(0x0f,LD_NONE,r3,im,r1)
 829 #define LD1_BIAS_inc(r1,r3,im)          M3(0x10,LD_NONE,r3,im,r1)
 830 #define LD2_BIAS_inc(r1,r3,im)          M3(0x11,LD_NONE,r3,im,r1)
 831 #define LD4_BIAS_inc(r1,r3,im)          M3(0x12,LD_NONE,r3,im,r1)
 832 #define LD8_BIAS_inc(r1,r3,im)          M3(0x13,LD_NONE,r3,im,r1)
 833 #define LD1_ACQ_inc(r1,r3,im)           M3(0x14,LD_NONE,r3,im,r1)
 834 #define LD2_ACQ_inc(r1,r3,im)           M3(0x15,LD_NONE,r3,im,r1)
 835 #define LD4_ACQ_inc(r1,r3,im)           M3(0x16,LD_NONE,r3,im,r1)
 836 #define LD8_AVQ_inc(r1,r3,im)           M3(0x17,LD_NONE,r3,im,r1)
 837 #define LD8_FILL_inc(r1,r3,im)          M3(0x1b,LD_NONE,r3,im,r1)
 838 #define LD1_C_CLR_inc(r1,r3,im)         M3(0x20,LD_NONE,r3,im,r1)
 839 #define LD2_C_CLR_inc(r1,r3,im)         M3(0x21,LD_NONE,r3,im,r1)
 840 #define LD4_C_CLR_inc(r1,r3,im)         M3(0x22,LD_NONE,r3,im,r1)
 841 #define LD8_C_CLR_inc(r1,r3,im)         M3(0x23,LD_NONE,r3,im,r1)
 842 #define LD1_C_NC_inc(r1,r3,im)          M3(0x24,LD_NONE,r3,im,r1)
 843 #define LD2_C_NC_inc(r1,r3,im)          M3(0x25,LD_NONE,r3,im,r1)
 844 #define LD4_C_NC_inc(r1,r3,im)          M3(0x26,LD_NONE,r3,im,r1)
 845 #define LD8_C_NC_inc(r1,r3,im)          M3(0x27,LD_NONE,r3,im,r1)
 846 #define LD1_C_CLR_ACQ_inc(r1,r3,im)     M3(0x28,LD_NONE,r3,im,r1)
 847 #define LD2_C_CLR_ACQ_inc(r1,r3,im)     M3(0x29,LD_NONE,r3,im,r1)
 848 #define LD4_C_CLR_ACQ_inc(r1,r3,im)     M3(0x2a,LD_NONE,r3,im,r1)
 849 #define LD8_C_CLR_ACQ_inc(r1,r3,im)     M3(0x2b,LD_NONE,r3,im,r1)
 850 #define LDX1(r1,r3,r2)                  M2(0x00,LD_NONE,r3,r2,r1)
 851 #define LDX2(r1,r3,r2)                  M2(0x01,LD_NONE,r3,r2,r1)
 852 #define LDX4(r1,r3,r2)                  M2(0x02,LD_NONE,r3,r2,r1)
 853 #define LDX8(r1,r3,r2)                  M2(0x03,LD_NONE,r3,r2,r1)
 854 #define LDX1_S(r1,r3,r2)                M2(0x04,LD_NONE,r3,r2,r1)
 855 #define LDX2_S(r1,r3,r2)                M2(0x05,LD_NONE,r3,r2,r1)
 856 #define LDX4_S(r1,r3,r2)                M2(0x06,LD_NONE,r3,r2,r1)
 857 #define LDX8_S(r1,r3,r2)                M2(0x07,LD_NONE,r3,r2,r1)
 858 #define LDX1_A(r1,r3,r2)                M2(0x08,LD_NONE,r3,r2,r1)
 859 #define LDX2_A(r1,r3,r2)                M2(0x09,LD_NONE,r3,r2,r1)
 860 #define LDX4_A(r1,r3,r2)                M2(0x0a,LD_NONE,r3,r2,r1)
 861 #define LDX8_A(r1,r3,r2)                M2(0x0b,LD_NONE,r3,r2,r1)
 862 #define LDX1_SA(r1,r3,r2)               M2(0x0c,LD_NONE,r3,r2,r1)
 863 #define LDX2_SA(r1,r3,r2)               M2(0x0d,LD_NONE,r3,r2,r1)
 864 #define LDX4_SA(r1,r3,r2)               M2(0x0e,LD_NONE,r3,r2,r1)
 865 #define LDX8_SA(r1,r3,r2)               M2(0x0f,LD_NONE,r3,r2,r1)
 866 #define LDX1_BIAS(r1,r3,r2)             M2(0x10,LD_NONE,r3,r2,r1)
 867 #define LDX2_BIAS(r1,r3,r2)             M2(0x11,LD_NONE,r3,r2,r1)
 868 #define LDX4_BIAS(r1,r3,r2)             M2(0x12,LD_NONE,r3,r2,r1)
 869 #define LDX8_BIAS(r1,r3,r2)             M2(0x13,LD_NONE,r3,r2,r1)
 870 #define LDX1_ACQ(r1,r3,r2)              M2(0x14,LD_NONE,r3,r2,r1)
 871 #define LDX2_ACQ(r1,r3,r2)              M2(0x15,LD_NONE,r3,r2,r1)
 872 #define LDX4_ACQ(r1,r3,r2)              M2(0x16,LD_NONE,r3,r2,r1)
 873 #define LDX8_ACQ(r1,r3,r2)              M2(0x17,LD_NONE,r3,r2,r1)
 874 #define LDX8_FILL(r1,r3,r2)             M2(0x1b,LD_NONE,r3,r2,r1)
 875 #define LDX1_C_CLR(r1,r3,r2)            M2(0x20,LD_NONE,r3,r2,r1)
 876 #define LDX2_C_CLR(r1,r3,r2)            M2(0x21,LD_NONE,r3,r2,r1)
 877 #define LDX4_C_CLR(r1,r3,r2)            M2(0x22,LD_NONE,r3,r2,r1)
 878 #define LDX8_C_CLR(r1,r3,r2)            M2(0x23,LD_NONE,r3,r2,r1)
 879 #define LDX1_C_NC(r1,r3,r2)             M2(0x24,LD_NONE,r3,r2,r1)
 880 #define LDX2_C_NC(r1,r3,r2)             M2(0x25,LD_NONE,r3,r2,r1)
 881 #define LDX4_C_NC(r1,r3,r2)             M2(0x26,LD_NONE,r3,r2,r1)
 882 #define LDX8_C_NC(r1,r3,r2)             M2(0x27,LD_NONE,r3,r2,r1)
 883 #define LDX1_C_CLR_ACQ(r1,r3,r2)        M2(0x28,LD_NONE,r3,r2,r1)
 884 #define LDX2_C_CLR_ACQ(r1,r3,r2)        M2(0x29,LD_NONE,r3,r2,r1)
 885 #define LDX4_C_CLR_ACQ(r1,r3,r2)        M2(0x2a,LD_NONE,r3,r2,r1)
 886 #define LDX8_C_CLR_ACQ(r1,r3,r2)        M2(0x2b,LD_NONE,r3,r2,r1)
 887 /* lfetch */
 888 #define LFETCH_EXCL(r3)                 M13(0x2d,LF_NONE,r3,GR_0)
 889 #define LFETCH_FAULT(r3)                M13(0x2e,LF_NONE,r3,GR_0)
 890 #define LFETCH_FAULT_EXCL(r3)           M13(0x2f,LF_NONE,r3,GR_0)
 891 #define LXFETCH(r3,r2)                  M14(0x2c,LF_NONE,r3,r2)
 892 #define LXFETCH_EXCL(r3,r2)             M14(0x2d,LF_NONE,r3,r2)
 893 #define LXFETCH_FAULT(r3,r2)            M14(0x2e,LF_NONE,r3,r2)
 894 #define LXFETCH_FAULT_EXCL(r3,r2)       M14(0x2f,LF_NONE,r3,r2)
 895 #define LFETCHI(r3,im)                  M15(0x2c,LF_NONE,r3,im)
 896 #define LFETCHI_EXCL(r3,im)             M15(0x2d,LF_NONE,r3,im)
 897 #define LFETCHI_FAULT(r3,im)            M15(0x2e,LF_NONE,r3,im)
 898 #define LFETCHI_FAULT_EXCL(r3,im)       M15(0x2f,LF_NONE,r3,im)
 899 /* loadrs */
 900 #define LOADRS()                        M25(0xa)
 901 /* mf */
 902 #define MF()                            M24(2,2)
 903 #define MF_A()                          M24(2,3)
 904 /* mix */
 905 #define MIX1_R(r1,r2,r3)                I2(0,2,0,2,0,r3,r2,r1)
 906 #define MIX2_R(r1,r2,r3)                I2(0,2,1,2,0,r3,r2,r1)
 907 #define MIX4_R(r1,r2,r3)                I2(1,2,0,2,0,r3,r2,r1)
 908 #define MIX1_L(r1,r2,r3)                I2(0,2,0,2,2,r3,r2,r1)
 909 #define MIX2_L(r1,r2,r3)                I2(0,2,1,2,2,r3,r2,r1)
 910 #define MIX4_L(r1,r2,r3)                I2(1,2,0,2,2,r3,r2,r1)
 911 /* mov - Move Application Register */
 912 #define MOV_I_rn_ar(r1,ar)              I28(ar,r1)
 913 #define MOV_I_ar_rn(ar,r2)              I26(ar,r2)
 914 #define MOV_I_ar_im(ar,im)              I27(ar,im)
 915 #define MOV_M_rn_a(r1,ar)               M31(r1,ar)
 916 #define MOV_M_ar_rn(ar,r2)              M29(ar,r2)
 917 #define MOV_M_ar_im(ar,im)              M30(ar,im)
 918 /* mov - Move Branch Register */
 919 #define MOV_rn_br(r1,b2)                I22(b2,r1)
 920 #define MOV_br_rn_tg(b1,r2,tag)         I21(tag,IH_NONE,0,MWH_NONE,r2,b1)
 921 #define MOV_br_rn(b1,r2)                MOV_br_rn_tg(b1,r2,0)
 922 #define MOV_RET_br_rn_tg(b1,r2,tag)     I21(tag,IH_NONE,1,MWH_NONE,r2,b1)
 923 /* mov - Move Control Register */
 924 #define MOV_rn_cr(cr,r1)                M33(cr,r1)
 925 #define MOV_cr_rr(cr,r2)                M32(cr,r2)
 926 /* mov - Move General Register */
 927 #define MOV(r0,r1)                      ADDS(r0,0,r1)
 928 #define MOV_p(r0,r1,_p)                 ADDS_p(r0,0,r1,_p)
 929 /* mov - Move Immediate */
 930 #define MOVI(r1,im)                     ADDL(r1,im,GR_0)
 931 #define MOVI_p(r1,im,_p)                ADDL_p(r1,im,GR_0,_p)
 932 /* mov - Move Indirect Register */
 933 #define MOV_rn_RR(r1,r3)                M43(0x10,r3,r1)
 934 #define MOV_rn_DBR(r1,r3)               M43(0x11,r3,r1)
 935 #define MOV_rn_IBR(r1,r3)               M43(0x012,r3,r1)
 936 #define MOV_rn_PKR(r1,r3)               M43(0x13,r3,r1)
 937 #define MOV_rn_PMC(r1,r3)               M43(0x14,r3,r1)
 938 #define MOV_rn_PMD(r1,r3)               M43(0x15,r3,r1)
 939 #define MOV_rn_CPUID(r1,r3)             M43(0x17,r3,r1)
 940 #define MOV_RR_rn(r3,r2)                M42(0x00,r3,r2)
 941 #define MOV_DBR_rn(r3,r2)               M42(0x01,r3,r2)
 942 #define MOV_IBR_rn(r3,r2)               M42(0x02,r3,r2)
 943 #define MOV_PKR_rn(r3,r2)               M42(0x03,r3,r2)
 944 #define MOV_PMC_rn(r3,r2)               M42(0x04,r3,r2)
 945 #define MOV_PMD_rn(r3,r2)               M42(0x05,r3,r2)
 946 /* mov - Move Instruction Pointer */
 947 #define MOV_rn_ip(r1)                   I25(0x30,r1)
 948 /* mov - Move Predicates */
 949 #define MOV_rn_pr(r1)                   I25(0x33,r1)
 950 #define MOV_pr_rn(r2,im)                I23(r2,im)
 951 #define MOVI_pr(im)                     I24(im)
 952 /* mov - Move Processor Status Register */
 953 #define MOV_rn_psr(r1)                  M36(0x25,r1)
 954 #define MOV_psr_l_rn(r2)                M35(0x2d,r2)
 955 /* mov - Move User Mask */
 956 #define MOV_rn_psr_um(r1)               M36(0x21,r1)
 957 #define MOV_psr_um_rn(r2)               M35(0x29,r2)
 958 /* movl */
 959 #define MOVL(r1,im)                     X2(r1,im)
 960 /* mpy4 */
 961 #define MPY4(r1,r2,r3)                  I2(1,0,0,3,1,r3,r2,r1)
 962 /* mpyshl4 */
 963 #define MPYSHL4(r1,r2,r3)               I2(1,0,0,3,3,r3,r2,r1)
 964 /* mux */
 965 #define MUX1(r1,r2,mbt)                 I3(mbt,r2,r1)
 966 #define MUX2(r1,r2,mht)                 I4(mht,r2,r1)
 967 /* nop */
 968 #define NOP_I(im)                       I18(im,0)
 969 #define NOP_M(im)                       M48(0,im)
 970 #define NOP_B(im)                       B9(2,0,im)
 971 #define NOP_X(im)                       X5(0,im)
 972 /* or */
 973 #define OR(r1,r2,r3)                    A1(3,2,r3,r2,r1)
 974 #define ORI(r1,im,r3)                   A3(0xb,2,r3,im,r1)
 975 /* pack */
 976 #define PACK2_USS(r1,r2,r3)             I2(0,2,1,0,0,r3,r2,r1)
 977 #define PACK2_SSS(r1,r2,r3)             I2(0,2,1,0,2,r3,r2,r1)
 978 #define PACK4_SSS(r1,r2,r3)             I2(1,2,0,0,2,r3,r2,r1)
 979 /* padd */
 980 #define PADD1(r1,r2,r3)                 A9(0,0,0,0,r3,r2,r1)
 981 #define PADD1_SSS(r1,r2,r3)             A9(0,0,0,1,r3,r2,r1)
 982 #define PADD1_UUU(r1,r2,r3)             A9(0,0,0,2,r3,r2,r1)
 983 #define PADD1_UUS(r1,r2,r3)             A9(0,0,0,3,r3,r2,r1)
 984 #define PADD2(r1,r2,r3)                 A9(0,1,0,0,r3,r2,r1)
 985 #define PADD2_SSS(r1,r2,r3)             A9(0,1,0,1,r3,r2,r1)
 986 #define PADD2_UUU(r1,r2,r3)             A9(0,1,0,2,r3,r2,r1)
 987 #define PADD2_UUS(r1,r2,r3)             A9(0,1,0,3,r3,r2,r1)
 988 #define PADD4(r1,r2,r3)                 A9(1,0,0,0,r3,r2,r1)
 989 /* pavg */
 990 #define PAVG1(r1,r2,r3)                 A9(0,0,2,2,r3,r2,r1)
 991 #define PAVG2(r1,r2,r3)                 A9(0,1,2,2,r3,r2,r1)
 992 #define PAVG1_RAZ(r1,r2,r3)             A9(0,0,2,3,r3,r2,r1)
 993 #define PAVG2_RAZ(r1,r2,r3)             A9(0,1,2,3,r3,r2,r1)
 994 /* pavgsub */
 995 #define PAVGSUB1(r1,r2,r3)              A9(0,0,3,2,r3,r2,r1)
 996 #define PAVGSUB2(r1,r2,r3)              A9(0,1,3,2,r3,r2,r1)
 997 /* pcmp */
 998 #define PCMP1_EQ(r1,r2,r3)              A9(0,0,9,0,r3,r2,r1)
 999 #define PCMP2_EQ(r1,r2,r3)              A9(0,1,9,0,r3,r2,r1)
1000 #define PCMP4_EQ(r1,r2,r3)              A9(1,0,9,0,r3,r2,r1)
1001 #define PCMP1_GT(r1,r2,r3)              A9(0,0,9,1,r3,r2,r1)
1002 #define PCMP2_GT(r1,r2,r3)              A9(0,1,9,1,r3,r2,r1)
1003 #define PCMP4_GT(r1,r2,r3)              A9(1,0,9,1,r3,r2,r1)
1004 /* pmax */
1005 #define PMAX1_U(r1,r2,r3)               I2(0,2,0,1,1,r3,r2,r1)
1006 #define PMAX2(r1,r2,r3)                 I2(0,2,1,1,3,r3,r2,r1)
1007 /* pmin */
1008 #define PMIN1_U(r1,r2,r3)               I2(0,2,0,0,1,r3,r2,r1)
1009 #define PMIN2(r1,r2,r3)                 I2(0,2,1,0,3,r3,r2,r1)
1010 /* pmpy */
1011 #define PMPY2_R(r1,r2,r3)               I2(0,2,1,3,1,r3,r2,r1)
1012 #define PMPY2_L(r1,r2,r3)               I2(0,2,1,3,3,r3,r2,r1)
1013 /* pmpyshr */
1014 #define PMPYSHR2(r1,r2,r3,im)           I1(im,3,r3,r2,r1)
1015 #define PMPYSHR2_U(r1,r2,r3,im)         I1(im,1,r3,r2,r1)
1016 /* popcnt */
1017 #define POPCNT(r1,r3)                   I9(2,r3,r1)
1018 /* probe */
1019 #define PROBE_R(r1,r3,r2)               M38(0x38,r3,r2,r1)
1020 #define PROBE_W(r1,r3,r2)               M38(0x39,r3,r2,r1)
1021 #define PROBEI_R(r1,r3,im)              M39(0x18,r3,im,r1)
1022 #define PROBEI_W(r1,r3,im)              M39(0x19,r3,im,r1)
1023 #define PROBE_RW_FAULT(r3,im)           M40(0x31,r3,im)
1024 #define PROBE_R_FAULT(r3,im)            M40(0x32,r3,im)
1025 #define PROBE_W_FAULT(r3,im)            M40(0x33,r3,im)
1026 /* psad */
1027 #define PSAD1(r1,r2,r3)                 I2(0,2,0,2,3,r3,r2,r1)
1028 /* pshl */
1029 #define PSHL2(r1,r2,r3)                 I7(0,1,r3,r2,r1)
1030 #define PSHL4(r1,r2,r3)                 I7(1,0,r3,r2,r1)
1031 #define PSHL2I(r1,r2,im)                I8(0,1,im,r2,r1)
1032 #define PSHL4I(r1,r2,im)                I8(1,0,im,r2,r1)
1033 /* pshladd */
1034 #define PSHLADD2(r1,r2,im,r3)           A10(4,im,r3,r2,r1)
1035 /* pshr */
1036 #define PSHR2(r1,r3,r2)                 I5(0,1,2,r3,r2,r1)
1037 #define PSHR2I(r1,r3,im)                I6(0,1,3,r3,im,r1)
1038 #define PSHR2_U(r1,r3,r2)               I5(0,1,0,r3,r2,r1)
1039 #define PSHR2I_U(r1,r3,im)              I6(0,1,1,r3,im,r1)
1040 #define PSHR4(r1,r3,r2)                 I5(1,0,2,r3,r2,r1)
1041 #define PSHR4I(r1,r3,im)                I6(1,0,3,r3,im,r1)
1042 #define PSHR4_U(r1,r3,r2)               I5(1,0,0,r3,r2,r1)
1043 #define PSHR4I_U(r1,r3,im)              I6(1,0,1,r3,im,r1)
1044 /* pshradd */
1045 #define PSHRADD2(r1,r2,im,r3)           A10(6,im,r3,r2,r1)
1046 /* psub */
1047 #define PSUB1(r1,r2,r3)                 A9(0,0,1,0,r3,r2,r1)
1048 #define PSUB1_SSS(r1,r2,r3)             A9(0,0,1,1,r3,r2,r1)
1049 #define PSUB1_UUU(r1,r2,r3)             A9(0,0,1,2,r3,r2,r1)
1050 #define PSUB1_UUS(r1,r2,r3)             A9(0,0,1,3,r3,r2,r1)
1051 #define PSUB2(r1,r2,r3)                 A9(0,1,1,0,r3,r2,r1)
1052 #define PSUB2_SSS(r1,r2,r3)             A9(0,1,1,1,r3,r2,r1)
1053 #define PSUB2_UUU(r1,r2,r3)             A9(0,1,1,2,r3,r2,r1)
1054 #define PSUB2_UUS(r1,r2,r3)             A9(0,1,1,3,r3,r2,r1)
1055 #define PSUB4(r1,r2,r3)                 A9(1,0,1,0,r3,r2,r1)
1056 /* ptc.e */
1057 #define PTC_E(r3)                       M47(0x34,r3)
1058 /* ptc.g, ptc.ga */
1059 #define PTC_G(r3,r2)                    M45(0xa,r3,r2)
1060 #define PTC_GA(r3,r2)                   M45(0xb,r3,r2)
1061 /* ptc.l */
1062 #define PTC_L(r3,r2)                    M45(0x9,r3,r2)
1063 /* ptr */
1064 #define PTR_D(r3,r2)                    M45(0xc,r3,r2)
1065 #define PTR_I(r3,r2)                    M45(0xd,r3,r2)
1066 /* rfi */
1067 #define RFI()                           B8(0x08)
1068 /* rsm */
1069 #define RSM(im)                         M44(7,im)
1070 /* rum */
1071 #define RUM(im)                         M44(5,im)
1072 /* shl */
1073 #define SHL(r1,r2,r3)                   I7(1,1,r3,r2,r1)
1074 /* shladd */
1075 #define SHLADD(r1,r2,im,r3)             A2(4,im,r3,r2,r1)
1076 /* shladdp4 */
1077 #define SHLADDP4(r1,r2,im,r3)           A2(6,im,r3,r2,r1)
1078 /* shr */
1079 #define SHR(r1,r3,r2)                   I5(1,1,2,r3,r2,r1)
1080 #define SHR_U(r1,r3,r2)                 I5(1,1,0,r3,r2,r1)
1081 /* shrp */
1082 #define SHRP(r1,r2,r3,im)               I10(im,r3,r2,r1)
1083 /* srlz */
1084 #define SRLZ_I()                        M24(3,1)
1085 #define SRLZ_D()                        M24(3,0)
1086 /* ssm */
1087 #define SSM(im)                         M44(6,im)
1088 /* st */
1089 #define ST1(r3,r2)                      M6(0x30,ST_NONE,0,r3,r2)
1090 #define ST2(r3,r2)                      M6(0x31,ST_NONE,0,r3,r2)
1091 #define ST4(r3,r2)                      M6(0x32,ST_NONE,0,r3,r2)
1092 #define ST8(r3,r2)                      M6(0x33,ST_NONE,0,r3,r2)
1093 #define ST1_REL(r3,r2)                  M6(0x34,ST_NONE,0,r3,r2)
1094 #define ST2_REL(r3,r2)                  M6(0x35,ST_NONE,0,r3,r2)
1095 #define ST4_REL(r3,r2)                  M6(0x36,ST_NONE,0,r3,r2)
1096 #define ST8_REL(r3,r2)                  M6(0x37,ST_NONE,0,r3,r2)
1097 #define ST8_SPILL(r3,r2)                M6(0x3b,ST_NONE,0,r3,r2)
1098 #define ST16(r3,r2)                     M6(0x30,ST_NONE,1,r3,r2)
1099 #define ST16_REL(r3,r2)                 M6(0x34,ST_NONE,1,r3,r2)
1100 #define ST1_inc(r3,r2,im)               M5(0x30,ST_NONE,r3,r2,im)
1101 #define ST2_inc(r3,r2,im)               M5(0x31,ST_NONE,r3,r2,im)
1102 #define ST4_inc(r3,r2,im)               M5(0x32,ST_NONE,r3,r2,im)
1103 #define ST8_inc(r3,r2,im)               M5(0x33,ST_NONE,r3,r2,im)
1104 #define ST1_REL_inc(r3,r2,im)           M5(0x34,ST_NONE,r3,r2,im)
1105 #define ST2_REL_inc(r3,r2,im)           M5(0x35,ST_NONE,r3,r2,im)
1106 #define ST4_REL_inc(r3,r2,im)           M5(0x36,ST_NONE,r3,r2,im)
1107 #define ST8_REL_inc(r3,r2,im)           M5(0x37,ST_NONE,r3,r2,im)
1108 #define ST8_SPILL_inc(r3,r2,im)         M5(0x3b,ST_NONE,r3,r2,im)
1109 /* sub */
1110 #define SUB(r1,r2,r3)                   A1(1,1,r3,r2,r1)
1111 #define SUB1(r1,r2,r3)                  A1(1,0,r3,r2,r1)
1112 #define SUBI(r1,im,r3)                  A3(9,1,r3,im,r1)
1113 /* sum */
1114 #define SUM(im)                         M44(4,im)
1115 /* sxt */
1116 #define SXT1(r1,r3)                     I29(0x14,r3,r1)
1117 #define SXT2(r1,r3)                     I29(0x15,r3,r1)
1118 #define SXT4(r1,r3)                     I29(0x16,r3,r1)
1119 /* sync */
1120 #define SYNC_I()                        M24(3,3)
1121 /* tak */
1122 #define TAK(r1,r3)                      M46(0x1f,r3,r1)
1123 /* tbit */
1124 #define TBIT_Z(p1,p2,r3,pos)            I16(0,0,p2,r3,pos,0,p1)
1125 #define TBIT_Z_UNC(p1,p2,r3,pos)        I16(0,0,p2,r3,pos,1,p1)
1126 #define TBIT_Z_AND(p1,p2,r3,pos)        I16(1,0,p2,r3,pos,0,p1)
1127 #define TBIT_NZ_AND(p1,p2,r3,pos)       I16(1,0,p2,r3,pos,1,p1)
1128 #define TBIT_Z_OR(p1,p2,r3,pos)         I16(0,1,p2,r3,pos,0,p1)
1129 #define TBIT_NZ_OR(p1,p2,r3,pos)        I16(0,1,p2,r3,pos,1,p1)
1130 #define TBIT_Z_ANDCM(p1,p2,r3,pos)      I16(1,1,p2,r3,pos,0,p1)
1131 #define TBIT_NZ_ANDCM(p1,p2,r3,pos)     I16(1,1,p2,r3,pos,1,p1)
1132 /* tf */
1133 #define TF_Z(p1,p2,im)                  I30(0,0,p2,im,0,p1)
1134 #define TF_Z_UNC(p1,p2,im)              I30(0,0,p2,im,1,p1)
1135 #define TF_Z_AND(p1,p2,im)              I30(1,0,p2,im,0,p1)
1136 #define TF_NZ_AND(p1,p2,im)             I30(1,0,p2,im,1,p1)
1137 #define TF_Z_OR(p1,p2,im)               I30(0,1,p2,im,0,p1)
1138 #define TF_NZ_OR(p1,p2,im)              I30(0,1,p2,im,1,p1)
1139 #define TF_Z_ANDCM(p1,p2,im)            I30(1,1,p2,im,0,p1)
1140 #define TF_NZ_ANDCM(p1,p2,im)           I30(1,1,p2,im,1,p1)
1141 /* thash */
1142 #define THASH(r1,r3)                    M46(0x1a,r3,r1)
1143 /* tnat */
1144 #define TNAT_Z(p1,p2,r3)                I17(0,0,p2,r3,0,p1)
1145 #define TNAT_Z_UNC(p1,p2,r3)            I17(0,0,p2,r3,1,p1)
1146 #define TNAT_Z_AND(p1,p2,r3)            I17(1,0,p2,r3,0,p1)
1147 #define TNAT_NZ_AND(p1,p2,r3)           I17(1,0,p2,r3,1,p1)
1148 #define TNAT_Z_OR(p1,p2,r3)             I17(0,1,p2,r3,0,p1)
1149 #define TNAT_NZ_OR(p1,p2,r3)            I17(0,1,p2,r3,1,p1)
1150 #define TNAT_Z_ANDCM(p1,p2,r3)          I17(1,1,p2,r3,0,p1)
1151 #define TNAT_NZ_ANDCM(p1,p2,r3)         I17(1,1,p2,r3,1,p1)
1152 /* tpa */
1153 #define TPA(r1,r3)                      M46(0x1e,r3,r1)
1154 /* ttag */
1155 #define TTAG(r1,r3)                     M46(0x1b,r3,r1)
1156 /* unpack */
1157 #define UNPACK1_H(r1,r2,r3)             I2(0,2,0,1,0,r3,r2,r1)
1158 #define UNPACK2_H(r1,r2,r3)             I2(0,2,1,1,0,r3,r2,r1)
1159 #define UNPACK4_H(r1,r2,r3)             I2(1,2,0,1,0,r3,r2,r1)
1160 #define UNPACK1_L(r1,r2,r3)             I2(0,2,0,1,2,r3,r2,r1)
1161 #define UNPACK2_L(r1,r2,r3)             I2(0,2,1,1,2,r3,r2,r1)
1162 #define UNPACK4_L(r1,r2,r3)             I2(1,2,0,1,2,r3,r2,r1)
1163 /* vmsw */
1164 #define VMSW_0()                        B8(0x18)
1165 #define VMSW_1()                        B8(0x19)
1166 /* xchg */
1167 #define XCHG1_ACQ(r1,r3,r2)             M16(0x08,LD_NONE,r3,r2,r1)
1168 #define XCHG2_ACQ(r1,r3,r2)             M16(0x09,LD_NONE,r3,r2,r1)
1169 #define XCHG4_ACQ(r1,r3,r2)             M16(0x0a,LD_NONE,r3,r2,r1)
1170 #define XCHG8_ACQ(r1,r3,r2)             M16(0x0b,LD_NONE,r3,r2,r1)
1171 /* xor */
1172 #define XOR(r1,r2,r3)                   A1(3,3,r3,r2,r1)
1173 #define XORI(r1,im,r3)                  A3(0xb,3,r3,im,r1)
1174 /* zxt */
1175 #define ZXT1(r1,r3)                     I29(0x10,r3,r1)
1176 #define ZXT2(r1,r3)                     I29(0x11,r3,r1)
1177 #define ZXT4(r1,r3)                     I29(0x12,r3,r1)
1178
1179 #  define nop(i0)                       _nop(_jit,i0)
1180 static void _nop(jit_state_t*, jit_int32_t);
1181 #define addr(r0,r1,r2)                  ADD(r0,r1,r2)
1182 #define addi(r0,r1,i0)                  _addi(_jit,r0,r1,i0)
1183 static void _addi(jit_state_t*,jit_int32_t,jit_int32_t,jit_word_t);
1184 #define addcr(r0, r1, r2)               _addcr(_jit, r0, r1, r2)
1185 static void _addcr(jit_state_t*, jit_int32_t, jit_int32_t, jit_int32_t);
1186 #define addci(r0, r1, i0)               _addci(_jit, r0, r1, i0)
1187 static void _addci(jit_state_t*, jit_int32_t, jit_int32_t, jit_word_t);
1188 #  define addxr(r0, r1, r2)             _addxr(_jit, r0, r1, r2)
1189 static void _addxr(jit_state_t*, jit_int32_t, jit_int32_t, jit_int32_t);
1190 #  define addxi(r0, r1, i0)             _addxi(_jit, r0, r1, i0)
1191 static void _addxi(jit_state_t*, jit_int32_t, jit_int32_t, jit_word_t);
1192 #define subr(r0,r1,r2)                  SUB(r0,r1,r2)
1193 #define subi(r0,r1,i0)                  _subi(_jit,r0,r1,i0)
1194 static void _subi(jit_state_t*,jit_int32_t,jit_int32_t,jit_word_t);
1195 #  define subcr(r0, r1, r2)             _subcr(_jit, r0, r1, r2)
1196 static void _subcr(jit_state_t*,jit_int32_t,jit_int32_t,jit_int32_t);
1197 #  define subci(r0, r1, i0)             _subci(_jit, r0, r1, i0)
1198 static void _subci(jit_state_t*,jit_int32_t,jit_int32_t,jit_word_t);
1199 #  define subxr(r0, r1, r2)             _subxr(_jit, r0, r1, r2)
1200 static void _subxr(jit_state_t*,jit_int32_t,jit_int32_t,jit_int32_t);
1201 #  define subxi(r0, r1, i0)             _subxi(_jit, r0, r1, i0)
1202 static void _subxi(jit_state_t*,jit_int32_t,jit_int32_t,jit_word_t);
1203 #  define rsbi(r0, r1, i0)              _rsbi(_jit, r0, r1, i0)
1204 static void _rsbi(jit_state_t*,jit_int32_t,jit_int32_t,jit_word_t);
1205 #define mulr(r0,r1,r2)                  _mulr(_jit,r0,r1,r2)
1206 static void _mulr(jit_state_t*,jit_int32_t,jit_int32_t,jit_int32_t);
1207 #define muli(r0,r1,i0)                  _muli(_jit,r0,r1,i0)
1208 static void _muli(jit_state_t*,jit_int32_t,jit_int32_t,jit_word_t);
1209 #define hmulr(r0,r1,r2)                 mulh(r0,r1,r2,1)
1210 #define hmuli(r0,r1,i0)                 _hmuli(_jit,r0,r1,i0)
1211 static void _hmuli(jit_state_t*,jit_int32_t,jit_int32_t,jit_word_t);
1212 #define hmulr_u(r0,r1,r2)               mulh(r0,r1,r2,0)
1213 #define hmuli_u(r0,r1,i0)               _hmuli_u(_jit,r0,r1,i0)
1214 static void _hmuli_u(jit_state_t*,jit_int32_t,jit_int32_t,jit_word_t);
1215 #define divr(r0,r1,r2)                  _divr(_jit,r0,r1,r2)
1216 static void _divr(jit_state_t*,jit_int32_t,jit_int32_t,jit_int32_t);
1217 #define divi(r0,r1,i0)                  _divi(_jit,r0,r1,i0)
1218 static void _divi(jit_state_t*,jit_int32_t,jit_int32_t,jit_word_t);
1219 #define divr_u(r0,r1,r2)                _divr_u(_jit,r0,r1,r2)
1220 static void _divr_u(jit_state_t*,jit_int32_t,jit_int32_t,jit_int32_t);
1221 #define divi_u(r0,r1,i0)                _divi_u(_jit,r0,r1,i0)
1222 static void _divi_u(jit_state_t*,jit_int32_t,jit_int32_t,jit_word_t);
1223 #define remr(r0,r1,r2)                  _remr(_jit,r0,r1,r2)
1224 static void _remr(jit_state_t*,jit_int32_t,jit_int32_t,jit_int32_t);
1225 #define remi(r0,r1,i0)                  _remi(_jit,r0,r1,i0)
1226 static void _remi(jit_state_t*,jit_int32_t,jit_int32_t,jit_word_t);
1227 #define remr_u(r0,r1,r2)                _remr_u(_jit,r0,r1,r2)
1228 static void _remr_u(jit_state_t*,jit_int32_t,jit_int32_t,jit_int32_t);
1229 #define remi_u(r0,r1,i0)                _remi_u(_jit,r0,r1,i0)
1230 static void _remi_u(jit_state_t*,jit_int32_t,jit_int32_t,jit_word_t);
1231 #define mulh(r0,r1,r2,sign)             _mulh(_jit,r0,r1,r2,sign)
1232 static void _mulh(jit_state_t*,jit_bool_t,jit_int32_t,jit_int32_t,jit_int32_t);
1233 #define qmulr(r0,r1,r2,r3)              iqmulr(r0,r1,r2,r3,1)
1234 #define qmulr_u(r0,r1,r2,r3)            iqmulr(r0,r1,r2,r3,0)
1235 #define iqmulr(r0,r1,r2,r3,sign)        _iqmulr(_jit,r0,r1,r2,r3,sign)
1236 static void _iqmulr(jit_state_t*,jit_int32_t,jit_int32_t,
1237                     jit_int32_t,jit_int32_t,jit_bool_t);
1238 #define qmuli(r0,r1,r2,i0)              iqmuli(r0,r1,r2,i0,1)
1239 #define qmuli_u(r0,r1,r2,i0)            iqmuli(r0,r1,r2,i0,0)
1240 #define iqmuli(r0,r1,r2,i0,sign)        _iqmuli(_jit,r0,r1,r2,i0,sign)
1241 static void _iqmuli(jit_state_t*,jit_int32_t,jit_int32_t,
1242                     jit_int32_t,jit_word_t,jit_bool_t);
1243 #define qdivr(r0,r1,r2,r3)              iqdivr(r0,r1,r2,r3,1)
1244 #define qdivr_u(r0,r1,r2,r3)            iqdivr(r0,r1,r2,r3,0)
1245 #define iqdivr(r0,r1,r2,r3,sign)        _iqdivr(_jit,r0,r1,r2,r3,sign)
1246 static void _iqdivr(jit_state_t*,jit_int32_t,jit_int32_t,
1247                     jit_int32_t,jit_int32_t,jit_bool_t);
1248 #define qdivi(r0,r1,r2,i0)              iqdivi(r0,r1,r2,i0,1)
1249 #define qdivi_u(r0,r1,r2,i0)            iqdivi(r0,r1,r2,i0,0)
1250 #define iqdivi(r0,r1,r2,i0,sign)        _iqdivi(_jit,r0,r1,r2,i0,sign)
1251 static void _iqdivi(jit_state_t*,jit_int32_t,jit_int32_t,
1252                     jit_int32_t,jit_word_t,jit_bool_t);
1253 #define andr(r0,r1,r2)                  AND(r0,r1,r2)
1254 #define andi(r0,r1,i0)                  _andi(_jit,r0,r1,i0)
1255 static void _andi(jit_state_t*,jit_int32_t,jit_int32_t,jit_word_t);
1256 #define orr(r0,r1,r2)                   OR(r0,r1,r2)
1257 #define ori(r0,r1,i0)                   _ori(_jit,r0,r1,i0)
1258 static void _ori(jit_state_t*,jit_int32_t,jit_int32_t,jit_word_t);
1259 #define xorr(r0,r1,r2)                  XOR(r0,r1,r2)
1260 #define xori(r0,r1,i0)                  _xori(_jit,r0,r1,i0)
1261 static void _xori(jit_state_t*,jit_int32_t,jit_int32_t,jit_word_t);
1262 #define lshr(r0,r1,r2)                  SHL(r0,r1,r2)
1263 #define lshi(r0,r1,i0)                  _lshi(_jit,r0,r1,i0)
1264 static void _lshi(jit_state_t*,jit_int32_t,jit_int32_t,jit_word_t);
1265 #define rshr(r0,r1,r2)                  SHR(r0,r1,r2)
1266 #define rshi(r0,r1,i0)                  _rshi(_jit,r0,r1,i0)
1267 static void _rshi(jit_state_t*,jit_int32_t,jit_int32_t,jit_word_t);
1268 #define rshr_u(r0,r1,r2)                SHR_U(r0,r1,r2)
1269 #define rshi_u(r0,r1,i0)                _rshi_u(_jit,r0,r1,i0)
1270 static void _rshi_u(jit_state_t*,jit_int32_t,jit_int32_t,jit_word_t);
1271 #  define qlshr(r0,r1,r2,r3)            xlshr(1,r0,r1,r2,r3)
1272 #  define qlshr_u(r0, r1, r2, r3)       xlshr(0, r0, r1, r2, r3)
1273 #  define xlshr(s,r0,r1,r2,r3)          _xlshr(_jit,s,r0,r1,r2,r3)
1274 static void
1275 _xlshr(jit_state_t*,jit_bool_t,jit_int32_t,jit_int32_t,jit_int32_t,jit_int32_t);
1276 #  define qlshi(r0, r1, r2, i0)         xlshi(1, r0, r1, r2, i0)
1277 #  define qlshi_u(r0, r1, r2, i0)       xlshi(0, r0, r1, r2, i0)
1278 #  define xlshi(s, r0, r1, r2, i0)      _xlshi(_jit, s, r0, r1, r2, i0)
1279 static void
1280 _xlshi(jit_state_t*,jit_bool_t,jit_int32_t,jit_int32_t,jit_int32_t,jit_word_t);
1281 #  define qrshr(r0, r1, r2, r3)         xrshr(1, r0, r1, r2, r3)
1282 #  define qrshr_u(r0, r1, r2, r3)       xrshr(0, r0, r1, r2, r3)
1283 #  define xrshr(s, r0, r1, r2, r3)      _xrshr(_jit, s, r0, r1, r2, r3)
1284 static void
1285 _xrshr(jit_state_t*,jit_bool_t,jit_int32_t,jit_int32_t,jit_int32_t,jit_int32_t);
1286 #  define qrshi(r0, r1, r2, i0)         xrshi(1, r0, r1, r2, i0)
1287 #  define qrshi_u(r0, r1, r2, i0)       xrshi(0, r0, r1, r2, i0)
1288 #  define xrshi(s, r0, r1, r2, i0)      _xrshi(_jit, s, r0, r1, r2, i0)
1289 static void
1290 _xrshi(jit_state_t*,jit_bool_t,jit_int32_t,jit_int32_t,jit_int32_t,jit_word_t);
1291 #define ltr(r0,r1,r2)                   _ltr(_jit,r0,r1,r2)
1292 static void _ltr(jit_state_t*,jit_int32_t,jit_int32_t,jit_int32_t);
1293 #define lti(r0,r1,i0)                   _lti(_jit,r0,r1,i0)
1294 static void _lti(jit_state_t*,jit_int32_t,jit_int32_t,jit_word_t);
1295 #define ltr_u(r0,r1,r2)                 _ltr_u(_jit,r0,r1,r2)
1296 static void _ltr_u(jit_state_t*,jit_int32_t,jit_int32_t,jit_int32_t);
1297 #define lti_u(r0,r1,i0)                 _lti_u(_jit,r0,r1,i0)
1298 static void _lti_u(jit_state_t*,jit_int32_t,jit_int32_t,jit_word_t);
1299 #define ler(r0,r1,r2)                   _ler(_jit,r0,r1,r2)
1300 static void _ler(jit_state_t*,jit_int32_t,jit_int32_t,jit_int32_t);
1301 #define lei(r0,r1,i0)                   _lei(_jit,r0,r1,i0)
1302 static void _lei(jit_state_t*,jit_int32_t,jit_int32_t,jit_word_t);
1303 #define ler_u(r0,r1,r2)                 _ler_u(_jit,r0,r1,r2)
1304 static void _ler_u(jit_state_t*,jit_int32_t,jit_int32_t,jit_int32_t);
1305 #define lei_u(r0,r1,i0)                 _lei_u(_jit,r0,r1,i0)
1306 static void _lei_u(jit_state_t*,jit_int32_t,jit_int32_t,jit_word_t);
1307 #define eqr(r0,r1,r2)                   _eqr(_jit,r0,r1,r2)
1308 static void _eqr(jit_state_t*,jit_int32_t,jit_int32_t,jit_int32_t);
1309 #define eqi(r0,r1,i0)                   _eqi(_jit,r0,r1,i0)
1310 static void _eqi(jit_state_t*,jit_int32_t,jit_int32_t,jit_word_t);
1311 #define ger(r0,r1,r2)                   _ger(_jit,r0,r1,r2)
1312 static void _ger(jit_state_t*,jit_int32_t,jit_int32_t,jit_int32_t);
1313 #define gei(r0,r1,i0)                   _gei(_jit,r0,r1,i0)
1314 static void _gei(jit_state_t*,jit_int32_t,jit_int32_t,jit_word_t);
1315 #define ger_u(r0,r1,r2)                 _ger_u(_jit,r0,r1,r2)
1316 static void _ger_u(jit_state_t*,jit_int32_t,jit_int32_t,jit_int32_t);
1317 #define gei_u(r0,r1,i0)                 _gei_u(_jit,r0,r1,i0)
1318 static void _gei_u(jit_state_t*,jit_int32_t,jit_int32_t,jit_word_t);
1319 #define gtr(r0,r1,r2)                   _gtr(_jit,r0,r1,r2)
1320 static void _gtr(jit_state_t*,jit_int32_t,jit_int32_t,jit_int32_t);
1321 #define gti(r0,r1,i0)                   _gti(_jit,r0,r1,i0)
1322 static void _gti(jit_state_t*,jit_int32_t,jit_int32_t,jit_word_t);
1323 #define gtr_u(r0,r1,r2)                 _gtr_u(_jit,r0,r1,r2)
1324 static void _gtr_u(jit_state_t*,jit_int32_t,jit_int32_t,jit_int32_t);
1325 #define gti_u(r0,r1,i0)                 _gti_u(_jit,r0,r1,i0)
1326 static void _gti_u(jit_state_t*,jit_int32_t,jit_int32_t,jit_word_t);
1327 #define ner(r0,r1,r2)                   _ner(_jit,r0,r1,r2)
1328 static void _ner(jit_state_t*,jit_int32_t,jit_int32_t,jit_int32_t);
1329 #define nei(r0,r1,i0)                   _nei(_jit,r0,r1,i0)
1330 static void _nei(jit_state_t*,jit_int32_t,jit_int32_t,jit_word_t);
1331 #define clor(r0, r1)                    _clor(_jit, r0, r1)
1332 static void _clor(jit_state_t*, jit_int32_t, jit_int32_t);
1333 #define clzr(r0, r1)                    _clzr(_jit, r0, r1)
1334 static void _clzr(jit_state_t*, jit_int32_t, jit_int32_t);
1335 #define ctor(r0, r1)                    _ctor(_jit, r0, r1)
1336 static void _ctor(jit_state_t*, jit_int32_t, jit_int32_t);
1337 #define ctzr(r0, r1)                    _ctzr(_jit, r0, r1)
1338 static void _ctzr(jit_state_t*, jit_int32_t, jit_int32_t);
1339 #define popcntr(r0, r1)                 POPCNT(r0, r1)
1340 #define negr(r0,r1)                     subr(r0,0,r1)
1341 #define comr(r0,r1)                     ANDCMI(r0,-1,r1)
1342 #define movr(r0,r1)                     _movr(_jit,r0,r1)
1343 static void _movr(jit_state_t*,jit_int32_t,jit_int32_t);
1344 #define movi(r0,i0)                     _movi(_jit,r0,i0)
1345 static void _movi(jit_state_t*,jit_int32_t,jit_word_t);
1346 #define movi_p(r0,i0)                   _movi_p(_jit,r0,i0)
1347 static jit_word_t _movi_p(jit_state_t*,jit_int32_t,jit_word_t);
1348 #  define movnr(r0,r1,r2)               _movnr(_jit,r0,r1,r2)
1349 static void _movnr(jit_state_t*,jit_int32_t,jit_int32_t,jit_int32_t);
1350 #  define movzr(r0,r1,r2)               _movzr(_jit,r0,r1,r2)
1351 static void _movzr(jit_state_t*,jit_int32_t,jit_int32_t,jit_int32_t);
1352 #  define casx(r0, r1, r2, r3, i0)      _casx(_jit, r0, r1, r2, r3, i0)
1353 static void _casx(jit_state_t *_jit,jit_int32_t,jit_int32_t,
1354                   jit_int32_t,jit_int32_t,jit_word_t);
1355 #define casr(r0, r1, r2, r3)            casx(r0, r1, r2, r3, 0)
1356 #define casi(r0, i0, r1, r2)            casx(r0, _NOREG, r1, r2, i0)
1357 #  define bswapr_us(r0,r1)              _bswapr_us(_jit,r0,r1)
1358 static void _bswapr_us(jit_state_t*,jit_int32_t,jit_int32_t);
1359 #  define bswapr_ui(r0,r1)              _bswapr_ui(_jit,r0,r1)
1360 static void _bswapr_ui(jit_state_t*,jit_int32_t,jit_int32_t);
1361 #  define bswapr_ul(r0,r1)              MUX1(r0,r1,MUX_REV)
1362 #define extr(r0,r1,i0,i1)               _extr(_jit,r0,r1,i0,i1)
1363 static void _extr(jit_state_t*,jit_int32_t,jit_int32_t,jit_word_t,jit_word_t);
1364 #define extr_u(r0,r1,i0,i1)             _extr_u(_jit,r0,r1,i0,i1)
1365 static void _extr_u(jit_state_t*,jit_int32_t,jit_int32_t,jit_word_t,jit_word_t);
1366 #define depr(r0,r1,i0,i1)               _depr(_jit,r0,r1,i0,i1)
1367 static void _depr(jit_state_t*,jit_int32_t,jit_int32_t,jit_word_t,jit_word_t);
1368 #define extr_c(r0,r1)                   SXT1(r0,r1)
1369 #define extr_uc(r0,r1)                  ZXT1(r0,r1)
1370 #define extr_s(r0,r1)                   SXT2(r0,r1)
1371 #define extr_us(r0,r1)                  ZXT2(r0,r1)
1372 #define extr_i(r0,r1)                   SXT4(r0,r1)
1373 #define extr_ui(r0,r1)                  ZXT4(r0,r1)
1374 #define bltr(i0,r0,r1)                  _bltr(_jit,i0,r0,r1)
1375 static jit_word_t _bltr(jit_state_t*,jit_word_t,jit_int32_t,jit_int32_t);
1376 #define blti(i0,r0,i1)                  _blti(_jit,i0,r0,i1)
1377 static jit_word_t _blti(jit_state_t*,jit_word_t,jit_int32_t,jit_word_t);
1378 #define bltr_u(i0,r0,r1)                _bltr_u(_jit,i0,r0,r1)
1379 static jit_word_t _bltr_u(jit_state_t*,jit_word_t,jit_int32_t,jit_int32_t);
1380 #define blti_u(i0,r0,i1)                _blti_u(_jit,i0,r0,i1)
1381 static jit_word_t _blti_u(jit_state_t*,jit_word_t,jit_int32_t,jit_word_t);
1382 #define bler(i0,r0,r1)                  _bler(_jit,i0,r0,r1)
1383 static jit_word_t _bler(jit_state_t*,jit_word_t,jit_int32_t,jit_int32_t);
1384 #define blei(i0,r0,i1)                  _blei(_jit,i0,r0,i1)
1385 static jit_word_t _blei(jit_state_t*,jit_word_t,jit_int32_t,jit_word_t);
1386 #define bler_u(i0,r0,r1)                _bler_u(_jit,i0,r0,r1)
1387 static jit_word_t _bler_u(jit_state_t*,jit_word_t,jit_int32_t,jit_int32_t);
1388 #define blei_u(i0,r0,i1)                _blei_u(_jit,i0,r0,i1)
1389 static jit_word_t _blei_u(jit_state_t*,jit_word_t,jit_int32_t,jit_word_t);
1390 #define beqr(i0,r0,r1)                  _beqr(_jit,i0,r0,r1)
1391 static jit_word_t _beqr(jit_state_t*,jit_word_t,jit_int32_t,jit_int32_t);
1392 #define beqi(i0,r0,i1)                  _beqi(_jit,i0,r0,i1)
1393 static jit_word_t _beqi(jit_state_t*,jit_word_t,jit_int32_t,jit_word_t);
1394 #define bger(i0,r0,r1)                  _bger(_jit,i0,r0,r1)
1395 static jit_word_t _bger(jit_state_t*,jit_word_t,jit_int32_t,jit_int32_t);
1396 #define bgei(i0,r0,i1)                  _bgei(_jit,i0,r0,i1)
1397 static jit_word_t _bgei(jit_state_t*,jit_word_t,jit_int32_t,jit_word_t);
1398 #define bger_u(i0,r0,r1)                _bger_u(_jit,i0,r0,r1)
1399 static jit_word_t _bger_u(jit_state_t*,jit_word_t,jit_int32_t,jit_int32_t);
1400 #define bgei_u(i0,r0,i1)                _bgei_u(_jit,i0,r0,i1)
1401 static jit_word_t _bgei_u(jit_state_t*,jit_word_t,jit_int32_t,jit_word_t);
1402 #define bgtr(i0,r0,r1)                  _bgtr(_jit,i0,r0,r1)
1403 static jit_word_t _bgtr(jit_state_t*,jit_word_t,jit_int32_t,jit_int32_t);
1404 #define bgti(i0,r0,i1)                  _bgti(_jit,i0,r0,i1)
1405 static jit_word_t _bgti(jit_state_t*,jit_word_t,jit_int32_t,jit_word_t);
1406 #define bgtr_u(i0,r0,r1)                _bgtr_u(_jit,i0,r0,r1)
1407 static jit_word_t _bgtr_u(jit_state_t*,jit_word_t,jit_int32_t,jit_int32_t);
1408 #define bgti_u(i0,r0,i1)                _bgti_u(_jit,i0,r0,i1)
1409 static jit_word_t _bgti_u(jit_state_t*,jit_word_t,jit_int32_t,jit_word_t);
1410 #define bner(i0,r0,r1)                  _bner(_jit,i0,r0,r1)
1411 static jit_word_t _bner(jit_state_t*,jit_word_t,jit_int32_t,jit_int32_t);
1412 #define bnei(i0,r0,i1)                  _bnei(_jit,i0,r0,i1)
1413 static jit_word_t _bnei(jit_state_t*,jit_word_t,jit_int32_t,jit_word_t);
1414 #define bmsr(i0,r0,r1)                  _bmsr(_jit,i0,r0,r1)
1415 static jit_word_t _bmsr(jit_state_t*,jit_word_t,jit_int32_t,jit_int32_t);
1416 #define bmsi(i0,r0,i1)                  _bmsi(_jit,i0,r0,i1)
1417 static jit_word_t _bmsi(jit_state_t*,jit_word_t,jit_int32_t,jit_word_t);
1418 #define bmcr(i0,r0,r1)                  _bmcr(_jit,i0,r0,r1)
1419 static jit_word_t _bmcr(jit_state_t*,jit_word_t,jit_int32_t,jit_int32_t);
1420 #define bmci(i0,r0,i1)                  _bmci(_jit,i0,r0,i1)
1421 static jit_word_t _bmci(jit_state_t*,jit_word_t,jit_int32_t,jit_word_t);
1422 #define baddr(i0,r0,r1,cc)              _baddr(_jit,i0,r0,r1,cc)
1423 static jit_word_t _baddr(jit_state_t*,jit_word_t,
1424                          jit_int32_t,jit_int32_t,jit_bool_t);
1425 #define baddi(i0,r0,i1,cc)              _baddi(_jit,i0,r0,i1,cc)
1426 static jit_word_t _baddi(jit_state_t*,jit_word_t,
1427                          jit_int32_t,jit_word_t,jit_bool_t);
1428 #define baddr_u(i0,r0,r1,cc)            _baddr_u(_jit,i0,r0,r1,cc)
1429 static jit_word_t _baddr_u(jit_state_t*,jit_word_t,
1430                            jit_int32_t,jit_int32_t,jit_bool_t);
1431 #define baddi_u(i0,r0,i1,cc)            _baddi_u(_jit,i0,r0,i1,cc)
1432 static jit_word_t _baddi_u(jit_state_t*,jit_word_t,
1433                            jit_int32_t,jit_word_t,jit_bool_t);
1434 #define bsubr(i0,r0,r1,cc)              _bsubr(_jit,i0,r0,r1,cc)
1435 static jit_word_t _bsubr(jit_state_t*,jit_word_t,
1436                          jit_int32_t,jit_int32_t,jit_bool_t);
1437 #define bsubi(i0,r0,i1,cc)              _bsubi(_jit,i0,r0,i1,cc)
1438 static jit_word_t _bsubi(jit_state_t*,jit_word_t,
1439                          jit_int32_t,jit_word_t,jit_bool_t);
1440 #define bsubr_u(i0,r0,r1,cc)            _bsubr_u(_jit,i0,r0,r1,cc)
1441 static jit_word_t _bsubr_u(jit_state_t*,jit_word_t,
1442                            jit_int32_t,jit_int32_t,jit_bool_t);
1443 #define bsubi_u(i0,r0,i1,cc)            _bsubi_u(_jit,i0,r0,i1,cc)
1444 static jit_word_t _bsubi_u(jit_state_t*,jit_word_t,
1445                            jit_int32_t,jit_word_t,jit_bool_t);
1446 #define boaddr(i0,r0,r1)                baddr(i0,r0,r1,1)
1447 #define boaddi(i0,r0,i1)                baddi(i0,r0,i1,1)
1448 #define boaddr_u(i0,r0,r1)              baddr_u(i0,r0,r1,1)
1449 #define boaddi_u(i0,r0,i1)              baddi_u(i0,r0,i1,1)
1450 #define bxaddr(i0,r0,r1)                baddr(i0,r0,r1,0)
1451 #define bxaddi(i0,r0,i1)                baddi(i0,r0,i1,0)
1452 #define bxaddr_u(i0,r0,r1)              baddr_u(i0,r0,r1,0)
1453 #define bxaddi_u(i0,r0,i1)              baddi_u(i0,r0,i1,0)
1454 #define bosubr(i0,r0,r1)                bsubr(i0,r0,r1,1)
1455 #define bosubi(i0,r0,i1)                bsubi(i0,r0,i1,1)
1456 #define bosubr_u(i0,r0,r1)              bsubr_u(i0,r0,r1,1)
1457 #define bosubi_u(i0,r0,i1)              bsubi_u(i0,r0,i1,1)
1458 #define bxsubr(i0,r0,r1)                bsubr(i0,r0,r1,0)
1459 #define bxsubi(i0,r0,i1)                bsubi(i0,r0,i1,0)
1460 #define bxsubr_u(i0,r0,r1)              bsubr_u(i0,r0,r1,0)
1461 #define bxsubi_u(i0,r0,i1)              bsubi_u(i0,r0,i1,0)
1462 #define ldr_c(r0,r1)                    _ldr_c(_jit,r0,r1)
1463 static void _ldr_c(jit_state_t*,jit_int32_t,jit_int32_t);
1464 #define ldi_c(r0,i0)                    _ldi_c(_jit,r0,i0)
1465 static void _ldi_c(jit_state_t*,jit_int32_t,jit_word_t);
1466 #define ldr_uc(r0,r1)                   LD1(r0,r1)
1467 #define ldi_uc(r0,i0)                   _ldi_uc(_jit,r0,i0)
1468 static void _ldi_uc(jit_state_t*,jit_int32_t,jit_word_t);
1469 #define ldr_s(r0,r1)                    _ldr_s(_jit,r0,r1)
1470 static void _ldr_s(jit_state_t*,jit_int32_t,jit_int32_t);
1471 #define ldi_s(r0,i0)                    _ldi_s(_jit,r0,i0)
1472 static void _ldi_s(jit_state_t*,jit_int32_t,jit_word_t);
1473 #define ldr_us(r0,r1)                   LD2(r0,r1)
1474 #define ldi_us(r0,i0)                   _ldi_us(_jit,r0,i0)
1475 static void _ldi_us(jit_state_t*,jit_int32_t,jit_word_t);
1476 #define ldr_i(r0,r1)                    _ldr_i(_jit,r0,r1)
1477 static void _ldr_i(jit_state_t*,jit_int32_t,jit_int32_t);
1478 #define ldi_i(r0,i0)                    _ldi_i(_jit,r0,i0)
1479 static void _ldi_i(jit_state_t*,jit_int32_t,jit_word_t);
1480 #define ldr_ui(r0,r1)                   LD4(r0,r1)
1481 #define ldi_ui(r0,i0)                   _ldi_ui(_jit,r0,i0)
1482 static void _ldi_ui(jit_state_t*,jit_int32_t,jit_word_t);
1483 #define ldr_l(r0,r1)                    LD8(r0,r1)
1484 #define ldi_l(r0,i0)                    _ldi_l(_jit,r0,i0)
1485 static void _ldi_l(jit_state_t*,jit_int32_t,jit_word_t);
1486 #define ldxr_c(r0,r1,r2)                _ldxr_c(_jit,r0,r1,r2)
1487 static void _ldxr_c(jit_state_t*,jit_int32_t,jit_int32_t,jit_int32_t);
1488 #define ldxi_c(r0,r1,i0)                _ldxi_c(_jit,r0,r1,i0)
1489 static void _ldxi_c(jit_state_t*,jit_int32_t,jit_int32_t,jit_word_t);
1490 #define ldxr_uc(r0,r1,r2)               _ldxr_uc(_jit,r0,r1,r2)
1491 static void _ldxr_uc(jit_state_t*,jit_int32_t,jit_int32_t,jit_int32_t);
1492 #define ldxi_uc(r0,r1,i0)               _ldxi_uc(_jit,r0,r1,i0)
1493 static void _ldxi_uc(jit_state_t*,jit_int32_t,jit_int32_t,jit_word_t);
1494 #define ldxr_s(r0,r1,r2)                _ldxr_s(_jit,r0,r1,r2)
1495 static void _ldxr_s(jit_state_t*,jit_int32_t,jit_int32_t,jit_int32_t);
1496 #define ldxi_s(r0,r1,i0)                _ldxi_s(_jit,r0,r1,i0)
1497 static void _ldxi_s(jit_state_t*,jit_int32_t,jit_int32_t,jit_word_t);
1498 #define ldxr_us(r0,r1,r2)               _ldxr_us(_jit,r0,r1,r2)
1499 static void _ldxr_us(jit_state_t*,jit_int32_t,jit_int32_t,jit_int32_t);
1500 #define ldxi_us(r0,r1,i0)               _ldxi_us(_jit,r0,r1,i0)
1501 static void _ldxi_us(jit_state_t*,jit_int32_t,jit_int32_t,jit_word_t);
1502 #define ldxr_i(r0,r1,r2)                _ldxr_i(_jit,r0,r1,r2)
1503 static void _ldxr_i(jit_state_t*,jit_int32_t,jit_int32_t,jit_int32_t);
1504 #define ldxi_i(r0,r1,i0)                _ldxi_i(_jit,r0,r1,i0)
1505 static void _ldxi_i(jit_state_t*,jit_int32_t,jit_int32_t,jit_word_t);
1506 #define ldxr_ui(r0,r1,r2)               _ldxr_ui(_jit,r0,r1,r2)
1507 static void _ldxr_ui(jit_state_t*,jit_int32_t,jit_int32_t,jit_int32_t);
1508 #define ldxi_ui(r0,r1,i0)               _ldxi_ui(_jit,r0,r1,i0)
1509 static void _ldxi_ui(jit_state_t*,jit_int32_t,jit_int32_t,jit_word_t);
1510 #define ldxr_l(r0,r1,r2)                _ldxr_l(_jit,r0,r1,r2)
1511 static void _ldxr_l(jit_state_t*,jit_int32_t,jit_int32_t,jit_int32_t);
1512 #define ldxi_l(r0,r1,i0)                _ldxi_l(_jit,r0,r1,i0)
1513 static void _ldxi_l(jit_state_t*,jit_int32_t,jit_int32_t,jit_word_t);
1514 #  define unldr(r0, r1, i0)             _unldr(_jit, r0, r1, i0)
1515 static void _unldr(jit_state_t*, jit_int32_t, jit_int32_t, jit_word_t);
1516 #  define unldi(r0, i0, i1)             _unldi(_jit, r0, i0, i1)
1517 static void _unldi(jit_state_t*, jit_int32_t, jit_word_t, jit_word_t);
1518 #  define unldr_u(r0, r1, i0)           _unldr_u(_jit, r0, r1, i0)
1519 static void _unldr_u(jit_state_t*, jit_int32_t, jit_int32_t, jit_word_t);
1520 #  define unldi_u(r0, i0, i1)           _unldi_u(_jit, r0, i0, i1)
1521 static void _unldi_u(jit_state_t*, jit_int32_t, jit_word_t, jit_word_t);
1522 #define str_c(r0,r1)                    ST1(r0,r1)
1523 #define sti_c(i0,r0)                    _sti_c(_jit,i0,r0)
1524 static void _sti_c(jit_state_t*,jit_word_t,jit_int32_t);
1525 #define str_s(r0,r1)                    ST2(r0,r1)
1526 #define sti_s(i0,r0)                    _sti_s(_jit,i0,r0)
1527 static void _sti_s(jit_state_t*,jit_word_t,jit_int32_t);
1528 #define str_i(r0,r1)                    ST4(r0,r1)
1529 #define sti_i(i0,r0)                    _sti_i(_jit,i0,r0)
1530 static void _sti_i(jit_state_t*,jit_word_t,jit_int32_t);
1531 #define str_l(r0,r1)                    ST8(r0,r1)
1532 #define sti_l(i0,r0)                    _sti_l(_jit,i0,r0)
1533 static void _sti_l(jit_state_t*,jit_word_t,jit_int32_t);
1534 #define stxr_c(r0,r1,r2)                _stxr_c(_jit,r0,r1,r2)
1535 static void _stxr_c(jit_state_t*,jit_int32_t,jit_int32_t,jit_int32_t);
1536 #define stxi_c(i0,r0,r1)                _stxi_c(_jit,i0,r0,r1)
1537 static void _stxi_c(jit_state_t*,jit_word_t,jit_int32_t,jit_int32_t);
1538 #define stxr_s(r0,r1,r2)                _stxr_s(_jit,r0,r1,r2)
1539 static void _stxr_s(jit_state_t*,jit_int32_t,jit_int32_t,jit_int32_t);
1540 #define stxi_s(i0,r0,r1)                _stxi_s(_jit,i0,r0,r1)
1541 static void _stxi_s(jit_state_t*,jit_word_t,jit_int32_t,jit_int32_t);
1542 #define stxr_i(r0,r1,r2)                _stxr_i(_jit,r0,r1,r2)
1543 static void _stxr_i(jit_state_t*,jit_int32_t,jit_int32_t,jit_int32_t);
1544 #define stxi_i(i0,r0,r1)                _stxi_i(_jit,i0,r0,r1)
1545 static void _stxi_i(jit_state_t*,jit_word_t,jit_int32_t,jit_int32_t);
1546 #define stxr_l(r0,r1,r2)                _stxr_l(_jit,r0,r1,r2)
1547 static void _stxr_l(jit_state_t*,jit_int32_t,jit_int32_t,jit_int32_t);
1548 #define stxi_l(i0,r0,r1)                _stxi_l(_jit,i0,r0,r1)
1549 static void _stxi_l(jit_state_t*,jit_word_t,jit_int32_t,jit_int32_t);
1550 #define unstr(r0, r1, i0)               _unstr(_jit, r0, r1, i0)
1551 static void _unstr(jit_state_t*, jit_int32_t, jit_int32_t, jit_word_t);
1552 #define unsti(i0, r0, i1)               _unsti(_jit, i0, r0, i1)
1553 static void _unsti(jit_state_t*, jit_word_t, jit_int32_t, jit_word_t);
1554 #define jmpr(r0)                        _jmpr(_jit,r0)
1555 static void _jmpr(jit_state_t*,jit_int32_t);
1556 #define jmpi(i0)                        _jmpi(_jit,i0)
1557 static jit_word_t _jmpi(jit_state_t*,jit_word_t);
1558 #define jmpi_p(i0)                      _jmpi_p(_jit,i0)
1559 static jit_word_t _jmpi_p(jit_state_t*,jit_word_t);
1560 #define callr(r0)                       _callr(_jit,r0)
1561 static void _callr(jit_state_t*,jit_int32_t);
1562 #define calli(i0)                       _calli(_jit,i0)
1563 static void _calli(jit_state_t*,jit_word_t);
1564 #define calli_p(i0)                     _calli_p(_jit,i0)
1565 static jit_word_t _calli_p(jit_state_t*,jit_word_t);
1566 #define prolog(node)                    _prolog(_jit,node)
1567 static void _prolog(jit_state_t*,jit_node_t*);
1568 #define epilog(node)                    _epilog(_jit,node)
1569 static void _epilog(jit_state_t*,jit_node_t*);
1570 #  define vastart(r0)                   _vastart(_jit, r0)
1571 static void _vastart(jit_state_t*, jit_int32_t);
1572 #  define vaarg(r0, r1)                 _vaarg(_jit, r0, r1)
1573 static void _vaarg(jit_state_t*, jit_int32_t, jit_int32_t);
1574 #define patch_at(code,instr,label)      _patch_at(_jit,code,instr,label)
1575 static void _patch_at(jit_state_t*,jit_code_t,jit_word_t,jit_word_t);
1576 #endif
1577
1578 #if CODE
1579 #if __BYTE_ORDER == __BIG_ENDIAN
1580 static jit_word_t
1581 byte_swap_if_big_endian(jit_word_t w)
1582 {
1583     union {
1584         char    c[8];
1585         long    w;
1586     } l, h;
1587     l.w = w;
1588     h.c[0] = l.c[7];
1589     h.c[1] = l.c[6];
1590     h.c[2] = l.c[5];
1591     h.c[3] = l.c[4];
1592     h.c[4] = l.c[3];
1593     h.c[5] = l.c[2];
1594     h.c[6] = l.c[1];
1595     h.c[7] = l.c[0];
1596     return (h.w);
1597 }
1598 #else
1599 #define byte_swap_if_big_endian(w)              (w)
1600 #endif
1601
1602 static void
1603 _out(jit_state_t *_jit, int n, int tm,
1604      jit_word_t s0, jit_word_t s1, jit_word_t s2)
1605 {
1606     int                  i;
1607     jit_word_t           l, h, *w;
1608     set_bundle(_jit->pc.ul, l, h, tm, s0, s1, s2);
1609     _jit->pc.ul += 2;
1610     w = (jit_word_t *)_jitc->inst;
1611     for (i = n; i < _jitc->ioff; i++)
1612         w[i - n] = w[i];
1613     _jitc->ioff -= n;
1614 }
1615
1616 #define nop_m           0x0008000000L
1617 #define nop_i           0x0008000000L
1618 #define nop_b           0x4000000000L
1619 static void
1620 _stop(jit_state_t *_jit)
1621 {
1622     /* Clear set of live registers */
1623     jit_regset_set_ui(&_jitc->regs, 0);
1624     _jitc->pred = 0;
1625     /* Flag a stop is required */
1626     if (_jitc->ioff)
1627         _jitc->inst[_jitc->ioff - 1].t |= INST_STOP;
1628     else
1629         inst(nop_m, INST_Ms);
1630 }
1631
1632 static void
1633 _sync(jit_state_t *_jit)
1634 {
1635     /* Taken branches are supposed to not need a stop, so, it
1636      * should not be required to stop if no registers live in
1637      * sequential code */
1638     if (jit_regset_cmp_ui(&_jitc->regs, 0) != 0 || _jitc->pred)
1639         stop();
1640     do
1641         flush();
1642     while (_jitc->ioff);
1643 }
1644
1645 #define A_0             INST_A
1646 #define As0             INST_As
1647 #define I_0             INST_I
1648 #define Is0             INST_Is
1649 #define M_0             INST_M
1650 #define Ms0             INST_Ms
1651 #define F_0             INST_F
1652 #define Fs0             INST_Fs
1653 #define B_0             INST_B
1654 #define Bs0             INST_Bs
1655 #define L_0             INST_L
1656 #define Ls0             INST_Ls
1657 #define X_0             INST_X
1658 #define Xs0             INST_Xs
1659 #define A_1             (INST_A<<4)
1660 #define As1             (INST_As<<4)
1661 #define I_1             (INST_I<<4)
1662 #define Is1             (INST_Is<<4)
1663 #define M_1             (INST_M<<4)
1664 #define Ms1             (INST_Ms<<4)
1665 #define F_1             (INST_F<<4)
1666 #define Fs1             (INST_Fs<<4)
1667 #define B_1             (INST_B<<4)
1668 #define Bs1             (INST_Bs<<4)
1669 #define L_1             (INST_L<<4)
1670 #define Ls1             (INST_Ls<<4)
1671 #define X_1             (INST_X<<4)
1672 #define Xs1             (INST_Xs<<4)
1673 #define A_2             (INST_A<<8)
1674 #define As2             (INST_As<<8)
1675 #define I_2             (INST_I<<8)
1676 #define Is2             (INST_Is<<8)
1677 #define M_2             (INST_M<<8)
1678 #define Ms2             (INST_Ms<<8)
1679 #define F_2             (INST_F<<8)
1680 #define Fs2             (INST_Fs<<8)
1681 #define B_2             (INST_B<<8)
1682 #define Bs2             (INST_Bs<<8)
1683 #define L_2             (INST_L<<8)
1684 #define Ls2             (INST_Ls<<8)
1685 #define X_2             (INST_X<<8)
1686 #define Xs2             (INST_Xs<<8)
1687
1688 #define I_              I_0
1689 #define I_I_            I_0|I_1
1690 #define I_Is            I_0|Is1
1691 #define I_B_            I_0|B_1
1692 #define I_Bs            I_0|Bs1
1693 #define Is              Is0
1694 #define IsI_            Is0|I_1
1695 #define IsIs            Is0|Is1
1696 #define M_              M_0
1697 #define M_I_            M_0|I_1
1698 #define M_Is            M_0|Is1
1699 #define M_M_            M_0|M_1
1700 #define M_Ms            M_0|Ms1
1701 #define M_F_            M_0|F_1
1702 #define M_Fs            M_0|Fs1
1703 #define M_B_            M_0|B_1
1704 #define M_Bs            M_0|Bs1
1705 #define M_I_I_          M_0|I_1|I_2
1706 #define M_I_Is          M_0|I_1|Is2
1707 #define M_I_B_          M_0|I_1|B_2
1708 #define M_I_Bs          M_0|I_1|Bs2
1709 #define M_IsI_          M_0|Is1|I_2
1710 #define M_IsIs          M_0|Is1|Is2
1711 #define M_M_I_          M_0|M_1|I_2
1712 #define M_M_Is          M_0|M_1|Is2
1713 #define M_M_F_          M_0|M_1|F_2
1714 #define M_M_Fs          M_0|M_1|Fs2
1715 #define M_M_B_          M_0|M_1|B_2
1716 #define M_M_Bs          M_0|M_1|Bs2
1717 #define M_F_I_          M_0|F_1|I_2
1718 #define M_F_Is          M_0|F_1|Is2
1719 #define M_F_B_          M_0|F_1|B_2
1720 #define M_F_Bs          M_0|F_1|Bs2
1721 #define M_B_B_          M_0|B_1|B_2
1722 #define M_B_Bs          M_0|B_1|Bs2
1723 #define M_L_X_          M_0|L_1|X_2
1724 #define M_L_Xs          M_0|L_1|Xs2
1725 #define Ms              Ms0
1726 #define MsI_            Ms0|I_1
1727 #define MsIs            Ms0|Is1
1728 #define MsM_            Ms0|M_1
1729 #define MsMs            Ms0|Ms1
1730 #define MsM_I_          Ms0|M_1|I_2
1731 #define MsM_Is          Ms0|M_1|Is2
1732 #define F_              F_0
1733 #define F_I_            F_0|I_1
1734 #define F_Is            F_0|Is1
1735 #define F_B_            F_0|B_1
1736 #define F_Bs            F_0|Bs1
1737 #define Fs              Fs0
1738 #define B_              B_0
1739 #define B_B_            B_0|B_1
1740 #define B_Bs            B_0|Bs1
1741 #define B_B_B_          B_0|B_1|B_2
1742 #define B_B_Bs          B_0|B_1|Bs2
1743 #define Bs              Bs0
1744 #define L_X_            L_0|X_1
1745 #define L_Xs            L_0|Xs1
1746
1747 static jit_word_t
1748 templat(jit_word_t cc)
1749 {
1750     switch (cc) {
1751         case I_:
1752         case I_I_:      case I_Is:
1753         case I_B_:      case I_Bs:
1754         case Is:
1755         case IsI_:      case IsIs:
1756         case M_:
1757         case M_I_:      case M_Is:
1758         case M_M_:      case M_Ms:
1759         case M_F_:      case M_Fs:
1760         case M_B_:      case M_Bs:
1761         case M_I_I_:    case M_I_Is:
1762         case M_I_B_:    case M_I_Bs:
1763         case M_IsI_:    case M_IsIs:
1764         case M_M_I_:    case M_M_Is:
1765         case M_M_F_:    case M_M_Fs:
1766         case M_M_B_:    case M_M_Bs:
1767         case M_F_I_:    case M_F_Is:
1768         case M_F_B_:    case M_F_Bs:
1769         case M_B_B_:    case M_B_Bs:
1770         case M_L_X_:    case M_L_Xs:
1771         case Ms:
1772         case MsI_:      case MsIs:
1773         case MsM_:      case MsMs:
1774         case MsM_I_:    case MsM_Is:
1775         case F_:
1776         case F_I_:      case F_Is:
1777         case F_B_:      case F_Bs:
1778         case Fs:
1779         case B_:
1780         case B_B_:      case B_Bs:
1781         case B_B_B_:    case B_B_Bs:
1782         case Bs:
1783         case L_X_:      case L_Xs:
1784             return (cc);
1785         default:
1786             return (0);
1787     }
1788 }
1789
1790 /* match* functions recurse attempting to find a template for A-
1791  * instructions, that may be executed in M- or I- unit.
1792  * It also uses an heuristic of trying first M- for slot 0 and 2,
1793  * and I- for slot 1, but tries all possible matches.
1794  */
1795 static jit_word_t
1796 match_2(jit_word_t cc)
1797 {
1798     jit_word_t          t;
1799     if ((t = templat(cc | M_2)))
1800         return (t);
1801     if ((t = templat(cc | I_2)))
1802         return (t);
1803     return (0);
1804 }
1805
1806 static jit_word_t
1807 matchs2(jit_word_t cc)
1808 {
1809     jit_word_t          t;
1810     if ((t = templat(cc | Ms2)))
1811         return (t);
1812     if ((t = templat(cc | Is2)))
1813         return (t);
1814     return (0);
1815 }
1816
1817 static jit_word_t
1818 match2(jit_word_t cc)
1819 {
1820     if ((cc & 0xf00) == A_2)
1821         return (match_2(cc & ~0xf00));
1822     if ((cc & 0xf00) == As2)
1823         return (matchs2(cc & ~0xf00));
1824     return (0);
1825 }
1826
1827 static jit_word_t
1828 match_1(jit_word_t cc)
1829 {
1830     jit_word_t          t;
1831     if ((t = templat(cc | I_1)))
1832         return (t);
1833     if ((t = templat(cc | M_1)))
1834         return (t);
1835     if ((t = match2(cc | I_1)))
1836         return (t);
1837     if ((t = match2(cc | M_1)))
1838         return (t);
1839     return (0);
1840 }
1841
1842 static jit_word_t
1843 matchs1(jit_word_t cc)
1844 {
1845     jit_word_t          t;
1846     if ((t = templat(cc | Is1)))
1847         return (t);
1848     if ((t = templat(cc | Ms1)))
1849         return (t);
1850     if ((t = match2(cc | Is1)))
1851         return (t);
1852     if ((t = match2(cc | Ms1)))
1853         return (t);
1854     return (0);
1855 }
1856
1857 static jit_word_t
1858 match1(jit_word_t cc)
1859 {
1860     if ((cc & 0x0f0) == A_1)
1861         return (match_1(cc & ~0x0f0));
1862     if ((cc & 0x0f0) == As1)
1863         return (matchs1(cc & ~0x0f0));
1864     return (0);
1865 }
1866
1867 static jit_word_t
1868 match_0(jit_word_t cc)
1869 {
1870     jit_word_t          t;
1871     if ((t = templat(cc | M_0)))
1872         return (t);
1873     if ((t = templat(cc | I_0)))
1874         return (t);
1875     if ((t = match1(cc | M_0)))
1876         return (t);
1877     if ((t = match1(cc | I_0)))
1878         return (t);
1879     return (0);
1880 }
1881
1882 static jit_word_t
1883 matchs0(jit_word_t cc)
1884 {
1885     jit_word_t          t;
1886     if ((t = templat(cc | Ms0)))
1887         return (t);
1888     if ((t = templat(cc | Is0)))
1889         return (t);
1890     if ((t = match1(cc | Ms0)))
1891         return (t);
1892     if ((t = match1(cc | Is0)))
1893         return (t);
1894     return (0);
1895 }
1896
1897 static jit_word_t
1898 match0(jit_word_t cc)
1899 {
1900     if ((cc & 0x00f) == A_0)
1901         return (match_0(cc & ~0x00f));
1902     if ((cc & 0x00f) == As0)
1903         return (matchs0(cc & ~0x00f));
1904     return (0);
1905 }
1906
1907 static void
1908 _flush(jit_state_t *_jit)
1909 {
1910     int                 n, soff;
1911     jit_word_t          t, cc, tm, s0, s1, s2;
1912
1913     if (!_jitc->ioff)
1914         return;
1915     for (cc = 0, n = soff = 0; n < _jitc->ioff; n++, soff += 4)
1916         cc |= (jit_uword_t)(_jitc->inst[n].t) << soff;
1917
1918     soff = 0xf00;
1919     while (soff) {
1920         /* Try to find a template, or reduce down
1921          * to one instruction if no template match */
1922         if ((t = templat(cc))) {
1923             cc = t;
1924             break;
1925         }
1926         /* A- instructions may be execute in M- or I- unit */
1927         if ((t = match0(cc))) {
1928             cc = t;
1929             break;
1930         }
1931         cc &= ~soff;
1932         soff >>= 4;
1933     }
1934     assert(soff);
1935
1936     /* Prefer tail nop if need to add some nop, so that patching is easier */
1937 #define ii(n)           _jitc->inst[n].i
1938     switch (cc) {
1939         case I_:
1940             n = 1;              tm = TM_M_I_I_;
1941             s0 = nop_m;         s1 = ii(0);             s2 = nop_i;
1942             break;
1943         case I_I_:
1944             n = 2;              tm = TM_M_I_I_;
1945             s0 = nop_m;         s1 = ii(0);             s2 = ii(1);
1946             break;
1947         case I_Is:
1948             n = 2;              tm = TM_M_I_Is;
1949             s0 = nop_m;         s1 = ii(0);             s2 = ii(1);
1950             break;
1951         case I_B_:
1952             n = 2;              tm = TM_M_I_B_;
1953             s0 = nop_m;         s1 = ii(0);             s2 = ii(1);
1954             break;
1955         case I_Bs:
1956             n = 2;              tm = TM_M_I_Bs;
1957             s0 = nop_m;         s1 = ii(0);             s2 = ii(1);
1958             break;
1959         case Is:
1960             n = 1;              tm = TM_M_IsI_;
1961             s0 = nop_m;         s1 = ii(0);             s2 = nop_i;
1962             break;
1963         case IsI_:
1964             n = 2;              tm = TM_M_IsI_;
1965             s0 = nop_m;         s1 = ii(0);             s2 = ii(1);
1966             break;
1967         case IsIs:
1968             n = 2;              tm = TM_M_IsIs;
1969             s0 = nop_m;         s1 = ii(0);             s2 = ii(1);
1970             break;
1971         case M_:
1972             n = 1;              tm = TM_M_I_I_;
1973             s0 = ii(0);         s1 = nop_i;             s2 = nop_i;
1974             break;
1975         case M_I_:
1976             n = 2;              tm = TM_M_I_I_;
1977             s0 = ii(0);         s1 = ii(1);             s2 = nop_i;
1978             break;
1979         case M_Is:
1980             n = 2;              tm = TM_M_IsI_;
1981             s0 = ii(0);         s1 = ii(1);             s2 = nop_i;
1982             break;
1983         case M_M_:
1984             n = 2;              tm = TM_M_M_I_;
1985             s0 = ii(0);         s1 = ii(1);             s2 = nop_i;
1986             break;
1987         case M_Ms:
1988             n = 2;              tm = TM_M_M_Is;
1989             s0 = ii(0);         s1 = ii(1);             s2 = nop_i;
1990             break;
1991         case M_F_:
1992             n = 2;              tm = TM_M_F_I_;
1993             s0 = ii(0);         s1 = ii(1);             s2 = nop_i;
1994             break;
1995         case M_Fs:
1996             n = 2;              tm = TM_M_F_Is;
1997             s0 = ii(0);         s1 = ii(1);             s2 = nop_i;
1998             break;
1999         case M_B_:
2000             n = 2;              tm = TM_M_B_B_;
2001             s0 = ii(0);         s1 = ii(1);             s2 = nop_b;
2002             break;
2003         case M_Bs:
2004             n = 2;              tm = TM_M_B_Bs;
2005             s0 = ii(0);         s1 = ii(1);             s2 = nop_b;
2006             break;
2007         case M_I_I_:
2008             n = 3;              tm = TM_M_I_I_;
2009             s0 = ii(0);         s1 = ii(1);             s2 = ii(2);
2010             break;
2011         case M_I_Is:
2012             n = 3;              tm = TM_M_I_Is;
2013             s0 = ii(0);         s1 = ii(1);             s2 = ii(2);
2014             break;
2015         case M_I_B_:
2016             n = 3;              tm = TM_M_I_B_;
2017             s0 = ii(0);         s1 = ii(1);             s2 = ii(2);
2018             break;
2019         case M_I_Bs:
2020             n = 3;              tm = TM_M_I_Bs;
2021             s0 = ii(0);         s1 = ii(1);             s2 = ii(2);
2022             break;
2023         case M_IsI_:
2024             n = 3;              tm = TM_M_IsI_;
2025             s0 = ii(0);         s1 = ii(1);             s2 = ii(2);
2026             break;
2027         case M_IsIs:
2028             n = 3;              tm = TM_M_IsIs;
2029             s0 = ii(0);         s1 = ii(1);             s2 = ii(2);
2030             break;
2031         case M_M_I_:
2032             n = 3;              tm = TM_M_M_I_;
2033             s0 = ii(0);         s1 = ii(1);             s2 = ii(2);
2034             break;
2035         case M_M_Is:
2036             n = 3;              tm = TM_M_M_Is;
2037             s0 = ii(0);         s1 = ii(1);             s2 = ii(2);
2038             break;
2039         case M_M_F_:
2040             n = 3;              tm = TM_M_M_F_;
2041             s0 = ii(0);         s1 = ii(1);             s2 = ii(2);
2042             break;
2043         case M_M_Fs:
2044             n = 3;              tm = TM_M_M_Fs;
2045             s0 = ii(0);         s1 = ii(1);             s2 = ii(2);
2046             break;
2047         case M_M_B_:
2048             n = 3;              tm = TM_M_M_B_;
2049             s0 = ii(0);         s1 = ii(1);             s2 = ii(2);
2050             break;
2051         case M_M_Bs:
2052             n = 3;              tm = TM_M_M_Bs;
2053             s0 = ii(0);         s1 = ii(1);             s2 = ii(2);
2054             break;
2055         case M_F_I_:
2056             n = 3;              tm = TM_M_F_I_;
2057             s0 = ii(0);         s1 = ii(1);             s2 = ii(2);
2058             break;
2059         case M_F_Is:
2060             n = 3;              tm = TM_M_F_Is;
2061             s0 = ii(0);         s1 = ii(1);             s2 = ii(2);
2062             break;
2063         case M_F_B_:
2064             n = 3;              tm = TM_M_F_B_;
2065             s0 = ii(0);         s1 = ii(1);             s2 = ii(2);
2066             break;
2067         case M_F_Bs:
2068             n = 3;              tm = TM_M_F_Bs;
2069             s0 = ii(0);         s1 = ii(1);             s2 = ii(2);
2070             break;
2071         case M_B_B_:
2072             n = 3;              tm = TM_M_B_B_;
2073             s0 = ii(0);         s1 = ii(1);             s2 = ii(2);
2074             break;
2075         case M_B_Bs:
2076             n = 3;              tm = TM_M_B_Bs;
2077             s0 = ii(0);         s1 = ii(1);             s2 = ii(2);
2078             break;
2079         case M_L_X_:
2080             n = 3;              tm = TM_M_L_X_;
2081             s0 = ii(0);         s1 = ii(1);             s2 = ii(2);
2082             break;
2083         case M_L_Xs:
2084             n = 3;              tm = TM_M_L_Xs;
2085             s0 = ii(0);         s1 = ii(1);             s2 = ii(2);
2086             break;
2087         case Ms:
2088             n = 1;              tm = TM_MsM_I_;
2089             s0 = ii(0);         s1 = nop_m;             s2 = nop_i;
2090             break;
2091         case MsI_:
2092             n = 2;              tm = TM_MsM_I_;
2093             s0 = ii(0);         s1 = nop_m;             s2 = ii(1);
2094             break;
2095         case MsIs:
2096             n = 2;              tm = TM_MsM_Is;
2097             s0 = ii(0);         s1 = nop_m;             s2 = ii(1);
2098             break;
2099         case MsM_:
2100             n = 2;              tm = TM_MsM_I_;
2101             s0 = ii(0);         s1 = ii(1);             s2 = nop_i;
2102             break;
2103         case MsMs:
2104             n = 2;              tm = TM_MsM_Is;
2105             s0 = ii(0);         s1 = ii(1);             s2 = nop_i;
2106             break;
2107         case MsM_I_:
2108             n = 3;              tm = TM_MsM_I_;
2109             s0 = ii(0);         s1 = ii(1);             s2 = ii(2);
2110             break;
2111         case MsM_Is:
2112             n = 3;              tm = TM_MsM_Is;
2113             s0 = ii(0);         s1 = ii(1);             s2 = ii(2);
2114             break;
2115         case F_:
2116             n = 1;              tm = TM_M_F_I_;
2117             s0 = nop_m;         s1 = ii(0);             s2 = nop_i;
2118             break;
2119         case F_I_:
2120             n = 2;              tm = TM_M_F_I_;
2121             s0 = nop_m;         s1 = ii(0);             s2 = ii(1);
2122             break;
2123         case F_Is:
2124             n = 2;              tm = TM_M_F_Is;
2125             s0 = nop_m;         s1 = ii(0);             s2 = ii(1);
2126             break;
2127         case F_B_:
2128             n = 2;              tm = TM_M_F_B_;
2129             s0 = nop_m;         s1 = ii(0);             s2 = ii(1);
2130             break;
2131         case F_Bs:
2132             n = 2;              tm = TM_M_F_Bs;
2133             s0 = nop_m;         s1 = ii(0);             s2 = ii(1);
2134             break;
2135         case Fs:
2136             n = 1;              tm = TM_M_F_Is;
2137             s0 = nop_m;         s1 = ii(0);             s2 = nop_i;
2138             break;
2139         case B_:
2140             n = 1;              tm = TM_B_B_B_;
2141             s0 = ii(0);         s1 = nop_b;             s2 = nop_b;
2142             break;
2143         case B_B_:
2144             n = 2;              tm = TM_B_B_B_;
2145             s0 = ii(0);         s1 = ii(1);             s2 = nop_b;
2146             break;
2147         case B_Bs:
2148             n = 2;              tm = TM_B_B_Bs;
2149             s0 = ii(0);         s1 = ii(1);             s2 = nop_b;
2150             break;
2151         case B_B_B_:
2152             n = 3;              tm = TM_B_B_B_;
2153             s0 = ii(0);         s1 = ii(1);             s2 = ii(2);
2154             break;
2155         case B_B_Bs:
2156             n = 3;              tm = TM_B_B_Bs;
2157             s0 = ii(0);         s1 = ii(1);             s2 = ii(2);
2158             break;
2159         case Bs:
2160             n = 1;              tm = TM_B_B_Bs;
2161             s0 = ii(0);         s1 = nop_b;             s2 = nop_b;
2162             break;
2163         case L_X_:
2164             n = 2;              tm = TM_M_L_X_;
2165             s0 = nop_m;         s1 = ii(0);             s2 = ii(1);
2166             break;
2167         case L_Xs:
2168             n = 2;              tm = TM_M_L_Xs;
2169             s0 = nop_m;         s1 = ii(0);             s2 = ii(1);
2170             break;
2171         default:
2172             abort();
2173     }
2174     out(n, tm, s0, s1, s2);
2175 }
2176
2177 static void
2178 _inst(jit_state_t *_jit, jit_word_t i, jit_uint8_t t)
2179 {
2180     if (_jitc->ioff > 2)
2181         flush();
2182     assert(!(i & 0x11111e0000000000L));
2183     _jitc->inst[_jitc->ioff].i = i;
2184     _jitc->inst[_jitc->ioff].t = t;
2185     ++_jitc->ioff;
2186 }
2187
2188 static void
2189 _A1(jit_state_t *_jit, jit_word_t _p,
2190     jit_word_t x4, jit_word_t x2, jit_word_t r3, jit_word_t r2, jit_word_t r1)
2191 {
2192     assert(!(_p & ~0x3fL));
2193     assert(!(x4 &  ~0xfL));
2194     assert(!(x2 &  ~0x3L));
2195     assert(!(r3 & ~0x7fL));
2196     assert(!(r2 & ~0x7fL));
2197     assert(!(r1 & ~0x7fL));
2198     TSTREG2(r2, r3);
2199     TSTPRED(_p);
2200     TSTREG1(r1);
2201     inst((8L<<37)|(x4<<29)|(x2<<27)|(r3<<20)|(r2<<13)|(r1<<6)|_p, INST_A);
2202     SETREG(r1);
2203 }
2204
2205 static void
2206 _A3(jit_state_t *_jit, jit_word_t _p,
2207     jit_word_t x4, jit_word_t x2, jit_word_t r3, jit_word_t im, jit_word_t r1)
2208 {
2209     assert(!(_p & ~0x3fL));
2210     assert(!(x4 &  ~0xfL));
2211     assert(!(x2 &  ~0x3L));
2212     assert(!(r3 & ~0x7fL));
2213     assert(im >= -128 && im <= 127);
2214     assert(!(r1 & ~0x7f));
2215     TSTREG1(r3);
2216     TSTPRED(_p);
2217     TSTREG1(r1);
2218     inst((8L<<37)|(((im>>7)&1L)<<36)|(x4<<29)|(x2<<27)|
2219          (r3<<20)|((im&0x7fL)<<13)|(r1<<6)|_p, INST_A);
2220     SETREG(r1);
2221 }
2222
2223 static void
2224 _A4(jit_state_t *_jit, jit_word_t _p,
2225     jit_word_t x2, jit_word_t r3, jit_word_t im, jit_word_t r1)
2226 {
2227     assert(!(_p & ~0x3fL));
2228     assert(!(x2 &  ~0x3L));
2229     assert(!(r3 & ~0x7fL));
2230     assert(im >= -8192 && im <= 8191);
2231     assert(!(r1  & ~0x7f));
2232     TSTREG1(r3);
2233     TSTPRED(_p);
2234     TSTREG1(r1);
2235     inst((8L<<37)|(((im>>13)&1L)<<36)|(x2<<34)|(((im>>7)&0x3fL)<<27)|
2236          (r3<<20)|((im&0x7fL)<<13)|(r1<<6)|_p, INST_A);
2237     SETREG(r1);
2238 }
2239
2240 static void
2241 _A5(jit_state_t *_jit, jit_word_t _p,
2242     jit_word_t r3, jit_word_t im, jit_word_t r1)
2243 {
2244     jit_word_t          s, i5, i9, i7;
2245     assert(!(_p & ~0x3fL));
2246     assert(!(r3  & ~0x3L));
2247     assert(im >= -2097152 && im <= 2097151);
2248     assert(!(r1  & ~0x7fL));
2249     /* imm22 = sign_ext(s << 21 | imm5c << 16 | imm9d << 7 | imm7b, 22) */
2250     s  = (im & 0x200000) >> 21;
2251     i5 = (im & 0x1f0000) >> 16;
2252     i9 = (im &   0xff80) >>  7;
2253     i7 =  im &     0x7f;
2254     TSTREG1(r3);
2255     TSTPRED(_p);
2256     TSTREG1(r1);
2257     inst((9L<<37)|(s<<36)|(i9<<27)|(i5<<22)|
2258          (r3<<20)|(i7<<13)|(r1<<6)|_p, INST_A);
2259     SETREG(r1);
2260 }
2261
2262 static void
2263 _A6(jit_state_t *_jit, jit_word_t _p,
2264     jit_word_t o, jit_word_t x2, jit_word_t ta, jit_word_t p2,
2265     jit_word_t r3, jit_word_t r2, jit_word_t c, jit_word_t p1)
2266 {
2267     assert(!(_p & ~0x3fL));
2268     assert(!(o  &  ~0xfL));
2269     assert(!(x2 &  ~0x3L));
2270     assert(!(ta &  ~0x1L));
2271     assert(!(p2 & ~0x7fL));
2272     assert(!(r3 & ~0x7fL));
2273     assert(!(r2 & ~0x7fL));
2274     assert(!(c  &  ~0x1L));
2275     assert(!(p1 & ~0x3fL));
2276     TSTREG2(r2, r3);
2277     TSTPRED(_p);
2278     inst((o<<37)|(x2<<34)|(ta<<33)|(p2<<27)|(r3<<20)|
2279          (r2<<13)|(c<<12)|(p1<<6)|_p, INST_A);
2280     if (p1)
2281         _jitc->pred |= 1 << p1;
2282     if (p2)
2283         _jitc->pred |= 1 << p2;
2284 }
2285
2286 static void
2287 _A7(jit_state_t *_jit, jit_word_t _p,
2288     jit_word_t o, jit_word_t x2, jit_word_t ta,
2289     jit_word_t p2, jit_word_t r3, jit_word_t c, jit_word_t p1)
2290 {
2291     assert(!(_p & ~0x3fL));
2292     assert(!(o  &  ~0xfL));
2293     assert(!(x2 &  ~0x3L));
2294     assert(!(ta &  ~0x1L));
2295     assert(!(p2 & ~0x7fL));
2296     assert(!(r3 & ~0x7fL));
2297     assert(!(c  &  ~0x1L));
2298     assert(!(p1 & ~0x3fL));
2299     TSTREG1(r3);
2300     TSTPRED(_p);
2301     inst((o<<37)|(1L<<36)|(x2<<34)|(ta<<33)|
2302          (p2<<27)|(r3<<20)|(c<<12)|(p1<<6)|_p, INST_A);
2303     if (p1)
2304         _jitc->pred |= 1 << p1;
2305     if (p2)
2306         _jitc->pred |= 1 << p2;
2307 }
2308
2309 static void
2310 _A8(jit_state_t *_jit, jit_word_t _p,
2311     jit_word_t o, jit_word_t x2, jit_word_t ta, jit_word_t p2,
2312     jit_word_t r3, jit_word_t im, jit_word_t c, jit_word_t p1)
2313 {
2314     assert(!(_p & ~0x3fL));
2315     assert(!(o  &  ~0xfL));
2316     assert(!(x2 &  ~0x3L));
2317     assert(!(ta &  ~0x1L));
2318     assert(!(p2 & ~0x7fL));
2319     assert(!(r3 & ~0x7fL));
2320     assert(im >= -128 && im <= 127);
2321     assert(!(c  &  ~0x1L));
2322     assert(!(p1 & ~0x3fL));
2323     TSTREG1(r3);
2324     TSTPRED(_p);
2325     inst((o<<37)|(((im>>7)&1L)<<36)|(x2<<34)|(ta<<33)|(p2<<27)|(r3<<20)|
2326          ((im&0x7fL)<<13)|(c<<12)|(p1<<6)|_p, INST_A);
2327     if (p1)
2328         _jitc->pred |= 1 << p1;
2329     if (p2)
2330         _jitc->pred |= 1 << p2;
2331 }
2332
2333 static void
2334 _A9(jit_state_t *_jit, jit_word_t _p,
2335     jit_word_t za, jit_word_t zb, jit_word_t x4,
2336     jit_word_t x2, jit_word_t r3, jit_word_t r2, jit_word_t r1)
2337 {
2338     assert(!(_p & ~0x3fL));
2339     assert(!(za &  ~0x1L));
2340     assert(!(zb &  ~0x1L));
2341     assert(!(x4 &  ~0xfL));
2342     assert(!(x2 &  ~0x3L));
2343     assert(!(r3 & ~0x7fL));
2344     assert(!(r2 & ~0x7fL));
2345     assert(!(r1 & ~0x7fL));
2346     TSTREG2(r2, r3);
2347     TSTPRED(_p);
2348     TSTREG1(r1);
2349     inst((8L<<37)|(za<<36)|(1L<<34)|(zb<<33)|(x4<<29)|(x2<<27)|
2350          (r3<<20)|(r2<<13)|(r1<<6)|_p, INST_A);
2351     SETREG(r1);
2352 }
2353
2354 static void
2355 _I1(jit_state_t *_jit, jit_word_t _p,
2356     jit_word_t ct, jit_word_t x2, jit_word_t r3, jit_word_t r2, jit_word_t r1)
2357 {
2358     assert(!(_p & ~0x3fL));
2359     assert(!(ct &  ~0x3L));
2360     assert(!(x2 &  ~0x3L));
2361     assert(!(r3 & ~0x7fL));
2362     assert(!(r2 & ~0x7fL));
2363     assert(!(r1 & ~0x7fL));
2364     TSTREG2(r2, r3);
2365     TSTPRED(_p);
2366     TSTREG1(r1);
2367     inst((7L<<37)|(1L<<33)|(ct<<30)|(x2<<28)|
2368          (r3<<20)|(r2<<13)|(r1<<6)|_p, INST_I);
2369     SETREG(r1);
2370 }
2371
2372 static void
2373 _I2(jit_state_t *_jit, jit_word_t _p,
2374     jit_word_t za, jit_word_t xa, jit_word_t zb, jit_word_t xc,
2375     jit_word_t xb ,jit_word_t r3, jit_word_t r2, jit_word_t r1)
2376 {
2377     assert(!(_p & ~0x3fL));
2378     assert(!(za &  ~0x1L));
2379     assert(!(xa &  ~0x3L));
2380     assert(!(zb &  ~0x1L));
2381     assert(!(xc &  ~0x3L));
2382     assert(!(xb &  ~0x3L));
2383     assert(!(r3 & ~0x7fL));
2384     assert(!(r2 & ~0x7fL));
2385     assert(!(r1 & ~0x7fL));
2386     TSTREG2(r2, r3);
2387     TSTPRED(_p);
2388     TSTREG1(r1);
2389     inst((7L<<37)|(za<<36)|(xa<<34)|(zb<<33)|(xc<<30)|
2390          (xb<<28)|(r3<<20)|(r2<<13)|(r1<<6), INST_I);
2391     SETREG(r1);
2392 }
2393
2394 static void
2395 _I3(jit_state_t *_jit, jit_word_t _p,
2396     jit_word_t mb, jit_word_t r2, jit_word_t r1)
2397 {
2398     assert(!(_p & ~0x3fL));
2399     assert(!(mb &  ~0xfL));
2400     assert(!(r2 & ~0x7fL));
2401     assert(!(r1 & ~0x7fL));
2402     TSTREG1(r2);
2403     TSTPRED(_p);
2404     TSTREG1(r1);
2405     inst((7L<<37)|(3L<<34)|(2L<<30)|(2L<<28)|
2406          (mb<<20)|(r2<<13)|(r1<<6)|_p, INST_I);
2407     SETREG(r1);
2408 }
2409
2410 static void
2411 _I4(jit_state_t *_jit, jit_word_t _p,
2412     jit_word_t mh, jit_word_t r2, jit_word_t r1)
2413 {
2414     assert(!(_p & ~0x3fL));
2415     assert(!(mh & ~0xffL));
2416     assert(!(r2 & ~0x7fL));
2417     assert(!(r1 & ~0x7fL));
2418     TSTREG1(r2);
2419     TSTPRED(_p);
2420     TSTREG1(r1);
2421     inst((7L<<37)|(3L<<34)|(1L<<33)|(2L<<30)|
2422          (2L<<28)|(mh<<20)|(r2<<13)|(r1<<6)|_p, INST_I);
2423     SETREG(r1);
2424 }
2425
2426 static void
2427 _I5(jit_state_t *_jit, jit_word_t _p,
2428     jit_word_t za, jit_word_t zb, jit_word_t x2,
2429     jit_word_t r3, jit_word_t r2, jit_word_t r1)
2430 {
2431     assert(!(_p & ~0x3fL));
2432     assert(!(za &  ~0x1L));
2433     assert(!(zb &  ~0x1L));
2434     assert(!(x2 &  ~0x3L));
2435     assert(!(r3 & ~0x7fL));
2436     assert(!(r2 & ~0x7fL));
2437     assert(!(r1 & ~0x7fL));
2438     TSTREG2(r2, r3);
2439     TSTPRED(_p);
2440     TSTREG1(r1);
2441     inst((7L<<37)|(za<<36)|(zb<<33)|(x2<<28)|
2442          (r3<<20)|(r2<<13)|(r1<<6)|_p, INST_I);
2443     SETREG(r1);
2444 }
2445
2446 static void
2447 _I6(jit_state_t *_jit, jit_word_t _p,
2448     jit_word_t za, jit_word_t zb, jit_word_t x2,
2449     jit_word_t r3, jit_word_t ct, jit_word_t r1)
2450 {
2451     assert(!(_p & ~0x3fL));
2452     assert(!(za &  ~0x1L));
2453     assert(!(zb &  ~0x1L));
2454     assert(!(x2 &  ~0x3L));
2455     assert(!(r3 & ~0x7fL));
2456     assert(!(ct & ~0x1fL));
2457     assert(!(r1 & ~0x7fL));
2458     TSTREG1(r3);
2459     TSTPRED(_p);
2460     TSTREG1(r1);
2461     inst((7L<<37)|(za<<36)|(1L<<34)|(zb<<33)|
2462          (x2<<28)|(r3<<20)|(ct<<14)|(r1<<6)|_p, INST_I);
2463     SETREG(r1);
2464 }
2465
2466 static void
2467 _I7(jit_state_t *_jit, jit_word_t _p,
2468     jit_word_t za, jit_word_t zb, jit_word_t r3, jit_word_t r2, jit_word_t r1)
2469 {
2470     assert(!(_p & ~0x3fL));
2471     assert(!(za &  ~0x1L));
2472     assert(!(zb &  ~0x1L));
2473     assert(!(r3 & ~0x7fL));
2474     assert(!(r2 & ~0x7fL));
2475     assert(!(r1 & ~0x7fL));
2476     TSTREG2(r2, r3);
2477     TSTPRED(_p);
2478     TSTREG1(r1);
2479     inst((7L<<37)|(za<<36)|(zb<<33)|(1L<<30)|
2480          (r3<<20)|(r2<<13)|(r1<<6)|_p, INST_I);
2481     SETREG(r1);
2482 }
2483
2484 static void
2485 _I8(jit_state_t *_jit, jit_word_t _p,
2486     jit_word_t za, jit_word_t zb, jit_word_t im, jit_word_t r2, jit_word_t r1)
2487 {
2488     assert(!(_p & ~0x3fL));
2489     assert(!(za &  ~0x1L));
2490     assert(!(zb &  ~0x1L));
2491     assert(!(im & ~0x1fL));
2492     assert(!(r2 & ~0x7fL));
2493     assert(!(r1 & ~0x7fL));
2494     TSTREG1(r2);
2495     TSTPRED(_p);
2496     TSTREG1(r1);
2497     inst((7L<<37)|(za<<36)|(3L<<34)|(zb<<33)|(1L<<30)|(1L<<28)|
2498          (im<<20)|(r2<<13)|(r1<<6), INST_I);
2499     SETREG(r1);
2500 }
2501
2502 static void
2503 _I9(jit_state_t *_jit, jit_word_t _p,
2504     jit_word_t x2, jit_word_t r3, jit_word_t r1)
2505 {
2506     assert(!(_p & ~0x3fL));
2507     assert(!(x2 &  ~0x3L));
2508     assert(!(r3 & ~0x7fL));
2509     assert(!(r1 & ~0x7fL));
2510     TSTREG1(r3);
2511     TSTPRED(_p);
2512     TSTREG1(r1);
2513     inst((7L<<37)|(1L<<34)|(1L<<33)|
2514          (x2<<30)|(1L<<28)|(r3<<20)|(r1<<6)|_p, INST_I);
2515     SETREG(r1);
2516 }
2517
2518 static void
2519 _I10(jit_state_t *_jit, jit_word_t _p,
2520      jit_word_t ct, jit_word_t r3, jit_word_t r2, jit_word_t r1)
2521 {
2522     assert(!(_p & ~0x3fL));
2523     assert(!(ct & ~0x3fL));
2524     assert(!(r3 & ~0x7fL));
2525     assert(!(r2 & ~0x7fL));
2526     assert(!(r1 & ~0x7fL));
2527     TSTREG2(r2, r3);
2528     TSTPRED(_p);
2529     TSTREG1(r1);
2530     inst((5L<<37)|(3L<<34)|(ct<<27)|(r3<<20)|(r2<<13)|(r1<<6)|_p, INST_I);
2531     SETREG(r1);
2532 }
2533
2534 static void
2535 _I11(jit_state_t *_jit, jit_word_t _p,
2536      jit_word_t len, jit_word_t r3,
2537      jit_word_t pos, jit_word_t y, jit_word_t r1)
2538 {
2539     assert(!(_p  & ~0x3fL));
2540     assert(!(len & ~0x3fL));
2541     assert(!(r3  & ~0x7fL));
2542     assert(!(pos & ~0x3fL));
2543     assert(!(y   &  ~0x1L));
2544     assert(!(r1  & ~0x7fL));
2545     TSTREG1(r3);
2546     TSTPRED(_p);
2547     TSTREG1(r1);
2548     inst((5L<<37)|(1L<<34)|(len<<27)|(r3<<20)|
2549          (pos<<14)|(y<<13)|(r1<<6)|_p, INST_I);
2550     SETREG(r1);
2551 }
2552
2553 static void
2554 _I12(jit_state_t *_jit, jit_word_t _p,
2555      jit_word_t len, jit_word_t pos, jit_word_t r2, jit_word_t r1)
2556 {
2557     assert(!(_p  & ~0x3fL));
2558     assert(!(len & ~0x3fL));
2559     assert(!(pos & ~0x3fL));
2560     assert(!(r2  & ~0x7fL));
2561     assert(!(r1  & ~0x7fL));
2562     TSTREG1(r2);
2563     TSTPRED(_p);
2564     TSTREG1(r1);
2565     inst((5L<<37)|(1L<<34)|(1L<<33)|(len<<27)|
2566          (pos<<20)|(r2<<13)|(r1<<6)|_p, INST_I);
2567     SETREG(r1);
2568 }
2569
2570 static void
2571 _I13(jit_state_t *_jit, jit_word_t _p,
2572      jit_word_t len, jit_word_t pos, jit_word_t im, jit_word_t r1)
2573 {
2574     assert(!(_p  & ~0x3fL));
2575     assert(!(len & ~0x3fL));
2576     assert(!(pos & ~0x3fL));
2577     assert(!(im  & ~0x7fL));
2578     assert(!(r1  & ~0x7fL));
2579     TSTPRED(_p);
2580     TSTREG1(r1);
2581     inst((5L<<37)|(((im>>7)&1L)<<36)|(1L<<34)|(1L<<33)|(len<<27)|
2582          (1L<<26)|(pos<<20)|((im&0x7fL)<<13)|(r1<<6)|_p, INST_I);
2583     SETREG(r1);
2584 }
2585
2586 static void
2587 _I14(jit_state_t *_jit, jit_word_t _p,
2588      jit_word_t s, jit_word_t len, jit_word_t r3, jit_word_t pos, jit_word_t r1)
2589 {
2590     assert(!(_p  & ~0x3fL));
2591     assert(!(s   &  ~0x1L));
2592     assert(!(len & ~0x3fL));
2593     assert(!(r3  & ~0x7fL));
2594     assert(!(pos & ~0x1fL));
2595     assert(!(r1  & ~0x7fL));
2596     TSTREG1(r3);
2597     TSTPRED(_p);
2598     TSTREG1(r1);
2599     inst((5L<<37)|(s<<36)|(3L<<34)|(1L<<33)|
2600          (len<<27)|(r3<<20)|(pos<<14)|(r1<<6)|_p, INST_I);
2601     SETREG(r1);
2602 }
2603
2604 static void
2605 _I15(jit_state_t *_jit, jit_word_t _p,
2606      jit_word_t pos, jit_word_t len,
2607      jit_word_t r3, jit_word_t r2, jit_word_t r1)
2608 {
2609     assert(!(_p  & ~0x3fL));
2610     assert(!(pos & ~0x3fL));
2611     assert(!(len &  ~0xfL));
2612     assert(!(r3  & ~0x7fL));
2613     assert(!(r2  & ~0x7fL));
2614     assert(!(r1  & ~0x7fL));
2615     TSTREG2(r2, r3);
2616     TSTPRED(_p);
2617     TSTREG1(r1);
2618     inst((4L<<37)|(pos<<31)|(len<<27)|(r3<<20)|(r2<<13)|(r1<<6)|_p, INST_I);
2619     SETREG(r1);
2620 }
2621
2622 static void
2623 _I16(jit_state_t *_jit, jit_word_t _p,
2624      jit_word_t tb, jit_word_t ta, jit_word_t p2,
2625      jit_word_t r3, jit_word_t ps, jit_word_t c, jit_word_t p1)
2626 {
2627     assert(!(_p & ~0x3fL));
2628     assert(!(tb &  ~0x1L));
2629     assert(!(ta &  ~0x1L));
2630     assert(!(p2 & ~0x7fL));
2631     assert(!(r3 & ~0x7fL));
2632     assert(!(ps & ~0x3fL));
2633     assert(!(c  &  ~0x1L));
2634     assert(!(p1 & ~0x3fL));
2635     TSTREG1(r3);
2636     TSTPRED(_p);
2637     inst((5L<<37)|(tb<<36)|(ta<<33)|(p2<<27)|
2638          (r3<<20)|(ps<<14)|(c<<12)|(p1<<6), INST_I);
2639     if (p1)
2640         _jitc->pred |= 1 << p1;
2641     if (p2)
2642         _jitc->pred |= 1 << p2;
2643 }
2644
2645 static void
2646 _I17(jit_state_t *_jit, jit_word_t _p,
2647      jit_word_t tb, jit_word_t ta, jit_word_t p2,
2648      jit_word_t r3, jit_word_t c, jit_word_t p1)
2649 {
2650     assert(!(_p & ~0x3fL));
2651     assert(!(tb &  ~0x1L));
2652     assert(!(ta &  ~0x1L));
2653     assert(!(p2 & ~0x7fL));
2654     assert(!(r3 & ~0x7fL));
2655     assert(!(c  &  ~0x1L));
2656     assert(!(p1 & ~0x3fL));
2657     TSTREG1(r3);
2658     TSTPRED(_p);
2659     inst((5L<<37)|(tb<<36)|(ta<<33)|(p2<<27)|
2660          (r3<<20)|(1L<<13)|(c<<12)|(p1<<6)|_p, INST_I);
2661     if (p1)
2662         _jitc->pred |= 1 << p1;
2663     if (p2)
2664         _jitc->pred |= 1 << p2;
2665 }
2666
2667 static void
2668 _I18(jit_state_t *_jit, jit_word_t _p,
2669      jit_word_t im, jit_word_t y)
2670 {
2671     assert(!(_p &     ~0x3fL));
2672     assert(!(im & ~0x1fffffL));
2673     assert(!(y  &      ~0x1L));
2674     TSTPRED(_p);
2675     inst((((im>>20)&1L)<<26)|(1L<<27)|(y<<26)|((im&0xffffL)<<6)|_p, INST_I);
2676 }
2677
2678 static void
2679 _I19(jit_state_t *_jit, jit_word_t _p,
2680      jit_word_t im)
2681 {
2682     assert(!(_p &     ~0x3fL));
2683     assert(!(im & ~0x1fffffL));
2684     TSTPRED(_p);
2685     inst(((im>>20)&1L)|((im&0xffffL)<<6)|_p, INST_I);
2686 }
2687
2688 static void
2689 _I20(jit_state_t *_jit, jit_word_t _p,
2690      jit_word_t r2, jit_word_t im)
2691 {
2692     assert(!(_p &     ~0x3fL));
2693     assert(!(r2 &     ~0x7fL));
2694     assert(!(im & ~0x1fffffL));
2695     TSTREG1(r2);
2696     TSTPRED(_p);
2697     inst(((im>>20)&1L)|(1L<<33)|(((im>>7)&0x1fffL)<<20)|
2698          (r2<<13)|((im&0x7fL)<<6)|_p, INST_I);
2699 }
2700
2701 static void
2702 _I21(jit_state_t *_jit, jit_word_t _p,
2703      jit_word_t im, jit_word_t ih, jit_word_t x,
2704      jit_word_t wh, jit_word_t r2, jit_word_t b1)
2705 {
2706     assert(!(_p &     ~0x3fL));
2707     assert(!(im &    ~0x1ffL));
2708     assert(!(ih &      ~0x1L));
2709     assert(!(x  &      ~0x1L));
2710     assert(!(wh &      ~0x3L));
2711     assert(!(r2 &     ~0x7fL));
2712     assert(!(b1 &      ~0x7L));
2713     TSTREG1(r2);
2714     TSTPRED(_p);
2715     inst((7L<<33)|(im<<24)|(ih<<23)|(x<<22)|(wh<<20)|
2716          (r2<<13)|(b1<<6), INST_I);
2717 }
2718
2719 static void
2720 _I22(jit_state_t *_jit, jit_word_t _p,
2721      jit_word_t b2, jit_word_t r1)
2722 {
2723     assert(!(_p & ~0x3fL));
2724     assert(!(b2 &  ~0x7L));
2725     assert(!(r1 & ~0x7fL));
2726     TSTPRED(_p);
2727     inst((0x31L<<27)|(b2<<13)|(r1<<6)|_p, INST_I);
2728 }
2729
2730 static void
2731 _I23(jit_state_t *_jit, jit_word_t _p,
2732      jit_word_t r2, jit_word_t im)
2733 {
2734     assert(!(_p &   ~0x3fL));
2735     assert(!(r2 &   ~0x7fL));
2736     assert(!(im & ~0xffffL));
2737     TSTREG1(r2);
2738     TSTPRED(_p);
2739     inst((((im>>15)&1L)<<36)|(3L<<33)|(((im>>7)&0xffL)<<24)|
2740          (r2<<13)|(im&0x7fL)|_p, INST_I);
2741 }
2742
2743 static void
2744 _I24(jit_state_t *_jit, jit_word_t _p,
2745      jit_word_t im)
2746 {
2747     jit_uint8_t         cc = INST_I;
2748     assert(!(_p &      ~0x3fL));
2749     assert(!(im & ~0xfffffffL));
2750     TSTPRED(_p);
2751     inst((((im>>27)&1L)<<36)|(2L<<33)|((im&0x7ffffffL)<<6)|_p, cc);
2752 }
2753
2754 static void
2755 _I25(jit_state_t *_jit, jit_word_t _p,
2756      jit_word_t x6, jit_word_t r1)
2757 {
2758     assert(!(_p & ~0x3fL));
2759     assert(!(x6 & ~0x3fL));
2760     assert(!(r1 & ~0x7fL));
2761     TSTPRED(_p);
2762     TSTREG1(r1);
2763     inst((x6<<27)|(r1<<6)|_p, INST_I);
2764     SETREG(r1);
2765 }
2766
2767 static void
2768 _I26(jit_state_t *_jit, jit_word_t _p,
2769      jit_word_t ar,jit_word_t r2)
2770 {
2771     assert(!(_p & ~0x3fL));
2772     assert(!(ar & ~0x7fL));
2773     assert(!(r2 & ~0x7fL));
2774     TSTREG1(r2);
2775     TSTPRED(_p);
2776     inst((0x2aL<<27)|(ar<<20)|(r2<<13)|_p, INST_I);
2777 }
2778
2779 static void
2780 _I27(jit_state_t *_jit, jit_word_t _p,
2781      jit_word_t ar,jit_word_t im)
2782 {
2783     assert(!(_p & ~0x3fL));
2784     assert(!(ar & ~0x7fL));
2785     assert(!(im & ~0xffL));
2786     TSTPRED(_p);
2787     inst((((im>>7)&1L)<<36)|(0xaL<<27)|(ar<<20)|((im&0x7fL)<<13)|_p, INST_I);
2788 }
2789
2790 static void
2791 _I28(jit_state_t *_jit, jit_word_t _p,
2792      jit_word_t ar, jit_word_t r1)
2793 {
2794     assert(!(_p & ~0x3fL));
2795     assert(!(ar & ~0x7fL));
2796     assert(!(r1 & ~0x7fL));
2797     TSTPRED(_p);
2798     TSTREG1(r1);
2799     inst((0x32L<<27)|(ar<<20)|(r1<<6)|_p, INST_I);
2800     SETREG(r1);
2801 }
2802
2803 static void
2804 _I29(jit_state_t *_jit, jit_word_t _p,
2805      jit_word_t x6, jit_word_t r3,jit_word_t r1)
2806 {
2807     assert(!(_p & ~0x3fL));
2808     assert(!(x6 & ~0x3fL));
2809     assert(!(r3 & ~0x7fL));
2810     assert(!(r1 & ~0x7fL));
2811     TSTREG1(r3);
2812     TSTPRED(_p);
2813     TSTREG1(r1);
2814     inst((x6<<27)|(r3<<20)|(r1<<6)|_p, INST_I);
2815     SETREG(r1);
2816 }
2817
2818 static void
2819 _I30(jit_state_t *_jit, jit_word_t _p,
2820      jit_word_t ta, jit_word_t tb, jit_word_t p2,
2821      jit_word_t im, jit_word_t c, jit_word_t p1)
2822 {
2823     assert(!(_p & ~0x3fL));
2824     assert(!(ta &  ~0x1L));
2825     assert(!(tb &  ~0x1L));
2826     assert(!(p2 & ~0x3fL));
2827     assert(!(im & ~0x1fL));
2828     assert(!(c  &  ~0x1L));
2829     assert(!(p1 & ~0x1fL));
2830     TSTPRED(_p);
2831     inst((5L<<37)|(tb<<36)|(ta<<33)|(1L<<19)|(im<<14)|
2832          (1L<<13)|(c<<12)|(p1<<6)|_p, INST_I);
2833     if (p1)
2834         _jitc->pred |= 1 << p1;
2835     if (p2)
2836         _jitc->pred |= 1 << p2;
2837 }
2838
2839 static void
2840 _M1(jit_state_t *_jit, jit_word_t _p,
2841     jit_word_t x6, jit_word_t ht, jit_word_t x, jit_word_t r3, jit_word_t r1)
2842 {
2843     assert(!(_p & ~0x3fL));
2844     assert(!(x6 & ~0x3fL));
2845     assert(!(ht &  ~0x3L));
2846     assert(!(x  &  ~0x1L));
2847     assert(!(r3 & ~0x7fL));
2848     assert(!(r1 & ~0x7fL));
2849     TSTREG1(r3);
2850     TSTPRED(_p);
2851     TSTREG1(r1);
2852     inst((4L<<37)|(x6<<30)|(ht<<28)|(x<<27)|(r3<<20)|(r1<<6)|_p, INST_M);
2853     SETREG(r1);
2854 }
2855
2856 static void
2857 _M2(jit_state_t *_jit, jit_word_t _p,
2858     jit_word_t x6, jit_word_t ht, jit_word_t r3, jit_word_t r2, jit_word_t r1)
2859 {
2860     assert(!(_p & ~0x3fL));
2861     assert(!(x6 & ~0x3fL));
2862     assert(!(ht &  ~0x3L));
2863     assert(!(r3 & ~0x7fL));
2864     assert(!(r2 & ~0x7fL));
2865     assert(!(r1 & ~0x7fL));
2866     TSTREG2(r2, r3);
2867     TSTPRED(_p);
2868     TSTREG1(r1);
2869     inst((4L<<37)|(1L<<36)|(x6<<30)|(ht<<28)|
2870          (r3<<20)|(r2<<13)|(r1<<6)|_p, INST_M);
2871     SETREG(r1);
2872     SETREG(r3);
2873 }
2874
2875 static void
2876 _M3(jit_state_t *_jit, jit_word_t _p,
2877     jit_word_t x6, jit_word_t ht, jit_word_t r3, jit_word_t im, jit_word_t r1)
2878 {
2879     jit_uint8_t         cc = INST_M;
2880     assert(!(_p &  ~0x3fL));
2881     assert(!(x6 &  ~0x3fL));
2882     assert(!(ht &   ~0x3L));
2883     assert(!(r3 &  ~0x7fL));
2884     assert(im > -256 && im <= 255);
2885     assert(!(r1 &  ~0x7fL));
2886     TSTREG1(r3);
2887     TSTPRED(_p);
2888     TSTREG1(r1);
2889     inst((5L<<37)|(((im>>8)&1L)<<36)|(x6<<30)|(ht<<28)|
2890          (((im>>7)&1L)<<27)|(r3<<20)|((im&0x7fL)<<13)|(r1<<6)|_p, cc);
2891     SETREG(r1);
2892     SETREG(r3);
2893 }
2894
2895 static void
2896 _M5(jit_state_t *_jit, jit_word_t _p,
2897     jit_word_t x6, jit_word_t ht, jit_word_t r3, jit_word_t r2, jit_word_t im)
2898 {
2899     assert(!(_p &  ~0x3fL));
2900     assert(!(x6 &  ~0x3fL));
2901     assert(!(ht &   ~0x3L));
2902     assert(!(r3 &  ~0x7fL));
2903     assert(!(r2 &  ~0x7fL));
2904     assert(im > -256 && im <= 255);
2905     TSTREG2(r2, r3);
2906     TSTPRED(_p);
2907     inst((5L<<37)|(((im>>8)&1L)<<36)|(x6<<30)|(ht<<28)|
2908          (((im>>7)&1L)<<27)|(r3<<20)|(r2<<13)|((im&0x7fL)<<6)|_p, INST_M);
2909     SETREG(r3);
2910 }
2911
2912 static void
2913 _M6(jit_state_t *_jit, jit_word_t _p,
2914     jit_word_t x6, jit_word_t ht, jit_word_t x, jit_word_t r3, jit_word_t r2)
2915 {
2916     assert(!(_p & ~0x3fL));
2917     assert(!(x6 & ~0x3fL));
2918     assert(!(ht &  ~0x3L));
2919     assert(!(x  &  ~0x1L));
2920     assert(!(r3 & ~0x7fL));
2921     assert(!(r2 & ~0x7fL));
2922     TSTREG2(r2, r3);
2923     TSTPRED(_p);
2924     inst((4L<<37)|(x6<<30)|(ht<<28)|(x<<27)|(r3<<20)|(r2<<13)|_p, INST_M);
2925 }
2926
2927 static void
2928 _M13(jit_state_t *_jit, jit_word_t _p,
2929      jit_word_t x6, jit_word_t ht, jit_word_t r3, jit_word_t r2)
2930 {
2931     assert(!(_p & ~0x3fL));
2932     assert(!(x6 & ~0x3fL));
2933     assert(!(ht &  ~0x3L));
2934     assert(!(r3 & ~0x7fL));
2935     assert(!(r2 & ~0x7fL));
2936     TSTREG1(r3);
2937     if (r2)
2938         TSTFREG1(r2);
2939     TSTPRED(_p);
2940     inst((6L<<37)|(x6<<30)|(ht<<28)|(r3<<20)|(r2<<13)|_p, INST_M);
2941 }
2942
2943 static void
2944 _M14(jit_state_t *_jit, jit_word_t _p,
2945      jit_word_t x6, jit_word_t ht, jit_word_t r3, jit_word_t r2)
2946 {
2947     assert(!(_p & ~0x3fL));
2948     assert(!(x6 & ~0x3fL));
2949     assert(!(ht &  ~0x3L));
2950     assert(!(r3 & ~0x7fL));
2951     assert(!(r2 & ~0x7fL));
2952     TSTREG2(r2, r3);
2953     TSTPRED(_p);
2954     inst((6L<<37)|(1L<<36)|(x6<<30)|(ht<<28)|(r3<<20)|(r2<<13)|_p, INST_M);
2955 }
2956
2957 static void
2958 _M15(jit_state_t *_jit, jit_word_t _p,
2959      jit_word_t x6, jit_word_t ht, jit_word_t r3, jit_word_t im)
2960 {
2961     assert(!(_p &  ~0x3fL));
2962     assert(!(x6 &  ~0x3fL));
2963     assert(!(ht &   ~0x3L));
2964     assert(!(r3 &  ~0x7fL));
2965     assert(!(im & ~0x1ffL));
2966     TSTREG1(r3);
2967     TSTPRED(_p);
2968     inst((7L<<37)|(((im>>8)&1L)<<36)|(x6<<30)|(ht<<28)|
2969          (((im>>7)&1L)<<27)|(r3<<20)|((im&0x7fL)<<13)|_p, INST_M);
2970 }
2971
2972 static void
2973 _M16(jit_state_t *_jit, jit_word_t _p,
2974      jit_word_t x6, jit_word_t ht, jit_word_t r3, jit_word_t r2, jit_word_t r1)
2975 {
2976     assert(!(_p & ~0x3fL));
2977     assert(!(x6 & ~0x3fL));
2978     assert(!(ht &  ~0x3L));
2979     assert(!(r3 & ~0x7fL));
2980     assert(!(r2 & ~0x7fL));
2981     assert(!(r1 & ~0x7fL));
2982     TSTREG2(r2, r3);
2983     TSTPRED(_p);
2984     TSTREG1(r1);
2985     inst((4L<<37)|(x6<<30)|(ht<<28)|(1L<<27)|
2986          (r3<<20)|(r2<<13)|(r1<<6)|_p, INST_M);
2987     SETREG(r1);
2988 }
2989
2990 static void
2991 _M17(jit_state_t *_jit, jit_word_t _p,
2992      jit_word_t x6, jit_word_t ht, jit_word_t r3, jit_word_t im, jit_word_t r1)
2993 {
2994     assert(!(_p & ~0x3fL));
2995     assert(!(x6 & ~0x3fL));
2996     assert(!(ht &  ~0x3L));
2997     assert(!(r3 & ~0x7fL));
2998     assert(!(im &  ~0x7L));
2999     assert(!(r1 & ~0x7fL));
3000     TSTREG1(r3);
3001     TSTPRED(_p);
3002     TSTREG1(r1);
3003     inst((4L<<37)|(x6<<30)|(ht<<28)|(1L<<27)|
3004          (r3<<20)|(im<<13)|(r1<<6)|_p, INST_M);
3005     SETREG(r1);
3006 }
3007
3008 static void
3009 _M20x(jit_state_t *_jit, jit_word_t _p,
3010       jit_word_t x3, jit_word_t r2, jit_word_t im)
3011 {
3012     assert(!(_p &     ~0x3fL));
3013     assert(!(x3 &      ~0x7L));
3014     assert(!(r2 &     ~0x7fL));
3015     assert(!(im & ~0x1fffffL));
3016     if (x3 == 1)
3017         TSTREG1(r2);
3018     else
3019         TSTFREG1(r2);
3020     TSTPRED(_p);
3021     inst((1L<<37)|(((im>>20)&1L)<<36)|(x3<<33)|
3022          (((im>>7)&0x1fffL)<<20)|(r2<<13)|((im&0x7fL)<<6)|_p, INST_M);
3023 }
3024
3025 static void
3026 _M22x(jit_state_t *_jit, jit_word_t _p,
3027       jit_word_t x3, jit_word_t im, jit_word_t r1)
3028 {
3029     assert(!(_p &     ~0x3fL));
3030     assert(!(x3 &      ~0x7L));
3031     assert(!(im & ~0x1fffffL));
3032     assert(!(r1 &     ~0x7fL));
3033     TSTPRED(_p);
3034     if (x3 < 6)
3035         TSTREG1(r1);
3036     else
3037         TSTFREG1(r1);
3038     inst((((im>>20)&1L)<<36)|(x3<<33)|((im&0xffffL)<<13)|(r1<<6)|_p, INST_M);
3039     if (x3 < 6)
3040         SETREG(r1);
3041     else
3042         SETFREG(r1);
3043 }
3044
3045 static void
3046 _M24(jit_state_t *_jit, jit_word_t _p,
3047      jit_word_t x2, jit_word_t x4)
3048 {
3049     assert(!(_p & ~0x3fL));
3050     assert(!(x2 &  ~0x3L));
3051     assert(!(x4 &  ~0xfL));
3052     TSTPRED(_p);
3053     inst((x2<<31)|(x4<<27)|_p, INST_M);
3054 }
3055
3056 static void
3057 _M26x(jit_state_t *_jit, jit_word_t _p,
3058       jit_word_t x4, jit_word_t r1)
3059 {
3060     assert(!(_p & ~0x3fL));
3061     assert(!(x4 &  ~0xfL));
3062     assert(!(r1 & ~0x7fL));
3063     if (x4 == 2)
3064         TSTREG1(r1);
3065     else
3066         TSTFREG1(r1);
3067     TSTPRED(_p);
3068     inst((1L<<31)|(x4<<27)|(r1<<6)|_p, INST_M);
3069 }
3070
3071 static void
3072 _M28(jit_state_t *_jit, jit_word_t _p,
3073      jit_word_t x, jit_word_t r3)
3074 {
3075     assert(!(_p & ~0x3fL));
3076     assert(!(x  &  ~0x1L));
3077     assert(!(r3 & ~0x7fL));
3078     TSTREG1(r3);
3079     TSTPRED(_p);
3080     inst((1L<<37)|(x<<36)|(0x30L<<27)|(r3<<20)|_p, INST_M);
3081 }
3082
3083 static void
3084 _M29(jit_state_t *_jit, jit_word_t _p,
3085      jit_word_t ar, jit_word_t r2)
3086 {
3087     assert(!(_p & ~0x3fL));
3088     assert(!(ar & ~0x7fL));
3089     assert(!(r2 & ~0x7fL));
3090     TSTREG1(r2);
3091     TSTPRED(_p);
3092     inst((1L<<37)|(0x2aL<<27)|(ar<<20)|(r2<<13)|_p, INST_M);
3093 }
3094
3095 static void
3096 _M30(jit_state_t *_jit, jit_word_t _p,
3097      jit_word_t ar, jit_word_t im)
3098 {
3099     assert(!(_p & ~0x3fL));
3100     assert(!(ar  & ~0x7L));
3101     assert(!(im & ~0xffL));
3102     TSTPRED(_p);
3103     inst((((im>>7)&1L)<<36)|(2L<<31)|(0x8L<<27)|
3104          (ar<<20)|((im&0x7fL)<<13)|_p, INST_M);
3105 }
3106
3107 static void
3108 _M31(jit_state_t *_jit, jit_word_t _p,
3109      jit_word_t ar, jit_word_t r1)
3110 {
3111     assert(!(_p & ~0x3fL));
3112     assert(!(ar  & ~0x7L));
3113     assert(!(r1 & ~0x7fL));
3114     TSTPRED(_p);
3115     TSTREG1(r1);
3116     inst((1L<<37)|(0x22L<<27)|(ar<<20)|(r1<<6)|_p, INST_M);
3117     SETREG(r1);
3118 }
3119
3120 static void
3121 _M32(jit_state_t *_jit, jit_word_t _p,
3122      jit_word_t cr, jit_word_t r2)
3123 {
3124     assert(!(_p & ~0x3fL));
3125     assert(!(cr  & ~0x7L));
3126     assert(!(r2 & ~0x7fL));
3127     TSTREG1(r2);
3128     TSTPRED(_p);
3129     inst((1L<<37)|(0x2cL<<27)|(cr<<20)|(r2<<13)|_p, INST_M);
3130 }
3131
3132 static void
3133 _M33(jit_state_t *_jit, jit_word_t _p,
3134      jit_word_t cr, jit_word_t r1)
3135 {
3136     assert(!(_p & ~0x3fL));
3137     assert(!(cr  & ~0x7L));
3138     assert(!(r1 & ~0x7fL));
3139     TSTPRED(_p);
3140     TSTREG1(r1);
3141     inst((1L<<37)|(0x24L<<27)|(cr<<20)|(r1<<6)|_p, INST_M);
3142     SETREG(r1);
3143 }
3144
3145 static void
3146 _M34(jit_state_t *_jit, jit_word_t _p,
3147      jit_word_t sor, jit_word_t sol, jit_word_t sof, jit_word_t r1)
3148 {
3149     assert(!(_p  & ~0x3fL));
3150     assert(!(sor &  ~0xfL));
3151     assert(!(sol & ~0x7fL));
3152     assert(!(sof & ~0x7fL));
3153     assert(!(r1  & ~0x7fL));
3154     TSTPRED(_p);
3155     inst((1L<<37)|(6L<<33)|(sor<<27)|(sol<<20)|(sof<<13)|(r1<<6)|_p, INST_M);
3156 }
3157
3158 static void
3159 _M35(jit_state_t *_jit, jit_word_t _p,
3160      jit_word_t x6, jit_word_t r2)
3161 {
3162     assert(!(_p & ~0x3fL));
3163     assert(!(x6 & ~0x3fL));
3164     assert(!(r2 & ~0x7fL));
3165     TSTREG1(r2);
3166     TSTPRED(_p);
3167     inst((1L<<37)|(x6<<27)|(r2<<13)|_p, INST_M);
3168 }
3169
3170 static void
3171 _M36(jit_state_t *_jit, jit_word_t _p,
3172      jit_word_t x6, jit_word_t r1)
3173 {
3174     assert(!(_p & ~0x3fL));
3175     assert(!(x6 & ~0x3fL));
3176     assert(!(r1 & ~0x7fL));
3177     TSTPRED(_p);
3178     TSTREG1(r1);
3179     inst((1L<<37)|(x6<<27)|(r1<<6)|_p, INST_M);
3180     SETREG(r1);
3181 }
3182
3183 static void
3184 _M37(jit_state_t *_jit, jit_word_t _p,
3185      jit_word_t im)
3186 {
3187     assert(!(_p &    ~0x3fL));
3188     assert(!(im & ~0x1ffffL));
3189     TSTPRED(_p);
3190     inst((((im>>20)&1L)<<36)|((im&0xffffL)<<6)|_p, INST_M);
3191 }
3192
3193 static void
3194 _M38(jit_state_t *_jit, jit_word_t _p,
3195      jit_word_t x6, jit_word_t r3, jit_word_t r2, jit_word_t r1)
3196 {
3197     assert(!(_p &  ~0x3fL));
3198     assert(!(x6 &  ~0x3fL));
3199     assert(!(r3 &  ~0x7fL));
3200     assert(!(r2 &  ~0x7fL));
3201     assert(!(r1 &  ~0x7fL));
3202     TSTREG2(r2, r3);
3203     TSTPRED(_p);
3204     TSTREG1(r1);
3205     inst((1L<<37)|(x6<<27)|(r3<<20)|(r2<<13)|(r1<<6)|_p, INST_M);
3206     SETREG(r1);
3207 }
3208
3209 static void
3210 _M39(jit_state_t *_jit, jit_word_t _p,
3211      jit_word_t x6, jit_word_t r3, jit_word_t im, jit_word_t r1)
3212 {
3213     assert(!(_p &  ~0x3fL));
3214     assert(!(x6 &  ~0x3fL));
3215     assert(!(r3 &  ~0x7fL));
3216     assert(!(im &   ~0x7L));
3217     assert(!(r1 &  ~0x7fL));
3218     TSTREG1(r3);
3219     TSTPRED(_p);
3220     TSTREG1(r1);
3221     inst((1L<<37)|(x6<<27)|(r3<<20)|(im<<13)|(r1<<6)|_p, INST_M);
3222     SETREG(r1);
3223 }
3224
3225 static void
3226 _M40(jit_state_t *_jit, jit_word_t _p,
3227      jit_word_t x6, jit_word_t r3, jit_word_t im)
3228 {
3229     assert(!(_p &  ~0x3fL));
3230     assert(!(x6 &  ~0x3fL));
3231     assert(!(r3 &  ~0x7fL));
3232     assert(!(im &   ~0x7L));
3233     TSTREG1(r3);
3234     TSTPRED(_p);
3235     inst((1L<<37)|(x6<<27)|(r3<<20)|(im<<13)|_p, INST_M);
3236 }
3237
3238 static void
3239 _M41(jit_state_t *_jit, jit_word_t _p,
3240      jit_word_t x6, jit_word_t r2)
3241 {
3242     assert(!(_p &  ~0x3fL));
3243     assert(!(x6 &  ~0x3fL));
3244     assert(!(r2 &  ~0x7fL));
3245     TSTREG1(r2);
3246     TSTPRED(_p);
3247     inst((1L<<37)|(x6<<27)|(r2<<13)|_p, INST_M);
3248 }
3249
3250 static void
3251 _M42(jit_state_t *_jit, jit_word_t _p,
3252      jit_word_t x6, jit_word_t r3, jit_word_t r2)
3253 {
3254     assert(!(_p &  ~0x3fL));
3255     assert(!(x6 &  ~0x3fL));
3256     assert(!(r3 &  ~0x7fL));
3257     assert(!(r2 &  ~0x7fL));
3258     TSTREG1(r2);
3259     TSTPRED(_p);
3260     inst((1L<<37)|(x6<<27)|(r3<<20)|(r2<<13)|_p, INST_M);
3261 }
3262
3263 static void
3264 _M43(jit_state_t *_jit, jit_word_t _p,
3265      jit_word_t x6, jit_word_t r3, jit_word_t r1)
3266 {
3267     assert(!(_p &  ~0x3fL));
3268     assert(!(x6 &  ~0x3fL));
3269     assert(!(r3 &  ~0x7fL));
3270     assert(!(r1 &  ~0x7fL));
3271     TSTPRED(_p);
3272     TSTREG1(r1);
3273     inst((1L<<37)|(x6<<27)|(r3<<20)|(r1<<6)|_p, INST_M);
3274     SETREG(r1);
3275 }
3276
3277 static void
3278 _M44(jit_state_t *_jit, jit_word_t _p,
3279      jit_word_t x4, jit_word_t im)
3280 {
3281     assert(!(_p &    ~0x3fL));
3282     assert(!(x4 &     ~0xfL));
3283     assert(!(im & ~0xfffffL));
3284     TSTPRED(_p);
3285     inst((((im>>23)&1L)<<36)|(((im>>21)&3L)<<31)|
3286          (x4<<27)|((im&0x1ffffL)<<6)|_p, INST_M);
3287 }
3288
3289 static void
3290 _M45(jit_state_t *_jit, jit_word_t _p,
3291      jit_word_t x6, jit_word_t r3, jit_word_t r2)
3292 {
3293     assert(!(_p &  ~0x3fL));
3294     assert(!(x6 &  ~0x3fL));
3295     assert(!(r3 &  ~0x7fL));
3296     assert(!(r2 &  ~0x7fL));
3297     TSTREG2(r2, r3);
3298     TSTPRED(_p);
3299     inst((1L<<37)|(x6<<27)|(r3<<20)|(r2<<13)|_p, INST_M);
3300 }
3301
3302 static void
3303 _M46(jit_state_t *_jit, jit_word_t _p,
3304      jit_word_t x6, jit_word_t r3, jit_word_t r1)
3305 {
3306     assert(!(_p & ~0x3fL));
3307     assert(!(x6 & ~0x3fL));
3308     assert(!(r3 & ~0x7fL));
3309     assert(!(r1 & ~0x7fL));
3310     TSTREG1(r3);
3311     TSTPRED(_p);
3312     if (r1)     TSTREG1(r1);
3313     inst((1L<<37)|(x6<<27)|(r3<<20)|(r1<<6)|_p, INST_M);
3314     if (r1)     SETREG(r1);
3315 }
3316
3317 static void
3318 _M48(jit_state_t *_jit, jit_word_t _p,
3319      jit_word_t y, jit_word_t im)
3320 {
3321     assert(!(_p &    ~0x3fL));
3322     assert(!(y  &     ~0x1L));
3323     assert(!(im & ~0x1ffffL));
3324     TSTPRED(_p);
3325     inst((((im>>20)&1L)<<36)|(1L<<27)|(y<<26)|((im&0xffffL)<<6)|_p, INST_M);
3326 }
3327
3328 static void
3329 _B1(jit_state_t *_jit, jit_word_t _p,
3330     jit_word_t d, jit_word_t wh, jit_word_t im, jit_word_t p, jit_word_t tp)
3331 {
3332     assert(!(_p &    ~0x3fL));
3333     assert(!(d  &     ~0x1L));
3334     assert(!(wh &     ~0x3L));
3335     assert(im >= -1048576 && im <= 1048575);
3336     assert(!(p  &     ~0x1L));
3337     assert(!(tp &     ~0x7L));
3338     TSTPRED(_p);
3339     inst((4L<<37)|(((im>>20)&1L)<<36)|(d<<35)|(wh<<33)|
3340          ((im&0xfffffL)<<13)|(p<<12)|(tp<<6)|_p, INST_B);
3341 }
3342
3343 static void
3344 _B3(jit_state_t *_jit, jit_word_t _p,
3345     jit_word_t d, jit_word_t wh, jit_word_t im, jit_word_t p, jit_word_t b)
3346 {
3347     assert(!(_p &    ~0x3fL));
3348     assert(!(d  &     ~0x1L));
3349     assert(!(wh &     ~0x3L));
3350     assert(im >= -1048576 && im <= 1048575);
3351     assert(!(p  &     ~0x1L));
3352     assert(!(b  &     ~0x3L));
3353     TSTPRED(_p);
3354     inst((5L<<37)|(((im>>20)&1L)<<36)|(d<<35)|(wh<<33)|
3355          ((im&0xfffffL)<<13)|(p<<12)|(b<<6)|_p, INST_B);
3356 }
3357
3358 static void
3359 _B4(jit_state_t *_jit, jit_word_t _p,
3360     jit_word_t d, jit_word_t wh, jit_word_t x6,
3361     jit_word_t b, jit_word_t p, jit_word_t tp)
3362 {
3363     assert(!(_p & ~0x3fL));
3364     assert(!(d  &  ~0x1L));
3365     assert(!(wh &  ~0x3L));
3366     assert(!(x6 & ~0x3fL));
3367     assert(!(b  &  ~0x7L));
3368     assert(!(p  &  ~0x1L));
3369     assert(!(tp &  ~0x7L));
3370     TSTPRED(_p);
3371     inst((d<<35)|(wh<<33)|(x6<<27)|(b<<13)|(p<<12)|(tp<<6)|_p, INST_B);
3372 }
3373
3374 static void
3375 _B5(jit_state_t *_jit, jit_word_t _p,
3376     jit_word_t d, jit_word_t wh, jit_word_t b2, jit_word_t p, jit_word_t b1)
3377 {
3378     assert(!(_p & ~0x3fL));
3379     assert(!(d  &  ~0x1L));
3380     assert(!(wh &  ~0x3L));
3381     assert(!(b2 &  ~0x7L));
3382     assert(!(p  &  ~0x1L));
3383     assert(!(b1 &  ~0x7L));
3384     TSTPRED(_p);
3385     inst((1L<<37)|(d<<35)|(wh<<32)|(b2<<13)|(p<<12)|(b1<<6)|_p, INST_B);
3386 }
3387
3388 static void
3389 _B6(jit_state_t *_jit, jit_word_t _p,
3390     jit_word_t ih, jit_word_t im, jit_word_t tag, jit_word_t wh)
3391 {
3392     assert(!(_p &     ~0x3fL));
3393     assert(!(ih  &     ~0x1L));
3394     assert(!(im  & ~0x1ffffL));
3395     assert(!(tag &   ~0x1ffL));
3396     assert(!(wh  &     ~0x3L));
3397     TSTPRED(_p);
3398     inst((7L<<37)|(((im>>20)&1L)<<36)|(ih<<35)|(((tag>>7)&3L)<<33)|
3399          ((im&0xfffffL)<<13)|((tag&0x7fL)<<6)|(wh<<3)|_p, INST_B);
3400 }
3401
3402 static void
3403 _B7(jit_state_t *_jit, jit_word_t _p,
3404     jit_word_t ih, jit_word_t x6, jit_word_t b2, jit_word_t tag, jit_word_t wh)
3405 {
3406     assert(!(_p &   ~0x3fL));
3407     assert(!(ih  &   ~0x1L));
3408     assert(!(x6  &  ~0x3fL));
3409     assert(!(b2  &   ~0x7L));
3410     assert(!(tag & ~0x1ffL));
3411     assert(!(wh  &   ~0x3L));
3412     TSTPRED(_p);
3413     inst((2L<<37)|(ih<<35)|(((tag>>7)&3L)<<33)|(x6<<27)|
3414          (b2<<13)|((tag&0x7fL)<<6)|(wh<<3)|_p, INST_B);
3415 }
3416
3417 static void
3418 _B8(jit_state_t *_jit, jit_word_t _p,
3419     jit_word_t x6)
3420 {
3421     assert(!(_p & ~0x3fL));
3422     assert(!(x6 & ~0x3fL));
3423     TSTPRED(_p);
3424     inst((x6<<27)|_p, INST_B);
3425 }
3426
3427 static void
3428 _B9(jit_state_t *_jit, jit_word_t _p,
3429     jit_word_t op, jit_word_t x6, jit_word_t im)
3430 {
3431     assert(!(_p &    ~0x3fL));
3432     assert(!(op &     ~0xfL));
3433     assert(!(x6 &    ~0x3fL));
3434     assert(!(im & ~0x1ffffL));
3435     TSTPRED(_p);
3436     inst((op<<37)|(((im>>20)&1L)<<36)|(x6<<27)|((im&0xffffL)<<6)|_p, INST_B);
3437 }
3438
3439 static void
3440 _X1(jit_state_t *_jit, jit_word_t _p,
3441     jit_word_t im)
3442 {
3443     jit_word_t          i41, i1, i20;
3444     assert(!(_p &               ~0x3fL));
3445     assert(im > -0x2000000000000000 && im <= 0x1fffffffffffffff);
3446     i41 = (im >> 22) & 0x1ffffffffffL;
3447     i1  = (im >> 21) &           0x1L;
3448     i20 =  im        &       0xfffffL;
3449     TSTPRED(_p);
3450     inst(i41, INST_L);
3451     inst((i1<<36)|(i20<<6)|_p, INST_X);
3452 }
3453
3454 static void
3455 _X2(jit_state_t *_jit, jit_word_t _p,
3456     jit_word_t r1, jit_word_t im)
3457 {
3458     jit_word_t          i1, i41, ic, i5, i9, i7;
3459     assert(!(_p & ~0x3fL));
3460     assert(!(r1 & ~0x7fL));
3461     i1  = (im >> 63) &           0x1L;
3462     i41 = (im >> 22) & 0x1ffffffffffL;
3463     ic  = (im >> 21) &           0x1L;
3464     i5  = (im >> 16) &          0x1fL;
3465     i9  = (im >>  7) &         0x1ffL;
3466     i7  =  im        &          0x7fL;
3467     TSTPRED(_p);
3468     TSTREG1(r1);
3469     inst(i41, INST_L);
3470     inst((6L<<37)|(i1<<36)|(i9<<27)|(i5<<22)|
3471          (ic<<21)|(i7<<13)|(r1<<6)|_p, INST_X);
3472     SETREG(r1);
3473 }
3474
3475 static void
3476 _X3x(jit_state_t *_jit, jit_word_t _p,
3477      jit_word_t op, jit_word_t d, jit_word_t wh,
3478      jit_word_t p, jit_word_t tp, jit_word_t im)
3479 {
3480     /* target64 = IP + ((i1 << 59 | imm39 << 20 | imm20b) << 4) */
3481     jit_word_t          i1, i41, i20;
3482     assert(!(_p &               ~0x3fL));
3483     assert(!(op &                ~0xfL));
3484     assert(!(d  &                ~0x1L));
3485     assert(!(wh &                ~0x3L));
3486     assert(!(p  &                ~0x1L));
3487     assert(!(tp &                ~0x7L));
3488     i1  = (im >> 61) &           0x1L;
3489     i41 = (im >> 22) & 0x1ffffffffffL;
3490     i20 =  im        &       0xfffffL;
3491     TSTPRED(_p);
3492     inst(i41, INST_L);
3493     inst((op<<37)|(i1<<36)|(d<<35)|(wh<<33)|
3494          (i20<<13)|(p<<12)|(tp<<6)|_p, INST_X);
3495 }
3496
3497 static void
3498 _X5(jit_state_t *_jit, jit_word_t _p,
3499     jit_word_t y, jit_word_t im)
3500 {
3501     jit_word_t          i41, i1, i20;
3502     assert(!(_p &               ~0x3fL));
3503     assert(im > -0x2000000000000000 && im <= 0x1fffffffffffffff);
3504     i41 = (im >> 22) & 0x1ffffffffffL;
3505     i1  = (im >> 21) &           0x1L;
3506     i20 =  im        &       0xfffffL;
3507     TSTPRED(_p);
3508     inst(i41, INST_L);
3509     inst((i1<<36)|(1L<<27)|(y<<26)|(i20<<6)|_p, INST_X);
3510 }
3511
3512 static void
3513 _nop(jit_state_t *_jit, jit_int32_t i0)
3514 {
3515     for (; i0 > 0; i0 -= 8) {
3516         NOP_M(0);
3517         sync();
3518     }
3519     assert(i0 == 0);
3520 }
3521
3522 static void
3523 _clzr(jit_state_t *_jit, jit_int32_t r0, jit_int32_t r1)
3524 {
3525     if (jit_cpu.clz)
3526         CLZ(r0, r1);
3527     else
3528         fallback_clz(r0, r1);
3529 }
3530
3531 static void
3532 _clor(jit_state_t *_jit, jit_int32_t r0, jit_int32_t r1)
3533 {
3534     if (jit_cpu.clz) {
3535         comr(r0, r1);
3536         clzr(r0, r0);
3537     }
3538     else
3539         fallback_clo(r0, r1);
3540 }
3541
3542 static void
3543 _ctor(jit_state_t *_jit, jit_int32_t r0, jit_int32_t r1)
3544 {
3545     if (jit_cpu.clz) {
3546         comr(r0, r1);
3547         ctzr(r0, r0);
3548     }
3549     else
3550         fallback_cto(r0, r1);
3551 }
3552
3553 static void
3554 _ctzr(jit_state_t *_jit, jit_int32_t r0, jit_int32_t r1)
3555 {
3556     jit_int32_t         t0, t1;
3557     if (jit_cpu.clz) {
3558         t0 = jit_get_reg(jit_class_gpr);
3559         t1 = jit_get_reg(jit_class_gpr);
3560         negr(rn(t0), r1);
3561         andr(rn(t0), rn(t0), r1);
3562         clzr(r0, rn(t0));
3563         xori(rn(t1), r0, __WORDSIZE - 1);
3564         movnr(r0, rn(t1), rn(t0));
3565         jit_unget_reg(t0);
3566         jit_unget_reg(t1);
3567     }
3568     else
3569         fallback_ctz(r0, r1);
3570 }
3571
3572 static void
3573 _movr(jit_state_t *_jit, jit_int32_t r0, jit_int32_t r1)
3574 {
3575     /* patch pushargr */
3576     if (r0 >= 120)
3577         r0 = _jitc->rout + (r0 - 120);
3578
3579     MOV(r0, r1);
3580 }
3581
3582 static void
3583 _movi(jit_state_t *_jit, jit_int32_t r0, jit_word_t i0)
3584 {
3585     /* patch pushargi */
3586     if (r0 >= 120)
3587         r0 = _jitc->rout + (r0 - 120);
3588
3589     if (i0 >= -2097152 && i0 <= 2097151)
3590         MOVI(r0, i0);
3591     else
3592         MOVL(r0, i0);
3593 }
3594
3595 static jit_word_t
3596 _movi_p(jit_state_t *_jit, jit_int32_t r0, jit_word_t i0)
3597 {
3598     jit_word_t          w;
3599     sync();
3600     w = _jit->pc.w;
3601     MOVL(r0, i0);
3602     return (w);
3603 }
3604
3605 static void
3606 _movnr(jit_state_t *_jit, jit_int32_t r0, jit_int32_t r1, jit_int32_t r2)
3607 {
3608     CMP_EQ(PR_6, PR_7, r2, GR_0);
3609     MOV_p(r0, r1, PR_7);
3610 }
3611
3612 static void
3613 _movzr(jit_state_t *_jit, jit_int32_t r0, jit_int32_t r1, jit_int32_t r2)
3614 {
3615     CMP_EQ(PR_6, PR_7, r2, GR_0);
3616     MOV_p(r0, r1, PR_6);
3617 }
3618
3619 static void
3620 _extr(jit_state_t *_jit,
3621       jit_int32_t r0, jit_int32_t r1, jit_word_t i0, jit_word_t i1)
3622 {
3623     assert(i0 >= 0 && i1 >= 1 && i0 + i1 <= __WORDSIZE);
3624     if ( i1 == __WORDSIZE)
3625         movr(r0, r1);
3626     else {
3627 #  if __BYTE_ORDER == __BIG_ENDIAN
3628         i0 = __WORDSIZE - (i0 + i1);
3629 #  endif
3630         EXTR(r0, r1, i0, i1 - 1);
3631     }
3632 }
3633
3634 static void
3635 _extr_u(jit_state_t *_jit,
3636         jit_int32_t r0, jit_int32_t r1, jit_word_t i0, jit_word_t i1)
3637 {
3638     assert(i0 >= 0 && i1 >= 1 && i0 + i1 <= __WORDSIZE);
3639     if (i1 == __WORDSIZE)
3640         movr(r0, r1);
3641     else {
3642 #  if __BYTE_ORDER == __BIG_ENDIAN
3643         i0 = __WORDSIZE - (i0 + i1);
3644 #  endif
3645         EXTR_U(r0, r1, i0, i1 - 1);
3646     }
3647 }
3648
3649 static void
3650 _depr(jit_state_t *_jit,
3651       jit_int32_t r0, jit_int32_t r1, jit_word_t i0, jit_word_t i1)
3652 {
3653     jit_int32_t         t0;
3654     jit_word_t          mask;
3655     assert(i0 >= 0 && i1 >= 1 && i0 + i1 <= __WORDSIZE);
3656     if (i1 == __WORDSIZE)
3657         movr(r0, r1);
3658     else {
3659 #  if __BYTE_ORDER == __BIG_ENDIAN
3660         i0 = __WORDSIZE - (i0 + i1);
3661 #  endif
3662         if (i1 <16)
3663             DEP(r0, r1, r0, 63 - i0, i1 - 1);
3664         else {
3665             t0 = jit_get_reg(jit_class_gpr);
3666             mask = ((1L << i1) - 1) << i0;
3667             movr(rn(t0), r0);
3668             DEP_Z(r0, r1, 63 - i0, i1 - 1);
3669             andi(rn(t0), rn(t0), ~mask);
3670             orr(r0, r0, rn(t0));
3671             jit_unget_reg(t0);
3672         }
3673     }
3674 }
3675
3676 static void
3677 _casx(jit_state_t *_jit, jit_int32_t r0, jit_int32_t r1,
3678       jit_int32_t r2, jit_int32_t r3, jit_word_t i0)
3679 {
3680     jit_int32_t         r1_reg, iscasi;
3681     if ((iscasi = (r1 == _NOREG))) {
3682         r1_reg = jit_get_reg(jit_class_gpr);
3683         r1 = rn(r1_reg);
3684         movi(r1, i0);
3685     }
3686     sync();
3687     MOV_M_ar_rn(AR_CCV, r2);
3688     CMPXCHG8_ACQ(r0, r1, r3);
3689     eqr(r0, r0, r2);
3690     if (iscasi)
3691         jit_unget_reg(r1_reg);
3692 }
3693
3694
3695 static void
3696 _bswapr_us(jit_state_t *_jit, jit_int32_t r0, jit_int32_t r1)
3697 {
3698         bswapr_ul(r0, r1);
3699         rshi_u(r0, r0, 48);
3700 }
3701
3702 static void
3703 _bswapr_ui(jit_state_t *_jit, jit_int32_t r0, jit_int32_t r1)
3704 {
3705         bswapr_ul(r0, r1);
3706         rshi_u(r0, r0, 32);
3707 }
3708
3709 static void
3710 _addi(jit_state_t *_jit, jit_int32_t r0, jit_int32_t r1, jit_word_t i0)
3711 {
3712     jit_int32_t         reg;
3713     if (i0 >= -8192 && i0 <= 8191)
3714         ADDS(r0, i0, r1);
3715     else if (!(r1 & ~3) && i0 >= -2097152 && i0 <= 2097151)
3716         ADDL(r1, i0, r0);
3717     else {
3718         reg = jit_get_reg(jit_class_gpr);
3719         movi(rn(reg), i0);
3720         addr(r0, r1, rn(reg));
3721         jit_unget_reg(reg);
3722     }
3723 }
3724
3725 static void
3726 _addcr(jit_state_t *_jit, jit_int32_t r0, jit_int32_t r1, jit_int32_t r2)
3727 {
3728     jit_int32_t         reg;
3729     if (jit_carry == _NOREG)
3730         jit_carry = jit_get_reg(jit_class_gpr);
3731     if (r0 == r1) {
3732         reg = jit_get_reg(jit_class_gpr);
3733         addr(rn(reg), r1, r2);
3734         ltr_u(rn(jit_carry), rn(reg), r1);
3735         movr(r0, rn(reg));
3736         jit_unget_reg(reg);
3737     }
3738     else {
3739         addr(r0, r1, r2);
3740         ltr_u(rn(jit_carry), r0, r1);
3741     }
3742 }
3743
3744 static void
3745 _addci(jit_state_t *_jit, jit_int32_t r0, jit_int32_t r1, jit_word_t i0)
3746 {
3747     jit_int32_t         reg;
3748     if (jit_carry == _NOREG)
3749         jit_carry = jit_get_reg(jit_class_gpr);
3750     if (r0 == r1) {
3751         reg = jit_get_reg(jit_class_gpr);
3752         addi(rn(reg), r1, i0);
3753         ltr_u(rn(jit_carry), rn(reg), r1);
3754         movr(r0, rn(reg));
3755         jit_unget_reg(reg);
3756     }
3757     else {
3758         addi(r0, r1, i0);
3759         ltr_u(rn(jit_carry), r0, r1);
3760     }
3761 }
3762
3763 static void
3764 _addxr(jit_state_t *_jit, jit_int32_t r0, jit_int32_t r1, jit_int32_t r2)
3765 {
3766     jit_int32_t         reg;
3767     assert(jit_carry != _NOREG);
3768     reg = jit_get_reg(jit_class_gpr);
3769     movr(rn(reg), rn(jit_carry));
3770     addcr(r0, r1, r2);
3771     addcr(r0, r0, rn(reg));
3772     jit_unget_reg(reg);
3773 }
3774
3775 static void
3776 _addxi(jit_state_t *_jit, jit_int32_t r0, jit_int32_t r1, jit_word_t i0)
3777 {
3778     jit_int32_t         reg;
3779     assert(jit_carry != _NOREG);
3780     reg = jit_get_reg(jit_class_gpr);
3781     movr(rn(reg), rn(jit_carry));
3782     addci(r0, r1, i0);
3783     addcr(r0, r0, rn(reg));
3784     jit_unget_reg(reg);
3785 }
3786
3787 static void
3788 _subi(jit_state_t *_jit, jit_int32_t r0, jit_int32_t r1, jit_word_t i0)
3789 {
3790     jit_int32_t         reg;
3791     reg = jit_get_reg(jit_class_gpr);
3792     movi(rn(reg), i0);
3793     subr(r0, r1, rn(reg));
3794     jit_unget_reg(reg);
3795 }
3796
3797 static void
3798 _subcr(jit_state_t *_jit, jit_int32_t r0, jit_int32_t r1, jit_int32_t r2)
3799 {
3800     jit_int32_t         reg;
3801     if (jit_carry == _NOREG)
3802         jit_carry = jit_get_reg(jit_class_gpr);
3803     if (r0 == r1) {
3804         reg = jit_get_reg(jit_class_gpr);
3805         subr(rn(reg), r1, r2);
3806         ltr_u(rn(jit_carry), r1, rn(reg));
3807         movr(r0, rn(reg));
3808         jit_unget_reg(reg);
3809     }
3810     else {
3811         subr(r0, r1, r2);
3812         ltr_u(rn(jit_carry), r1, r0);
3813     }
3814 }
3815
3816 static void
3817 _subci(jit_state_t *_jit, jit_int32_t r0, jit_int32_t r1, jit_word_t i0)
3818 {
3819     jit_int32_t         reg;
3820     if (jit_carry == _NOREG)
3821         jit_carry = jit_get_reg(jit_class_gpr);
3822     if (r0 == r1) {
3823         reg = jit_get_reg(jit_class_gpr);
3824         addi(rn(reg), r1, -i0);
3825         ltr_u(rn(jit_carry), r1, rn(reg));
3826         movr(r0, rn(reg));
3827         jit_unget_reg(reg);
3828     }
3829     else {
3830         addi(r0, r1, -i0);
3831         ltr_u(rn(jit_carry), r1, r0);
3832     }
3833 }
3834
3835 static void
3836 _subxr(jit_state_t *_jit, jit_int32_t r0, jit_int32_t r1, jit_int32_t r2)
3837 {
3838     jit_int32_t         reg;
3839     assert(jit_carry != _NOREG);
3840     reg = jit_get_reg(jit_class_gpr);
3841     movr(rn(reg), rn(jit_carry));
3842     subcr(r0, r1, r2);
3843     subcr(r0, r0, rn(reg));
3844     jit_unget_reg(reg);
3845 }
3846
3847 static void
3848 _subxi(jit_state_t *_jit, jit_int32_t r0, jit_int32_t r1, jit_word_t i0)
3849 {
3850     jit_int32_t         reg;
3851     assert(jit_carry != _NOREG);
3852     reg = jit_get_reg(jit_class_gpr);
3853     movr(rn(reg), rn(jit_carry));
3854     subci(r0, r1, i0);
3855     subcr(r0, r0, rn(reg));
3856     jit_unget_reg(reg);
3857 }
3858
3859 static void
3860 _rsbi(jit_state_t *_jit, jit_int32_t r0, jit_int32_t r1, jit_word_t i0)
3861 {
3862     jit_int32_t         reg;
3863     if (i0 >= -128 && i0 <= 127)
3864         SUBI(r0, i0, r1);
3865     else if (!(r1 & ~3) && i0 >= -2097151 && i0 <= 2097152)
3866         ADDL(r1, -i0, r0);
3867     else {
3868         reg = jit_get_reg(jit_class_gpr);
3869         movi(rn(reg), i0);
3870         subr(r0, rn(reg), r1);
3871         jit_unget_reg(reg);
3872     }
3873 }
3874
3875 static void
3876 _mulr(jit_state_t *_jit, jit_int32_t r0, jit_int32_t r1, jit_int32_t r2)
3877 {
3878     jit_int32_t         f0, f1;
3879     f0 = jit_get_reg(jit_class_fpr);
3880     f1 = jit_get_reg(jit_class_fpr);
3881     SETF_SIG(rn(f0), r1);
3882     SETF_SIG(rn(f1), r2);
3883     XMPY_L(rn(f0), rn(f0), rn(f1));
3884     GETF_SIG(r0, rn(f0));
3885     jit_unget_reg(f0);
3886     jit_unget_reg(f1);
3887 }
3888
3889 static void
3890 _muli(jit_state_t *_jit, jit_int32_t r0, jit_int32_t r1, jit_word_t i0)
3891 {
3892     jit_int32_t         reg;
3893     reg = jit_get_reg(jit_class_gpr);
3894     movi(rn(reg), i0);
3895     mulr(r0, r1, rn(reg));
3896     jit_unget_reg(reg);
3897 }
3898
3899 static void
3900 _hmuli(jit_state_t *_jit, jit_int32_t r0, jit_int32_t r1, jit_word_t i0)
3901 {
3902     jit_int32_t         reg;
3903     reg = jit_get_reg(jit_class_gpr);
3904     movi(rn(reg), i0);
3905     hmulr(r0, r1, rn(reg));
3906     jit_unget_reg(reg);
3907 }
3908
3909 static void
3910 _hmuli_u(jit_state_t *_jit, jit_int32_t r0, jit_int32_t r1, jit_word_t i0)
3911 {
3912     jit_int32_t         reg;
3913     reg = jit_get_reg(jit_class_gpr);
3914     movi(rn(reg), i0);
3915     hmulr_u(r0, r1, rn(reg));
3916     jit_unget_reg(reg);
3917 }
3918
3919 #if !defined(__GNUC__)
3920 static long
3921 __divdi3(long u, long v)
3922 {
3923     return (u / v);
3924 }
3925
3926 static unsigned long
3927 __udivdi3(unsigned long u, unsigned long v)
3928 {
3929     return (u / v);
3930 }
3931
3932 static long
3933 __moddi3(long u, long v)
3934 {
3935     return (u % v);
3936 }
3937
3938 static unsigned long
3939 __umoddi3(unsigned long u, unsigned long v)
3940 {
3941     return (u % v);
3942 }
3943 #endif
3944
3945 static void
3946 _divr(jit_state_t *_jit, jit_int32_t r0, jit_int32_t r1, jit_int32_t r2)
3947 {
3948     MOV(_jitc->rout, r1);
3949     MOV(_jitc->rout + 1, r2);
3950     calli((jit_word_t)__divdi3);
3951     MOV(r0, GR_8);
3952 }
3953
3954 static void
3955 _divi(jit_state_t *_jit, jit_int32_t r0, jit_int32_t r1, jit_word_t i0)
3956 {
3957     jit_int32_t         reg;
3958     switch (i0) {
3959         case 1:
3960             movr(r0, r1);
3961             return;
3962         case -1:
3963             negr(r0, r1);
3964             return;
3965         default:
3966             if (i0 > 0 && !(i0 & (i0 - 1))) {
3967                 movr(r0, r1);
3968                 rshi(r0, r0, ffsl(i0) - 1);
3969                 return;
3970             }
3971             break;
3972     }
3973     reg = jit_get_reg(jit_class_gpr);
3974     movi(rn(reg), i0);
3975     divr(r0, r1, rn(reg));
3976     jit_unget_reg(reg);
3977 }
3978
3979 static void
3980 _divr_u(jit_state_t *_jit, jit_int32_t r0, jit_int32_t r1, jit_int32_t r2)
3981 {
3982     MOV(_jitc->rout, r1);
3983     MOV(_jitc->rout + 1, r2);
3984     calli((jit_word_t)__udivdi3);
3985     MOV(r0, GR_8);
3986 }
3987
3988 static void
3989 _divi_u(jit_state_t *_jit, jit_int32_t r0, jit_int32_t r1, jit_word_t i0)
3990 {
3991     jit_int32_t         reg;
3992     if (i0 == 1) {
3993         movr(r0, r1);
3994         return;
3995     }
3996     else if (i0 > 0 && !(i0 & (i0 - 1))) {
3997         movr(r0, r1);
3998         rshi_u(r0, r0, ffsl(i0) - 1);
3999         return;
4000     }
4001     reg = jit_get_reg(jit_class_gpr);
4002     movi(rn(reg), i0);
4003     divr_u(r0, r1, rn(reg));
4004     jit_unget_reg(reg);
4005 }
4006
4007 static void
4008 _remr(jit_state_t *_jit, jit_int32_t r0, jit_int32_t r1, jit_int32_t r2)
4009 {
4010     MOV(_jitc->rout, r1);
4011     MOV(_jitc->rout + 1, r2);
4012     calli((jit_word_t)__moddi3);
4013     MOV(r0, GR_8);
4014 }
4015
4016 static void
4017 _remi(jit_state_t *_jit, jit_int32_t r0, jit_int32_t r1, jit_word_t i0)
4018 {
4019     jit_int32_t         reg;
4020     if (i0 == 1 || i0 == -1) {
4021         MOV(r0, GR_0);
4022         return;
4023     }
4024     reg = jit_get_reg(jit_class_gpr);
4025     movi(rn(reg), i0);
4026     remr(r0, r1, rn(reg));
4027     jit_unget_reg(reg);
4028 }
4029
4030 static void
4031 _remr_u(jit_state_t *_jit, jit_int32_t r0, jit_int32_t r1, jit_int32_t r2)
4032 {
4033     MOV(_jitc->rout, r1);
4034     MOV(_jitc->rout + 1, r2);
4035     calli((jit_word_t)__umoddi3);
4036     MOV(r0, GR_8);
4037 }
4038
4039 static void
4040 _remi_u(jit_state_t *_jit, jit_int32_t r0, jit_int32_t r1, jit_word_t i0)
4041 {
4042     jit_int32_t         reg;
4043     if (i0 == 1) {
4044         MOV(r0, GR_0);
4045         return;
4046     }
4047     else if (i0 > 0 && !(i0 & (i0 - 1))) {
4048         andi(r0, r1, i0 - 1);
4049         return;
4050     }
4051     reg = jit_get_reg(jit_class_gpr);
4052     movi(rn(reg), i0);
4053     remr_u(r0, r1, rn(reg));
4054     jit_unget_reg(reg);
4055 }
4056
4057 static void
4058 _mulh(jit_state_t *_jit,
4059       jit_int32_t r0, jit_int32_t r1, jit_int32_t r2, jit_bool_t sign)
4060 {
4061     jit_int32_t         f0, f1;
4062     f0 = jit_get_reg(jit_class_fpr);
4063     f1 = jit_get_reg(jit_class_fpr);
4064     SETF_SIG(rn(f0), r1);
4065     SETF_SIG(rn(f1), r2);
4066     if (sign)
4067         XMPY_H(rn(f0), rn(f0), rn(f1));
4068     else
4069         XMPY_HU(rn(f0), rn(f0), rn(f1));
4070     GETF_SIG(r0, rn(f0));
4071     jit_unget_reg(f0);
4072     jit_unget_reg(f1);
4073 }
4074
4075 static void
4076 _iqmulr(jit_state_t *_jit, jit_int32_t r0, jit_int32_t r1,
4077         jit_int32_t r2, jit_int32_t r3, jit_bool_t sign)
4078 {
4079     jit_int32_t         reg;
4080     if (r0 == r2 || r0 == r3) {
4081         reg = jit_get_reg(jit_class_gpr);
4082         mulr(rn(reg), r2, r3);
4083     }
4084     else
4085         mulr(r0, r2, r3);
4086     mulh(r1, r2, r3, sign);
4087     if (r0 == r2 || r0 == r3) {
4088         movr(r0, rn(reg));
4089         jit_unget_reg(reg);
4090     }
4091 }
4092
4093 static void
4094 _iqmuli(jit_state_t *_jit, jit_int32_t r0, jit_int32_t r1,
4095         jit_int32_t r2, jit_word_t i0, jit_bool_t sign)
4096 {
4097     jit_int32_t         reg;
4098     reg = jit_get_reg(jit_class_gpr);
4099     movi(rn(reg), i0);
4100     iqmulr(r0, r1, r2, rn(reg), sign);
4101     jit_unget_reg(reg);
4102 }
4103
4104 static void
4105 _iqdivr(jit_state_t *_jit, jit_int32_t r0, jit_int32_t r1,
4106         jit_int32_t r2, jit_int32_t r3, jit_bool_t sign)
4107 {
4108     jit_int32_t         sv0, rg0;
4109     jit_int32_t         sv1, rg1;
4110
4111     if (r0 == r2 || r0 == r3) {
4112         sv0 = jit_get_reg(jit_class_gpr);
4113         rg0 = rn(sv0);
4114     }
4115     else
4116         rg0 = r0;
4117     if (r1 == r2 || r1 == r3) {
4118         sv1 = jit_get_reg(jit_class_gpr);
4119         rg1 = rn(sv1);
4120     }
4121     else
4122         rg1 = r1;
4123
4124     if (sign)
4125         divr(rg0, r2, r3);
4126     else
4127         divr_u(rg0, r2, r3);
4128     mulr(rg1, r3, rg0);
4129     subr(rg1, r2, rg1);
4130     if (rg0 != r0) {
4131         movr(r0, rg0);
4132         jit_unget_reg(sv0);
4133     }
4134     if (rg1 != r1) {
4135         movr(r1, rg1);
4136         jit_unget_reg(sv1);
4137     }
4138 }
4139
4140 static void
4141 _iqdivi(jit_state_t *_jit, jit_int32_t r0, jit_int32_t r1,
4142         jit_int32_t r2, jit_word_t i0, jit_bool_t sign)
4143 {
4144     jit_int32_t         reg;
4145     reg = jit_get_reg(jit_class_gpr);
4146     movi(rn(reg), i0);
4147     iqdivr(r0, r1, r2, rn(reg), sign);
4148     jit_unget_reg(reg);
4149 }
4150
4151 static void
4152 _andi(jit_state_t *_jit, jit_int32_t r0, jit_int32_t r1, jit_word_t i0)
4153 {
4154     jit_int32_t         reg;
4155     if (i0 >= -128 && i0 <= 127)
4156         ANDI(r0, i0, r1);
4157     else if (~i0 >= -128 && ~i0 <= 127)
4158         ANDCMI(r0, ~i0, r1);
4159     else {
4160         reg = jit_get_reg(jit_class_gpr);
4161         movi(rn(reg), i0);
4162         andr(r0, r1, rn(reg));
4163         jit_unget_reg(reg);
4164     }
4165 }
4166
4167 static void
4168 _ori(jit_state_t *_jit, jit_int32_t r0, jit_int32_t r1, jit_word_t i0)
4169 {
4170     jit_int32_t         reg;
4171     if (i0 >= -128 && i0 <= 127)
4172         ORI(r0, i0, r1);
4173     else {
4174         reg = jit_get_reg(jit_class_gpr);
4175         movi(rn(reg), i0);
4176         orr(r0, r1, rn(reg));
4177         jit_unget_reg(reg);
4178     }
4179 }
4180
4181 static void
4182 _xori(jit_state_t *_jit, jit_int32_t r0, jit_int32_t r1, jit_word_t i0)
4183 {
4184     jit_int32_t         reg;
4185     if (i0 >= -128 && i0 <= 127)
4186         XORI(r0, i0, r1);
4187     else {
4188         reg = jit_get_reg(jit_class_gpr);
4189         movi(rn(reg), i0);
4190         xorr(r0, r1, rn(reg));
4191         jit_unget_reg(reg);
4192     }
4193 }
4194
4195 static void
4196 _lshi(jit_state_t *_jit, jit_int32_t r0, jit_int32_t r1, jit_word_t i0)
4197 {
4198     jit_int32_t         reg;
4199     reg = jit_get_reg(jit_class_gpr);
4200     movi(rn(reg), i0);
4201     lshr(r0, r1, rn(reg));
4202     jit_unget_reg(reg);
4203 }
4204
4205 static void
4206 _rshi(jit_state_t *_jit, jit_int32_t r0, jit_int32_t r1, jit_word_t i0)
4207 {
4208     jit_int32_t         reg;
4209     reg = jit_get_reg(jit_class_gpr);
4210     movi(rn(reg), i0);
4211     rshr(r0, r1, rn(reg));
4212     jit_unget_reg(reg);
4213 }
4214
4215 static void
4216 _rshi_u(jit_state_t *_jit, jit_int32_t r0, jit_int32_t r1, jit_word_t i0)
4217 {
4218     jit_int32_t         reg;
4219     reg = jit_get_reg(jit_class_gpr);
4220     movi(rn(reg), i0);
4221     rshr_u(r0, r1, rn(reg));
4222     jit_unget_reg(reg);
4223 }
4224
4225 static void
4226 _xlshr(jit_state_t *_jit, jit_bool_t sign,
4227        jit_int32_t r0, jit_int32_t r1, jit_int32_t r2, jit_int32_t r3)
4228 {
4229     jit_bool_t          branch;
4230     jit_word_t          over, zero, done, done_over;
4231     jit_int32_t         t0, s0, t1, s1, t2, s2, t3, s3;
4232     s0 = jit_get_reg(jit_class_gpr);
4233     t0 = rn(s0);
4234     if (r0 == r2 || r1 == r2) {
4235         s2 = jit_get_reg(jit_class_gpr);
4236         t2 = rn(s2);
4237         movr(t2, r2);
4238     }
4239     else
4240         t2 = r2;
4241     if (r0 == r3 || r1 == r3) {
4242         s3 = jit_get_reg(jit_class_gpr);
4243         t3 = rn(s3);
4244         movr(t3, r3);
4245     }
4246     else
4247         t3 = r3;
4248     if ((s1 = jit_get_reg(jit_class_gpr|jit_class_nospill|jit_class_chk))) {
4249         t1 = rn(s1);
4250         branch = 0;
4251     }
4252     else
4253         branch = 1;
4254     rsbi(t0, t3, __WORDSIZE);
4255     lshr(r0, t2, t3);
4256     if (sign)
4257         rshr(r1, t2, t0);
4258     else
4259         rshr_u(r1, t2, t0);
4260     if (branch) {
4261         zero = beqi(_jit->pc.w, t3, 0);
4262         over = beqi(_jit->pc.w, t3, __WORDSIZE);
4263         done = jmpi(_jit->pc.w);
4264         patch_at(jit_code_beqi, over, _jit->pc.w);
4265         /* overflow */
4266         movi(r0, 0);
4267         done_over = jmpi(_jit->pc.w);
4268         /* zero */
4269         patch_at(jit_code_beqi, zero, _jit->pc.w);
4270         if (sign)
4271             rshi(r1, t2, __WORDSIZE - 1);
4272         else
4273             movi(r1, 0);
4274         patch_at(jit_code_jmpi, done, _jit->pc.w);
4275         patch_at(jit_code_jmpi, done_over, _jit->pc.w);
4276     }
4277     else {
4278         if (sign)
4279             rshi(t0, t2, __WORDSIZE - 1);
4280         else
4281             movi(t0, 0);
4282         /* zero? */
4283         movzr(r1, t0, t3);
4284         /* Branchless but 4 bytes longer than branching fallback */
4285         if (sign)
4286             movi(t0, 0);
4287         /* overflow? */
4288         eqi(t1, t3, __WORDSIZE);
4289         movnr(r0, t0, t1);
4290         jit_unget_reg(s1);
4291     }
4292     jit_unget_reg(s0);
4293     if (t2 != r2)
4294         jit_unget_reg(s2);
4295     if (t3 != r3)
4296         jit_unget_reg(s3);
4297 }
4298
4299 static void
4300 _xlshi(jit_state_t *_jit, jit_bool_t sign,
4301        jit_int32_t r0, jit_int32_t r1, jit_int32_t r2, jit_word_t i0)
4302 {
4303     if (i0 == 0) {
4304         movr(r0, r2);
4305         if (sign)
4306             rshi(r1, r2, __WORDSIZE - 1);
4307         else
4308             movi(r1, 0);
4309     }
4310     else if (i0 == __WORDSIZE) {
4311         movr(r1, r2);
4312         movi(r0, 0);
4313     }
4314     else {
4315         assert((jit_uword_t)i0 <= __WORDSIZE);
4316         if (sign)
4317             rshi(r1, r2, __WORDSIZE - i0);
4318         else
4319             rshi_u(r1, r2, __WORDSIZE - i0);
4320         lshi(r0, r2, i0);
4321     }
4322 }
4323
4324 static void
4325 _xrshr(jit_state_t *_jit, jit_bool_t sign,
4326        jit_int32_t r0, jit_int32_t r1, jit_int32_t r2, jit_int32_t r3)
4327 {
4328     jit_bool_t          branch;
4329     jit_word_t          over, zero, done, done_over;
4330     jit_int32_t         t0, s0, t1, s1, t2, s2, t3, s3;
4331     s0 = jit_get_reg(jit_class_gpr);
4332     t0 = rn(s0);
4333     if (r0 == r2 || r1 == r2) {
4334         s2 = jit_get_reg(jit_class_gpr);
4335         t2 = rn(s2);
4336         movr(t2, r2);
4337     }
4338     else
4339         t2 = r2;
4340     if (r0 == r3 || r1 == r3) {
4341         s3 = jit_get_reg(jit_class_gpr);
4342         t3 = rn(s3);
4343         movr(t3, r3);
4344     }
4345     else
4346         t3 = r3;
4347     if ((s1 = jit_get_reg(jit_class_gpr|jit_class_nospill|jit_class_chk))) {
4348         t1 = rn(s1);
4349         branch = 0;
4350     }
4351     else
4352         branch = 1;
4353     rsbi(t0, t3, __WORDSIZE);
4354     if (sign)
4355         rshr(r0, t2, t3);
4356     else
4357         rshr_u(r0, t2, t3);
4358     lshr(r1, t2, t0);
4359     if (branch) {
4360         zero = beqi(_jit->pc.w, t3, 0);
4361         over = beqi(_jit->pc.w, t3, __WORDSIZE);
4362         done = jmpi(_jit->pc.w);
4363         patch_at(jit_code_beqi, over, _jit->pc.w);
4364         /* underflow */
4365         if (sign)
4366             rshi(r0, t2, __WORDSIZE - 1);
4367         else
4368             movi(r0, 0);
4369         done_over = jmpi(_jit->pc.w);
4370         /* zero */
4371         patch_at(jit_code_beqi, zero, _jit->pc.w);
4372         movi(r1, 0);
4373         patch_at(jit_code_jmpi, done, _jit->pc.w);
4374         patch_at(jit_code_jmpi, done_over, _jit->pc.w);
4375         jit_unget_reg(s1);
4376     }
4377     else {
4378         /* zero? */
4379         movi(t0, 0);
4380         movzr(r1, t0, t3);
4381         /* underflow? */
4382         eqi(t1, t3, __WORDSIZE);
4383         if (sign)
4384             rshi(t0, t2, __WORDSIZE - 1);
4385         movnr(r0, t0, t1);
4386         jit_unget_reg(s1);
4387     }
4388     jit_unget_reg(s0);
4389     if (t2 != r2)
4390         jit_unget_reg(s2);
4391     if (t3 != r3)
4392         jit_unget_reg(s3);
4393 }
4394
4395 static void
4396 _xrshi(jit_state_t *_jit, jit_bool_t sign,
4397        jit_int32_t r0, jit_int32_t r1, jit_int32_t r2, jit_word_t i0)
4398 {
4399     if (i0 == 0) {
4400         movr(r0, r2);
4401         movi(r1, 0);
4402     }
4403     else if (i0 == __WORDSIZE) {
4404         movr(r1, r2);
4405         if (sign)
4406             rshi(r0, r2, __WORDSIZE - 1);
4407         else
4408             movi(r0, 0);
4409     }
4410     else {
4411         assert((jit_uword_t)i0 <= __WORDSIZE);
4412         lshi(r1, r2, __WORDSIZE - i0);
4413         if (sign)
4414             rshi(r0, r2, i0);
4415         else
4416             rshi_u(r0, r2, i0);
4417     }
4418 }
4419
4420 static void
4421 _ltr(jit_state_t *_jit, jit_int32_t r0, jit_int32_t r1, jit_int32_t r2)
4422 {
4423     CMP_LT(PR_6, PR_7, r1, r2);
4424     MOVI_p(r0, 1, PR_6);
4425     MOV_p(r0, GR_0, PR_7);
4426 }
4427
4428 static void
4429 _lti(jit_state_t *_jit, jit_int32_t r0, jit_int32_t r1, jit_word_t i0)
4430 {
4431     jit_int32_t         reg;
4432     if (i0 >= -127 && i0 <= 128)
4433         CMPI_LT(PR_7, PR_6, i0 - 1, r1);
4434     else {
4435         reg = jit_get_reg(jit_class_gpr);
4436         movi(rn(reg), i0);
4437         CMP_LT(PR_6, PR_7, r1, rn(reg));
4438         jit_unget_reg(reg);
4439     }
4440     MOVI_p(r0, 1, PR_6);
4441     MOV_p(r0, GR_0, PR_7);
4442 }
4443
4444 static void
4445 _ltr_u(jit_state_t *_jit, jit_int32_t r0, jit_int32_t r1, jit_int32_t r2)
4446 {
4447     CMP_LTU(PR_6, PR_7, r1, r2);
4448     MOVI_p(r0, 1, PR_6);
4449     MOV_p(r0, GR_0, PR_7);
4450 }
4451
4452 static void
4453 _lti_u(jit_state_t *_jit, jit_int32_t r0, jit_int32_t r1, jit_word_t i0)
4454 {
4455     jit_int32_t         reg;
4456     if (i0 >= -127 && i0 <= 128)
4457         CMPI_LTU(PR_7, PR_6, i0 - 1, r1);
4458     else {
4459         reg = jit_get_reg(jit_class_gpr);
4460         movi(rn(reg), i0);
4461         CMP_LTU(PR_6, PR_7, r1, rn(reg));
4462         jit_unget_reg(reg);
4463     }
4464     MOVI_p(r0, 1, PR_6);
4465     MOV_p(r0, GR_0, PR_7);
4466 }
4467
4468 static void
4469 _ler(jit_state_t *_jit, jit_int32_t r0, jit_int32_t r1, jit_int32_t r2)
4470 {
4471     CMP_LT(PR_6, PR_7, r2, r1);
4472     MOV_p(r0, GR_0, PR_6);
4473     MOVI_p(r0, 1, PR_7);
4474 }
4475
4476 static void
4477 _lei(jit_state_t *_jit, jit_int32_t r0, jit_int32_t r1, jit_word_t i0)
4478 {
4479     jit_int32_t         reg;
4480     reg = jit_get_reg(jit_class_gpr);
4481     movi(rn(reg), i0);
4482     ler(r0, r1, rn(reg));
4483     jit_unget_reg(reg);
4484 }
4485
4486 static void
4487 _ler_u(jit_state_t *_jit, jit_int32_t r0, jit_int32_t r1, jit_int32_t r2)
4488 {
4489     CMP_LTU(PR_6, PR_7, r2, r1);
4490     MOV_p(r0, GR_0, PR_6);
4491     MOVI_p(r0, 1, PR_7);
4492 }
4493
4494 static void
4495 _lei_u(jit_state_t *_jit, jit_int32_t r0, jit_int32_t r1, jit_word_t i0)
4496 {
4497     jit_int32_t         reg;
4498     reg = jit_get_reg(jit_class_gpr);
4499     movi(rn(reg), i0);
4500     ler_u(r0, r1, rn(reg));
4501     jit_unget_reg(reg);
4502 }
4503
4504 static void
4505 _eqr(jit_state_t *_jit, jit_int32_t r0, jit_int32_t r1, jit_int32_t r2)
4506 {
4507     CMP_EQ(PR_6, PR_7, r1, r2);
4508     MOVI_p(r0, 1, PR_6);
4509     MOV_p(r0, GR_0, PR_7);
4510 }
4511
4512 static void
4513 _eqi(jit_state_t *_jit, jit_int32_t r0, jit_int32_t r1, jit_word_t i0)
4514 {
4515     jit_int32_t         reg;
4516     if (i0 >= -128 && i0 <= 127)
4517         CMPI_EQ(PR_6, PR_7, i0, r1);
4518     else {
4519         reg = jit_get_reg(jit_class_gpr);
4520         movi(rn(reg), i0);
4521         CMP_EQ(PR_6, PR_7, r1, rn(reg));
4522         jit_unget_reg(reg);
4523     }
4524     MOVI_p(r0, 1, PR_6);
4525     MOV_p(r0, GR_0, PR_7);
4526 }
4527
4528 static void
4529 _ger(jit_state_t *_jit, jit_int32_t r0, jit_int32_t r1, jit_int32_t r2)
4530 {
4531     CMP_LT(PR_6, PR_7, r1, r2);
4532     MOV_p(r0, GR_0, PR_6);
4533     MOVI_p(r0, 1, PR_7);
4534 }
4535
4536 static void
4537 _gei(jit_state_t *_jit, jit_int32_t r0, jit_int32_t r1, jit_word_t i0)
4538 {
4539     jit_int32_t         reg;
4540     if (i0 >= -127 && i0 <= 128)
4541         CMPI_LT(PR_7, PR_6, i0 - 1, r1);
4542     else {
4543         reg = jit_get_reg(jit_class_gpr);
4544         movi(rn(reg), i0);
4545         CMP_LT(PR_6, PR_7, r1, rn(reg));
4546         jit_unget_reg(reg);
4547     }
4548     MOV_p(r0, GR_0, PR_6);
4549     MOVI_p(r0, 1, PR_7);
4550 }
4551
4552 static void
4553 _ger_u(jit_state_t *_jit, jit_int32_t r0, jit_int32_t r1, jit_int32_t r2)
4554 {
4555     CMP_LTU(PR_6, PR_7, r1, r2);
4556     MOV_p(r0, GR_0, PR_6);
4557     MOVI_p(r0, 1, PR_7);
4558 }
4559
4560 static void
4561 _gei_u(jit_state_t *_jit, jit_int32_t r0, jit_int32_t r1, jit_word_t i0)
4562 {
4563     jit_int32_t         reg;
4564     if (i0 >= -127 && i0 <= 128)
4565         CMPI_LTU(PR_7, PR_6, i0 - 1, r1);
4566     else {
4567         reg = jit_get_reg(jit_class_gpr);
4568         movi(rn(reg), i0);
4569         CMP_LTU(PR_6, PR_7, r1, rn(reg));
4570         jit_unget_reg(reg);
4571     }
4572     MOV_p(r0, GR_0, PR_6);
4573     MOVI_p(r0, 1, PR_7);
4574 }
4575
4576 static void
4577 _gtr(jit_state_t *_jit, jit_int32_t r0, jit_int32_t r1, jit_int32_t r2)
4578 {
4579     CMP_LT(PR_6, PR_7, r2, r1);
4580     MOVI_p(r0, 1, PR_6);
4581     MOV_p(r0, GR_0, PR_7);
4582 }
4583
4584 static void
4585 _gti(jit_state_t *_jit, jit_int32_t r0, jit_int32_t r1, jit_word_t i0)
4586 {
4587     jit_int32_t         reg;
4588     reg = jit_get_reg(jit_class_gpr);
4589     movi(rn(reg), i0);
4590     gtr(r0, r1, rn(reg));
4591     jit_unget_reg(reg);
4592 }
4593
4594 static void
4595 _gtr_u(jit_state_t *_jit, jit_int32_t r0, jit_int32_t r1, jit_int32_t r2)
4596 {
4597     CMP_LTU(PR_6, PR_7, r2, r1);
4598     MOVI_p(r0, 1, PR_6);
4599     MOV_p(r0, GR_0, PR_7);
4600 }
4601
4602 static void
4603 _gti_u(jit_state_t *_jit, jit_int32_t r0, jit_int32_t r1, jit_word_t i0)
4604 {
4605     jit_int32_t         reg;
4606     reg = jit_get_reg(jit_class_gpr);
4607     movi(rn(reg), i0);
4608     gtr_u(r0, r1, rn(reg));
4609     jit_unget_reg(reg);
4610 }
4611
4612 static void
4613 _ner(jit_state_t *_jit, jit_int32_t r0, jit_int32_t r1, jit_int32_t r2)
4614 {
4615     CMP_EQ(PR_6, PR_7, r1, r2);
4616     MOV_p(r0, GR_0, PR_6);
4617     MOVI_p(r0, 1, PR_7);
4618 }
4619
4620 static void
4621 _nei(jit_state_t *_jit, jit_int32_t r0, jit_int32_t r1, jit_word_t i0)
4622 {
4623     jit_int32_t         reg;
4624     if (i0 >= -128 && i0 <= 127)
4625         CMPI_EQ(PR_6, PR_7, i0, r1);
4626     else {
4627         reg = jit_get_reg(jit_class_gpr);
4628         movi(rn(reg), i0);
4629         CMP_EQ(PR_6, PR_7, r1, rn(reg));
4630         jit_unget_reg(reg);
4631     }
4632     MOV_p(r0, GR_0, PR_6);
4633     MOVI_p(r0, 1, PR_7);
4634 }
4635
4636 static void
4637 _ldr_c(jit_state_t *_jit, jit_int32_t r0, jit_int32_t r1)
4638 {
4639     ldr_uc(r0, r1);
4640     extr_c(r0, r0);
4641 }
4642
4643 static void
4644 _ldi_c(jit_state_t *_jit, jit_int32_t r0, jit_word_t i0)
4645 {
4646     jit_int32_t         reg;
4647     reg = jit_get_reg(jit_class_gpr);
4648     movi(rn(reg), i0);
4649     ldr_c(r0, rn(reg));
4650     jit_unget_reg(reg);
4651 }
4652
4653 static void
4654 _ldi_uc(jit_state_t *_jit, jit_int32_t r0, jit_word_t i0)
4655 {
4656     jit_int32_t         reg;
4657     reg = jit_get_reg(jit_class_gpr);
4658     movi(rn(reg), i0);
4659     ldr_uc(r0, rn(reg));
4660     jit_unget_reg(reg);
4661 }
4662
4663 static void
4664 _ldr_s(jit_state_t *_jit, jit_int32_t r0, jit_int32_t r1)
4665 {
4666     ldr_us(r0, r1);
4667     extr_s(r0, r0);
4668 }
4669
4670 static void
4671 _ldi_s(jit_state_t *_jit, jit_int32_t r0, jit_word_t i0)
4672 {
4673     jit_int32_t         reg;
4674     reg = jit_get_reg(jit_class_gpr);
4675     movi(rn(reg), i0);
4676     ldr_s(r0, rn(reg));
4677     jit_unget_reg(reg);
4678 }
4679
4680 static void
4681 _ldi_us(jit_state_t *_jit, jit_int32_t r0, jit_word_t i0)
4682 {
4683     jit_int32_t         reg;
4684     reg = jit_get_reg(jit_class_gpr);
4685     movi(rn(reg), i0);
4686     ldr_us(r0, rn(reg));
4687     jit_unget_reg(reg);
4688 }
4689
4690 static void
4691 _ldr_i(jit_state_t *_jit, jit_int32_t r0, jit_int32_t r1)
4692 {
4693     ldr_ui(r0, r1);
4694     extr_i(r0, r0);
4695 }
4696
4697 static void
4698 _ldi_i(jit_state_t *_jit, jit_int32_t r0, jit_word_t i0)
4699 {
4700     jit_int32_t         reg;
4701     reg = jit_get_reg(jit_class_gpr);
4702     movi(rn(reg), i0);
4703     ldr_i(r0, rn(reg));
4704     jit_unget_reg(reg);
4705 }
4706
4707 static void
4708 _ldi_ui(jit_state_t *_jit, jit_int32_t r0, jit_word_t i0)
4709 {
4710     jit_int32_t         reg;
4711     reg = jit_get_reg(jit_class_gpr);
4712     movi(rn(reg), i0);
4713     ldr_ui(r0, rn(reg));
4714     jit_unget_reg(reg);
4715 }
4716
4717 static void
4718 _ldi_l(jit_state_t *_jit, jit_int32_t r0, jit_word_t i0)
4719 {
4720     jit_int32_t         reg;
4721     reg = jit_get_reg(jit_class_gpr);
4722     movi(rn(reg), i0);
4723     ldr_l(r0, rn(reg));
4724     jit_unget_reg(reg);
4725 }
4726
4727 static void
4728 _ldxr_c(jit_state_t *_jit, jit_int32_t r0, jit_int32_t r1, jit_int32_t r2)
4729 {
4730     jit_int32_t         reg;
4731     reg = jit_get_reg(jit_class_gpr);
4732     addr(rn(reg), r1, r2);
4733     ldr_c(r0, rn(reg));
4734     jit_unget_reg(reg);
4735 }
4736
4737 static void
4738 _ldxi_c(jit_state_t *_jit, jit_int32_t r0, jit_int32_t r1, jit_word_t i0)
4739 {
4740     jit_int32_t         reg;
4741     if (i0) {
4742         reg = jit_get_reg(jit_class_gpr);
4743         movi(rn(reg), i0);
4744         ldxr_c(r0, r1, rn(reg));
4745         jit_unget_reg(reg);
4746     }
4747     else
4748         ldr_c(r0, r1);
4749 }
4750
4751 static void
4752 _ldxr_uc(jit_state_t *_jit, jit_int32_t r0, jit_int32_t r1, jit_int32_t r2)
4753 {
4754     jit_int32_t         reg;
4755     reg = jit_get_reg(jit_class_gpr);
4756     addr(rn(reg), r1, r2);
4757     ldr_uc(r0, rn(reg));
4758     jit_unget_reg(reg);
4759 }
4760
4761 static void
4762 _ldxi_uc(jit_state_t *_jit, jit_int32_t r0, jit_int32_t r1, jit_word_t i0)
4763 {
4764     jit_int32_t         reg;
4765     if (i0) {
4766         reg = jit_get_reg(jit_class_gpr);
4767         movi(rn(reg), i0);
4768         ldxr_uc(r0, r1, rn(reg));
4769         jit_unget_reg(reg);
4770     }
4771     else
4772         ldr_uc(r0, r1);
4773 }
4774
4775 static void
4776 _ldxr_s(jit_state_t *_jit, jit_int32_t r0, jit_int32_t r1, jit_int32_t r2)
4777 {
4778     jit_int32_t         reg;
4779     reg = jit_get_reg(jit_class_gpr);
4780     addr(rn(reg), r1, r2);
4781     ldr_s(r0, rn(reg));
4782     jit_unget_reg(reg);
4783 }
4784
4785 static void
4786 _ldxi_s(jit_state_t *_jit, jit_int32_t r0, jit_int32_t r1, jit_word_t i0)
4787 {
4788     jit_int32_t         reg;
4789     if (i0) {
4790         reg = jit_get_reg(jit_class_gpr);
4791         movi(rn(reg), i0);
4792         ldxr_s(r0, r1, rn(reg));
4793         jit_unget_reg(reg);
4794     }
4795     else
4796         ldr_s(r0, r1);
4797 }
4798
4799 static void
4800 _ldxr_us(jit_state_t *_jit, jit_int32_t r0, jit_int32_t r1, jit_int32_t r2)
4801 {
4802     jit_int32_t         reg;
4803     reg = jit_get_reg(jit_class_gpr);
4804     addr(rn(reg), r1, r2);
4805     ldr_us(r0, rn(reg));
4806     jit_unget_reg(reg);
4807 }
4808
4809 static void
4810 _ldxi_us(jit_state_t *_jit, jit_int32_t r0, jit_int32_t r1, jit_word_t i0)
4811 {
4812     jit_int32_t         reg;
4813     if (i0) {
4814         reg = jit_get_reg(jit_class_gpr);
4815         movi(rn(reg), i0);
4816         ldxr_us(r0, r1, rn(reg));
4817         jit_unget_reg(reg);
4818     }
4819     else
4820         ldr_us(r0, r1);
4821 }
4822
4823 static void
4824 _ldxr_i(jit_state_t *_jit, jit_int32_t r0, jit_int32_t r1, jit_int32_t r2)
4825 {
4826     jit_int32_t         reg;
4827     reg = jit_get_reg(jit_class_gpr);
4828     addr(rn(reg), r1, r2);
4829     ldr_i(r0, rn(reg));
4830     jit_unget_reg(reg);
4831 }
4832
4833 static void
4834 _ldxi_i(jit_state_t *_jit, jit_int32_t r0, jit_int32_t r1, jit_word_t i0)
4835 {
4836     jit_int32_t         reg;
4837     if (i0) {
4838         reg = jit_get_reg(jit_class_gpr);
4839         movi(rn(reg), i0);
4840         ldxr_i(r0, r1, rn(reg));
4841         jit_unget_reg(reg);
4842     }
4843     else
4844         ldr_i(r0, r1);
4845 }
4846
4847 static void
4848 _ldxr_ui(jit_state_t *_jit, jit_int32_t r0, jit_int32_t r1, jit_int32_t r2)
4849 {
4850     jit_int32_t         reg;
4851     reg = jit_get_reg(jit_class_gpr);
4852     addr(rn(reg), r1, r2);
4853     ldr_ui(r0, rn(reg));
4854     jit_unget_reg(reg);
4855 }
4856
4857 static void
4858 _ldxi_ui(jit_state_t *_jit, jit_int32_t r0, jit_int32_t r1, jit_word_t i0)
4859 {
4860     jit_int32_t         reg;
4861     if (i0) {
4862         reg = jit_get_reg(jit_class_gpr);
4863         movi(rn(reg), i0);
4864         ldxr_ui(r0, r1, rn(reg));
4865         jit_unget_reg(reg);
4866     }
4867     else
4868         ldr_ui(r0, r1);
4869 }
4870
4871 static void
4872 _ldxr_l(jit_state_t *_jit, jit_int32_t r0, jit_int32_t r1, jit_int32_t r2)
4873 {
4874     jit_int32_t         reg;
4875     reg = jit_get_reg(jit_class_gpr);
4876     addr(rn(reg), r1, r2);
4877     ldr_l(r0, rn(reg));
4878     jit_unget_reg(reg);
4879 }
4880
4881 static void
4882 _ldxi_l(jit_state_t *_jit, jit_int32_t r0, jit_int32_t r1, jit_word_t i0)
4883 {
4884     jit_int32_t         reg;
4885     if (i0) {
4886         reg = jit_get_reg(jit_class_gpr);
4887         movi(rn(reg), i0);
4888         ldxr_l(r0, r1, rn(reg));
4889         jit_unget_reg(reg);
4890     }
4891     else
4892         ldr_l(r0, r1);
4893 }
4894
4895 static void
4896 _unldr(jit_state_t *_jit, jit_int32_t r0, jit_int32_t r1, jit_word_t i0)
4897 {
4898     if (jit_unaligned_p())
4899         fallback_unldr(r0, r1, i0);
4900     else
4901         generic_unldr(r0, r1, i0);
4902 }
4903
4904 static void
4905 _unldi(jit_state_t *_jit, jit_int32_t r0, jit_word_t i0, jit_word_t i1)
4906 {
4907     jit_int32_t         t0, r2;
4908     if (jit_unaligned_p())
4909         fallback_unldi(r0, i0, i1);
4910     else
4911         generic_unldi(r0, i0, i1);
4912 }
4913
4914 static void
4915 _unldr_u(jit_state_t *_jit, jit_int32_t r0, jit_int32_t r1, jit_word_t i0)
4916 {
4917     if (jit_unaligned_p())
4918         fallback_unldr_u(r0, r1, i0);
4919     else
4920         generic_unldr_u(r0, r1, i0);
4921 }
4922
4923 static void
4924 _unldi_u(jit_state_t *_jit, jit_int32_t r0, jit_word_t i0, jit_word_t i1)
4925 {
4926     jit_int32_t         t0, r2;
4927     if (jit_unaligned_p())
4928         fallback_unldi_u(r0, i0, i1);
4929     else
4930         generic_unldi_u(r0, i0, i1);
4931 }
4932
4933 static void
4934 _sti_c(jit_state_t *_jit, jit_word_t i0, jit_int32_t r0)
4935 {
4936     jit_int32_t         reg;
4937     reg = jit_get_reg(jit_class_gpr);
4938     movi(rn(reg), i0);
4939     str_c(rn(reg), r0);
4940     jit_unget_reg(reg);
4941 }
4942
4943 static void
4944 _sti_s(jit_state_t *_jit, jit_word_t i0, jit_int32_t r0)
4945 {
4946     jit_int32_t         reg;
4947     reg = jit_get_reg(jit_class_gpr);
4948     movi(rn(reg), i0);
4949     str_s(rn(reg), r0);
4950     jit_unget_reg(reg);
4951 }
4952
4953 static void
4954 _sti_i(jit_state_t *_jit, jit_word_t i0, jit_int32_t r0)
4955 {
4956     jit_int32_t         reg;
4957     reg = jit_get_reg(jit_class_gpr);
4958     movi(rn(reg), i0);
4959     str_i(rn(reg), r0);
4960     jit_unget_reg(reg);
4961 }
4962
4963 static void
4964 _sti_l(jit_state_t *_jit, jit_word_t i0, jit_int32_t r0)
4965 {
4966     jit_int32_t         reg;
4967     reg = jit_get_reg(jit_class_gpr);
4968     movi(rn(reg), i0);
4969     str_l(rn(reg), r0);
4970     jit_unget_reg(reg);
4971 }
4972
4973 static void
4974 _stxr_c(jit_state_t *_jit, jit_int32_t r0, jit_int32_t r1, jit_int32_t r2)
4975 {
4976     jit_int32_t         reg;
4977     reg = jit_get_reg(jit_class_gpr);
4978     addr(rn(reg), r0, r1);
4979     str_c(rn(reg), r2);
4980     jit_unget_reg(reg);
4981 }
4982
4983 static void
4984 _stxi_c(jit_state_t *_jit, jit_word_t i0, jit_int32_t r0, jit_int32_t r1)
4985 {
4986     jit_int32_t         reg;
4987     if (i0) {
4988         reg = jit_get_reg(jit_class_gpr);
4989         addi(rn(reg), r0, i0);
4990         str_c(rn(reg), r1);
4991         jit_unget_reg(reg);
4992     }
4993     else
4994         str_c(r0, r1);
4995 }
4996
4997 static void
4998 _stxr_s(jit_state_t *_jit, jit_int32_t r0, jit_int32_t r1, jit_int32_t r2)
4999 {
5000     jit_int32_t         reg;
5001     reg = jit_get_reg(jit_class_gpr);
5002     addr(rn(reg), r0, r1);
5003     str_s(rn(reg), r2);
5004     jit_unget_reg(reg);
5005 }
5006
5007 static void
5008 _stxi_s(jit_state_t *_jit, jit_word_t i0, jit_int32_t r0, jit_int32_t r1)
5009 {
5010     jit_int32_t         reg;
5011     if (i0) {
5012         reg = jit_get_reg(jit_class_gpr);
5013         addi(rn(reg), r0, i0);
5014         str_s(rn(reg), r1);
5015         jit_unget_reg(reg);
5016     }
5017     else
5018         str_s(r0, r1);
5019 }
5020
5021 static void
5022 _stxr_i(jit_state_t *_jit, jit_int32_t r0, jit_int32_t r1, jit_int32_t r2)
5023 {
5024     jit_int32_t         reg;
5025     reg = jit_get_reg(jit_class_gpr);
5026     addr(rn(reg), r0, r1);
5027     str_i(rn(reg), r2);
5028     jit_unget_reg(reg);
5029 }
5030
5031 static void
5032 _stxi_i(jit_state_t *_jit, jit_word_t i0, jit_int32_t r0, jit_int32_t r1)
5033 {
5034     jit_int32_t         reg;
5035     if (i0) {
5036         reg = jit_get_reg(jit_class_gpr);
5037         addi(rn(reg), r0, i0);
5038         str_i(rn(reg), r1);
5039         jit_unget_reg(reg);
5040     }
5041     else
5042         str_i(r0, r1);
5043 }
5044
5045 static void
5046 _stxr_l(jit_state_t *_jit, jit_int32_t r0, jit_int32_t r1, jit_int32_t r2)
5047 {
5048     jit_int32_t         reg;
5049     reg = jit_get_reg(jit_class_gpr);
5050     addr(rn(reg), r0, r1);
5051     str_l(rn(reg), r2);
5052     jit_unget_reg(reg);
5053 }
5054
5055 static void
5056 _stxi_l(jit_state_t *_jit, jit_word_t i0, jit_int32_t r0, jit_int32_t r1)
5057 {
5058     jit_int32_t         reg;
5059     if (i0) {
5060         reg = jit_get_reg(jit_class_gpr);
5061         addi(rn(reg), r0, i0);
5062         str_l(rn(reg), r1);
5063         jit_unget_reg(reg);
5064     }
5065     else
5066         str_l(r0, r1);
5067 }
5068
5069 static void
5070 _unstr(jit_state_t *_jit, jit_int32_t r0, jit_int32_t r1, jit_word_t i0)
5071 {
5072     if (jit_unaligned_p())
5073         fallback_unstr(r0, r1, i0);
5074     else
5075         generic_unstr(r0, r1, i0);
5076 }
5077
5078 static void
5079 _unsti(jit_state_t *_jit, jit_word_t i0, jit_int32_t r0, jit_word_t i1)
5080 {
5081     if (jit_unaligned_p())
5082         fallback_unsti(i0, r0, i1);
5083     else
5084         generic_unsti(i0, r0, i1);
5085 }
5086
5087 static jit_word_t
5088 _bltr(jit_state_t *_jit, jit_word_t i0, jit_int32_t r0, jit_int32_t r1)
5089 {
5090     jit_word_t          w;
5091     CMP_LT(PR_6, PR_7, r0, r1);
5092     sync();
5093     w = _jit->pc.w;
5094     BRI_COND((i0 - w) >> 4, PR_6);
5095     return (w);
5096 }
5097
5098 static jit_word_t
5099 _blti(jit_state_t *_jit, jit_word_t i0, jit_int32_t r0, jit_word_t i1)
5100 {
5101     jit_word_t          w;
5102     jit_int32_t         reg;
5103     if (i1 >= -127 && i1 <= 128)
5104         CMPI_LT(PR_7, PR_6, i1 - 1, r0);
5105     else {
5106         reg = jit_get_reg(jit_class_gpr);
5107         movi(rn(reg), i1);
5108         CMP_LT(PR_6, PR_7, r0, rn(reg));
5109         jit_unget_reg(reg);
5110     }
5111     sync();
5112     w = _jit->pc.w;
5113     BRI_COND((i0 - w) >> 4, PR_6);
5114     return (w);
5115 }
5116
5117 static jit_word_t
5118 _bltr_u(jit_state_t *_jit, jit_word_t i0, jit_int32_t r0, jit_int32_t r1)
5119 {
5120     jit_word_t          w;
5121     CMP_LTU(PR_6, PR_7, r0, r1);
5122     sync();
5123     w = _jit->pc.w;
5124     BRI_COND((i0 - w) >> 4, PR_6);
5125     return (w);
5126 }
5127
5128 static jit_word_t
5129 _blti_u(jit_state_t *_jit, jit_word_t i0, jit_int32_t r0, jit_word_t i1)
5130 {
5131     jit_word_t          w;
5132     jit_int32_t         reg;
5133     if (i1 >= -127 && i1 <= 128)
5134         CMPI_LTU(PR_7, PR_6, i1 - 1, r0);
5135     else {
5136         reg = jit_get_reg(jit_class_gpr);
5137         movi(rn(reg), i1);
5138         CMP_LTU(PR_6, PR_7, r0, rn(reg));
5139         jit_unget_reg(reg);
5140     }
5141     sync();
5142     w = _jit->pc.w;
5143     BRI_COND((i0 - w) >> 4, PR_6);
5144     return (w);
5145 }
5146
5147 static jit_word_t
5148 _bler(jit_state_t *_jit, jit_word_t i0, jit_int32_t r0, jit_int32_t r1)
5149 {
5150     jit_word_t          w;
5151     CMP_LT(PR_6, PR_7, r1, r0);
5152     sync();
5153     w = _jit->pc.w;
5154     BRI_COND((i0 - w) >> 4, PR_7);
5155     return (w);
5156 }
5157
5158 static jit_word_t
5159 _blei(jit_state_t *_jit, jit_word_t i0, jit_int32_t r0, jit_word_t i1)
5160 {
5161     jit_word_t          w;
5162     jit_int32_t         reg;
5163     reg = jit_get_reg(jit_class_gpr|jit_class_nospill);
5164     movi(rn(reg), i1);
5165     w = bler(i0, r0, rn(reg));
5166     jit_unget_reg(reg);
5167     return (w);
5168 }
5169
5170 static jit_word_t
5171 _bler_u(jit_state_t *_jit, jit_word_t i0, jit_int32_t r0, jit_int32_t r1)
5172 {
5173     jit_word_t          w;
5174     CMP_LTU(PR_6, PR_7, r1, r0);
5175     sync();
5176     w = _jit->pc.w;
5177     BRI_COND((i0 - w) >> 4, PR_7);
5178     return (w);
5179 }
5180
5181 static jit_word_t
5182 _blei_u(jit_state_t *_jit, jit_word_t i0, jit_int32_t r0, jit_word_t i1)
5183 {
5184     jit_word_t          w;
5185     jit_int32_t         reg;
5186     reg = jit_get_reg(jit_class_gpr|jit_class_nospill);
5187     movi(rn(reg), i1);
5188     w = bler_u(i0, r0, rn(reg));
5189     jit_unget_reg(reg);
5190     return (w);
5191 }
5192
5193 static jit_word_t
5194 _beqr(jit_state_t *_jit, jit_word_t i0, jit_int32_t r0, jit_int32_t r1)
5195 {
5196     jit_word_t          w;
5197     CMP_EQ(PR_6, PR_7, r0, r1);
5198     sync();
5199     w = _jit->pc.w;
5200     BRI_COND((i0 - w) >> 4, PR_6);
5201     return (w);
5202 }
5203
5204 static jit_word_t
5205 _beqi(jit_state_t *_jit, jit_word_t i0, jit_int32_t r0, jit_word_t i1)
5206 {
5207     jit_word_t          w;
5208     jit_int32_t         reg;
5209     if (i1 >= -128 && i1 <= 127)
5210         CMPI_EQ(PR_6, PR_7, i1, r0);
5211     else {
5212         reg = jit_get_reg(jit_class_gpr);
5213         movi(rn(reg), i1);
5214         CMP_EQ(PR_6, PR_7, r0, rn(reg));
5215         jit_unget_reg(reg);
5216     }
5217     sync();
5218     w = _jit->pc.w;
5219     BRI_COND((i0 - w) >> 4, PR_6);
5220     return (w);
5221 }
5222
5223 static jit_word_t
5224 _bger(jit_state_t *_jit, jit_word_t i0, jit_int32_t r0, jit_int32_t r1)
5225 {
5226     jit_word_t          w;
5227     CMP_LT(PR_6, PR_7, r0, r1);
5228     sync();
5229     w = _jit->pc.w;
5230     BRI_COND((i0 - w) >> 4, PR_7);
5231     return (w);
5232 }
5233
5234 static jit_word_t
5235 _bgei(jit_state_t *_jit, jit_word_t i0, jit_int32_t r0, jit_word_t i1)
5236 {
5237     jit_word_t          w;
5238     jit_int32_t         reg;
5239     if (i1 >= -127 && i1 <= 128)
5240         CMPI_LT(PR_7, PR_6, i1 - 1, r0);
5241     else {
5242         reg = jit_get_reg(jit_class_gpr);
5243         movi(rn(reg), i1);
5244         CMP_LT(PR_6, PR_7, r0, rn(reg));
5245         jit_unget_reg(reg);
5246     }
5247     sync();
5248     w = _jit->pc.w;
5249     BRI_COND((i0 - w) >> 4, PR_7);
5250     return (w);
5251 }
5252
5253 static jit_word_t
5254 _bger_u(jit_state_t *_jit, jit_word_t i0, jit_int32_t r0, jit_int32_t r1)
5255 {
5256     jit_word_t          w;
5257     CMP_LTU(PR_6, PR_7, r0, r1);
5258     sync();
5259     w = _jit->pc.w;
5260     BRI_COND((i0 - w) >> 4, PR_7);
5261     return (w);
5262 }
5263
5264 static jit_word_t
5265 _bgei_u(jit_state_t *_jit, jit_word_t i0, jit_int32_t r0, jit_word_t i1)
5266 {
5267     jit_word_t          w;
5268     jit_int32_t         reg;
5269     if (i1 >= -127 && i1 <= 128)
5270         CMPI_LTU(PR_7, PR_6, i1 - 1, r0);
5271     else {
5272         reg = jit_get_reg(jit_class_gpr);
5273         movi(rn(reg), i1);
5274         CMP_LTU(PR_6, PR_7, r0, rn(reg));
5275         jit_unget_reg(reg);
5276     }
5277     sync();
5278     w = _jit->pc.w;
5279     BRI_COND((i0 - w) >> 4, PR_7);
5280     return (w);
5281 }
5282
5283 static jit_word_t
5284 _bgtr(jit_state_t *_jit, jit_word_t i0, jit_int32_t r0, jit_int32_t r1)
5285 {
5286     jit_word_t          w;
5287     CMP_LT(PR_6, PR_7, r1, r0);
5288     sync();
5289     w = _jit->pc.w;
5290     BRI_COND((i0 - w) >> 4, PR_6);
5291     return (w);
5292 }
5293
5294 static jit_word_t
5295 _bgti(jit_state_t *_jit, jit_word_t i0, jit_int32_t r0, jit_word_t i1)
5296 {
5297     jit_word_t          w;
5298     jit_int32_t         reg;
5299     reg = jit_get_reg(jit_class_gpr|jit_class_nospill);
5300     movi(rn(reg), i1);
5301     w = bgtr(i0, r0, rn(reg));
5302     jit_unget_reg(reg);
5303     return (w);
5304 }
5305
5306 static jit_word_t
5307 _bgtr_u(jit_state_t *_jit, jit_word_t i0, jit_int32_t r0, jit_int32_t r1)
5308 {
5309     jit_word_t          w;
5310     CMP_LTU(PR_6, PR_7, r1, r0);
5311     sync();
5312     w = _jit->pc.w;
5313     BRI_COND((i0 - w) >> 4, PR_6);
5314     return (w);
5315 }
5316
5317 static jit_word_t
5318 _bgti_u(jit_state_t *_jit, jit_word_t i0, jit_int32_t r0, jit_word_t i1)
5319 {
5320     jit_word_t          w;
5321     jit_int32_t         reg;
5322     reg = jit_get_reg(jit_class_gpr|jit_class_nospill);
5323     movi(rn(reg), i1);
5324     w = bgtr_u(i0, r0, rn(reg));
5325     jit_unget_reg(reg);
5326     return (w);
5327 }
5328
5329 static jit_word_t
5330 _bner(jit_state_t *_jit, jit_word_t i0, jit_int32_t r0, jit_int32_t r1)
5331 {
5332     jit_word_t          w;
5333     CMP_EQ(PR_6, PR_7, r0, r1);
5334     sync();
5335     w = _jit->pc.w;
5336     BRI_COND((i0 - w) >> 4, PR_7);
5337     return (w);
5338 }
5339
5340 static jit_word_t
5341 _bnei(jit_state_t *_jit, jit_word_t i0, jit_int32_t r0, jit_word_t i1)
5342 {
5343     jit_word_t          w;
5344     jit_int32_t         reg;
5345     if (i1 >= -128 && i1 <= 127)
5346         CMPI_EQ(PR_6, PR_7, i1, r0);
5347     else {
5348         reg = jit_get_reg(jit_class_gpr);
5349         movi(rn(reg), i1);
5350         CMP_EQ(PR_6, PR_7, r0, rn(reg));
5351         jit_unget_reg(reg);
5352     }
5353     sync();
5354     w = _jit->pc.w;
5355     BRI_COND((i0 - w) >> 4, PR_7);
5356     return (w);
5357 }
5358
5359 static jit_word_t
5360 _bmsr(jit_state_t *_jit, jit_word_t i0, jit_int32_t r0, jit_int32_t r1)
5361 {
5362     jit_word_t          w;
5363     jit_int32_t         reg;
5364     reg = jit_get_reg(jit_class_gpr);
5365     andr(rn(reg), r0, r1);
5366     CMPI_EQ(PR_6, PR_7, 0, rn(reg));
5367     jit_unget_reg(reg);
5368     sync();
5369     w = _jit->pc.w;
5370     BRI_COND((i0 - w) >> 4, PR_7);
5371     return (w);
5372 }
5373
5374 static jit_word_t
5375 _bmsi(jit_state_t *_jit, jit_word_t i0, jit_int32_t r0, jit_word_t i1)
5376 {
5377     jit_word_t          w;
5378     jit_int32_t         reg;
5379     reg = jit_get_reg(jit_class_gpr);
5380     movi(rn(reg), i1);
5381     andr(rn(reg), r0, rn(reg));
5382     CMPI_EQ(PR_6, PR_7, 0, rn(reg));
5383     jit_unget_reg(reg);
5384     sync();
5385     w = _jit->pc.w;
5386     BRI_COND((i0 - w) >> 4, PR_7);
5387     return (w);
5388 }
5389
5390 static jit_word_t
5391 _bmcr(jit_state_t *_jit, jit_word_t i0, jit_int32_t r0, jit_int32_t r1)
5392 {
5393     jit_word_t          w;
5394     jit_int32_t         reg;
5395     reg = jit_get_reg(jit_class_gpr);
5396     andr(rn(reg), r0, r1);
5397     CMPI_EQ(PR_6, PR_7, 0, rn(reg));
5398     jit_unget_reg(reg);
5399     sync();
5400     w = _jit->pc.w;
5401     BRI_COND((i0 - w) >> 4, PR_6);
5402     return (w);
5403 }
5404
5405 static jit_word_t
5406 _bmci(jit_state_t *_jit, jit_word_t i0, jit_int32_t r0, jit_word_t i1)
5407 {
5408     jit_word_t          w;
5409     jit_int32_t         reg;
5410     reg = jit_get_reg(jit_class_gpr);
5411     movi(rn(reg), i1);
5412     andr(rn(reg), r0, rn(reg));
5413     CMPI_EQ(PR_6, PR_7, 0, rn(reg));
5414     jit_unget_reg(reg);
5415     sync();
5416     w = _jit->pc.w;
5417     BRI_COND((i0 - w) >> 4, PR_6);
5418     return (w);
5419 }
5420
5421 static jit_word_t
5422 _baddr(jit_state_t *_jit, jit_word_t i0, jit_int32_t r0, jit_int32_t r1,
5423        jit_bool_t carry)
5424 {
5425     jit_word_t          w;
5426     jit_int32_t         t0;
5427     jit_int32_t         t1;
5428     jit_int32_t         t2;
5429     /* t1 = r0 + r1;    overflow = r1 < 0 ? r0 < t1 : t1 < r0 */
5430     t0 = jit_get_reg(jit_class_gpr|jit_class_nospill);
5431     t1 = jit_get_reg(jit_class_gpr|jit_class_nospill);
5432     t2 = jit_get_reg(jit_class_gpr|jit_class_nospill);
5433     lti(rn(t0), r1, 0);                 /* t0 = r1 < 0 */
5434     addr(rn(t1), r0, r1);               /* t1 = r0 + r1 */
5435     ltr(rn(t2), rn(t1), r0);            /* t2 = t1 < r0 */
5436     ltr(rn(t1), r0, rn(t1));            /* t1 = r0 < t1 */
5437     CMPI_EQ(PR_6, PR_7, 0, rn(t0));
5438     CMPI_EQ_p(PR_8, PR_9, 0, rn(t2), PR_6);/* if (t0==0) p8=t2==0,p9=t2!=0; */
5439     CMPI_EQ_p(PR_8, PR_9, 0, rn(t1), PR_7);/* if (t0!=0) p8=t1==0,p9=t1!=0; */
5440     addr(r0, r0, r1);
5441     sync();
5442     w = _jit->pc.w;
5443     BRI_COND((i0 - w) >> 4, carry ? PR_9 : PR_8);
5444     jit_unget_reg(t2);
5445     jit_unget_reg(t1);
5446     jit_unget_reg(t0);
5447     return (w);
5448 }
5449
5450 static jit_word_t
5451 _baddi(jit_state_t *_jit, jit_word_t i0, jit_int32_t r0, jit_word_t i1,
5452        jit_bool_t carry)
5453 {
5454     jit_word_t          w;
5455     jit_int32_t         reg;
5456     reg = jit_get_reg(jit_class_gpr|jit_class_nospill);
5457     movi(rn(reg), i1);
5458     w = baddr(i0, r0, rn(reg), carry);
5459     jit_unget_reg(reg);
5460     return (w);
5461 }
5462
5463 static jit_word_t
5464 _baddr_u(jit_state_t *_jit, jit_word_t i0, jit_int32_t r0, jit_int32_t r1,
5465          jit_bool_t carry)
5466 {
5467     jit_word_t          w;
5468     jit_int32_t         t0;
5469     jit_int32_t         t1;
5470     t0 = jit_get_reg(jit_class_gpr|jit_class_nospill);
5471     t1 = jit_get_reg(jit_class_gpr|jit_class_nospill);
5472     addr(rn(t0), r0, r1);
5473     ltr_u(rn(t1), rn(t0), r0);
5474     CMPI_EQ(PR_6, PR_7, 0, rn(t1));
5475     MOV(r0, rn(t0));
5476     sync();
5477     w = _jit->pc.w;
5478     BRI_COND((i0 - w) >> 4, carry ? PR_7 : PR_6);
5479     jit_unget_reg(t1);
5480     jit_unget_reg(t0);
5481     return (w);
5482 }
5483
5484 static jit_word_t
5485 _baddi_u(jit_state_t *_jit, jit_word_t i0, jit_int32_t r0, jit_word_t i1,
5486          jit_bool_t carry)
5487 {
5488     jit_word_t          w;
5489     jit_int32_t         reg;
5490     reg = jit_get_reg(jit_class_gpr|jit_class_nospill);
5491     movi(rn(reg), i1);
5492     w = baddr_u(i0, r0, rn(reg), carry);
5493     jit_unget_reg(reg);
5494     return (w);
5495 }
5496
5497 static jit_word_t
5498 _bsubr(jit_state_t *_jit, jit_word_t i0, jit_int32_t r0, jit_int32_t r1,
5499        jit_bool_t carry)
5500 {
5501     jit_word_t          w;
5502     jit_int32_t         t0;
5503     jit_int32_t         t1;
5504     jit_int32_t         t2;
5505     /* t1 = r0 - r1;    overflow = 0 < r1 ? r0 < t1 : t1 < r0 */
5506     t0 = jit_get_reg(jit_class_gpr|jit_class_nospill);
5507     t1 = jit_get_reg(jit_class_gpr|jit_class_nospill);
5508     t2 = jit_get_reg(jit_class_gpr|jit_class_nospill);
5509     gti(rn(t0), r1, 0);                 /* t0 = r1 > 0 */
5510     subr(rn(t1), r0, r1);               /* t1 = r0 - r1 */
5511     ltr(rn(t2), rn(t1), r0);            /* t2 = t1 < r0 */
5512     ltr(rn(t1), r0, rn(t1));            /* t1 = r0 < t1 */
5513     CMPI_EQ(PR_6, PR_7, 0, rn(t0));
5514     CMPI_EQ_p(PR_8, PR_9, 0, rn(t2), PR_6);/* if (t0==0) p4=t2==0,p5=t2!=0; */
5515     CMPI_EQ_p(PR_8, PR_9, 0, rn(t1), PR_7);/* if (t0!=0) p4=t1==0,p5=t1!=0; */
5516     subr(r0, r0, r1);
5517     sync();
5518     w = _jit->pc.w;
5519     BRI_COND((i0 - w) >> 4, carry ? PR_9 : PR_8);
5520     jit_unget_reg(t2);
5521     jit_unget_reg(t1);
5522     jit_unget_reg(t0);
5523     return (w);
5524 }
5525
5526 static jit_word_t
5527 _bsubi(jit_state_t *_jit, jit_word_t i0, jit_int32_t r0, jit_word_t i1,
5528        jit_bool_t carry)
5529 {
5530     jit_word_t          w;
5531     jit_int32_t         reg;
5532     reg = jit_get_reg(jit_class_gpr|jit_class_nospill);
5533     movi(rn(reg), i1);
5534     w = bsubr(i0, r0, rn(reg), carry);
5535     jit_unget_reg(reg);
5536     return (w);
5537 }
5538
5539 static jit_word_t
5540 _bsubr_u(jit_state_t *_jit, jit_word_t i0, jit_int32_t r0, jit_int32_t r1,
5541          jit_bool_t carry)
5542 {
5543     jit_word_t          w;
5544     jit_int32_t         t0;
5545     jit_int32_t         t1;
5546     t0 = jit_get_reg(jit_class_gpr|jit_class_nospill);
5547     t1 = jit_get_reg(jit_class_gpr|jit_class_nospill);
5548     subr(rn(t0), r0, r1);
5549     ltr_u(rn(t1), r0, rn(t0));
5550     CMPI_EQ(PR_6, PR_7, 0, rn(t1));
5551     MOV(r0, rn(t0));
5552     sync();
5553     w = _jit->pc.w;
5554     BRI_COND((i0 - w) >> 4, carry ? PR_7 : PR_6);
5555     jit_unget_reg(t1);
5556     jit_unget_reg(t0);
5557     return (w);
5558 }
5559
5560 static jit_word_t
5561 _bsubi_u(jit_state_t *_jit, jit_word_t i0, jit_int32_t r0, jit_word_t i1,
5562          jit_bool_t carry)
5563 {
5564     jit_word_t          w;
5565     jit_int32_t         reg;
5566     reg = jit_get_reg(jit_class_gpr|jit_class_nospill);
5567     movi(rn(reg), i1);
5568     w = bsubr_u(i0, r0, rn(reg), carry);
5569     jit_unget_reg(reg);
5570     return (w);
5571 }
5572
5573 static void
5574 _jmpr(jit_state_t *_jit, jit_int32_t r0)
5575 {
5576     MOV_br_rn(BR_6, r0);
5577     BR(BR_6);
5578 }
5579
5580 static jit_word_t
5581 _jmpi(jit_state_t *_jit, jit_word_t i0)
5582 {
5583     jit_word_t          d, w;
5584     sync();
5585     w = _jit->pc.w;
5586     d = ((jit_word_t)i0 - w) >> 4;
5587     if (d >= -16777216 && d <= 16777215)
5588         BRI(d);
5589     else
5590         BRL(d);
5591     return (w);
5592 }
5593
5594 static jit_word_t
5595 _jmpi_p(jit_state_t *_jit, jit_word_t i0)
5596 {
5597     jit_word_t          d, w;
5598     sync();
5599     w = _jit->pc.w;
5600     d = ((jit_word_t)i0 - w) >> 4;
5601     BRL(d);
5602     return (w);
5603 }
5604
5605 static void
5606 _callr(jit_state_t *_jit, jit_int32_t r0)
5607 {
5608     jit_int32_t         reg;
5609     reg = jit_get_reg(jit_class_gpr);
5610     LD8_inc(rn(reg), r0, 8);
5611     MOV_br_rn(BR_6, rn(reg));
5612     jit_unget_reg(reg);
5613     LD8(GR_1, r0);
5614     BR_CALL(BR_0, BR_6);
5615 }
5616
5617 static void
5618 _calli(jit_state_t *_jit, jit_word_t i0)
5619 {
5620     jit_int32_t         reg;
5621     reg = jit_get_reg(jit_class_gpr);
5622     movi(rn(reg), i0);
5623     callr(rn(reg));
5624     jit_unget_reg(reg);
5625 }
5626
5627 static jit_word_t
5628 _calli_p(jit_state_t *_jit, jit_word_t i0)
5629 {
5630     jit_word_t          w;
5631     jit_int32_t         reg;
5632     reg = jit_get_reg(jit_class_gpr);
5633     w = movi_p(rn(reg), i0);
5634     callr(rn(reg));
5635     jit_unget_reg(reg);
5636     return (w);
5637 }
5638
5639 static void
5640 _prolog(jit_state_t *_jit, jit_node_t *node)
5641 {
5642     jit_int32_t         reg, ruse, rout;
5643
5644     if (_jitc->function->define_frame || _jitc->function->assume_frame) {
5645         jit_int32_t     frame = -_jitc->function->frame;
5646         assert(_jitc->function->self.aoff >= frame);
5647         if (_jitc->function->assume_frame)
5648             return;
5649         _jitc->function->self.aoff = frame;
5650     }
5651     if (_jitc->function->allocar)
5652         _jitc->function->self.aoff &= -16;
5653     _jitc->function->stack = ((_jitc->function->self.alen -
5654                                _jitc->function->self.aoff) + 15) & -16;
5655
5656     /* First lowest unused register is first output register */
5657     for (reg = _R115; reg >= _R40; reg--) {
5658         if (jit_regset_tstbit(&_jitc->function->regset, reg))
5659             break;
5660     }
5661     _jitc->breg = rn(reg) + 1;
5662     _jitc->rout = _jitc->breg + 5;
5663     ruse = _jitc->rout - GR_32;
5664
5665     /* How many out argument registers required? */
5666     if (!_jitc->function->define_frame) {
5667         for (reg = _OUT7; reg >= _OUT0; --reg) {
5668             if (jit_regset_tstbit(&_jitc->function->regset, reg))
5669                 break;
5670         }
5671         rout = (reg + 1) - _OUT0;
5672     }
5673     else
5674         rout = 8;
5675
5676     /* Do not know if will call div/mod functions (sqrt* needs one) */
5677     if (rout < 2)
5678         rout = 2;
5679
5680     /* Match gcc prolog */
5681     ALLOC(_jitc->breg + 1, ruse, rout);
5682     MOV(_jitc->breg + 2, GR_12);
5683     MOV_rn_br(_jitc->breg, BR_0);
5684     MOV(_jitc->breg + 3, GR_1);
5685
5686     /* lightning specific, use r4 as frame pointer */
5687     MOV(_jitc->breg + 4, GR_4);
5688     addi(GR_4, GR_12, -(stack_framesize + params_offset));
5689
5690     /* adjust stack pointer */
5691     addi(GR_12, GR_12, -(stack_framesize +
5692                          (params_offset << 1) + _jitc->function->stack));
5693
5694     if (jit_regset_tstbit(&_jitc->function->regset, JIT_F0))
5695         STF_SPILL(GR_4, rn(JIT_F0));
5696     if (jit_regset_tstbit(&_jitc->function->regset, JIT_F1)) {
5697         addi(GR_2, GR_4, 16);
5698         STF_SPILL(GR_2, rn(JIT_F1));
5699     }
5700     if (jit_regset_tstbit(&_jitc->function->regset, JIT_F2)) {
5701         addi(GR_2, GR_4, 32);
5702         STF_SPILL(GR_2, rn(JIT_F2));
5703     }
5704     if (jit_regset_tstbit(&_jitc->function->regset, JIT_F3)) {
5705         addi(GR_2, GR_4, 48);
5706         STF_SPILL(GR_2, rn(JIT_F3));
5707     }
5708     if (jit_regset_tstbit(&_jitc->function->regset, JIT_F4)) {
5709         addi(GR_2, GR_4, 64);
5710         STF_SPILL(GR_2, rn(JIT_F4));
5711     }
5712     if (jit_regset_tstbit(&_jitc->function->regset, JIT_F5)) {
5713         addi(GR_2, GR_4, 80);
5714         STF_SPILL(GR_2, rn(JIT_F5));
5715     }
5716
5717     if (_jitc->function->allocar) {
5718         reg = jit_get_reg(jit_class_gpr);
5719         movi(rn(reg), _jitc->function->self.aoff);
5720         stxi_i(_jitc->function->aoffoff, GR_4, rn(reg));
5721         jit_unget_reg(reg);
5722     }
5723
5724     if (_jitc->function->self.call & jit_call_varargs) {
5725         for (reg = _jitc->function->vagp; reg < 8; ++reg)
5726             stxi(112 + reg * 8, GR_4, GR_32 + reg);
5727     }
5728 }
5729
5730 static void
5731 _epilog(jit_state_t *_jit, jit_node_t *node)
5732 {
5733     if (_jitc->function->assume_frame)
5734         return;
5735     if (jit_regset_tstbit(&_jitc->function->regset, JIT_F0))
5736         LDF_FILL(rn(JIT_F0), GR_4);
5737     if (jit_regset_tstbit(&_jitc->function->regset, JIT_F1)) {
5738         addi(GR_2, GR_4, 16);
5739         LDF_FILL(rn(JIT_F1), GR_2);
5740     }
5741     if (jit_regset_tstbit(&_jitc->function->regset, JIT_F2)) {
5742         addi(GR_2, GR_4, 32);
5743         LDF_FILL(rn(JIT_F2), GR_2);
5744     }
5745     if (jit_regset_tstbit(&_jitc->function->regset, JIT_F3)) {
5746         addi(GR_2, GR_4, 48);
5747         LDF_FILL(rn(JIT_F3), GR_2);
5748     }
5749     if (jit_regset_tstbit(&_jitc->function->regset, JIT_F4)) {
5750         addi(GR_2, GR_4, 64);
5751         LDF_FILL(rn(JIT_F4), GR_2);
5752     }
5753     if (jit_regset_tstbit(&_jitc->function->regset, JIT_F5)) {
5754         addi(GR_2, GR_4, 80);
5755         LDF_FILL(rn(JIT_F5), GR_2);
5756     }
5757     /* Match gcc epilog */
5758     MOV(GR_1, _jitc->breg + 3);
5759     MOV_I_ar_rn(AR_PFS, _jitc->breg + 1);
5760     MOV_br_rn(BR_0, _jitc->breg);
5761     MOV(GR_12, _jitc->breg + 2);
5762     /* Restore lightning specific r4 as frame pointer */
5763     MOV(GR_4, _jitc->breg + 4);
5764     BR_RET(BR_0);
5765     flush();
5766 }
5767
5768 static void
5769 _vastart(jit_state_t *_jit, jit_int32_t r0)
5770 {
5771     assert(_jitc->function->self.call & jit_call_varargs);
5772     /* Initialize va_list to the first stack argument. */
5773     if (_jitc->function->vagp < 8)
5774         addi(r0, GR_4, 112 + _jitc->function->vagp * 8);
5775     else
5776         addi(r0, GR_4, _jitc->function->self.size);
5777 }
5778
5779 static void
5780 _vaarg(jit_state_t *_jit, jit_int32_t r0, jit_int32_t r1)
5781 {
5782     assert(_jitc->function->self.call & jit_call_varargs);
5783     /* Load argument. */
5784     ldr(r0, r1);
5785     /* Update va_list. */
5786     addi(r1, r1, 8);
5787 }
5788
5789 static void
5790 _patch_at(jit_state_t *_jit, jit_code_t code,
5791           jit_word_t instr, jit_word_t label)
5792 {
5793     jit_word_t           l, h, tm, s0, s1, s2;
5794     union {
5795         jit_word_t       w;
5796         jit_word_t      *p;
5797     } c;
5798     jit_word_t           i1, i41, i20, ic, i5, i9, i7;
5799     c.w = instr;
5800     get_bundle(c.p, l, h, tm, s0, s1, s2);
5801     switch (code) {
5802         case jit_code_movi:
5803             /* Handle jit functions as C function, so that jit function
5804              * pointers can be passed to C code, and jit code does not
5805              * need to try to differentiate them. */
5806         case jit_code_calli:
5807             i1  = (label >> 63) &           0x1L;
5808             i41 = (label >> 22) & 0x1ffffffffffL;
5809             ic  = (label >> 21) &           0x1L;
5810             i5  = (label >> 16) &          0x1fL;
5811             i9  = (label >>  7) &         0x1ffL;
5812             i7  =  label        &          0x7fL;
5813             s1 = i41;
5814             assert((tm & ~1) == TM_M_L_X_ &&
5815                    (s2 & 0xfL<<37) == (6L<<37) &&
5816                    s0 == nop_m);
5817             s2 &= (6L<<37)|(0x7fL<<6);
5818             s2 |= (i1<<36)|(i9<<27)|(i5<<22)|(ic<<21)|(i7<<13);
5819             break;
5820         case jit_code_jmpi:
5821             if (_jitc->jump) {
5822                 /* kludge to hide function descriptors; check that gp
5823                  * is zero, what is done for all jit functions */
5824                 if (((long *)label)[1] == 0) {
5825                     for (ic = 0; ic < _jitc->prolog.offset; ic++) {
5826                         if (_jitc->prolog.ptr[ic] == label) {
5827                             label += 16;
5828                             break;
5829                         }
5830                     }
5831                 }
5832             }
5833             ic = (label - instr) >> 4;
5834             i1  = (ic >> 61) &           0x1L;
5835             i41 = (ic >> 22) & 0x1ffffffffffL;
5836             i20 =  ic        &       0xfffffL;
5837             if (!((tm & ~1) == TM_M_L_X_ &&
5838                    (s2 & 0xfL<<37) == (0xcL<<37) &&
5839                   s0 == nop_m))
5840                 goto short_jump;
5841             s1 = i41;
5842             s2 &= (0xcL<<37)|(0x7L<<33)|(1L<<12);
5843             s2 |= (i1<<36)|(i20<<13);
5844             break;
5845         default:
5846         short_jump:
5847             /* Only B1 in slot 0 expected due to need to either
5848              * a stop to update predicates, or a sync before
5849              * unconditional short branch */
5850             ic = (label - instr) >> 4;
5851             assert((s0 >> 37) == 4 && (s0 & (7 << 6)) == 0);
5852             s0 &= (4L<<37)|(7L<<33)|(1L<<12)|0x1f;
5853             s0 |= (((ic>>20)&1L)<<36)|((ic&0xfffffL)<<13);
5854             break;
5855     }
5856     set_bundle(c.p, l, h, tm, s0, s1, s2);
5857 }
5858 #endif