deps/lightning/lib/jit_ia64-cpu.c

   1 /*
   2  * Copyright (C) 2013-2022  Free Software Foundation, Inc.
   3  *
   4  * This file is part of GNU lightning.
   5  *
   6  * GNU lightning is free software; you can redistribute it and/or modify it
   7  * under the terms of the GNU Lesser General Public License as published
   8  * by the Free Software Foundation; either version 3, or (at your option)
   9  * any later version.
  10  *
  11  * GNU lightning is distributed in the hope that it will be useful, but
  12  * WITHOUT ANY WARRANTY; without even the implied warranty of MERCHANTABILITY
  13  * or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU Lesser General Public
  14  * License for more details.
  15  *
  16  * Authors:
  17  *      Paulo Cesar Pereira de Andrade
  18  */
  19
  20 #if PROTO
  21 #define stack_framesize                 144
  22 #define params_offset                   16
  23 #define INST_NONE                       0       /* should never be generated */
  24 #define INST_STOP                       1       /* or'ed if stop is required */
  25 #define INST_A                          2       /* M- or I- unit */
  26 #define INST_As                         3
  27 #define INST_I                          4
  28 #define INST_Is                         5
  29 #define INST_M                          6
  30 #define INST_Ms                         7
  31 #define INST_F                          8
  32 #define INST_Fs                         9
  33 #define INST_B                          10
  34 #define INST_Bs                         11
  35 #define INST_L                          12
  36 #define INST_Ls                         13
  37 #define INST_X                          14
  38 #define INST_Xs                         15
  39
  40 /* Data and instructions are referenced by 64-bit addresses. Instructions
  41  * are stored in memory in little endian byte order, in which the least
  42  * significant byte appears in the lowest addressed byte of a memory
  43  * location. For data, modes for both big and little endian byte order are
  44  * supported and can be controlled by a bit in the User Mask Register.
  45  */
  46 #define il(ii)                          *_jit->pc.ul++ = ii
  47 #define set_bundle(p, l, h, tm, s0, s1, s2)                             \
  48     do {                                                                \
  49         l = tm | ((s0 & 0x1ffffffffffL) << 5L) | (s1 << 46L);           \
  50         h = ((s1 >> 18L) & 0x7fffffLL) | (s2 << 23L);                   \
  51         p[0] = byte_swap_if_big_endian(l);                              \
  52         p[1] = byte_swap_if_big_endian(h);                              \
  53     } while (0)
  54 #define get_bundle(p, l, h, tm, s0, s1, s2)                             \
  55     do {                                                                \
  56         l = byte_swap_if_big_endian(p[0]);                              \
  57         h = byte_swap_if_big_endian(p[1]);                              \
  58         tm = l & 0x1f;                                                  \
  59         s0 = (l >> 5L) & 0x1ffffffffffL;                                \
  60         s1 = ((l >> 46L) & 0x3ffffL) | ((h & 0x7fffffL) << 18L);        \
  61         s2 = (h >> 23L) & 0x1ffffffffffL;                               \
  62     } while (0)
  63
  64 /*  Need to insert a stop if a modified register would (or could)
  65  *  be read in the same cycle.
  66  */
  67 #define TSTREG1(r0)                                                     \
  68     do {                                                                \
  69         if (jit_regset_tstbit(&_jitc->regs, r0))                        \
  70             stop();                                                     \
  71     } while (0)
  72 #define TSTREG2(r0, r1)                                                 \
  73     do {                                                                \
  74         if (jit_regset_tstbit(&_jitc->regs, r0) ||                      \
  75             jit_regset_tstbit(&_jitc->regs, r1))                        \
  76             stop();                                                     \
  77     } while (0)
  78 #define TSTPRED(p0)                                                     \
  79     do {                                                                \
  80         if (p0 && (_jitc->pred & (1 << p0)))                            \
  81             stop();                                                     \
  82     } while (0)
  83 /* Record register was modified */
  84 #define SETREG(r0)              jit_regset_setbit(&_jitc->regs, r0)
  85
  86 /* Avoid using constants in macros and code */
  87 typedef enum {
  88     GR_0,               GR_1,           GR_2,           GR_3,
  89     GR_4,               GR_5,           GR_6,           GR_7,
  90     GR_8,               GR_9,           GR_10,          GR_11,
  91     GR_12,              GR_13,          GR_14,          GR_15,
  92     GR_16,              GR_17,          GR_18,          GR_19,
  93     GR_20,              GR_21,          GR_22,          GR_23,
  94     GR_24,              GR_25,          GR_26,          GR_27,
  95     GR_28,              GR_29,          GR_30,          GR_31,
  96     GR_32,              GR_33,          GR_34,          GR_35,
  97     GR_36,              GR_37,          GR_38,          GR_39,
  98     GR_40,              GR_41,          GR_42,          GR_43,
  99     /* GR_44...GR_127 */
 100 } gr_reg_t;
 101
 102 typedef enum {
 103     PR_0,               /* constant - always 1 */
 104     /* p0-p5             -  preserved */
 105     PR_1,               PR_2,           PR_3,           PR_4,
 106     PR_5,
 107     /* p6-p15            - scratch */
 108     PR_6,               PR_7,           PR_8,           PR_9,
 109     PR_10,              PR_11,          PR_12,          PR_13,
 110     PR_14,              PR_15,
 111     /* p16-...           - preserved - rotating */
 112 } pr_reg_t;
 113
 114 typedef enum {
 115     BR_0,               /* scratch - Return link */
 116     /* b1-b5             - preserved */
 117     BR_1,               BR_2,           BR_3,           BR_4,
 118     BR_5,
 119     /* b6-b7            - scratch */
 120     BR_6,               BR_7
 121 } br_reg_t;
 122
 123 typedef enum {
 124     AR_KR0,             AR_KR1,         AR_KR2,         AR_KR3,
 125     AR_KR4,             AR_KR5,         AR_KR6,         AR_KR7,
 126     AR_8,               AR_9,           AR_10,          AR_11,
 127     AR_12,              AR_13,          AR_14,          AR_15,
 128     AR_RSC,             AR_BSP,         AR_BSPSTORE,    AR_RNAT,
 129     AR_20,              AR_FCR,         AR_22,          AR_23,
 130     AR_EFLAG,           AR_CSD,         AR_SSD,         AR_CFLG,
 131     AR_FSR,             AR_FIR,         AR_FDR,         AR_31,
 132     AR_CCV,             AR_33,          AR_34,          AR_35,
 133     AR_UNAT,            AR_37,          AR_38,          AR_39,
 134     AR_FPSR,            AR_41,          AR_42,          AR_43,
 135     AR_ITC,             AR_RUC,         AR_46,          AR_47,
 136     AR_48,              AR_49,          AR_50,          AR_51,
 137     AR_52,              AR_53,          AR_54,          AR_55,
 138     AR_56,              AR_57,          AR_58,          AR_59,
 139     AR_60,              AR_61,          AR_62,          AR_63,
 140     AR_PFS,             AR_LC,          AR_EC,
 141     /* AR_67 ... AR_127 */
 142 } ar_reg_t;
 143
 144 typedef enum {
 145     TM_M_I_I_,  TM_M_I_Is,      TM_M_IsI_,      TM_M_IsIs,
 146     TM_M_L_X_,  TM_M_L_Xs,      TM_ILL_06,      TM_ILL_07,
 147     TM_M_M_I_,  TM_M_M_Is,      TM_MsM_I_,      TM_MsM_Is,
 148     TM_M_F_I_,  TM_M_F_Is,      TM_M_M_F_,      TM_M_M_Fs,
 149     TM_M_I_B_,  TM_M_I_Bs,      TM_M_B_B_,      TM_M_B_Bs,
 150     TM_ILL_14,  TM_ILL_15,      TM_B_B_B_,      TM_B_B_Bs,
 151     TM_M_M_B_,  TM_M_M_Bs,      TM_ILL_1A,      TM_ILL_1B,
 152     TM_M_F_B_,  TM_M_F_Bs,      TM_ILL_1E,      TM_ILL_1F,
 153 } template_t;
 154
 155 #define MWH_SPTK                        0
 156 #define MWH_NONE                        1
 157 #define MWH_DPTK                        2
 158
 159 #define IH_NONE                         0
 160 #define IH_IMP                          1
 161
 162 #define LD_NONE                         0
 163 #define LD_NT1                          1
 164 #define LD_NTA                          3
 165
 166 #define ST_NONE                         0
 167 #define ST_NTA                          3
 168
 169 #define LF_NONE                         0
 170 #define LF_NT1                          1
 171 #define LF_NT2                          2
 172 #define LF_NTA                          3
 173
 174 #define BR_PH_FEW                       0
 175 #define BR_PH_MANY                      1
 176
 177 #define BR_BWH_SPTK                     0
 178 #define BR_BWH_SPNT                     1
 179 #define BR_BWH_DPTK                     2
 180 #define BR_BWH_DPNT                     3
 181
 182 #define BRI_BWH_SPTK                    1
 183 #define BRI_BWH_SPNT                    3
 184 #define BRI_BWH_DPTK                    5
 185 #define BRI_BWH_DPNT                    7
 186
 187 #define BR_DH_NONE                      0
 188 #define BR_DH_CLR                       1
 189
 190 #define BR_IH_NONE                      0
 191 #define BR_IH_IMP                       1
 192
 193 #define BR_IPWH_SPTK                    0
 194 #define BR_IPWH_LOOP                    1
 195 #define BR_IPWH_DPTK                    2
 196 #define BR_IPWH_EXIT                    3
 197
 198 #define BR_INDWH_SPTK                   0
 199 #define BR_INDWH_DPTK                   2
 200
 201 #define MUX_BRCST                       0
 202 #define MUX_REV                         11
 203 #define MUX_MIX                         8
 204 #define MUX_SHUF                        9
 205 #define MUX_ALT                         10
 206
 207 #define ldr(r0,r1)                      ldr_l(r0,r1)
 208 #define ldi(r0,i0)                      ldi_l(r0,i0)
 209 #define str(r0,r1)                      str_l(r0,r1)
 210 #define sti(i0,r0)                      str_l(i0,r0)
 211 #define ldxr(r0,r1,r2)                  ldxr_l(r0,r1,r2)
 212 #define ldxi(r0,r1,i0)                  ldxi_l(r0,r1,i0)
 213 #define stxr(r0,r1,r2)                  stxr_l(r0,r1,r2)
 214 #define stxi(i0,r0,r1)                  stxi_l(i0,r0,r1)
 215
 216 #if !HAVE_FFSL
 217 #  define ffsl(l)                       __builtin_ffsl(l)
 218 #endif
 219
 220 /* libgcc */
 221 #if defined(__GNUC__)
 222 extern long __divdi3(long,long);
 223 extern unsigned long __udivdi3(unsigned long,unsigned long);
 224 extern long __moddi3(long,long);
 225 extern unsigned long __umoddi3(unsigned long,unsigned long);
 226 #else
 227 static long __divdi3(long,long);
 228 static unsigned long __udivdi3(unsigned long,unsigned long);
 229 static long __moddi3(long,long);
 230 static unsigned long __umoddi3(unsigned long,unsigned long);
 231 #endif
 232 #define out(n,tm,s0,s1,s2)              _out(_jit,n,tm,s0,s1,s2)
 233 static void _out(jit_state_t*,int,int,jit_word_t,jit_word_t,jit_word_t);
 234 #define stop()                          _stop(_jit)
 235 static void _stop(jit_state_t*);
 236 #define sync()                          _sync(_jit)
 237 static void _sync(jit_state_t*);
 238 #define flush()                         _flush(_jit)
 239 static void _flush(jit_state_t*);
 240 #define inst(w, t)                      _inst(_jit, w, t)
 241 static void _inst(jit_state_t*, jit_word_t, jit_uint8_t);
 242 #define A1(x4,x2,r3,r2,r1)              _A1(_jit,0,x4,x2,r3,r2,r1)
 243 static void _A1(jit_state_t*, jit_word_t,
 244                 jit_word_t,jit_word_t,jit_word_t,jit_word_t,jit_word_t);
 245 #define A2(x4,ct,r3,r2,r1)              A1(x4,ct,r3,r2,r1)
 246 #define A3(x4,x2,r3,im,r1)              _A3(_jit,0,x4,x2,r3,im,r1)
 247 static void _A3(jit_state_t*, jit_word_t,
 248                 jit_word_t,jit_word_t,jit_word_t,jit_word_t,jit_word_t);
 249 #define A4(x2a,r3,im,r1)                _A4(_jit,0,x2a,r3,im,r1)
 250 static void _A4(jit_state_t*,jit_word_t,
 251                 jit_word_t,jit_word_t,jit_word_t,jit_word_t);
 252 #define A5(r3,im,r1)                    _A5(_jit,0,r3,im,r1)
 253 static void _A5(jit_state_t*,jit_word_t,
 254                 jit_word_t,jit_word_t,jit_word_t);
 255 #define A6(o,x2,ta,p2,r3,r2,c,p1)       _A6(_jit,0,o,x2,ta,p2,r3,r2,c,p1)
 256 static void _A6(jit_state_t*,jit_word_t,
 257                 jit_word_t,jit_word_t,jit_word_t,jit_word_t,
 258                 jit_word_t,jit_word_t,jit_word_t,jit_word_t);
 259 #define A7(o,x2,ta,p2,r3,c,p1)          _A7(_jit,0,o,x2,ta,p2,r3,c,p1)
 260 static void _A7(jit_state_t*,jit_word_t,
 261                 jit_word_t,jit_word_t,jit_word_t,
 262                 jit_word_t,jit_word_t,jit_word_t,jit_word_t)
 263     maybe_unused;
 264 #define A8(o,x2,ta,p2,r3,im,c,p1)       _A8(_jit,0,o,x2,ta,p2,r3,im,c,p1)
 265 static void _A8(jit_state_t*,jit_word_t,
 266                 jit_word_t,jit_word_t,jit_word_t,jit_word_t,
 267                 jit_word_t,jit_word_t,jit_word_t,jit_word_t);
 268 #define A9(za,zb,x4,x2,r3,r2,r1)        _A9(_jit,0,za,zb,x4,x2,r3,r2,r1)
 269 static void _A9(jit_state_t*,jit_word_t,
 270                 jit_word_t,jit_word_t,jit_word_t,
 271                 jit_word_t,jit_word_t,jit_word_t,jit_word_t)
 272     maybe_unused;
 273 #define A10(x4,ct,r3,r2,r1)             A9(0,1,x4,ct,r3,r2,r1)
 274 #define I1(ct,x2,r3,r2,r1)              _I1(_jit,0,ct,x2,r3,r2,r1)
 275 static void _I1(jit_state_t*,jit_word_t,
 276                 jit_word_t,jit_word_t,jit_word_t,jit_word_t,jit_word_t)
 277     maybe_unused;
 278 #define I2(za,x2a,zb,x2c,x2b,r3,r2,r1)  _I2(_jit,0,za,x2a,zb,x2c,x2b,r3,r2,r1)
 279 static void _I2(jit_state_t*,jit_word_t,
 280                 jit_word_t,jit_word_t,jit_word_t,jit_word_t,
 281                 jit_word_t,jit_word_t,jit_word_t,jit_word_t)
 282     maybe_unused;
 283 #define I3(mbt,r2,r1)                   _I3(_jit,0,mbt,r2,r1)
 284 static void _I3(jit_state_t*,jit_word_t,
 285                 jit_word_t,jit_word_t,jit_word_t)
 286 #if __BYTE_ORDER == __BIG_ENDIAN
 287     maybe_unused
 288 #endif
 289     ;
 290 #define I4(mht,r2,r1)                   _I4(_jit,0,mht,r2,r1)
 291 static void _I4(jit_state_t*,jit_word_t,
 292                 jit_word_t,jit_word_t,jit_word_t)
 293     maybe_unused;
 294 #define I5(za,zb,x2b,r3,r2,r1)          _I5(_jit,0,za,zb,x2b,r3,r2,r1)
 295 static void _I5(jit_state_t*,jit_word_t,
 296                 jit_word_t,jit_word_t,jit_word_t,
 297                 jit_word_t,jit_word_t,jit_word_t);
 298 #define I6(za,zb,x2b,r3,ct,r1)          _I6(_jit,0,za,zb,x2b,r3,ct,r1)
 299 static void _I6(jit_state_t*,jit_word_t,
 300                 jit_word_t,jit_word_t,jit_word_t,
 301                 jit_word_t,jit_word_t,jit_word_t)
 302     maybe_unused;
 303 #define I7(za,zb,r3,r2,r1)              _I7(_jit,0,za,zb,r3,r2,r1)
 304 static void _I7(jit_state_t*,jit_word_t,
 305                 jit_word_t,jit_word_t,jit_word_t,jit_word_t,jit_word_t);
 306 #define I8(za,zb,ct,r2,r1)              _I8(_jit,0,za,zb,ct,r2,r1)
 307 static void _I8(jit_state_t*,jit_word_t,
 308                 jit_word_t,jit_word_t,jit_word_t,jit_word_t,jit_word_t)
 309     maybe_unused;
 310 #define I9(x2c,r3,r1)                   _I9(_jit,0,x2c,r3,r1)
 311 static void _I9(jit_state_t*,jit_word_t,
 312                 jit_word_t,jit_word_t,jit_word_t)
 313     maybe_unused;
 314 #define I10(ct,r3,r2,r1)                _I10(_jit,0,ct,r3,r2,r1)
 315 static void _I10(jit_state_t*,jit_word_t,
 316                  jit_word_t,jit_word_t,jit_word_t,jit_word_t)
 317     maybe_unused;
 318 #define I11(len,r3,pos,y,r1)            _I11(_jit,0,len,r3,pos,y,r1)
 319 static void _I11(jit_state_t*,jit_word_t,
 320                  jit_word_t,jit_word_t,jit_word_t,jit_word_t,jit_word_t)
 321     maybe_unused;
 322 #define I12(len,pos,r2,r1)              _I12(_jit,0,len,pos,r2,r1)
 323 static void _I12(jit_state_t*,jit_word_t,
 324                  jit_word_t,jit_word_t,jit_word_t,jit_word_t)
 325     maybe_unused;
 326 #define I13(len,pos,im,r1)              _I13(_jit,0,len,pos,im,r1)
 327 static void _I13(jit_state_t*,jit_word_t,
 328                  jit_word_t,jit_word_t,jit_word_t,jit_word_t)
 329     maybe_unused;
 330 #define I14(s,len,r3,pos,r1)            _I14(_jit,0,s,len,r3,pos,r1)
 331 static void _I14(jit_state_t*,jit_word_t,
 332                  jit_word_t,jit_word_t,jit_word_t,jit_word_t,jit_word_t)
 333     maybe_unused;
 334 #define I15(pos,len,r3,r2,r1)           _I15(_jit,0,pos,len,r3,r2,r1)
 335 static void _I15(jit_state_t*,jit_word_t,
 336                  jit_word_t,jit_word_t,jit_word_t,jit_word_t,jit_word_t)
 337     maybe_unused;
 338 #define I16(tb,ta,p2,r3,pos,c,p1)       _I16(_jit,0,tb,ta,p2,r3,pos,c,p1)
 339 static void _I16(jit_state_t*,jit_word_t,
 340                  jit_word_t,jit_word_t,jit_word_t,
 341                  jit_word_t,jit_word_t,jit_word_t,jit_word_t)
 342     maybe_unused;
 343 #define I17(tb,ta,p2,r3,c,p1)           _I17(_jit,0,tb,ta,p2,r3,c,p1)
 344 static void _I17(jit_state_t*,jit_word_t,
 345                  jit_word_t,jit_word_t,jit_word_t,
 346                  jit_word_t,jit_word_t,jit_word_t)
 347     maybe_unused;
 348 #define I18(im,y)                       _I18(_jit,0,im,y)
 349 static void _I18(jit_state_t*,jit_word_t,
 350                  jit_word_t,jit_word_t)
 351     maybe_unused;
 352 #define I19(im)                         _I19(_jit,0,im)
 353 static void _I19(jit_state_t*,jit_word_t,
 354                  jit_word_t)
 355     maybe_unused;
 356 #define I20(r2,im)                      _I20(_jit,0,r2,im)
 357 static void _I20(jit_state_t*,jit_word_t,
 358                  jit_word_t,jit_word_t)
 359     maybe_unused;
 360 #define I21(im,ih,x,wh,r2,b1)           _I21(_jit,0,im,ih,x,wh,r2,b1)
 361 static void _I21(jit_state_t*,jit_word_t,
 362                  jit_word_t,jit_word_t,
 363                  jit_word_t,jit_word_t,jit_word_t,jit_word_t);
 364 #define I22(b2,r1)                      _I22(_jit,0,b2,r1)
 365 static void _I22(jit_state_t*,jit_word_t,
 366                  jit_word_t,jit_word_t);
 367 #define I23(r2,im)                      _I23(_jit,0,r2,im)
 368 static void _I23(jit_state_t*,jit_word_t,
 369                  jit_word_t,jit_word_t)
 370     maybe_unused;
 371 #define I24(im)                         _I24(_jit,0,im)
 372 static void _I24(jit_state_t*,jit_word_t,
 373                  jit_word_t)
 374     maybe_unused;
 375 #define I25(x6,r1)                      _I25(_jit,0,x6,r1)
 376 static void _I25(jit_state_t*,jit_word_t,
 377                  jit_word_t,jit_word_t)
 378     maybe_unused;
 379 #define I26(ar,r2)                      _I26(_jit,0,ar,r2)
 380 static void _I26(jit_state_t*,jit_word_t,
 381                  jit_word_t,jit_word_t);
 382 #define I27(ar,im)                      _I27(_jit,0,ar,im)
 383 static void _I27(jit_state_t*,jit_word_t,
 384                  jit_word_t,jit_word_t)
 385     maybe_unused;
 386 #define I28(ar,r1)                      _I28(_jit,0,ar,r1)
 387 static void _I28(jit_state_t*,jit_word_t,
 388                  jit_word_t,jit_word_t)
 389     maybe_unused;
 390 #define I29(x6,r3,r1)                   _I29(_jit,0,x6,r3,r1)
 391 static void _I29(jit_state_t*,jit_word_t,
 392                  jit_word_t,jit_word_t,jit_word_t);
 393 #define I30(tb,ta,p2,im,c,p1)           _I30(_jit,0,ta,tb,p2,im,c,p1)
 394 static void _I30(jit_state_t*,jit_word_t,
 395                  jit_word_t,jit_word_t,jit_word_t,
 396                  jit_word_t,jit_word_t,jit_word_t)
 397     maybe_unused;
 398 #define M1(x6,ht,x,r3,r1)               _M1(_jit,0,x6,ht,x,r3,r1)
 399 static void _M1(jit_state_t*,jit_word_t,
 400                 jit_word_t,jit_word_t,jit_word_t,jit_word_t,jit_word_t);
 401 #define M2(x6,ht,r3,r2,r1)              _M2(_jit,0,x6,ht,r3,r2,r1)
 402 static void _M2(jit_state_t*,jit_word_t,
 403                 jit_word_t,jit_word_t,jit_word_t,jit_word_t,jit_word_t)
 404     maybe_unused;
 405 #define M3(x6,ht,r3,im,r1)              _M3(_jit,0,x6,ht,r3,im,r1)
 406 static void _M3(jit_state_t*,jit_word_t,
 407                 jit_word_t,jit_word_t,jit_word_t,jit_word_t,jit_word_t);
 408 #define M5(x6,ht,r3,r2,im)              _M5(_jit,0,x6,ht,r3,r2,im)
 409 static void _M5(jit_state_t*,jit_word_t,
 410                 jit_word_t,jit_word_t,jit_word_t,jit_word_t,jit_word_t)
 411     maybe_unused;
 412 #define M6(x6,ht,x,r3,r2)               _M6(_jit,0,x6,ht,x,r3,r2)
 413 static void _M6(jit_state_t*,jit_word_t,
 414                 jit_word_t,jit_word_t,jit_word_t,jit_word_t,jit_word_t);
 415 #define M13(x6,ht,r3,f2)                _M13(_jit,0,x6,ht,r3,f2)
 416 static void _M13(jit_state_t*,jit_word_t,
 417                  jit_word_t,jit_word_t,jit_word_t,jit_word_t);
 418 #define M14(x6,ht,r3,r2)                _M14(_jit,0,x6,ht,r3,r2)
 419 static void _M14(jit_state_t*,jit_word_t,
 420                  jit_word_t,jit_word_t,jit_word_t,jit_word_t)
 421     maybe_unused;
 422 #define M15(x6,ht,r3,im)                _M15(_jit,0,x6,ht,r3,im)
 423 static void _M15(jit_state_t*,jit_word_t,
 424                  jit_word_t,jit_word_t,jit_word_t,jit_word_t)
 425     maybe_unused;
 426 #define M16(x6,ht,r3,r2,r1)             _M16(_jit,0,x6,ht,r3,r2,r1)
 427 static void _M16(jit_state_t*,jit_word_t,
 428                  jit_word_t,jit_word_t,jit_word_t,jit_word_t,jit_word_t)
 429     maybe_unused;
 430 #define M17(x6,ht,r3,im,r1)             _M17(_jit,0,x6,ht,r3,im,r1)
 431 static void _M17(jit_state_t*,jit_word_t,
 432                  jit_word_t,jit_word_t,jit_word_t,jit_word_t,jit_word_t)
 433     maybe_unused;
 434 #define M20(r2,im)                      M20x(0x1,r2,im)
 435 #define M20x(x3,r2,im)                  _M20x(_jit,0,x3,r2,im)
 436 static void _M20x(jit_state_t*,jit_word_t,
 437                   jit_word_t,jit_word_t,jit_word_t)
 438     maybe_unused;
 439 #define M22(x3,im,r1)                   M22x(x3,im,r1)
 440 #define M22x(x3,im,r1)                  _M22x(_jit,0,x3,im,r1)
 441 static void _M22x(jit_state_t*,jit_word_t,
 442                   jit_word_t,jit_word_t,jit_word_t)
 443     maybe_unused;
 444 #define M24(x2,x4)                      _M24(_jit,0,x2,x4)
 445 #define M25(x4)                         M24(0,x4)
 446 static void _M24(jit_state_t*,jit_word_t,
 447                  jit_word_t,jit_word_t)
 448     maybe_unused;
 449 #define M26(r1)                         M26x(2,r1)
 450 #define M26x(x4,r1)                     _M26x(_jit,0,x4,r1)
 451 static void _M26x(jit_state_t*,jit_word_t,
 452                   jit_word_t,jit_word_t)
 453     maybe_unused;
 454 #define M28(x,r3)                       _M28(_jit,0,x,r3)
 455 static void _M28(jit_state_t*,jit_word_t,
 456                  jit_word_t,jit_word_t)
 457     maybe_unused;
 458 #define M29(ar,r2)                      _M29(_jit,0,ar,r2)
 459 static void _M29(jit_state_t*,jit_word_t,
 460                  jit_word_t,jit_word_t)
 461     maybe_unused;
 462 #define M30(ar,im)                      _M30(_jit,0,ar,im)
 463 static void _M30(jit_state_t*,jit_word_t,jit_word_t,jit_word_t)
 464     maybe_unused;
 465 #define M31(ar,r1)                      _M31(_jit,0,ar,r1)
 466 static void _M31(jit_state_t*,jit_word_t,
 467                  jit_word_t,jit_word_t)
 468     maybe_unused;
 469 #define M32(cr,r2)                      _M32(_jit,0,cr,r2)
 470 static void _M32(jit_state_t*,jit_word_t,
 471                  jit_word_t,jit_word_t)
 472     maybe_unused;
 473 #define M33(cr,r1)                      _M33(_jit,0,cr,r1)
 474 static void _M33(jit_state_t*,jit_word_t,
 475                  jit_word_t,jit_word_t)
 476     maybe_unused;
 477 #define M34(sor,sol,sof,r1)             _M34(_jit,0,sor,sol,sof,r1)
 478 static void _M34(jit_state_t*,jit_word_t,
 479                  jit_word_t,jit_word_t,jit_word_t,jit_word_t);
 480 #define M35(x6,r2)                      _M35(_jit,0,x6,r2)
 481 static void _M35(jit_state_t*,jit_word_t,
 482                  jit_word_t,jit_word_t)
 483     maybe_unused;
 484 #define M36(x6,r1)                      _M36(_jit,0,x6,r1)
 485 static void _M36(jit_state_t*,jit_word_t,
 486                  jit_word_t,jit_word_t)
 487     maybe_unused;
 488 #define M37(im)                         _M37(_jit,0,im)
 489 static void _M37(jit_state_t*,jit_word_t,
 490                  jit_word_t)
 491     maybe_unused;
 492 #define M38(x6,r3,r2,r1)                _M38(_jit,0,x6,r3,r2,r1)
 493 static void _M38(jit_state_t*,jit_word_t,
 494                  jit_word_t,jit_word_t,jit_word_t,jit_word_t)
 495     maybe_unused;
 496 #define M39(x6,r3,im,r1)                _M39(_jit,0,x6,r3,im,r1)
 497 static void _M39(jit_state_t*,jit_word_t,
 498                  jit_word_t,jit_word_t,jit_word_t,jit_word_t)
 499     maybe_unused;
 500 #define M40(x6,r3,im)                   _M40(_jit,0,x6,r3,im)
 501 static void _M40(jit_state_t*,jit_word_t,
 502                  jit_word_t,jit_word_t,jit_word_t)
 503     maybe_unused;
 504 #define M41(x6,r2)                      _M41(_jit,0,x6,r2)
 505 static void _M41(jit_state_t*,jit_word_t,
 506                  jit_word_t,jit_word_t)
 507     maybe_unused;
 508 #define M42(x6,r3,r2)                   _M42(_jit,0,x6,r3,r2)
 509 static void _M42(jit_state_t*,jit_word_t,
 510                  jit_word_t,jit_word_t,jit_word_t)
 511     maybe_unused;
 512 #define M43(x6,r3,r1)                   _M43(_jit,0,x6,r3,r1)
 513 static void _M43(jit_state_t*,jit_word_t,
 514                  jit_word_t,jit_word_t,jit_word_t)
 515     maybe_unused;
 516 #define M44(x4,im)                      _M44(_jit,0,x4,im)
 517 static void _M44(jit_state_t*,jit_word_t,
 518                  jit_word_t,jit_word_t)
 519     maybe_unused;
 520 #define M45(x6,r3,r2)                   _M45(_jit,0,x6,r3,r2)
 521 static void _M45(jit_state_t*,jit_word_t,
 522                  jit_word_t,jit_word_t,jit_word_t)
 523     maybe_unused;
 524 #define M46(x6,r3,r1)                   _M46(_jit,0,x6,r3,r1)
 525 #define M47(x6,r3)                      M46(x6,r3,0)
 526 static void _M46(jit_state_t*,jit_word_t,
 527                  jit_word_t,jit_word_t,jit_word_t)
 528     maybe_unused;
 529 #define M48(y,im)                       _M48(_jit,0,y,im)
 530 static void _M48(jit_state_t*,jit_word_t,
 531                  jit_word_t,jit_word_t)
 532     maybe_unused;
 533 #define B1(d,wh,im,p,tp)                _B1(_jit,0,d,wh,im,p,tp)
 534 #define B2(d,wh,im,p,tp)                B1(d,wh,im,p,tp)
 535 static void _B1(jit_state_t*,jit_word_t,
 536                 jit_word_t,jit_word_t,jit_word_t,jit_word_t,jit_word_t);
 537 #define B3(d,wh,im,p,b)                 _B3(_jit,0,d,wh,im,p,b)
 538 static void _B3(jit_state_t*,jit_word_t,
 539                 jit_word_t,jit_word_t,jit_word_t,jit_word_t,jit_word_t)
 540     maybe_unused;
 541 #define B4(d,wh,x6,b,p,tp)              _B4(_jit,0,d,wh,x6,b,p,tp)
 542 static void _B4(jit_state_t*,jit_word_t,
 543                 jit_word_t,jit_word_t,jit_word_t,
 544                 jit_word_t,jit_word_t,jit_word_t);
 545 #define B5(d,wh,b2,p,b1)                _B5(_jit,0,d,wh,b2,p,b1)
 546 static void _B5(jit_state_t*,jit_word_t,
 547                 jit_word_t,jit_word_t,jit_word_t,jit_word_t,jit_word_t);
 548 #define B6(ih,im,tag,wh)                _B6(_jit,0,ih,im,tag,wh)
 549 static void _B6(jit_state_t*,jit_word_t,
 550                 jit_word_t,jit_word_t,jit_word_t,jit_word_t)
 551     maybe_unused;
 552 #define B7(ih,x6,b2,tag,wh)             _B7(_jit,0,ih,x6,b2,tag,wh)
 553 static void _B7(jit_state_t*,jit_word_t,
 554                 jit_word_t,jit_word_t,jit_word_t,jit_word_t,jit_word_t)
 555     maybe_unused;
 556 #define B8(x6)                          _B8(_jit,0,x6)
 557 static void _B8(jit_state_t*,jit_word_t,
 558                 jit_word_t)
 559     maybe_unused;
 560 #define B9(op,x6,im)                    _B9(_jit,0,op,x6,im)
 561 static void _B9(jit_state_t*,jit_word_t,
 562                 jit_word_t,jit_word_t,jit_word_t)
 563     maybe_unused;
 564 #define X1(im)                          _X1(_jit,0,im)
 565 static void _X1(jit_state_t*,jit_word_t,
 566                 jit_word_t)
 567     maybe_unused;
 568 #define X2(r1,im)                       _X2(_jit,0,r1,im)
 569 static void _X2(jit_state_t*,jit_word_t,
 570                 jit_word_t,jit_word_t);
 571 #define X3x(o,d,wh,p,tp,im)             _X3x(_jit,0,o,d,wh,p,tp,im)
 572 #define X3(d,wh,p,tp,im)                X3x(0xc,d,wh,p,tp,im)
 573 #define X4(d,wh,p,tp,im)                X3x(0xd,d,wh,p,tp,im)
 574 static void _X3x(jit_state_t*,jit_word_t,
 575                  jit_word_t,jit_word_t,jit_word_t,
 576                  jit_word_t,jit_word_t,jit_word_t);
 577 #define X5(y,im)                        _X5(_jit,0,y,im)
 578 static void _X5(jit_state_t*,jit_word_t,
 579                 jit_word_t,jit_word_t)
 580     maybe_unused;
 581
 582 /* add */
 583 #define ADD(r1,r2,r3)                   A1(0,0,r3,r2,r1)
 584 #define ADD1(r1,r2,r3)                  A1(0,1,r3,r2,r1)
 585 #define ADDS(r1,im,r3)                  A4(2,r3,im,r1)
 586 #define ADDS_p(r1,im,r3,_p)             _A4(_jit,_p,2,r3,im,r1)
 587 #define ADDL(r1,im,r3)                  A5(r3,im,r1)
 588 #define ADDL_p(r1,im,r3,_p)             _A5(_jit,_p,r3,im,r1)
 589 /* addp4 */
 590 #define ADDP4(r1,r2,r3)                 A1(2,0,r3,r2,r1)
 591 #define ADDIP4(r1,im,r3)                A4(3,r3,im,r1)
 592 /* alloc */
 593 #define ALLOCR(r1,i,l,o,r)              M34((r)>>3,(i)+(l),(i)+(l)+(o),r1)
 594 #define ALLOC(r1,i,o)                   ALLOCR(r1,i,0,o,0)
 595 /* and */
 596 #define AND(r1,r2,r3)                   A1(3,0,r3,r2,r1)
 597 #define ANDI(r1,im,r3)                  A3(0xb,0,r3,im,r1)
 598 /* andcm */
 599 #define ANDCM(r1,r2,r3)                 A1(3,1,r3,r2,r1)
 600 #define ANDCMI(r1,im,r3)                A3(0xb,1,r3,im,r1)
 601 /* br */
 602 #define BRI(im)                         B1(BR_DH_NONE,BR_BWH_SPTK,im,BR_PH_FEW,0)
 603 #define BRI_COND(im,_p)                 _B1(_jit,_p,BR_DH_NONE,BR_BWH_SPTK,im,BR_PH_FEW,0)
 604 #define BRI_WEXIT(im)                   B1(BR_DH_NONE,BR_BWH_SPTK,im,BR_PH_MANY,2)
 605 #define BRI_WTOP(im)                    B1(BR_DH_NONE,BR_BWH_SPTK,im,BR_PH_MANY,3)
 606 #define BRI_CALL(b,im)                  B3(BR_DH_NONE,BR_BWH_SPTK,im,BR_PH_MANY,b)
 607 #define BRI_CLOOP(im)                   B2(BR_DH_NONE,BR_BWH_SPTK,im,BR_PH_MANY,5)
 608 #define BRI_CEXIT(im)                   B2(BR_DH_NONE,BR_BWH_SPTK,im,BR_PH_MANY,6)
 609 #define BRI_CTOP(im)                    B2(BR_DH_NONE,BR_BWH_SPTK,im,BR_PH_MANY,7)
 610 #define BR_COND(b,_p)                   _B4(_jit,_p,BR_DH_NONE,BR_BWH_SPTK,0x20,b,BR_PH_FEW,0)
 611 #define BR(b)                           B4(BR_DH_NONE,BR_BWH_SPTK,0x20,b,BR_PH_FEW,0)
 612 #define BR_IA(b)                        B4(BR_DH_NONE,BR_BWH_SPTK,0x20,b,BR_PH_MANY,1)
 613 #define BR_RET(b)                       B4(BR_DH_NONE,BR_BWH_SPTK,0x21,b,BR_PH_MANY,4)
 614 #define BR_CALL(b1,b2)                  B5(BR_DH_NONE,BRI_BWH_SPTK,b2,BR_PH_MANY,b1)
 615 /* break */
 616 #define BREAK_I(im)                     I19(im)
 617 #define BREAK_M(im)                     M37(im)
 618 #define BREAK_B(im)                     B9(0,0,im)
 619 #define BREAK_X(im)                     X1(im)
 620 /* brl */
 621 #define BRL(im)                         X3(BR_DH_NONE,BR_BWH_SPTK,BR_PH_MANY,0,im)
 622 #define BRL_COND(im,_p)                 _X3(_jit,_p,BR_DH_NONE,BR_BWH_SPTK,BR_PH_MANY,0,im)
 623 #define BRL_CALL(b1,im)                 X4(BR_DH_NONE,BR_BWH_SPTK,BR_PH_MANY,b1,im)
 624 /* brp */
 625 #define BRP(im,tag)                     B6(BR_IH_NONE,im,tag,BR_IPWH_SPTK)
 626 #define BRPI(b2,tag)                    B7(BR_IH_NONE,0x10,b2,tag,BR_INDWH_SPTK)
 627 #define BRPI_RET(b2,tag)                B7(BR_IH_NONE,0x11,b2,tag,BR_INDWH_SPTK)
 628 /* bsw */
 629 #define BSW_0()                         B8(0x0c)
 630 #define BSW_1()                         B8(0x0d)
 631 /* chk */
 632 #define CHK_S_I(r2,im)                  I20(r2,im)
 633 #define CHK_S_M(r2,im)                  M20(r2,im)
 634 #define CHK_A_NC(r1,im)                 M22(0x4,im,r1)
 635 #define CHK_A_CLR(r1,im)                M22(0x5,im,r1)
 636 /* clrrrb */
 637 #define CLRRRB()                        B8(0x04)
 638 #define CLRRRB_PR()                     B8(0x05)
 639 /* clz */
 640 #define CLZ(r1,r3)                      I9(3,r3,r1)
 641 /* cmp */
 642 #define CMP_LT(p1,p2,r2,r3)             A6(0xc,0,0,p2,r3,r2,0,p1)
 643 #define CMP_LT_p(p1,p2,r2,r3,_p)        A6(_jit,_p,0xc,0,0,p2,r3,r2,0,p1)
 644 #define CMP_LTU(p1,p2,r2,r3)            A6(0xd,0,0,p2,r3,r2,0,p1)
 645 #define CMP_EQ(p1,p2,r2,r3)             A6(0xe,0,0,p2,r3,r2,0,p1)
 646 #define CMP_LT_UNC(p1,p2,r2,r3)         A6(0xc,0,0,p2,r3,r2,1,p1)
 647 #define CMP_LTU_UNC(p1,p2,r2,r3)        A6(0xd,0,0,p2,r3,r2,1,p1)
 648 #define CMP_EQ_UNC(p1,p2,r2,r3)         A6(0xe,0,0,p2,r3,r2,1,p1)
 649 #define CMP_EQ_AND(p1,p2,r2,r3)         A6(0xc,0,1,p2,r3,r2,0,p1)
 650 #define CMP_EQ_OR(p1,p2,r2,r3)          A6(0xd,0,1,p2,r3,r2,0,p1)
 651 #define CMP_EQ_OR_ANDCM(p1,p2,r2,r3)    A6(0xe,0,1,p2,r3,r2,0,p1)
 652 #define CMP_NE_AND(p1,p2,r2,r3)         A6(0xc,0,1,p2,r3,r2,1,p1)
 653 #define CMP_NE_OR(p1,p2,r2,r3)          A6(0xd,0,1,p2,r3,r2,1,p1)
 654 #define CMP_NE_OR_ANDCM(p1,p2,r2,r3)    A6(0xe,0,1,p2,r3,r2,1,p1)
 655 #define CMPI_LT(p1,p2,im,r3)            A8(0xc,2,0,p2,r3,im,0,p1)
 656 #define CMPI_LTU(p1,p2,im,r3)           A8(0xd,2,0,p2,r3,im,0,p1)
 657 #define CMPI_EQ(p1,p2,im,r3)            A8(0xe,2,0,p2,r3,im,0,p1)
 658 #define CMPI_EQ_p(p1,p2,im,r3,_p)       _A8(_jit,_p,0xe,2,0,p2,r3,im,0,p1)
 659 #define CMPI_LT_UNC(p1,p2,im,r3)        A8(0xc,2,0,p2,r3,im,1,p1)
 660 #define CMPI_LTU_UNC(p1,p2,im,r3)       A8(0xd,2,0,p2,r3,im,1,p1)
 661 #define CMPI_EQ_UNC(p1,p2,im,r3)        A8(0xe,2,0,p2,r3,im,1,p1)
 662 #define CMPI_EQ_AND(p1,p2,im,r3)        A8(0xc,2,1,p2,r3,im,0,p1)
 663 #define CMPI_EQ_OR(p1,p2,im,r3)         A8(0xd,2,1,p2,r3,im,0,p1)
 664 #define CMPI_EQ_ANDCM(p1,p2,im,r3)      A8(0xe,2,1,p2,r3,im,0,p1)
 665 #define CMPI_NE_AND(p1,p2,im,r3)        A8(0xc,2,1,p2,r3,im,1,p1)
 666 #define CMPI_NE_OR(p1,p2,im,r3)         A8(0xd,2,1,p2,r3,im,1,p1)
 667 #define CMPI_NE_ANDCM(p1,p2,im,r3)      A8(0xe,2,1,p2,r3,im,1,p1)
 668 #define ZCMP_GT_AND(p1,p2,r3)           A7(0xc,0,0,p2,r3,0,p1)
 669 #define ZCMP_GT_OR(p1,p2,r3)            A7(0xd,0,0,p2,r3,0,p1)
 670 #define ZCMP_GT_ANDCM(p1,p2,r3)         A7(0xe,0,0,p2,r3,0,p1)
 671 #define ZCMP_LE_AND(p1,p2,r3)           A7(0xc,0,0,p2,r3,1,p1)
 672 #define ZCMP_LE_OR(p1,p2,r3)            A7(0xd,0,0,p2,r3,1,p1)
 673 #define ZCMP_LE_ANDCM(p1,p2,r3)         A7(0xe,0,0,p2,r3,1,p1)
 674 #define ZCMP_GE_AND(p1,p2,r3)           A7(0xc,0,1,p2,r3,0,p1)
 675 #define ZCMP_GE_OR(p1,p2,r3)            A7(0xd,0,1,p2,r3,0,p1)
 676 #define ZCMP_GE_ANDCM(p1,p2,r3)         A7(0xe,0,1,p2,r3,0,p1)
 677 #define ZCMP_LT_AND(p1,p2,r3)           A7(0xc,0,1,p2,r3,1,p1)
 678 #define ZCMP_LT_OR(p1,p2,r3)            A7(0xd,0,1,p2,r3,1,p1)
 679 #define ZCMP_LT_ANDCM(p1,p2,r3)         A7(0xe,0,1,p2,r3,1,p1)
 680 /* cmp4 */
 681 #define CMP4_LT(p1,p2,r2,r3)            A6(0xc,1,0,p2,r3,r2,0,p1)
 682 #define CMP4_LTU(p1,p2,r2,r3)           A6(0xd,1,0,p2,r3,r2,0,p1)
 683 #define CMP4_EQ(p1,p2,r2,r3)            A6(0xe,1,0,p2,r3,r2,0,p1)
 684 #define CMP4_LT_UNC(p1,p2,r2,r3)        A6(0xc,1,0,p2,r3,r2,1,p1)
 685 #define CMP4_LTU_UNC(p1,p2,r2,r3)       A6(0xd,1,0,p2,r3,r2,1,p1)
 686 #define CMP4_EQ_UNC(p1,p2,r2,r3)        A6(0xe,1,0,p2,r3,r2,1,p1)
 687 #define CMP4_EQ_AND(p1,p2,r2,r3)        A6(0xc,1,1,p2,r3,r2,0,p1)
 688 #define CMP4_EQ_OR(p1,p2,r2,r3)         A6(0xd,1,1,p2,r3,r2,0,p1)
 689 #define CMP4_EQ_XOR(p1,p2,r2,r3)        A6(0xe,1,1,p2,r3,r2,0,p1)
 690 #define CMP4_NE_AND(p1,p2,r2,r3)        A6(0xc,1,1,p2,r3,r2,1,p1)
 691 #define CMP4_NE_OR(p1,p2,r2,r3)         A6(0xd,1,1,p2,r3,r2,1,p1)
 692 #define CMP4_NE_XOR(p1,p2,r2,r3)        A6(0xe,1,1,p2,r3,r2,1,p1)
 693 #define CMP4I_LT(p1,p2,im,r3)           A8(0xc,3,0,p2,r3,im,0,p1)
 694 #define CMP4I_LTU(p1,p2,im,r3)          A8(0xd,3,0,p2,r3,im,0,p1)
 695 #define CMP4I_EQ(p1,p2,im,r3)           A8(0xe,3,0,p2,r3,im,0,p1)
 696 #define CMP4I_LT_UNC(p1,p2,im,r3)       A8(0xc,3,0,p2,r3,im,1,p1)
 697 #define CMP4I_LTU_UNC(p1,p2,im,r3)      A8(0xd,3,0,p2,r3,im,1,p1)
 698 #define CMP4I_EQ_UNC(p1,p2,im,r3)       A8(0xe,3,0,p2,r3,im,1,p1)
 699 #define CMP4I_EQ_AND(p1,p2,im,r3)       A8(0xc,3,1,p2,r3,im,0,p1)
 700 #define CMP4I_EQ_OR(p1,p2,im,r3)        A8(0xd,3,1,p2,r3,im,0,p1)
 701 #define CMP4I_EQ_ANDCM(p1,p2,im,r3)     A8(0xe,3,1,p2,r3,im,0,p1)
 702 #define CMP4I_NE_AND(p1,p2,im,r3)       A8(0xc,3,1,p2,r3,im,1,p1)
 703 #define CMP4I_NE_OR(p1,p2,im,r3)        A8(0xd,3,1,p2,r3,im,1,p1)
 704 #define CMP4I_NE_ANDCM(p1,p2,im,r3)     A8(0xe,3,1,p2,r3,im,1,p1)
 705 #define ZCMP4_GT_AND(p1,p2,r3)          A7(0xc,1,0,p2,r3,0,p1)
 706 #define ZCMP4_GT_OR(p1,p2,r3)           A7(0xd,1,0,p2,r3,0,p1)
 707 #define ZCMP4_GT_ANDCM(p1,p2,r3)        A7(0xe,1,0,p2,r3,0,p1)
 708 #define ZCMP4_LE_AND(p1,p2,r3)          A7(0xc,1,0,p2,r3,1,p1)
 709 #define ZCMP4_LE_OR(p1,p2,r3)           A7(0xd,1,0,p2,r3,1,p1)
 710 #define ZCMP4_LE_ANDCM(p1,p2,r3)        A7(0xe,1,0,p2,r3,1,p1)
 711 #define ZCMP4_GE_AND(p1,p2,r3)          A7(0xc,1,1,p2,r3,0,p1)
 712 #define ZCMP4_GE_OR(p1,p2,r3)           A7(0xd,1,1,p2,r3,0,p1)
 713 #define ZCMP4_GE_ANDCM(p1,p2,r3)        A7(0xe,1,1,p2,r3,0,p1)
 714 #define ZCMP4_LT_AND(p1,p2,r3)          A7(0xc,1,1,p2,r3,1,p1)
 715 #define ZCMP4_LT_OR(p1,p2,r3)           A7(0xd,1,1,p2,r3,1,p1)
 716 #define ZCMP4_LT_ANDCM(p1,p2,r3)        A7(0xe,1,1,p2,r3,1,p1)
 717 /* cmpxchg */
 718 #define CMPXCHG1_ACQ(r1,r3,r2)          M16(0x00,LD_NONE,r3,r2,r1)
 719 #define CMPXCHG2_ACQ(r1,r3,r2)          M16(0x01,LD_NONE,r3,r2,r1)
 720 #define CMPXCHG4_ACQ(r1,r3,r2)          M16(0x02,LD_NONE,r3,r2,r1)
 721 #define CMPXCHG8_ACQ(r1,r3,r2)          M16(0x03,LD_NONE,r3,r2,r1)
 722 #define CMPXCHG1_REL(r1,r3,r2)          M16(0x04,LD_NONE,r3,r2,r1)
 723 #define CMPXCHG2_REL(r1,r3,r2)          M16(0x05,LD_NONE,r3,r2,r1)
 724 #define CMPXCHG4_REL(r1,r3,r2)          M16(0x06,LD_NONE,r3,r2,r1)
 725 #define CMPXCHG8_REL(r1,r3,r2)          M16(0x07,LD_NONE,r3,r2,r1)
 726 #define CMP8XCHG16_ACQ(r1,r3,r2)        M16(0x20,LD_NONE,r3,r2,r1)
 727 #define CMP8XCHG16_REL(r1,r3,r2)        M16(0x24,LD_NONE,r3,r2,r1)
 728 /* cover */
 729 #define COVER()                         B8(0x02)
 730 /* cxz */
 731 #define CZX1_L(r1,r3)                   I29(0x18,r3,r1)
 732 #define CZX2_L(r1,r3)                   I29(0x19,r3,r1)
 733 #define CZX1_R(r1,r3)                   I29(0x1c,r3,r1)
 734 #define CZX2_R(r1,r3)                   I29(0x1d,r3,r1)
 735 /* dep */
 736 #define DEP_Z(r1,r2,pos,len)            I12(len,pos,r2,r1)
 737 #define DEPI_Z(r1,im,pos,len)           I13(len,pos,im,r1)
 738 #define DEPs(r1,r2,r3,pos,len)          I14(1,len,r3,pos,r1)
 739 #define DEPu(r1,r2,r3,pos,len)          I14(0,len,r3,pos,r1)
 740 #define DEP(r1,r2,r3,pos,len)           I15(pos,len,r3,r2,r1)
 741 /* epc */
 742 #define EPC()                           B8(0x10)
 743 /* extr */
 744 #define EXTR(r1,r3,pos,len)             I11(len,r3,pos,1,r1)
 745 #define EXTR_U(r1,r3,pos,len)           I11(len,r3,pos,0,r1)
 746 /* fc */
 747 #define FC(r3)                          M28(0,r3)
 748 #define FC_I(r3)                        M28(1,r3)
 749 /* fetchadd */
 750 #define FETCHADD4_ACQ(r1,r3,im)         M17(0x12,LD_NONE,r3,im,r1)
 751 #define FETCHADD8_ACQ(r1,r3,im)         M17(0x13,LD_NONE,r3,im,r1)
 752 #define FETCHADD4_REL(r1,r3,im)         M17(0x16,LD_NONE,r3,im,r1)
 753 #define FETCHADD8_REL(r1,r3,im)         M17(0x17,LD_NONE,r3,im,r1)
 754 /* flushrs */
 755 #define FLUSHRS()                       M25(0xc)
 756 /* fwb */
 757 #define FWB()                           M24(2,0)
 758 /* hint */
 759 #define HINT_I(im)                      I18(im,1)
 760 #define HINT_M(im)                      M48(1,im)
 761 #define HINT_B(im)                      B9(2,1,im)
 762 #define HINT_X(im)                      X5(1,im)
 763 /* invala */
 764 #define INVALA()                        M24(1,0)
 765 #define INVALA_E(r1)                    M26(r1)
 766 /* itc */
 767 #define ITC_I(r2)                       M41(0x2f,r2)
 768 #define ITC_D(r2)                       M41(0x2e,r2)
 769 /* itr */
 770 #define ITR_I(r3,r2)                    M42(0x0f,r3,r2)
 771 #define ITR_D(r3,r2)                    M42(0x0e,r3,r2)
 772 /* ld */
 773 #define LD1(r1,r3)                      M1(0x00,LD_NONE,0,r3,r1)
 774 #define LD2(r1,r3)                      M1(0x01,LD_NONE,0,r3,r1)
 775 #define LD4(r1,r3)                      M1(0x02,LD_NONE,0,r3,r1)
 776 #define LD8(r1,r3)                      M1(0x03,LD_NONE,0,r3,r1)
 777 #define LD1_S(r1,r3)                    M1(0x04,LD_NONE,0,r3,r1)
 778 #define LD2_S(r1,r3)                    M1(0x05,LD_NONE,0,r3,r1)
 779 #define LD4_S(r1,r3)                    M1(0x06,LD_NONE,0,r3,r1)
 780 #define LD8_S(r1,r3)                    M1(0x07,LD_NONE,0,r3,r1)
 781 #define LD1_A(r1,r3)                    M1(0x08,LD_NONE,0,r3,r1)
 782 #define LD2_A(r1,r3)                    M1(0x09,LD_NONE,0,r3,r1)
 783 #define LD4_A(r1,r3)                    M1(0x0a,LD_NONE,0,r3,r1)
 784 #define LD8_A(r1,r3)                    M1(0x0b,LD_NONE,0,r3,r1)
 785 #define LD1_SA(r1,r3)                   M1(0x0c,LD_NONE,0,r3,r1)
 786 #define LD2_SA(r1,r3)                   M1(0x0d,LD_NONE,0,r3,r1)
 787 #define LD4_SA(r1,r3)                   M1(0x0e,LD_NONE,0,r3,r1)
 788 #define LD8_SA(r1,r3)                   M1(0x0f,LD_NONE,0,r3,r1)
 789 #define LD1_BIAS(r1,r3)                 M1(0x10,LD_NONE,0,r3,r1)
 790 #define LD2_BIAS(r1,r3)                 M1(0x11,LD_NONE,0,r3,r1)
 791 #define LD4_BIAS(r1,r3)                 M1(0x12,LD_NONE,0,r3,r1)
 792 #define LD8_BIAS(r1,r3)                 M1(0x13,LD_NONE,0,r3,r1)
 793 #define LD1_ACQ(r1,r3)                  M1(0x14,LD_NONE,0,r3,r1)
 794 #define LD2_ACQ(r1,r3)                  M1(0x15,LD_NONE,0,r3,r1)
 795 #define LD4_ACQ(r1,r3)                  M1(0x16,LD_NONE,0,r3,r1)
 796 #define LD8_ACQ(r1,r3)                  M1(0x17,LD_NONE,0,r3,r1)
 797 #define LD8_FILL(r1,r3)                 M1(0x1b,LD_NONE,0,r3,r1)
 798 #define LD1_C_CLR(r1,r3)                M1(0x20,LD_NONE,0,r3,r1)
 799 #define LD2_C_CLR(r1,r3)                M1(0x21,LD_NONE,0,r3,r1)
 800 #define LD4_C_CLR(r1,r3)                M1(0x22,LD_NONE,0,r3,r1)
 801 #define LD8_C_CLR(r1,r3)                M1(0x23,LD_NONE,0,r3,r1)
 802 #define LD1_C_NC(r1,r3)                 M1(0x24,LD_NONE,0,r3,r1)
 803 #define LD2_C_NC(r1,r3)                 M1(0x25,LD_NONE,0,r3,r1)
 804 #define LD4_C_NC(r1,r3)                 M1(0x26,LD_NONE,0,r3,r1)
 805 #define LD8_C_NC(r1,r3)                 M1(0x27,LD_NONE,0,r3,r1)
 806 #define LD1_C_CLR_ACQ(r1,r3)            M1(0x28,LD_NONE,0,r3,r1)
 807 #define LD2_C_CLR_ACQ(r1,r3)            M1(0x29,LD_NONE,0,r3,r1)
 808 #define LD4_C_CLR_ACQ(r1,r3)            M1(0x2a,LD_NONE,0,r3,r1)
 809 #define LD8_C_CLR_ACQ(r1,r3)            M1(0x2b,LD_NONE,0,r3,r1)
 810 #define LD16(r1,r3)                     M1(0x28,LD_NONE,1,r3,r1)
 811 #define LD16_ACQ(r1,r3)                 M1(0x2c,LD_NONE,1,r3,r1)
 812 #define LD1_inc(r1,r3,im)               M3(0x00,LD_NONE,r3,im,r1)
 813 #define LD2_inc(r1,r3,im)               M3(0x01,LD_NONE,r3,im,r1)
 814 #define LD4_inc(r1,r3,im)               M3(0x02,LD_NONE,r3,im,r1)
 815 #define LD8_inc(r1,r3,im)               M3(0x03,LD_NONE,r3,im,r1)
 816 #define LD1_S_inc(r1,r3,im)             M3(0x04,LD_NONE,r3,im,r1)
 817 #define LD2_S_inc(r1,r3,im)             M3(0x05,LD_NONE,r3,im,r1)
 818 #define LD4_S_inc(r1,r3,im)             M3(0x06,LD_NONE,r3,im,r1)
 819 #define LD8_S_inc(r1,r3,im)             M3(0x07,LD_NONE,r3,im,r1)
 820 #define LD1_A_inc(r1,r3,im)             M3(0x08,LD_NONE,r3,im,r1)
 821 #define LD2_A_inc(r1,r3,im)             M3(0x09,LD_NONE,r3,im,r1)
 822 #define LD4_A_inc(r1,r3,im)             M3(0x0a,LD_NONE,r3,im,r1)
 823 #define LD8_A_inc(r1,r3,im)             M3(0x0b,LD_NONE,r3,im,r1)
 824 #define LD1_SA_inc(r1,r3,im)            M3(0x0c,LD_NONE,r3,im,r1)
 825 #define LD2_SA_inc(r1,r3,im)            M3(0x0d,LD_NONE,r3,im,r1)
 826 #define LD4_SA_inc(r1,r3,im)            M3(0x0e,LD_NONE,r3,im,r1)
 827 #define LD8_SA_inc(r1,r3,im)            M3(0x0f,LD_NONE,r3,im,r1)
 828 #define LD1_BIAS_inc(r1,r3,im)          M3(0x10,LD_NONE,r3,im,r1)
 829 #define LD2_BIAS_inc(r1,r3,im)          M3(0x11,LD_NONE,r3,im,r1)
 830 #define LD4_BIAS_inc(r1,r3,im)          M3(0x12,LD_NONE,r3,im,r1)
 831 #define LD8_BIAS_inc(r1,r3,im)          M3(0x13,LD_NONE,r3,im,r1)
 832 #define LD1_ACQ_inc(r1,r3,im)           M3(0x14,LD_NONE,r3,im,r1)
 833 #define LD2_ACQ_inc(r1,r3,im)           M3(0x15,LD_NONE,r3,im,r1)
 834 #define LD4_ACQ_inc(r1,r3,im)           M3(0x16,LD_NONE,r3,im,r1)
 835 #define LD8_AVQ_inc(r1,r3,im)           M3(0x17,LD_NONE,r3,im,r1)
 836 #define LD8_FILL_inc(r1,r3,im)          M3(0x1b,LD_NONE,r3,im,r1)
 837 #define LD1_C_CLR_inc(r1,r3,im)         M3(0x20,LD_NONE,r3,im,r1)
 838 #define LD2_C_CLR_inc(r1,r3,im)         M3(0x21,LD_NONE,r3,im,r1)
 839 #define LD4_C_CLR_inc(r1,r3,im)         M3(0x22,LD_NONE,r3,im,r1)
 840 #define LD8_C_CLR_inc(r1,r3,im)         M3(0x23,LD_NONE,r3,im,r1)
 841 #define LD1_C_NC_inc(r1,r3,im)          M3(0x24,LD_NONE,r3,im,r1)
 842 #define LD2_C_NC_inc(r1,r3,im)          M3(0x25,LD_NONE,r3,im,r1)
 843 #define LD4_C_NC_inc(r1,r3,im)          M3(0x26,LD_NONE,r3,im,r1)
 844 #define LD8_C_NC_inc(r1,r3,im)          M3(0x27,LD_NONE,r3,im,r1)
 845 #define LD1_C_CLR_ACQ_inc(r1,r3,im)     M3(0x28,LD_NONE,r3,im,r1)
 846 #define LD2_C_CLR_ACQ_inc(r1,r3,im)     M3(0x29,LD_NONE,r3,im,r1)
 847 #define LD4_C_CLR_ACQ_inc(r1,r3,im)     M3(0x2a,LD_NONE,r3,im,r1)
 848 #define LD8_C_CLR_ACQ_inc(r1,r3,im)     M3(0x2b,LD_NONE,r3,im,r1)
 849 #define LDX1(r1,r3,r2)                  M2(0x00,LD_NONE,r3,r2,r1)
 850 #define LDX2(r1,r3,r2)                  M2(0x01,LD_NONE,r3,r2,r1)
 851 #define LDX4(r1,r3,r2)                  M2(0x02,LD_NONE,r3,r2,r1)
 852 #define LDX8(r1,r3,r2)                  M2(0x03,LD_NONE,r3,r2,r1)
 853 #define LDX1_S(r1,r3,r2)                M2(0x04,LD_NONE,r3,r2,r1)
 854 #define LDX2_S(r1,r3,r2)                M2(0x05,LD_NONE,r3,r2,r1)
 855 #define LDX4_S(r1,r3,r2)                M2(0x06,LD_NONE,r3,r2,r1)
 856 #define LDX8_S(r1,r3,r2)                M2(0x07,LD_NONE,r3,r2,r1)
 857 #define LDX1_A(r1,r3,r2)                M2(0x08,LD_NONE,r3,r2,r1)
 858 #define LDX2_A(r1,r3,r2)                M2(0x09,LD_NONE,r3,r2,r1)
 859 #define LDX4_A(r1,r3,r2)                M2(0x0a,LD_NONE,r3,r2,r1)
 860 #define LDX8_A(r1,r3,r2)                M2(0x0b,LD_NONE,r3,r2,r1)
 861 #define LDX1_SA(r1,r3,r2)               M2(0x0c,LD_NONE,r3,r2,r1)
 862 #define LDX2_SA(r1,r3,r2)               M2(0x0d,LD_NONE,r3,r2,r1)
 863 #define LDX4_SA(r1,r3,r2)               M2(0x0e,LD_NONE,r3,r2,r1)
 864 #define LDX8_SA(r1,r3,r2)               M2(0x0f,LD_NONE,r3,r2,r1)
 865 #define LDX1_BIAS(r1,r3,r2)             M2(0x10,LD_NONE,r3,r2,r1)
 866 #define LDX2_BIAS(r1,r3,r2)             M2(0x11,LD_NONE,r3,r2,r1)
 867 #define LDX4_BIAS(r1,r3,r2)             M2(0x12,LD_NONE,r3,r2,r1)
 868 #define LDX8_BIAS(r1,r3,r2)             M2(0x13,LD_NONE,r3,r2,r1)
 869 #define LDX1_ACQ(r1,r3,r2)              M2(0x14,LD_NONE,r3,r2,r1)
 870 #define LDX2_ACQ(r1,r3,r2)              M2(0x15,LD_NONE,r3,r2,r1)
 871 #define LDX4_ACQ(r1,r3,r2)              M2(0x16,LD_NONE,r3,r2,r1)
 872 #define LDX8_ACQ(r1,r3,r2)              M2(0x17,LD_NONE,r3,r2,r1)
 873 #define LDX8_FILL(r1,r3,r2)             M2(0x1b,LD_NONE,r3,r2,r1)
 874 #define LDX1_C_CLR(r1,r3,r2)            M2(0x20,LD_NONE,r3,r2,r1)
 875 #define LDX2_C_CLR(r1,r3,r2)            M2(0x21,LD_NONE,r3,r2,r1)
 876 #define LDX4_C_CLR(r1,r3,r2)            M2(0x22,LD_NONE,r3,r2,r1)
 877 #define LDX8_C_CLR(r1,r3,r2)            M2(0x23,LD_NONE,r3,r2,r1)
 878 #define LDX1_C_NC(r1,r3,r2)             M2(0x24,LD_NONE,r3,r2,r1)
 879 #define LDX2_C_NC(r1,r3,r2)             M2(0x25,LD_NONE,r3,r2,r1)
 880 #define LDX4_C_NC(r1,r3,r2)             M2(0x26,LD_NONE,r3,r2,r1)
 881 #define LDX8_C_NC(r1,r3,r2)             M2(0x27,LD_NONE,r3,r2,r1)
 882 #define LDX1_C_CLR_ACQ(r1,r3,r2)        M2(0x28,LD_NONE,r3,r2,r1)
 883 #define LDX2_C_CLR_ACQ(r1,r3,r2)        M2(0x29,LD_NONE,r3,r2,r1)
 884 #define LDX4_C_CLR_ACQ(r1,r3,r2)        M2(0x2a,LD_NONE,r3,r2,r1)
 885 #define LDX8_C_CLR_ACQ(r1,r3,r2)        M2(0x2b,LD_NONE,r3,r2,r1)
 886 /* lfetch */
 887 #define LFETCH_EXCL(r3)                 M13(0x2d,LF_NONE,r3,GR_0)
 888 #define LFETCH_FAULT(r3)                M13(0x2e,LF_NONE,r3,GR_0)
 889 #define LFETCH_FAULT_EXCL(r3)           M13(0x2f,LF_NONE,r3,GR_0)
 890 #define LXFETCH(r3,r2)                  M14(0x2c,LF_NONE,r3,r2)
 891 #define LXFETCH_EXCL(r3,r2)             M14(0x2d,LF_NONE,r3,r2)
 892 #define LXFETCH_FAULT(r3,r2)            M14(0x2e,LF_NONE,r3,r2)
 893 #define LXFETCH_FAULT_EXCL(r3,r2)       M14(0x2f,LF_NONE,r3,r2)
 894 #define LFETCHI(r3,im)                  M15(0x2c,LF_NONE,r3,im)
 895 #define LFETCHI_EXCL(r3,im)             M15(0x2d,LF_NONE,r3,im)
 896 #define LFETCHI_FAULT(r3,im)            M15(0x2e,LF_NONE,r3,im)
 897 #define LFETCHI_FAULT_EXCL(r3,im)       M15(0x2f,LF_NONE,r3,im)
 898 /* loadrs */
 899 #define LOADRS()                        M25(0xa)
 900 /* mf */
 901 #define MF()                            M24(2,2)
 902 #define MF_A()                          M24(2,3)
 903 /* mix */
 904 #define MIX1_R(r1,r2,r3)                I2(0,2,0,2,0,r3,r2,r1)
 905 #define MIX2_R(r1,r2,r3)                I2(0,2,1,2,0,r3,r2,r1)
 906 #define MIX4_R(r1,r2,r3)                I2(1,2,0,2,0,r3,r2,r1)
 907 #define MIX1_L(r1,r2,r3)                I2(0,2,0,2,2,r3,r2,r1)
 908 #define MIX2_L(r1,r2,r3)                I2(0,2,1,2,2,r3,r2,r1)
 909 #define MIX4_L(r1,r2,r3)                I2(1,2,0,2,2,r3,r2,r1)
 910 /* mov - Move Application Register */
 911 #define MOV_I_rn_ar(r1,ar)              I28(ar,r1)
 912 #define MOV_I_ar_rn(ar,r2)              I26(ar,r2)
 913 #define MOV_I_ar_im(ar,im)              I27(ar,im)
 914 #define MOV_M_rn_a(r1,ar)               M31(r1,ar)
 915 #define MOV_M_ar_rn(ar,r2)              M29(ar,r2)
 916 #define MOV_M_ar_im(ar,im)              M30(ar,im)
 917 /* mov - Move Branch Register */
 918 #define MOV_rn_br(r1,b2)                I22(b2,r1)
 919 #define MOV_br_rn_tg(b1,r2,tag)         I21(tag,IH_NONE,0,MWH_NONE,r2,b1)
 920 #define MOV_br_rn(b1,r2)                MOV_br_rn_tg(b1,r2,0)
 921 #define MOV_RET_br_rn_tg(b1,r2,tag)     I21(tag,IH_NONE,1,MWH_NONE,r2,b1)
 922 /* mov - Move Control Register */
 923 #define MOV_rn_cr(cr,r1)                M33(cr,r1)
 924 #define MOV_cr_rr(cr,r2)                M32(cr,r2)
 925 /* mov - Move General Register */
 926 #define MOV(r0,r1)                      ADDS(r0,0,r1)
 927 #define MOV_p(r0,r1,_p)                 ADDS_p(r0,0,r1,_p)
 928 /* mov - Move Immediate */
 929 #define MOVI(r1,im)                     ADDL(r1,im,GR_0)
 930 #define MOVI_p(r1,im,_p)                ADDL_p(r1,im,GR_0,_p)
 931 /* mov - Move Indirect Register */
 932 #define MOV_rn_RR(r1,r3)                M43(0x10,r3,r1)
 933 #define MOV_rn_DBR(r1,r3)               M43(0x11,r3,r1)
 934 #define MOV_rn_IBR(r1,r3)               M43(0x012,r3,r1)
 935 #define MOV_rn_PKR(r1,r3)               M43(0x13,r3,r1)
 936 #define MOV_rn_PMC(r1,r3)               M43(0x14,r3,r1)
 937 #define MOV_rn_PMD(r1,r3)               M43(0x15,r3,r1)
 938 #define MOV_rn_CPUID(r1,r3)             M43(0x17,r3,r1)
 939 #define MOV_RR_rn(r3,r2)                M42(0x00,r3,r2)
 940 #define MOV_DBR_rn(r3,r2)               M42(0x01,r3,r2)
 941 #define MOV_IBR_rn(r3,r2)               M42(0x02,r3,r2)
 942 #define MOV_PKR_rn(r3,r2)               M42(0x03,r3,r2)
 943 #define MOV_PMC_rn(r3,r2)               M42(0x04,r3,r2)
 944 #define MOV_PMD_rn(r3,r2)               M42(0x05,r3,r2)
 945 /* mov - Move Instruction Pointer */
 946 #define MOV_rn_ip(r1)                   I25(0x30,r1)
 947 /* mov - Move Predicates */
 948 #define MOV_rn_pr(r1)                   I25(0x33,r1)
 949 #define MOV_pr_rn(r2,im)                I23(r2,im)
 950 #define MOVI_pr(im)                     I24(im)
 951 /* mov - Move Processor Status Register */
 952 #define MOV_rn_psr(r1)                  M36(0x25,r1)
 953 #define MOV_psr_l_rn(r2)                M35(0x2d,r2)
 954 /* mov - Move User Mask */
 955 #define MOV_rn_psr_um(r1)               M36(0x21,r1)
 956 #define MOV_psr_um_rn(r2)               M35(0x29,r2)
 957 /* movl */
 958 #define MOVL(r1,im)                     X2(r1,im)
 959 /* mpy4 */
 960 #define MPY4(r1,r2,r3)                  I2(1,0,0,3,1,r3,r2,r1)
 961 /* mpyshl4 */
 962 #define MPYSHL4(r1,r2,r3)               I2(1,0,0,3,3,r3,r2,r1)
 963 /* mux */
 964 #define MUX1(r1,r2,mbt)                 I3(mbt,r2,r1)
 965 #define MUX2(r1,r2,mht)                 I4(mht,r2,r1)
 966 /* nop */
 967 #define NOP_I(im)                       I18(im,0)
 968 #define NOP_M(im)                       M48(0,im)
 969 #define NOP_B(im)                       B9(2,0,im)
 970 #define NOP_X(im)                       X5(0,im)
 971 /* or */
 972 #define OR(r1,r2,r3)                    A1(3,2,r3,r2,r1)
 973 #define ORI(r1,im,r3)                   A3(0xb,2,r3,im,r1)
 974 /* pack */
 975 #define PACK2_USS(r1,r2,r3)             I2(0,2,1,0,0,r3,r2,r1)
 976 #define PACK2_SSS(r1,r2,r3)             I2(0,2,1,0,2,r3,r2,r1)
 977 #define PACK4_SSS(r1,r2,r3)             I2(1,2,0,0,2,r3,r2,r1)
 978 /* padd */
 979 #define PADD1(r1,r2,r3)                 A9(0,0,0,0,r3,r2,r1)
 980 #define PADD1_SSS(r1,r2,r3)             A9(0,0,0,1,r3,r2,r1)
 981 #define PADD1_UUU(r1,r2,r3)             A9(0,0,0,2,r3,r2,r1)
 982 #define PADD1_UUS(r1,r2,r3)             A9(0,0,0,3,r3,r2,r1)
 983 #define PADD2(r1,r2,r3)                 A9(0,1,0,0,r3,r2,r1)
 984 #define PADD2_SSS(r1,r2,r3)             A9(0,1,0,1,r3,r2,r1)
 985 #define PADD2_UUU(r1,r2,r3)             A9(0,1,0,2,r3,r2,r1)
 986 #define PADD2_UUS(r1,r2,r3)             A9(0,1,0,3,r3,r2,r1)
 987 #define PADD4(r1,r2,r3)                 A9(1,0,0,0,r3,r2,r1)
 988 /* pavg */
 989 #define PAVG1(r1,r2,r3)                 A9(0,0,2,2,r3,r2,r1)
 990 #define PAVG2(r1,r2,r3)                 A9(0,1,2,2,r3,r2,r1)
 991 #define PAVG1_RAZ(r1,r2,r3)             A9(0,0,2,3,r3,r2,r1)
 992 #define PAVG2_RAZ(r1,r2,r3)             A9(0,1,2,3,r3,r2,r1)
 993 /* pavgsub */
 994 #define PAVGSUB1(r1,r2,r3)              A9(0,0,3,2,r3,r2,r1)
 995 #define PAVGSUB2(r1,r2,r3)              A9(0,1,3,2,r3,r2,r1)
 996 /* pcmp */
 997 #define PCMP1_EQ(r1,r2,r3)              A9(0,0,9,0,r3,r2,r1)
 998 #define PCMP2_EQ(r1,r2,r3)              A9(0,1,9,0,r3,r2,r1)
 999 #define PCMP4_EQ(r1,r2,r3)              A9(1,0,9,0,r3,r2,r1)
1000 #define PCMP1_GT(r1,r2,r3)              A9(0,0,9,1,r3,r2,r1)
1001 #define PCMP2_GT(r1,r2,r3)              A9(0,1,9,1,r3,r2,r1)
1002 #define PCMP4_GT(r1,r2,r3)              A9(1,0,9,1,r3,r2,r1)
1003 /* pmax */
1004 #define PMAX1_U(r1,r2,r3)               I2(0,2,0,1,1,r3,r2,r1)
1005 #define PMAX2(r1,r2,r3)                 I2(0,2,1,1,3,r3,r2,r1)
1006 /* pmin */
1007 #define PMIN1_U(r1,r2,r3)               I2(0,2,0,0,1,r3,r2,r1)
1008 #define PMIN2(r1,r2,r3)                 I2(0,2,1,0,3,r3,r2,r1)
1009 /* pmpy */
1010 #define PMPY2_R(r1,r2,r3)               I2(0,2,1,3,1,r3,r2,r1)
1011 #define PMPY2_L(r1,r2,r3)               I2(0,2,1,3,3,r3,r2,r1)
1012 /* pmpyshr */
1013 #define PMPYSHR2(r1,r2,r3,im)           I1(im,3,r3,r2,r1)
1014 #define PMPYSHR2_U(r1,r2,r3,im)         I1(im,1,r3,r2,r1)
1015 /* popcnt */
1016 #define POPCNT(r1,r3)                   I9(2,r3,r1)
1017 /* probe */
1018 #define PROBE_R(r1,r3,r2)               M38(0x38,r3,r2,r1)
1019 #define PROBE_W(r1,r3,r2)               M38(0x39,r3,r2,r1)
1020 #define PROBEI_R(r1,r3,im)              M39(0x18,r3,im,r1)
1021 #define PROBEI_W(r1,r3,im)              M39(0x19,r3,im,r1)
1022 #define PROBE_RW_FAULT(r3,im)           M40(0x31,r3,im)
1023 #define PROBE_R_FAULT(r3,im)            M40(0x32,r3,im)
1024 #define PROBE_W_FAULT(r3,im)            M40(0x33,r3,im)
1025 /* psad */
1026 #define PSAD1(r1,r2,r3)                 I2(0,2,0,2,3,r3,r2,r1)
1027 /* pshl */
1028 #define PSHL2(r1,r2,r3)                 I7(0,1,r3,r2,r1)
1029 #define PSHL4(r1,r2,r3)                 I7(1,0,r3,r2,r1)
1030 #define PSHL2I(r1,r2,im)                I8(0,1,im,r2,r1)
1031 #define PSHL4I(r1,r2,im)                I8(1,0,im,r2,r1)
1032 /* pshladd */
1033 #define PSHLADD2(r1,r2,im,r3)           A10(4,im,r3,r2,r1)
1034 /* pshr */
1035 #define PSHR2(r1,r3,r2)                 I5(0,1,2,r3,r2,r1)
1036 #define PSHR2I(r1,r3,im)                I6(0,1,3,r3,im,r1)
1037 #define PSHR2_U(r1,r3,r2)               I5(0,1,0,r3,r2,r1)
1038 #define PSHR2I_U(r1,r3,im)              I6(0,1,1,r3,im,r1)
1039 #define PSHR4(r1,r3,r2)                 I5(1,0,2,r3,r2,r1)
1040 #define PSHR4I(r1,r3,im)                I6(1,0,3,r3,im,r1)
1041 #define PSHR4_U(r1,r3,r2)               I5(1,0,0,r3,r2,r1)
1042 #define PSHR4I_U(r1,r3,im)              I6(1,0,1,r3,im,r1)
1043 /* pshradd */
1044 #define PSHRADD2(r1,r2,im,r3)           A10(6,im,r3,r2,r1)
1045 /* psub */
1046 #define PSUB1(r1,r2,r3)                 A9(0,0,1,0,r3,r2,r1)
1047 #define PSUB1_SSS(r1,r2,r3)             A9(0,0,1,1,r3,r2,r1)
1048 #define PSUB1_UUU(r1,r2,r3)             A9(0,0,1,2,r3,r2,r1)
1049 #define PSUB1_UUS(r1,r2,r3)             A9(0,0,1,3,r3,r2,r1)
1050 #define PSUB2(r1,r2,r3)                 A9(0,1,1,0,r3,r2,r1)
1051 #define PSUB2_SSS(r1,r2,r3)             A9(0,1,1,1,r3,r2,r1)
1052 #define PSUB2_UUU(r1,r2,r3)             A9(0,1,1,2,r3,r2,r1)
1053 #define PSUB2_UUS(r1,r2,r3)             A9(0,1,1,3,r3,r2,r1)
1054 #define PSUB4(r1,r2,r3)                 A9(1,0,1,0,r3,r2,r1)
1055 /* ptc.e */
1056 #define PTC_E(r3)                       M47(0x34,r3)
1057 /* ptc.g, ptc.ga */
1058 #define PTC_G(r3,r2)                    M45(0xa,r3,r2)
1059 #define PTC_GA(r3,r2)                   M45(0xb,r3,r2)
1060 /* ptc.l */
1061 #define PTC_L(r3,r2)                    M45(0x9,r3,r2)
1062 /* ptr */
1063 #define PTR_D(r3,r2)                    M45(0xc,r3,r2)
1064 #define PTR_I(r3,r2)                    M45(0xd,r3,r2)
1065 /* rfi */
1066 #define RFI()                           B8(0x08)
1067 /* rsm */
1068 #define RSM(im)                         M44(7,im)
1069 /* rum */
1070 #define RUM(im)                         M44(5,im)
1071 /* shl */
1072 #define SHL(r1,r2,r3)                   I7(1,1,r3,r2,r1)
1073 /* shladd */
1074 #define SHLADD(r1,r2,im,r3)             A2(4,im,r3,r2,r1)
1075 /* shladdp4 */
1076 #define SHLADDP4(r1,r2,im,r3)           A2(6,im,r3,r2,r1)
1077 /* shr */
1078 #define SHR(r1,r3,r2)                   I5(1,1,2,r3,r2,r1)
1079 #define SHR_U(r1,r3,r2)                 I5(1,1,0,r3,r2,r1)
1080 /* shrp */
1081 #define SHRP(r1,r2,r3,im)               I10(im,r3,r2,r1)
1082 /* srlz */
1083 #define SRLZ_I()                        M24(3,1)
1084 #define SRLZ_D()                        M24(3,0)
1085 /* ssm */
1086 #define SSM(im)                         M44(6,im)
1087 /* st */
1088 #define ST1(r3,r2)                      M6(0x30,ST_NONE,0,r3,r2)
1089 #define ST2(r3,r2)                      M6(0x31,ST_NONE,0,r3,r2)
1090 #define ST4(r3,r2)                      M6(0x32,ST_NONE,0,r3,r2)
1091 #define ST8(r3,r2)                      M6(0x33,ST_NONE,0,r3,r2)
1092 #define ST1_REL(r3,r2)                  M6(0x34,ST_NONE,0,r3,r2)
1093 #define ST2_REL(r3,r2)                  M6(0x35,ST_NONE,0,r3,r2)
1094 #define ST4_REL(r3,r2)                  M6(0x36,ST_NONE,0,r3,r2)
1095 #define ST8_REL(r3,r2)                  M6(0x37,ST_NONE,0,r3,r2)
1096 #define ST8_SPILL(r3,r2)                M6(0x3b,ST_NONE,0,r3,r2)
1097 #define ST16(r3,r2)                     M6(0x30,ST_NONE,1,r3,r2)
1098 #define ST16_REL(r3,r2)                 M6(0x34,ST_NONE,1,r3,r2)
1099 #define ST1_inc(r3,r2,im)               M5(0x30,ST_NONE,r3,r2,im)
1100 #define ST2_inc(r3,r2,im)               M5(0x31,ST_NONE,r3,r2,im)
1101 #define ST4_inc(r3,r2,im)               M5(0x32,ST_NONE,r3,r2,im)
1102 #define ST8_inc(r3,r2,im)               M5(0x33,ST_NONE,r3,r2,im)
1103 #define ST1_REL_inc(r3,r2,im)           M5(0x34,ST_NONE,r3,r2,im)
1104 #define ST2_REL_inc(r3,r2,im)           M5(0x35,ST_NONE,r3,r2,im)
1105 #define ST4_REL_inc(r3,r2,im)           M5(0x36,ST_NONE,r3,r2,im)
1106 #define ST8_REL_inc(r3,r2,im)           M5(0x37,ST_NONE,r3,r2,im)
1107 #define ST8_SPILL_inc(r3,r2,im)         M5(0x3b,ST_NONE,r3,r2,im)
1108 /* sub */
1109 #define SUB(r1,r2,r3)                   A1(1,1,r3,r2,r1)
1110 #define SUB1(r1,r2,r3)                  A1(1,0,r3,r2,r1)
1111 #define SUBI(r1,im,r3)                  A3(9,1,r3,im,r1)
1112 /* sum */
1113 #define SUM(im)                         M44(4,im)
1114 /* sxt */
1115 #define SXT1(r1,r3)                     I29(0x14,r3,r1)
1116 #define SXT2(r1,r3)                     I29(0x15,r3,r1)
1117 #define SXT4(r1,r3)                     I29(0x16,r3,r1)
1118 /* sync */
1119 #define SYNC_I()                        M24(3,3)
1120 /* tak */
1121 #define TAK(r1,r3)                      M46(0x1f,r3,r1)
1122 /* tbit */
1123 #define TBIT_Z(p1,p2,r3,pos)            I16(0,0,p2,r3,pos,0,p1)
1124 #define TBIT_Z_UNC(p1,p2,r3,pos)        I16(0,0,p2,r3,pos,1,p1)
1125 #define TBIT_Z_AND(p1,p2,r3,pos)        I16(1,0,p2,r3,pos,0,p1)
1126 #define TBIT_NZ_AND(p1,p2,r3,pos)       I16(1,0,p2,r3,pos,1,p1)
1127 #define TBIT_Z_OR(p1,p2,r3,pos)         I16(0,1,p2,r3,pos,0,p1)
1128 #define TBIT_NZ_OR(p1,p2,r3,pos)        I16(0,1,p2,r3,pos,1,p1)
1129 #define TBIT_Z_ANDCM(p1,p2,r3,pos)      I16(1,1,p2,r3,pos,0,p1)
1130 #define TBIT_NZ_ANDCM(p1,p2,r3,pos)     I16(1,1,p2,r3,pos,1,p1)
1131 /* tf */
1132 #define TF_Z(p1,p2,im)                  I30(0,0,p2,im,0,p1)
1133 #define TF_Z_UNC(p1,p2,im)              I30(0,0,p2,im,1,p1)
1134 #define TF_Z_AND(p1,p2,im)              I30(1,0,p2,im,0,p1)
1135 #define TF_NZ_AND(p1,p2,im)             I30(1,0,p2,im,1,p1)
1136 #define TF_Z_OR(p1,p2,im)               I30(0,1,p2,im,0,p1)
1137 #define TF_NZ_OR(p1,p2,im)              I30(0,1,p2,im,1,p1)
1138 #define TF_Z_ANDCM(p1,p2,im)            I30(1,1,p2,im,0,p1)
1139 #define TF_NZ_ANDCM(p1,p2,im)           I30(1,1,p2,im,1,p1)
1140 /* thash */
1141 #define THASH(r1,r3)                    M46(0x1a,r3,r1)
1142 /* tnat */
1143 #define TNAT_Z(p1,p2,r3)                I17(0,0,p2,r3,0,p1)
1144 #define TNAT_Z_UNC(p1,p2,r3)            I17(0,0,p2,r3,1,p1)
1145 #define TNAT_Z_AND(p1,p2,r3)            I17(1,0,p2,r3,0,p1)
1146 #define TNAT_NZ_AND(p1,p2,r3)           I17(1,0,p2,r3,1,p1)
1147 #define TNAT_Z_OR(p1,p2,r3)             I17(0,1,p2,r3,0,p1)
1148 #define TNAT_NZ_OR(p1,p2,r3)            I17(0,1,p2,r3,1,p1)
1149 #define TNAT_Z_ANDCM(p1,p2,r3)          I17(1,1,p2,r3,0,p1)
1150 #define TNAT_NZ_ANDCM(p1,p2,r3)         I17(1,1,p2,r3,1,p1)
1151 /* tpa */
1152 #define TPA(r1,r3)                      M46(0x1e,r3,r1)
1153 /* ttag */
1154 #define TTAG(r1,r3)                     M46(0x1b,r3,r1)
1155 /* unpack */
1156 #define UNPACK1_H(r1,r2,r3)             I2(0,2,0,1,0,r3,r2,r1)
1157 #define UNPACK2_H(r1,r2,r3)             I2(0,2,1,1,0,r3,r2,r1)
1158 #define UNPACK4_H(r1,r2,r3)             I2(1,2,0,1,0,r3,r2,r1)
1159 #define UNPACK1_L(r1,r2,r3)             I2(0,2,0,1,2,r3,r2,r1)
1160 #define UNPACK2_L(r1,r2,r3)             I2(0,2,1,1,2,r3,r2,r1)
1161 #define UNPACK4_L(r1,r2,r3)             I2(1,2,0,1,2,r3,r2,r1)
1162 /* vmsw */
1163 #define VMSW_0()                        B8(0x18)
1164 #define VMSW_1()                        B8(0x19)
1165 /* xchg */
1166 #define XCHG1_ACQ(r1,r3,r2)             M16(0x08,LD_NONE,r3,r2,r1)
1167 #define XCHG2_ACQ(r1,r3,r2)             M16(0x09,LD_NONE,r3,r2,r1)
1168 #define XCHG4_ACQ(r1,r3,r2)             M16(0x0a,LD_NONE,r3,r2,r1)
1169 #define XCHG8_ACQ(r1,r3,r2)             M16(0x0b,LD_NONE,r3,r2,r1)
1170 /* xor */
1171 #define XOR(r1,r2,r3)                   A1(3,3,r3,r2,r1)
1172 #define XORI(r1,im,r3)                  A3(0xb,3,r3,im,r1)
1173 /* zxt */
1174 #define ZXT1(r1,r3)                     I29(0x10,r3,r1)
1175 #define ZXT2(r1,r3)                     I29(0x11,r3,r1)
1176 #define ZXT4(r1,r3)                     I29(0x12,r3,r1)
1177
1178 #  define nop(i0)                       _nop(_jit,i0)
1179 static void _nop(jit_state_t*, jit_int32_t);
1180 #define addr(r0,r1,r2)                  ADD(r0,r1,r2)
1181 #define addi(r0,r1,i0)                  _addi(_jit,r0,r1,i0)
1182 static void _addi(jit_state_t*,jit_int32_t,jit_int32_t,jit_word_t);
1183 #define addcr(r0, r1, r2)               _addcr(_jit, r0, r1, r2)
1184 static void _addcr(jit_state_t*, jit_int32_t, jit_int32_t, jit_int32_t);
1185 #define addci(r0, r1, i0)               _addci(_jit, r0, r1, i0)
1186 static void _addci(jit_state_t*, jit_int32_t, jit_int32_t, jit_word_t);
1187 #  define addxr(r0, r1, r2)             _addxr(_jit, r0, r1, r2)
1188 static void _addxr(jit_state_t*, jit_int32_t, jit_int32_t, jit_int32_t);
1189 #  define addxi(r0, r1, i0)             _addxi(_jit, r0, r1, i0)
1190 static void _addxi(jit_state_t*, jit_int32_t, jit_int32_t, jit_word_t);
1191 #define subr(r0,r1,r2)                  SUB(r0,r1,r2)
1192 #define subi(r0,r1,i0)                  _subi(_jit,r0,r1,i0)
1193 static void _subi(jit_state_t*,jit_int32_t,jit_int32_t,jit_word_t);
1194 #  define subcr(r0, r1, r2)             _subcr(_jit, r0, r1, r2)
1195 static void _subcr(jit_state_t*,jit_int32_t,jit_int32_t,jit_int32_t);
1196 #  define subci(r0, r1, i0)             _subci(_jit, r0, r1, i0)
1197 static void _subci(jit_state_t*,jit_int32_t,jit_int32_t,jit_word_t);
1198 #  define subxr(r0, r1, r2)             _subxr(_jit, r0, r1, r2)
1199 static void _subxr(jit_state_t*,jit_int32_t,jit_int32_t,jit_int32_t);
1200 #  define subxi(r0, r1, i0)             _subxi(_jit, r0, r1, i0)
1201 static void _subxi(jit_state_t*,jit_int32_t,jit_int32_t,jit_word_t);
1202 #  define rsbi(r0, r1, i0)              _rsbi(_jit, r0, r1, i0)
1203 static void _rsbi(jit_state_t*,jit_int32_t,jit_int32_t,jit_word_t);
1204 #define mulr(r0,r1,r2)                  _mulr(_jit,r0,r1,r2)
1205 static void _mulr(jit_state_t*,jit_int32_t,jit_int32_t,jit_int32_t);
1206 #define muli(r0,r1,i0)                  _muli(_jit,r0,r1,i0)
1207 static void _muli(jit_state_t*,jit_int32_t,jit_int32_t,jit_word_t);
1208 #define divr(r0,r1,r2)                  _divr(_jit,r0,r1,r2)
1209 static void _divr(jit_state_t*,jit_int32_t,jit_int32_t,jit_int32_t);
1210 #define divi(r0,r1,i0)                  _divi(_jit,r0,r1,i0)
1211 static void _divi(jit_state_t*,jit_int32_t,jit_int32_t,jit_word_t);
1212 #define divr_u(r0,r1,r2)                _divr_u(_jit,r0,r1,r2)
1213 static void _divr_u(jit_state_t*,jit_int32_t,jit_int32_t,jit_int32_t);
1214 #define divi_u(r0,r1,i0)                _divi_u(_jit,r0,r1,i0)
1215 static void _divi_u(jit_state_t*,jit_int32_t,jit_int32_t,jit_word_t);
1216 #define remr(r0,r1,r2)                  _remr(_jit,r0,r1,r2)
1217 static void _remr(jit_state_t*,jit_int32_t,jit_int32_t,jit_int32_t);
1218 #define remi(r0,r1,i0)                  _remi(_jit,r0,r1,i0)
1219 static void _remi(jit_state_t*,jit_int32_t,jit_int32_t,jit_word_t);
1220 #define remr_u(r0,r1,r2)                _remr_u(_jit,r0,r1,r2)
1221 static void _remr_u(jit_state_t*,jit_int32_t,jit_int32_t,jit_int32_t);
1222 #define remi_u(r0,r1,i0)                _remi_u(_jit,r0,r1,i0)
1223 static void _remi_u(jit_state_t*,jit_int32_t,jit_int32_t,jit_word_t);
1224 #define mulh(r0,r1,r2,sign)             _mulh(_jit,r0,r1,r2,sign)
1225 static void _mulh(jit_state_t*,jit_bool_t,jit_int32_t,jit_int32_t,jit_int32_t);
1226 #define qmulr(r0,r1,r2,r3)              iqmulr(r0,r1,r2,r3,1)
1227 #define qmulr_u(r0,r1,r2,r3)            iqmulr(r0,r1,r2,r3,0)
1228 #define iqmulr(r0,r1,r2,r3,sign)        _iqmulr(_jit,r0,r1,r2,r3,sign)
1229 static void _iqmulr(jit_state_t*,jit_int32_t,jit_int32_t,
1230                     jit_int32_t,jit_int32_t,jit_bool_t);
1231 #define qmuli(r0,r1,r2,i0)              iqmuli(r0,r1,r2,i0,1)
1232 #define qmuli_u(r0,r1,r2,i0)            iqmuli(r0,r1,r2,i0,0)
1233 #define iqmuli(r0,r1,r2,i0,sign)        _iqmuli(_jit,r0,r1,r2,i0,sign)
1234 static void _iqmuli(jit_state_t*,jit_int32_t,jit_int32_t,
1235                     jit_int32_t,jit_word_t,jit_bool_t);
1236 #define qdivr(r0,r1,r2,r3)              iqdivr(r0,r1,r2,r3,1)
1237 #define qdivr_u(r0,r1,r2,r3)            iqdivr(r0,r1,r2,r3,0)
1238 #define iqdivr(r0,r1,r2,r3,sign)        _iqdivr(_jit,r0,r1,r2,r3,sign)
1239 static void _iqdivr(jit_state_t*,jit_int32_t,jit_int32_t,
1240                     jit_int32_t,jit_int32_t,jit_bool_t);
1241 #define qdivi(r0,r1,r2,i0)              iqdivi(r0,r1,r2,i0,1)
1242 #define qdivi_u(r0,r1,r2,i0)            iqdivi(r0,r1,r2,i0,0)
1243 #define iqdivi(r0,r1,r2,i0,sign)        _iqdivi(_jit,r0,r1,r2,i0,sign)
1244 static void _iqdivi(jit_state_t*,jit_int32_t,jit_int32_t,
1245                     jit_int32_t,jit_word_t,jit_bool_t);
1246 #define andr(r0,r1,r2)                  AND(r0,r1,r2)
1247 #define andi(r0,r1,i0)                  _andi(_jit,r0,r1,i0)
1248 static void _andi(jit_state_t*,jit_int32_t,jit_int32_t,jit_word_t);
1249 #define orr(r0,r1,r2)                   OR(r0,r1,r2)
1250 #define ori(r0,r1,i0)                   _ori(_jit,r0,r1,i0)
1251 static void _ori(jit_state_t*,jit_int32_t,jit_int32_t,jit_word_t);
1252 #define xorr(r0,r1,r2)                  XOR(r0,r1,r2)
1253 #define xori(r0,r1,i0)                  _xori(_jit,r0,r1,i0)
1254 static void _xori(jit_state_t*,jit_int32_t,jit_int32_t,jit_word_t);
1255 #define lshr(r0,r1,r2)                  SHL(r0,r1,r2)
1256 #define lshi(r0,r1,i0)                  _lshi(_jit,r0,r1,i0)
1257 static void _lshi(jit_state_t*,jit_int32_t,jit_int32_t,jit_word_t);
1258 #define rshr(r0,r1,r2)                  SHR(r0,r1,r2)
1259 #define rshi(r0,r1,i0)                  _rshi(_jit,r0,r1,i0)
1260 static void _rshi(jit_state_t*,jit_int32_t,jit_int32_t,jit_word_t);
1261 #define rshr_u(r0,r1,r2)                SHR_U(r0,r1,r2)
1262 #define rshi_u(r0,r1,i0)                _rshi_u(_jit,r0,r1,i0)
1263 static void _rshi_u(jit_state_t*,jit_int32_t,jit_int32_t,jit_word_t);
1264 #define ltr(r0,r1,r2)                   _ltr(_jit,r0,r1,r2)
1265 static void _ltr(jit_state_t*,jit_int32_t,jit_int32_t,jit_int32_t);
1266 #define lti(r0,r1,i0)                   _lti(_jit,r0,r1,i0)
1267 static void _lti(jit_state_t*,jit_int32_t,jit_int32_t,jit_word_t);
1268 #define ltr_u(r0,r1,r2)                 _ltr_u(_jit,r0,r1,r2)
1269 static void _ltr_u(jit_state_t*,jit_int32_t,jit_int32_t,jit_int32_t);
1270 #define lti_u(r0,r1,i0)                 _lti_u(_jit,r0,r1,i0)
1271 static void _lti_u(jit_state_t*,jit_int32_t,jit_int32_t,jit_word_t);
1272 #define ler(r0,r1,r2)                   _ler(_jit,r0,r1,r2)
1273 static void _ler(jit_state_t*,jit_int32_t,jit_int32_t,jit_int32_t);
1274 #define lei(r0,r1,i0)                   _lei(_jit,r0,r1,i0)
1275 static void _lei(jit_state_t*,jit_int32_t,jit_int32_t,jit_word_t);
1276 #define ler_u(r0,r1,r2)                 _ler_u(_jit,r0,r1,r2)
1277 static void _ler_u(jit_state_t*,jit_int32_t,jit_int32_t,jit_int32_t);
1278 #define lei_u(r0,r1,i0)                 _lei_u(_jit,r0,r1,i0)
1279 static void _lei_u(jit_state_t*,jit_int32_t,jit_int32_t,jit_word_t);
1280 #define eqr(r0,r1,r2)                   _eqr(_jit,r0,r1,r2)
1281 static void _eqr(jit_state_t*,jit_int32_t,jit_int32_t,jit_int32_t);
1282 #define eqi(r0,r1,i0)                   _eqi(_jit,r0,r1,i0)
1283 static void _eqi(jit_state_t*,jit_int32_t,jit_int32_t,jit_word_t);
1284 #define ger(r0,r1,r2)                   _ger(_jit,r0,r1,r2)
1285 static void _ger(jit_state_t*,jit_int32_t,jit_int32_t,jit_int32_t);
1286 #define gei(r0,r1,i0)                   _gei(_jit,r0,r1,i0)
1287 static void _gei(jit_state_t*,jit_int32_t,jit_int32_t,jit_word_t);
1288 #define ger_u(r0,r1,r2)                 _ger_u(_jit,r0,r1,r2)
1289 static void _ger_u(jit_state_t*,jit_int32_t,jit_int32_t,jit_int32_t);
1290 #define gei_u(r0,r1,i0)                 _gei_u(_jit,r0,r1,i0)
1291 static void _gei_u(jit_state_t*,jit_int32_t,jit_int32_t,jit_word_t);
1292 #define gtr(r0,r1,r2)                   _gtr(_jit,r0,r1,r2)
1293 static void _gtr(jit_state_t*,jit_int32_t,jit_int32_t,jit_int32_t);
1294 #define gti(r0,r1,i0)                   _gti(_jit,r0,r1,i0)
1295 static void _gti(jit_state_t*,jit_int32_t,jit_int32_t,jit_word_t);
1296 #define gtr_u(r0,r1,r2)                 _gtr_u(_jit,r0,r1,r2)
1297 static void _gtr_u(jit_state_t*,jit_int32_t,jit_int32_t,jit_int32_t);
1298 #define gti_u(r0,r1,i0)                 _gti_u(_jit,r0,r1,i0)
1299 static void _gti_u(jit_state_t*,jit_int32_t,jit_int32_t,jit_word_t);
1300 #define ner(r0,r1,r2)                   _ner(_jit,r0,r1,r2)
1301 static void _ner(jit_state_t*,jit_int32_t,jit_int32_t,jit_int32_t);
1302 #define nei(r0,r1,i0)                   _nei(_jit,r0,r1,i0)
1303 static void _nei(jit_state_t*,jit_int32_t,jit_int32_t,jit_word_t);
1304 #define negr(r0,r1)                     subr(r0,0,r1)
1305 #define comr(r0,r1)                     ANDCMI(r0,-1,r1)
1306 #define movr(r0,r1)                     _movr(_jit,r0,r1)
1307 static void _movr(jit_state_t*,jit_int32_t,jit_int32_t);
1308 #define movi(r0,i0)                     _movi(_jit,r0,i0)
1309 static void _movi(jit_state_t*,jit_int32_t,jit_word_t);
1310 #define movi_p(r0,i0)                   _movi_p(_jit,r0,i0)
1311 static jit_word_t _movi_p(jit_state_t*,jit_int32_t,jit_word_t);
1312 #  define movnr(r0,r1,r2)               _movnr(_jit,r0,r1,r2)
1313 static void _movnr(jit_state_t*,jit_int32_t,jit_int32_t,jit_int32_t);
1314 #  define movzr(r0,r1,r2)               _movzr(_jit,r0,r1,r2)
1315 static void _movzr(jit_state_t*,jit_int32_t,jit_int32_t,jit_int32_t);
1316 #  define casx(r0, r1, r2, r3, i0)      _casx(_jit, r0, r1, r2, r3, i0)
1317 static void _casx(jit_state_t *_jit,jit_int32_t,jit_int32_t,
1318                   jit_int32_t,jit_int32_t,jit_word_t);
1319 #define casr(r0, r1, r2, r3)            casx(r0, r1, r2, r3, 0)
1320 #define casi(r0, i0, r1, r2)            casx(r0, _NOREG, r1, r2, i0)
1321 #  define bswapr_us(r0,r1)              _bswapr_us(_jit,r0,r1)
1322 static void _bswapr_us(jit_state_t*,jit_int32_t,jit_int32_t);
1323 #  define bswapr_ui(r0,r1)              _bswapr_ui(_jit,r0,r1)
1324 static void _bswapr_ui(jit_state_t*,jit_int32_t,jit_int32_t);
1325 #  define bswapr_ul(r0,r1)              MUX1(r0,r1,MUX_REV)
1326 #define extr_c(r0,r1)                   SXT1(r0,r1)
1327 #define extr_uc(r0,r1)                  ZXT1(r0,r1)
1328 #define extr_s(r0,r1)                   SXT2(r0,r1)
1329 #define extr_us(r0,r1)                  ZXT2(r0,r1)
1330 #define extr_i(r0,r1)                   SXT4(r0,r1)
1331 #define extr_ui(r0,r1)                  ZXT4(r0,r1)
1332 #define bltr(i0,r0,r1)                  _bltr(_jit,i0,r0,r1)
1333 static jit_word_t _bltr(jit_state_t*,jit_word_t,jit_int32_t,jit_int32_t);
1334 #define blti(i0,r0,i1)                  _blti(_jit,i0,r0,i1)
1335 static jit_word_t _blti(jit_state_t*,jit_word_t,jit_int32_t,jit_word_t);
1336 #define bltr_u(i0,r0,r1)                _bltr_u(_jit,i0,r0,r1)
1337 static jit_word_t _bltr_u(jit_state_t*,jit_word_t,jit_int32_t,jit_int32_t);
1338 #define blti_u(i0,r0,i1)                _blti_u(_jit,i0,r0,i1)
1339 static jit_word_t _blti_u(jit_state_t*,jit_word_t,jit_int32_t,jit_word_t);
1340 #define bler(i0,r0,r1)                  _bler(_jit,i0,r0,r1)
1341 static jit_word_t _bler(jit_state_t*,jit_word_t,jit_int32_t,jit_int32_t);
1342 #define blei(i0,r0,i1)                  _blei(_jit,i0,r0,i1)
1343 static jit_word_t _blei(jit_state_t*,jit_word_t,jit_int32_t,jit_word_t);
1344 #define bler_u(i0,r0,r1)                _bler_u(_jit,i0,r0,r1)
1345 static jit_word_t _bler_u(jit_state_t*,jit_word_t,jit_int32_t,jit_int32_t);
1346 #define blei_u(i0,r0,i1)                _blei_u(_jit,i0,r0,i1)
1347 static jit_word_t _blei_u(jit_state_t*,jit_word_t,jit_int32_t,jit_word_t);
1348 #define beqr(i0,r0,r1)                  _beqr(_jit,i0,r0,r1)
1349 static jit_word_t _beqr(jit_state_t*,jit_word_t,jit_int32_t,jit_int32_t);
1350 #define beqi(i0,r0,i1)                  _beqi(_jit,i0,r0,i1)
1351 static jit_word_t _beqi(jit_state_t*,jit_word_t,jit_int32_t,jit_word_t);
1352 #define bger(i0,r0,r1)                  _bger(_jit,i0,r0,r1)
1353 static jit_word_t _bger(jit_state_t*,jit_word_t,jit_int32_t,jit_int32_t);
1354 #define bgei(i0,r0,i1)                  _bgei(_jit,i0,r0,i1)
1355 static jit_word_t _bgei(jit_state_t*,jit_word_t,jit_int32_t,jit_word_t);
1356 #define bger_u(i0,r0,r1)                _bger_u(_jit,i0,r0,r1)
1357 static jit_word_t _bger_u(jit_state_t*,jit_word_t,jit_int32_t,jit_int32_t);
1358 #define bgei_u(i0,r0,i1)                _bgei_u(_jit,i0,r0,i1)
1359 static jit_word_t _bgei_u(jit_state_t*,jit_word_t,jit_int32_t,jit_word_t);
1360 #define bgtr(i0,r0,r1)                  _bgtr(_jit,i0,r0,r1)
1361 static jit_word_t _bgtr(jit_state_t*,jit_word_t,jit_int32_t,jit_int32_t);
1362 #define bgti(i0,r0,i1)                  _bgti(_jit,i0,r0,i1)
1363 static jit_word_t _bgti(jit_state_t*,jit_word_t,jit_int32_t,jit_word_t);
1364 #define bgtr_u(i0,r0,r1)                _bgtr_u(_jit,i0,r0,r1)
1365 static jit_word_t _bgtr_u(jit_state_t*,jit_word_t,jit_int32_t,jit_int32_t);
1366 #define bgti_u(i0,r0,i1)                _bgti_u(_jit,i0,r0,i1)
1367 static jit_word_t _bgti_u(jit_state_t*,jit_word_t,jit_int32_t,jit_word_t);
1368 #define bner(i0,r0,r1)                  _bner(_jit,i0,r0,r1)
1369 static jit_word_t _bner(jit_state_t*,jit_word_t,jit_int32_t,jit_int32_t);
1370 #define bnei(i0,r0,i1)                  _bnei(_jit,i0,r0,i1)
1371 static jit_word_t _bnei(jit_state_t*,jit_word_t,jit_int32_t,jit_word_t);
1372 #define bmsr(i0,r0,r1)                  _bmsr(_jit,i0,r0,r1)
1373 static jit_word_t _bmsr(jit_state_t*,jit_word_t,jit_int32_t,jit_int32_t);
1374 #define bmsi(i0,r0,i1)                  _bmsi(_jit,i0,r0,i1)
1375 static jit_word_t _bmsi(jit_state_t*,jit_word_t,jit_int32_t,jit_word_t);
1376 #define bmcr(i0,r0,r1)                  _bmcr(_jit,i0,r0,r1)
1377 static jit_word_t _bmcr(jit_state_t*,jit_word_t,jit_int32_t,jit_int32_t);
1378 #define bmci(i0,r0,i1)                  _bmci(_jit,i0,r0,i1)
1379 static jit_word_t _bmci(jit_state_t*,jit_word_t,jit_int32_t,jit_word_t);
1380 #define baddr(i0,r0,r1,cc)              _baddr(_jit,i0,r0,r1,cc)
1381 static jit_word_t _baddr(jit_state_t*,jit_word_t,
1382                          jit_int32_t,jit_int32_t,jit_bool_t);
1383 #define baddi(i0,r0,i1,cc)              _baddi(_jit,i0,r0,i1,cc)
1384 static jit_word_t _baddi(jit_state_t*,jit_word_t,
1385                          jit_int32_t,jit_word_t,jit_bool_t);
1386 #define baddr_u(i0,r0,r1,cc)            _baddr_u(_jit,i0,r0,r1,cc)
1387 static jit_word_t _baddr_u(jit_state_t*,jit_word_t,
1388                            jit_int32_t,jit_int32_t,jit_bool_t);
1389 #define baddi_u(i0,r0,i1,cc)            _baddi_u(_jit,i0,r0,i1,cc)
1390 static jit_word_t _baddi_u(jit_state_t*,jit_word_t,
1391                            jit_int32_t,jit_word_t,jit_bool_t);
1392 #define bsubr(i0,r0,r1,cc)              _bsubr(_jit,i0,r0,r1,cc)
1393 static jit_word_t _bsubr(jit_state_t*,jit_word_t,
1394                          jit_int32_t,jit_int32_t,jit_bool_t);
1395 #define bsubi(i0,r0,i1,cc)              _bsubi(_jit,i0,r0,i1,cc)
1396 static jit_word_t _bsubi(jit_state_t*,jit_word_t,
1397                          jit_int32_t,jit_word_t,jit_bool_t);
1398 #define bsubr_u(i0,r0,r1,cc)            _bsubr_u(_jit,i0,r0,r1,cc)
1399 static jit_word_t _bsubr_u(jit_state_t*,jit_word_t,
1400                            jit_int32_t,jit_int32_t,jit_bool_t);
1401 #define bsubi_u(i0,r0,i1,cc)            _bsubi_u(_jit,i0,r0,i1,cc)
1402 static jit_word_t _bsubi_u(jit_state_t*,jit_word_t,
1403                            jit_int32_t,jit_word_t,jit_bool_t);
1404 #define boaddr(i0,r0,r1)                baddr(i0,r0,r1,1)
1405 #define boaddi(i0,r0,i1)                baddi(i0,r0,i1,1)
1406 #define boaddr_u(i0,r0,r1)              baddr_u(i0,r0,r1,1)
1407 #define boaddi_u(i0,r0,i1)              baddi_u(i0,r0,i1,1)
1408 #define bxaddr(i0,r0,r1)                baddr(i0,r0,r1,0)
1409 #define bxaddi(i0,r0,i1)                baddi(i0,r0,i1,0)
1410 #define bxaddr_u(i0,r0,r1)              baddr_u(i0,r0,r1,0)
1411 #define bxaddi_u(i0,r0,i1)              baddi_u(i0,r0,i1,0)
1412 #define bosubr(i0,r0,r1)                bsubr(i0,r0,r1,1)
1413 #define bosubi(i0,r0,i1)                bsubi(i0,r0,i1,1)
1414 #define bosubr_u(i0,r0,r1)              bsubr_u(i0,r0,r1,1)
1415 #define bosubi_u(i0,r0,i1)              bsubi_u(i0,r0,i1,1)
1416 #define bxsubr(i0,r0,r1)                bsubr(i0,r0,r1,0)
1417 #define bxsubi(i0,r0,i1)                bsubi(i0,r0,i1,0)
1418 #define bxsubr_u(i0,r0,r1)              bsubr_u(i0,r0,r1,0)
1419 #define bxsubi_u(i0,r0,i1)              bsubi_u(i0,r0,i1,0)
1420 #define ldr_c(r0,r1)                    _ldr_c(_jit,r0,r1)
1421 static void _ldr_c(jit_state_t*,jit_int32_t,jit_int32_t);
1422 #define ldi_c(r0,i0)                    _ldi_c(_jit,r0,i0)
1423 static void _ldi_c(jit_state_t*,jit_int32_t,jit_word_t);
1424 #define ldr_uc(r0,r1)                   LD1(r0,r1)
1425 #define ldi_uc(r0,i0)                   _ldi_uc(_jit,r0,i0)
1426 static void _ldi_uc(jit_state_t*,jit_int32_t,jit_word_t);
1427 #define ldr_s(r0,r1)                    _ldr_s(_jit,r0,r1)
1428 static void _ldr_s(jit_state_t*,jit_int32_t,jit_int32_t);
1429 #define ldi_s(r0,i0)                    _ldi_s(_jit,r0,i0)
1430 static void _ldi_s(jit_state_t*,jit_int32_t,jit_word_t);
1431 #define ldr_us(r0,r1)                   LD2(r0,r1)
1432 #define ldi_us(r0,i0)                   _ldi_us(_jit,r0,i0)
1433 static void _ldi_us(jit_state_t*,jit_int32_t,jit_word_t);
1434 #define ldr_i(r0,r1)                    _ldr_i(_jit,r0,r1)
1435 static void _ldr_i(jit_state_t*,jit_int32_t,jit_int32_t);
1436 #define ldi_i(r0,i0)                    _ldi_i(_jit,r0,i0)
1437 static void _ldi_i(jit_state_t*,jit_int32_t,jit_word_t);
1438 #define ldr_ui(r0,r1)                   LD4(r0,r1)
1439 #define ldi_ui(r0,i0)                   _ldi_ui(_jit,r0,i0)
1440 static void _ldi_ui(jit_state_t*,jit_int32_t,jit_word_t);
1441 #define ldr_l(r0,r1)                    LD8(r0,r1)
1442 #define ldi_l(r0,i0)                    _ldi_l(_jit,r0,i0)
1443 static void _ldi_l(jit_state_t*,jit_int32_t,jit_word_t);
1444 #define ldxr_c(r0,r1,r2)                _ldxr_c(_jit,r0,r1,r2)
1445 static void _ldxr_c(jit_state_t*,jit_int32_t,jit_int32_t,jit_int32_t);
1446 #define ldxi_c(r0,r1,i0)                _ldxi_c(_jit,r0,r1,i0)
1447 static void _ldxi_c(jit_state_t*,jit_int32_t,jit_int32_t,jit_word_t);
1448 #define ldxr_uc(r0,r1,r2)               _ldxr_uc(_jit,r0,r1,r2)
1449 static void _ldxr_uc(jit_state_t*,jit_int32_t,jit_int32_t,jit_int32_t);
1450 #define ldxi_uc(r0,r1,i0)               _ldxi_uc(_jit,r0,r1,i0)
1451 static void _ldxi_uc(jit_state_t*,jit_int32_t,jit_int32_t,jit_word_t);
1452 #define ldxr_s(r0,r1,r2)                _ldxr_s(_jit,r0,r1,r2)
1453 static void _ldxr_s(jit_state_t*,jit_int32_t,jit_int32_t,jit_int32_t);
1454 #define ldxi_s(r0,r1,i0)                _ldxi_s(_jit,r0,r1,i0)
1455 static void _ldxi_s(jit_state_t*,jit_int32_t,jit_int32_t,jit_word_t);
1456 #define ldxr_us(r0,r1,r2)               _ldxr_us(_jit,r0,r1,r2)
1457 static void _ldxr_us(jit_state_t*,jit_int32_t,jit_int32_t,jit_int32_t);
1458 #define ldxi_us(r0,r1,i0)               _ldxi_us(_jit,r0,r1,i0)
1459 static void _ldxi_us(jit_state_t*,jit_int32_t,jit_int32_t,jit_word_t);
1460 #define ldxr_i(r0,r1,r2)                _ldxr_i(_jit,r0,r1,r2)
1461 static void _ldxr_i(jit_state_t*,jit_int32_t,jit_int32_t,jit_int32_t);
1462 #define ldxi_i(r0,r1,i0)                _ldxi_i(_jit,r0,r1,i0)
1463 static void _ldxi_i(jit_state_t*,jit_int32_t,jit_int32_t,jit_word_t);
1464 #define ldxr_ui(r0,r1,r2)               _ldxr_ui(_jit,r0,r1,r2)
1465 static void _ldxr_ui(jit_state_t*,jit_int32_t,jit_int32_t,jit_int32_t);
1466 #define ldxi_ui(r0,r1,i0)               _ldxi_ui(_jit,r0,r1,i0)
1467 static void _ldxi_ui(jit_state_t*,jit_int32_t,jit_int32_t,jit_word_t);
1468 #define ldxr_l(r0,r1,r2)                _ldxr_l(_jit,r0,r1,r2)
1469 static void _ldxr_l(jit_state_t*,jit_int32_t,jit_int32_t,jit_int32_t);
1470 #define ldxi_l(r0,r1,i0)                _ldxi_l(_jit,r0,r1,i0)
1471 static void _ldxi_l(jit_state_t*,jit_int32_t,jit_int32_t,jit_word_t);
1472 #define str_c(r0,r1)                    ST1(r0,r1)
1473 #define sti_c(i0,r0)                    _sti_c(_jit,i0,r0)
1474 static void _sti_c(jit_state_t*,jit_word_t,jit_int32_t);
1475 #define str_s(r0,r1)                    ST2(r0,r1)
1476 #define sti_s(i0,r0)                    _sti_s(_jit,i0,r0)
1477 static void _sti_s(jit_state_t*,jit_word_t,jit_int32_t);
1478 #define str_i(r0,r1)                    ST4(r0,r1)
1479 #define sti_i(i0,r0)                    _sti_i(_jit,i0,r0)
1480 static void _sti_i(jit_state_t*,jit_word_t,jit_int32_t);
1481 #define str_l(r0,r1)                    ST8(r0,r1)
1482 #define sti_l(i0,r0)                    _sti_l(_jit,i0,r0)
1483 static void _sti_l(jit_state_t*,jit_word_t,jit_int32_t);
1484 #define stxr_c(r0,r1,r2)                _stxr_c(_jit,r0,r1,r2)
1485 static void _stxr_c(jit_state_t*,jit_int32_t,jit_int32_t,jit_int32_t);
1486 #define stxi_c(i0,r0,r1)                _stxi_c(_jit,i0,r0,r1)
1487 static void _stxi_c(jit_state_t*,jit_word_t,jit_int32_t,jit_int32_t);
1488 #define stxr_s(r0,r1,r2)                _stxr_s(_jit,r0,r1,r2)
1489 static void _stxr_s(jit_state_t*,jit_int32_t,jit_int32_t,jit_int32_t);
1490 #define stxi_s(i0,r0,r1)                _stxi_s(_jit,i0,r0,r1)
1491 static void _stxi_s(jit_state_t*,jit_word_t,jit_int32_t,jit_int32_t);
1492 #define stxr_i(r0,r1,r2)                _stxr_i(_jit,r0,r1,r2)
1493 static void _stxr_i(jit_state_t*,jit_int32_t,jit_int32_t,jit_int32_t);
1494 #define stxi_i(i0,r0,r1)                _stxi_i(_jit,i0,r0,r1)
1495 static void _stxi_i(jit_state_t*,jit_word_t,jit_int32_t,jit_int32_t);
1496 #define stxr_l(r0,r1,r2)                _stxr_l(_jit,r0,r1,r2)
1497 static void _stxr_l(jit_state_t*,jit_int32_t,jit_int32_t,jit_int32_t);
1498 #define stxi_l(i0,r0,r1)                _stxi_l(_jit,i0,r0,r1)
1499 static void _stxi_l(jit_state_t*,jit_word_t,jit_int32_t,jit_int32_t);
1500 #define jmpr(r0)                        _jmpr(_jit,r0)
1501 static void _jmpr(jit_state_t*,jit_int32_t);
1502 #define jmpi(i0)                        _jmpi(_jit,i0)
1503 static void _jmpi(jit_state_t*,jit_word_t);
1504 #define jmpi_p(i0)                      _jmpi_p(_jit,i0)
1505 static jit_word_t _jmpi_p(jit_state_t*,jit_word_t);
1506 #define callr(r0)                       _callr(_jit,r0)
1507 static void _callr(jit_state_t*,jit_int32_t);
1508 #define calli(i0)                       _calli(_jit,i0)
1509 static void _calli(jit_state_t*,jit_word_t);
1510 #define calli_p(i0)                     _calli_p(_jit,i0)
1511 static jit_word_t _calli_p(jit_state_t*,jit_word_t);
1512 #define prolog(node)                    _prolog(_jit,node)
1513 static void _prolog(jit_state_t*,jit_node_t*);
1514 #define epilog(node)                    _epilog(_jit,node)
1515 static void _epilog(jit_state_t*,jit_node_t*);
1516 #  define vastart(r0)                   _vastart(_jit, r0)
1517 static void _vastart(jit_state_t*, jit_int32_t);
1518 #  define vaarg(r0, r1)                 _vaarg(_jit, r0, r1)
1519 static void _vaarg(jit_state_t*, jit_int32_t, jit_int32_t);
1520 #define patch_at(code,instr,label)      _patch_at(_jit,code,instr,label)
1521 static void _patch_at(jit_state_t*,jit_code_t,jit_word_t,jit_word_t);
1522 #endif
1523
1524 #if CODE
1525 #if __BYTE_ORDER == __BIG_ENDIAN
1526 static jit_word_t
1527 byte_swap_if_big_endian(jit_word_t w)
1528 {
1529     union {
1530         char    c[8];
1531         long    w;
1532     } l, h;
1533     l.w = w;
1534     h.c[0] = l.c[7];
1535     h.c[1] = l.c[6];
1536     h.c[2] = l.c[5];
1537     h.c[3] = l.c[4];
1538     h.c[4] = l.c[3];
1539     h.c[5] = l.c[2];
1540     h.c[6] = l.c[1];
1541     h.c[7] = l.c[0];
1542     return (h.w);
1543 }
1544 #else
1545 #define byte_swap_if_big_endian(w)              (w)
1546 #endif
1547
1548 static void
1549 _out(jit_state_t *_jit, int n, int tm,
1550      jit_word_t s0, jit_word_t s1, jit_word_t s2)
1551 {
1552     int                  i;
1553     jit_word_t           l, h, *w;
1554     set_bundle(_jit->pc.ul, l, h, tm, s0, s1, s2);
1555     _jit->pc.ul += 2;
1556     w = (jit_word_t *)_jitc->inst;
1557     for (i = n; i < _jitc->ioff; i++)
1558         w[i - n] = w[i];
1559     _jitc->ioff -= n;
1560 }
1561
1562 #define nop_m           0x0008000000L
1563 #define nop_i           0x0008000000L
1564 #define nop_b           0x4000000000L
1565 static void
1566 _stop(jit_state_t *_jit)
1567 {
1568     /* Clear set of live registers */
1569     jit_regset_set_ui(&_jitc->regs, 0);
1570     _jitc->pred = 0;
1571     /* Flag a stop is required */
1572     if (_jitc->ioff)
1573         _jitc->inst[_jitc->ioff - 1].t |= INST_STOP;
1574     else
1575         inst(nop_m, INST_Ms);
1576 }
1577
1578 static void
1579 _sync(jit_state_t *_jit)
1580 {
1581     /* Taken branches are supposed to not need a stop, so, it
1582      * should not be required to stop if no registers live in
1583      * sequential code */
1584     if (jit_regset_cmp_ui(&_jitc->regs, 0) != 0 || _jitc->pred)
1585         stop();
1586     do
1587         flush();
1588     while (_jitc->ioff);
1589 }
1590
1591 #define A_0             INST_A
1592 #define As0             INST_As
1593 #define I_0             INST_I
1594 #define Is0             INST_Is
1595 #define M_0             INST_M
1596 #define Ms0             INST_Ms
1597 #define F_0             INST_F
1598 #define Fs0             INST_Fs
1599 #define B_0             INST_B
1600 #define Bs0             INST_Bs
1601 #define L_0             INST_L
1602 #define Ls0             INST_Ls
1603 #define X_0             INST_X
1604 #define Xs0             INST_Xs
1605 #define A_1             (INST_A<<4)
1606 #define As1             (INST_As<<4)
1607 #define I_1             (INST_I<<4)
1608 #define Is1             (INST_Is<<4)
1609 #define M_1             (INST_M<<4)
1610 #define Ms1             (INST_Ms<<4)
1611 #define F_1             (INST_F<<4)
1612 #define Fs1             (INST_Fs<<4)
1613 #define B_1             (INST_B<<4)
1614 #define Bs1             (INST_Bs<<4)
1615 #define L_1             (INST_L<<4)
1616 #define Ls1             (INST_Ls<<4)
1617 #define X_1             (INST_X<<4)
1618 #define Xs1             (INST_Xs<<4)
1619 #define A_2             (INST_A<<8)
1620 #define As2             (INST_As<<8)
1621 #define I_2             (INST_I<<8)
1622 #define Is2             (INST_Is<<8)
1623 #define M_2             (INST_M<<8)
1624 #define Ms2             (INST_Ms<<8)
1625 #define F_2             (INST_F<<8)
1626 #define Fs2             (INST_Fs<<8)
1627 #define B_2             (INST_B<<8)
1628 #define Bs2             (INST_Bs<<8)
1629 #define L_2             (INST_L<<8)
1630 #define Ls2             (INST_Ls<<8)
1631 #define X_2             (INST_X<<8)
1632 #define Xs2             (INST_Xs<<8)
1633
1634 #define I_              I_0
1635 #define I_I_            I_0|I_1
1636 #define I_Is            I_0|Is1
1637 #define I_B_            I_0|B_1
1638 #define I_Bs            I_0|Bs1
1639 #define Is              Is0
1640 #define IsI_            Is0|I_1
1641 #define IsIs            Is0|Is1
1642 #define M_              M_0
1643 #define M_I_            M_0|I_1
1644 #define M_Is            M_0|Is1
1645 #define M_M_            M_0|M_1
1646 #define M_Ms            M_0|Ms1
1647 #define M_F_            M_0|F_1
1648 #define M_Fs            M_0|Fs1
1649 #define M_B_            M_0|B_1
1650 #define M_Bs            M_0|Bs1
1651 #define M_I_I_          M_0|I_1|I_2
1652 #define M_I_Is          M_0|I_1|Is2
1653 #define M_I_B_          M_0|I_1|B_2
1654 #define M_I_Bs          M_0|I_1|Bs2
1655 #define M_IsI_          M_0|Is1|I_2
1656 #define M_IsIs          M_0|Is1|Is2
1657 #define M_M_I_          M_0|M_1|I_2
1658 #define M_M_Is          M_0|M_1|Is2
1659 #define M_M_F_          M_0|M_1|F_2
1660 #define M_M_Fs          M_0|M_1|Fs2
1661 #define M_M_B_          M_0|M_1|B_2
1662 #define M_M_Bs          M_0|M_1|Bs2
1663 #define M_F_I_          M_0|F_1|I_2
1664 #define M_F_Is          M_0|F_1|Is2
1665 #define M_F_B_          M_0|F_1|B_2
1666 #define M_F_Bs          M_0|F_1|Bs2
1667 #define M_B_B_          M_0|B_1|B_2
1668 #define M_B_Bs          M_0|B_1|Bs2
1669 #define M_L_X_          M_0|L_1|X_2
1670 #define M_L_Xs          M_0|L_1|Xs2
1671 #define Ms              Ms0
1672 #define MsI_            Ms0|I_1
1673 #define MsIs            Ms0|Is1
1674 #define MsM_            Ms0|M_1
1675 #define MsMs            Ms0|Ms1
1676 #define MsM_I_          Ms0|M_1|I_2
1677 #define MsM_Is          Ms0|M_1|Is2
1678 #define F_              F_0
1679 #define F_I_            F_0|I_1
1680 #define F_Is            F_0|Is1
1681 #define F_B_            F_0|B_1
1682 #define F_Bs            F_0|Bs1
1683 #define Fs              Fs0
1684 #define B_              B_0
1685 #define B_B_            B_0|B_1
1686 #define B_Bs            B_0|Bs1
1687 #define B_B_B_          B_0|B_1|B_2
1688 #define B_B_Bs          B_0|B_1|Bs2
1689 #define Bs              Bs0
1690 #define L_X_            L_0|X_1
1691 #define L_Xs            L_0|Xs1
1692
1693 static jit_word_t
1694 templat(jit_word_t cc)
1695 {
1696     switch (cc) {
1697         case I_:
1698         case I_I_:      case I_Is:
1699         case I_B_:      case I_Bs:
1700         case Is:
1701         case IsI_:      case IsIs:
1702         case M_:
1703         case M_I_:      case M_Is:
1704         case M_M_:      case M_Ms:
1705         case M_F_:      case M_Fs:
1706         case M_B_:      case M_Bs:
1707         case M_I_I_:    case M_I_Is:
1708         case M_I_B_:    case M_I_Bs:
1709         case M_IsI_:    case M_IsIs:
1710         case M_M_I_:    case M_M_Is:
1711         case M_M_F_:    case M_M_Fs:
1712         case M_M_B_:    case M_M_Bs:
1713         case M_F_I_:    case M_F_Is:
1714         case M_F_B_:    case M_F_Bs:
1715         case M_B_B_:    case M_B_Bs:
1716         case M_L_X_:    case M_L_Xs:
1717         case Ms:
1718         case MsI_:      case MsIs:
1719         case MsM_:      case MsMs:
1720         case MsM_I_:    case MsM_Is:
1721         case F_:
1722         case F_I_:      case F_Is:
1723         case F_B_:      case F_Bs:
1724         case Fs:
1725         case B_:
1726         case B_B_:      case B_Bs:
1727         case B_B_B_:    case B_B_Bs:
1728         case Bs:
1729         case L_X_:      case L_Xs:
1730             return (cc);
1731         default:
1732             return (0);
1733     }
1734 }
1735
1736 /* match* functions recurse attempting to find a template for A-
1737  * instructions, that may be executed in M- or I- unit.
1738  * It also uses an heuristic of trying first M- for slot 0 and 2,
1739  * and I- for slot 1, but tries all possible matches.
1740  */
1741 static jit_word_t
1742 match_2(jit_word_t cc)
1743 {
1744     jit_word_t          t;
1745     if ((t = templat(cc | M_2)))
1746         return (t);
1747     if ((t = templat(cc | I_2)))
1748         return (t);
1749     return (0);
1750 }
1751
1752 static jit_word_t
1753 matchs2(jit_word_t cc)
1754 {
1755     jit_word_t          t;
1756     if ((t = templat(cc | Ms2)))
1757         return (t);
1758     if ((t = templat(cc | Is2)))
1759         return (t);
1760     return (0);
1761 }
1762
1763 static jit_word_t
1764 match2(jit_word_t cc)
1765 {
1766     if ((cc & 0xf00) == A_2)
1767         return (match_2(cc & ~0xf00));
1768     if ((cc & 0xf00) == As2)
1769         return (matchs2(cc & ~0xf00));
1770     return (0);
1771 }
1772
1773 static jit_word_t
1774 match_1(jit_word_t cc)
1775 {
1776     jit_word_t          t;
1777     if ((t = templat(cc | I_1)))
1778         return (t);
1779     if ((t = templat(cc | M_1)))
1780         return (t);
1781     if ((t = match2(cc | I_1)))
1782         return (t);
1783     if ((t = match2(cc | M_1)))
1784         return (t);
1785     return (0);
1786 }
1787
1788 static jit_word_t
1789 matchs1(jit_word_t cc)
1790 {
1791     jit_word_t          t;
1792     if ((t = templat(cc | Is1)))
1793         return (t);
1794     if ((t = templat(cc | Ms1)))
1795         return (t);
1796     if ((t = match2(cc | Is1)))
1797         return (t);
1798     if ((t = match2(cc | Ms1)))
1799         return (t);
1800     return (0);
1801 }
1802
1803 static jit_word_t
1804 match1(jit_word_t cc)
1805 {
1806     if ((cc & 0x0f0) == A_1)
1807         return (match_1(cc & ~0x0f0));
1808     if ((cc & 0x0f0) == As1)
1809         return (matchs1(cc & ~0x0f0));
1810     return (0);
1811 }
1812
1813 static jit_word_t
1814 match_0(jit_word_t cc)
1815 {
1816     jit_word_t          t;
1817     if ((t = templat(cc | M_0)))
1818         return (t);
1819     if ((t = templat(cc | I_0)))
1820         return (t);
1821     if ((t = match1(cc | M_0)))
1822         return (t);
1823     if ((t = match1(cc | I_0)))
1824         return (t);
1825     return (0);
1826 }
1827
1828 static jit_word_t
1829 matchs0(jit_word_t cc)
1830 {
1831     jit_word_t          t;
1832     if ((t = templat(cc | Ms0)))
1833         return (t);
1834     if ((t = templat(cc | Is0)))
1835         return (t);
1836     if ((t = match1(cc | Ms0)))
1837         return (t);
1838     if ((t = match1(cc | Is0)))
1839         return (t);
1840     return (0);
1841 }
1842
1843 static jit_word_t
1844 match0(jit_word_t cc)
1845 {
1846     if ((cc & 0x00f) == A_0)
1847         return (match_0(cc & ~0x00f));
1848     if ((cc & 0x00f) == As0)
1849         return (matchs0(cc & ~0x00f));
1850     return (0);
1851 }
1852
1853 static void
1854 _flush(jit_state_t *_jit)
1855 {
1856     int                 n, soff;
1857     jit_word_t          t, cc, tm, s0, s1, s2;
1858
1859     if (!_jitc->ioff)
1860         return;
1861     for (cc = 0, n = soff = 0; n < _jitc->ioff; n++, soff += 4)
1862         cc |= (jit_uword_t)(_jitc->inst[n].t) << soff;
1863
1864     soff = 0xf00;
1865     while (soff) {
1866         /* Try to find a template, or reduce down
1867          * to one instruction if no template match */
1868         if ((t = templat(cc))) {
1869             cc = t;
1870             break;
1871         }
1872         /* A- instructions may be execute in M- or I- unit */
1873         if ((t = match0(cc))) {
1874             cc = t;
1875             break;
1876         }
1877         cc &= ~soff;
1878         soff >>= 4;
1879     }
1880     assert(soff);
1881
1882     /* Prefer tail nop if need to add some nop, so that patching is easier */
1883 #define ii(n)           _jitc->inst[n].i
1884     switch (cc) {
1885         case I_:
1886             n = 1;              tm = TM_M_I_I_;
1887             s0 = nop_m;         s1 = ii(0);             s2 = nop_i;
1888             break;
1889         case I_I_:
1890             n = 2;              tm = TM_M_I_I_;
1891             s0 = nop_m;         s1 = ii(0);             s2 = ii(1);
1892             break;
1893         case I_Is:
1894             n = 2;              tm = TM_M_I_Is;
1895             s0 = nop_m;         s1 = ii(0);             s2 = ii(1);
1896             break;
1897         case I_B_:
1898             n = 2;              tm = TM_M_I_B_;
1899             s0 = nop_m;         s1 = ii(0);             s2 = ii(1);
1900             break;
1901         case I_Bs:
1902             n = 2;              tm = TM_M_I_Bs;
1903             s0 = nop_m;         s1 = ii(0);             s2 = ii(1);
1904             break;
1905         case Is:
1906             n = 1;              tm = TM_M_IsI_;
1907             s0 = nop_m;         s1 = ii(0);             s2 = nop_i;
1908             break;
1909         case IsI_:
1910             n = 2;              tm = TM_M_IsI_;
1911             s0 = nop_m;         s1 = ii(0);             s2 = ii(1);
1912             break;
1913         case IsIs:
1914             n = 2;              tm = TM_M_IsIs;
1915             s0 = nop_m;         s1 = ii(0);             s2 = ii(1);
1916             break;
1917         case M_:
1918             n = 1;              tm = TM_M_I_I_;
1919             s0 = ii(0);         s1 = nop_i;             s2 = nop_i;
1920             break;
1921         case M_I_:
1922             n = 2;              tm = TM_M_I_I_;
1923             s0 = ii(0);         s1 = ii(1);             s2 = nop_i;
1924             break;
1925         case M_Is:
1926             n = 2;              tm = TM_M_IsI_;
1927             s0 = ii(0);         s1 = ii(1);             s2 = nop_i;
1928             break;
1929         case M_M_:
1930             n = 2;              tm = TM_M_M_I_;
1931             s0 = ii(0);         s1 = ii(1);             s2 = nop_i;
1932             break;
1933         case M_Ms:
1934             n = 2;              tm = TM_M_M_Is;
1935             s0 = ii(0);         s1 = ii(1);             s2 = nop_i;
1936             break;
1937         case M_F_:
1938             n = 2;              tm = TM_M_F_I_;
1939             s0 = ii(0);         s1 = ii(1);             s2 = nop_i;
1940             break;
1941         case M_Fs:
1942             n = 2;              tm = TM_M_F_Is;
1943             s0 = ii(0);         s1 = ii(1);             s2 = nop_i;
1944             break;
1945         case M_B_:
1946             n = 2;              tm = TM_M_B_B_;
1947             s0 = ii(0);         s1 = ii(1);             s2 = nop_b;
1948             break;
1949         case M_Bs:
1950             n = 2;              tm = TM_M_B_Bs;
1951             s0 = ii(0);         s1 = ii(1);             s2 = nop_b;
1952             break;
1953         case M_I_I_:
1954             n = 3;              tm = TM_M_I_I_;
1955             s0 = ii(0);         s1 = ii(1);             s2 = ii(2);
1956             break;
1957         case M_I_Is:
1958             n = 3;              tm = TM_M_I_Is;
1959             s0 = ii(0);         s1 = ii(1);             s2 = ii(2);
1960             break;
1961         case M_I_B_:
1962             n = 3;              tm = TM_M_I_B_;
1963             s0 = ii(0);         s1 = ii(1);             s2 = ii(2);
1964             break;
1965         case M_I_Bs:
1966             n = 3;              tm = TM_M_I_Bs;
1967             s0 = ii(0);         s1 = ii(1);             s2 = ii(2);
1968             break;
1969         case M_IsI_:
1970             n = 3;              tm = TM_M_IsI_;
1971             s0 = ii(0);         s1 = ii(1);             s2 = ii(2);
1972             break;
1973         case M_IsIs:
1974             n = 3;              tm = TM_M_IsIs;
1975             s0 = ii(0);         s1 = ii(1);             s2 = ii(2);
1976             break;
1977         case M_M_I_:
1978             n = 3;              tm = TM_M_M_I_;
1979             s0 = ii(0);         s1 = ii(1);             s2 = ii(2);
1980             break;
1981         case M_M_Is:
1982             n = 3;              tm = TM_M_M_Is;
1983             s0 = ii(0);         s1 = ii(1);             s2 = ii(2);
1984             break;
1985         case M_M_F_:
1986             n = 3;              tm = TM_M_M_F_;
1987             s0 = ii(0);         s1 = ii(1);             s2 = ii(2);
1988             break;
1989         case M_M_Fs:
1990             n = 3;              tm = TM_M_M_Fs;
1991             s0 = ii(0);         s1 = ii(1);             s2 = ii(2);
1992             break;
1993         case M_M_B_:
1994             n = 3;              tm = TM_M_M_B_;
1995             s0 = ii(0);         s1 = ii(1);             s2 = ii(2);
1996             break;
1997         case M_M_Bs:
1998             n = 3;              tm = TM_M_M_Bs;
1999             s0 = ii(0);         s1 = ii(1);             s2 = ii(2);
2000             break;
2001         case M_F_I_:
2002             n = 3;              tm = TM_M_F_I_;
2003             s0 = ii(0);         s1 = ii(1);             s2 = ii(2);
2004             break;
2005         case M_F_Is:
2006             n = 3;              tm = TM_M_F_Is;
2007             s0 = ii(0);         s1 = ii(1);             s2 = ii(2);
2008             break;
2009         case M_F_B_:
2010             n = 3;              tm = TM_M_F_B_;
2011             s0 = ii(0);         s1 = ii(1);             s2 = ii(2);
2012             break;
2013         case M_F_Bs:
2014             n = 3;              tm = TM_M_F_Bs;
2015             s0 = ii(0);         s1 = ii(1);             s2 = ii(2);
2016             break;
2017         case M_B_B_:
2018             n = 3;              tm = TM_M_B_B_;
2019             s0 = ii(0);         s1 = ii(1);             s2 = ii(2);
2020             break;
2021         case M_B_Bs:
2022             n = 3;              tm = TM_M_B_Bs;
2023             s0 = ii(0);         s1 = ii(1);             s2 = ii(2);
2024             break;
2025         case M_L_X_:
2026             n = 3;              tm = TM_M_L_X_;
2027             s0 = ii(0);         s1 = ii(1);             s2 = ii(2);
2028             break;
2029         case M_L_Xs:
2030             n = 3;              tm = TM_M_L_Xs;
2031             s0 = ii(0);         s1 = ii(1);             s2 = ii(2);
2032             break;
2033         case Ms:
2034             n = 1;              tm = TM_MsM_I_;
2035             s0 = ii(0);         s1 = nop_m;             s2 = nop_i;
2036             break;
2037         case MsI_:
2038             n = 2;              tm = TM_MsM_I_;
2039             s0 = ii(0);         s1 = nop_m;             s2 = ii(1);
2040             break;
2041         case MsIs:
2042             n = 2;              tm = TM_MsM_Is;
2043             s0 = ii(0);         s1 = nop_m;             s2 = ii(1);
2044             break;
2045         case MsM_:
2046             n = 2;              tm = TM_MsM_I_;
2047             s0 = ii(0);         s1 = ii(1);             s2 = nop_i;
2048             break;
2049         case MsMs:
2050             n = 2;              tm = TM_MsM_Is;
2051             s0 = ii(0);         s1 = ii(1);             s2 = nop_i;
2052             break;
2053         case MsM_I_:
2054             n = 3;              tm = TM_MsM_I_;
2055             s0 = ii(0);         s1 = ii(1);             s2 = ii(2);
2056             break;
2057         case MsM_Is:
2058             n = 3;              tm = TM_MsM_Is;
2059             s0 = ii(0);         s1 = ii(1);             s2 = ii(2);
2060             break;
2061         case F_:
2062             n = 1;              tm = TM_M_F_I_;
2063             s0 = nop_m;         s1 = ii(0);             s2 = nop_i;
2064             break;
2065         case F_I_:
2066             n = 2;              tm = TM_M_F_I_;
2067             s0 = nop_m;         s1 = ii(0);             s2 = ii(1);
2068             break;
2069         case F_Is:
2070             n = 2;              tm = TM_M_F_Is;
2071             s0 = nop_m;         s1 = ii(0);             s2 = ii(1);
2072             break;
2073         case F_B_:
2074             n = 2;              tm = TM_M_F_B_;
2075             s0 = nop_m;         s1 = ii(0);             s2 = ii(1);
2076             break;
2077         case F_Bs:
2078             n = 2;              tm = TM_M_F_Bs;
2079             s0 = nop_m;         s1 = ii(0);             s2 = ii(1);
2080             break;
2081         case Fs:
2082             n = 1;              tm = TM_M_F_Is;
2083             s0 = nop_m;         s1 = ii(0);             s2 = nop_i;
2084             break;
2085         case B_:
2086             n = 1;              tm = TM_B_B_B_;
2087             s0 = ii(0);         s1 = nop_b;             s2 = nop_b;
2088             break;
2089         case B_B_:
2090             n = 2;              tm = TM_B_B_B_;
2091             s0 = ii(0);         s1 = ii(1);             s2 = nop_b;
2092             break;
2093         case B_Bs:
2094             n = 2;              tm = TM_B_B_Bs;
2095             s0 = ii(0);         s1 = ii(1);             s2 = nop_b;
2096             break;
2097         case B_B_B_:
2098             n = 3;              tm = TM_B_B_B_;
2099             s0 = ii(0);         s1 = ii(1);             s2 = ii(2);
2100             break;
2101         case B_B_Bs:
2102             n = 3;              tm = TM_B_B_Bs;
2103             s0 = ii(0);         s1 = ii(1);             s2 = ii(2);
2104             break;
2105         case Bs:
2106             n = 1;              tm = TM_B_B_Bs;
2107             s0 = ii(0);         s1 = nop_b;             s2 = nop_b;
2108             break;
2109         case L_X_:
2110             n = 2;              tm = TM_M_L_X_;
2111             s0 = nop_m;         s1 = ii(0);             s2 = ii(1);
2112             break;
2113         case L_Xs:
2114             n = 2;              tm = TM_M_L_Xs;
2115             s0 = nop_m;         s1 = ii(0);             s2 = ii(1);
2116             break;
2117         default:
2118             abort();
2119     }
2120     out(n, tm, s0, s1, s2);
2121 }
2122
2123 static void
2124 _inst(jit_state_t *_jit, jit_word_t i, jit_uint8_t t)
2125 {
2126     if (_jitc->ioff > 2)
2127         flush();
2128     assert(!(i & 0x11111e0000000000L));
2129     _jitc->inst[_jitc->ioff].i = i;
2130     _jitc->inst[_jitc->ioff].t = t;
2131     ++_jitc->ioff;
2132 }
2133
2134 static void
2135 _A1(jit_state_t *_jit, jit_word_t _p,
2136     jit_word_t x4, jit_word_t x2, jit_word_t r3, jit_word_t r2, jit_word_t r1)
2137 {
2138     assert(!(_p & ~0x3fL));
2139     assert(!(x4 &  ~0xfL));
2140     assert(!(x2 &  ~0x3L));
2141     assert(!(r3 & ~0x7fL));
2142     assert(!(r2 & ~0x7fL));
2143     assert(!(r1 & ~0x7fL));
2144     TSTREG2(r2, r3);
2145     TSTPRED(_p);
2146     TSTREG1(r1);
2147     inst((8L<<37)|(x4<<29)|(x2<<27)|(r3<<20)|(r2<<13)|(r1<<6)|_p, INST_A);
2148     SETREG(r1);
2149 }
2150
2151 static void
2152 _A3(jit_state_t *_jit, jit_word_t _p,
2153     jit_word_t x4, jit_word_t x2, jit_word_t r3, jit_word_t im, jit_word_t r1)
2154 {
2155     assert(!(_p & ~0x3fL));
2156     assert(!(x4 &  ~0xfL));
2157     assert(!(x2 &  ~0x3L));
2158     assert(!(r3 & ~0x7fL));
2159     assert(im >= -128 && im <= 127);
2160     assert(!(r1 & ~0x7f));
2161     TSTREG1(r3);
2162     TSTPRED(_p);
2163     TSTREG1(r1);
2164     inst((8L<<37)|(((im>>7)&1L)<<36)|(x4<<29)|(x2<<27)|
2165          (r3<<20)|((im&0x7fL)<<13)|(r1<<6)|_p, INST_A);
2166     SETREG(r1);
2167 }
2168
2169 static void
2170 _A4(jit_state_t *_jit, jit_word_t _p,
2171     jit_word_t x2, jit_word_t r3, jit_word_t im, jit_word_t r1)
2172 {
2173     assert(!(_p & ~0x3fL));
2174     assert(!(x2 &  ~0x3L));
2175     assert(!(r3 & ~0x7fL));
2176     assert(im >= -8192 && im <= 8191);
2177     assert(!(r1  & ~0x7f));
2178     TSTREG1(r3);
2179     TSTPRED(_p);
2180     TSTREG1(r1);
2181     inst((8L<<37)|(((im>>13)&1L)<<36)|(x2<<34)|(((im>>7)&0x3fL)<<27)|
2182          (r3<<20)|((im&0x7fL)<<13)|(r1<<6)|_p, INST_A);
2183     SETREG(r1);
2184 }
2185
2186 static void
2187 _A5(jit_state_t *_jit, jit_word_t _p,
2188     jit_word_t r3, jit_word_t im, jit_word_t r1)
2189 {
2190     jit_word_t          s, i5, i9, i7;
2191     assert(!(_p & ~0x3fL));
2192     assert(!(r3  & ~0x3L));
2193     assert(im >= -2097152 && im <= 2097151);
2194     assert(!(r1  & ~0x7fL));
2195     /* imm22 = sign_ext(s << 21 | imm5c << 16 | imm9d << 7 | imm7b, 22) */
2196     s  = (im & 0x200000) >> 21;
2197     i5 = (im & 0x1f0000) >> 16;
2198     i9 = (im &   0xff80) >>  7;
2199     i7 =  im &     0x7f;
2200     TSTREG1(r3);
2201     TSTPRED(_p);
2202     TSTREG1(r1);
2203     inst((9L<<37)|(s<<36)|(i9<<27)|(i5<<22)|
2204          (r3<<20)|(i7<<13)|(r1<<6)|_p, INST_A);
2205     SETREG(r1);
2206 }
2207
2208 static void
2209 _A6(jit_state_t *_jit, jit_word_t _p,
2210     jit_word_t o, jit_word_t x2, jit_word_t ta, jit_word_t p2,
2211     jit_word_t r3, jit_word_t r2, jit_word_t c, jit_word_t p1)
2212 {
2213     assert(!(_p & ~0x3fL));
2214     assert(!(o  &  ~0xfL));
2215     assert(!(x2 &  ~0x3L));
2216     assert(!(ta &  ~0x1L));
2217     assert(!(p2 & ~0x7fL));
2218     assert(!(r3 & ~0x7fL));
2219     assert(!(r2 & ~0x7fL));
2220     assert(!(c  &  ~0x1L));
2221     assert(!(p1 & ~0x3fL));
2222     TSTREG2(r2, r3);
2223     TSTPRED(_p);
2224     inst((o<<37)|(x2<<34)|(ta<<33)|(p2<<27)|(r3<<20)|
2225          (r2<<13)|(c<<12)|(p1<<6)|_p, INST_A);
2226     if (p1)
2227         _jitc->pred |= 1 << p1;
2228     if (p2)
2229         _jitc->pred |= 1 << p2;
2230 }
2231
2232 static void
2233 _A7(jit_state_t *_jit, jit_word_t _p,
2234     jit_word_t o, jit_word_t x2, jit_word_t ta,
2235     jit_word_t p2, jit_word_t r3, jit_word_t c, jit_word_t p1)
2236 {
2237     assert(!(_p & ~0x3fL));
2238     assert(!(o  &  ~0xfL));
2239     assert(!(x2 &  ~0x3L));
2240     assert(!(ta &  ~0x1L));
2241     assert(!(p2 & ~0x7fL));
2242     assert(!(r3 & ~0x7fL));
2243     assert(!(c  &  ~0x1L));
2244     assert(!(p1 & ~0x3fL));
2245     TSTREG1(r3);
2246     TSTPRED(_p);
2247     inst((o<<37)|(1L<<36)|(x2<<34)|(ta<<33)|
2248          (p2<<27)|(r3<<20)|(c<<12)|(p1<<6)|_p, INST_A);
2249     if (p1)
2250         _jitc->pred |= 1 << p1;
2251     if (p2)
2252         _jitc->pred |= 1 << p2;
2253 }
2254
2255 static void
2256 _A8(jit_state_t *_jit, jit_word_t _p,
2257     jit_word_t o, jit_word_t x2, jit_word_t ta, jit_word_t p2,
2258     jit_word_t r3, jit_word_t im, jit_word_t c, jit_word_t p1)
2259 {
2260     assert(!(_p & ~0x3fL));
2261     assert(!(o  &  ~0xfL));
2262     assert(!(x2 &  ~0x3L));
2263     assert(!(ta &  ~0x1L));
2264     assert(!(p2 & ~0x7fL));
2265     assert(!(r3 & ~0x7fL));
2266     assert(im >= -128 && im <= 127);
2267     assert(!(c  &  ~0x1L));
2268     assert(!(p1 & ~0x3fL));
2269     TSTREG1(r3);
2270     TSTPRED(_p);
2271     inst((o<<37)|(((im>>7)&1L)<<36)|(x2<<34)|(ta<<33)|(p2<<27)|(r3<<20)|
2272          ((im&0x7fL)<<13)|(c<<12)|(p1<<6)|_p, INST_A);
2273     if (p1)
2274         _jitc->pred |= 1 << p1;
2275     if (p2)
2276         _jitc->pred |= 1 << p2;
2277 }
2278
2279 static void
2280 _A9(jit_state_t *_jit, jit_word_t _p,
2281     jit_word_t za, jit_word_t zb, jit_word_t x4,
2282     jit_word_t x2, jit_word_t r3, jit_word_t r2, jit_word_t r1)
2283 {
2284     assert(!(_p & ~0x3fL));
2285     assert(!(za &  ~0x1L));
2286     assert(!(zb &  ~0x1L));
2287     assert(!(x4 &  ~0xfL));
2288     assert(!(x2 &  ~0x3L));
2289     assert(!(r3 & ~0x7fL));
2290     assert(!(r2 & ~0x7fL));
2291     assert(!(r1 & ~0x7fL));
2292     TSTREG2(r2, r3);
2293     TSTPRED(_p);
2294     TSTREG1(r1);
2295     inst((8L<<37)|(za<<36)|(1L<<34)|(zb<<33)|(x4<<29)|(x2<<27)|
2296          (r3<<20)|(r2<<13)|(r1<<6)|_p, INST_A);
2297     SETREG(r1);
2298 }
2299
2300 static void
2301 _I1(jit_state_t *_jit, jit_word_t _p,
2302     jit_word_t ct, jit_word_t x2, jit_word_t r3, jit_word_t r2, jit_word_t r1)
2303 {
2304     assert(!(_p & ~0x3fL));
2305     assert(!(ct &  ~0x3L));
2306     assert(!(x2 &  ~0x3L));
2307     assert(!(r3 & ~0x7fL));
2308     assert(!(r2 & ~0x7fL));
2309     assert(!(r1 & ~0x7fL));
2310     TSTREG2(r2, r3);
2311     TSTPRED(_p);
2312     TSTREG1(r1);
2313     inst((7L<<37)|(1L<<33)|(ct<<30)|(x2<<28)|
2314          (r3<<20)|(r2<<13)|(r1<<6)|_p, INST_I);
2315     SETREG(r1);
2316 }
2317
2318 static void
2319 _I2(jit_state_t *_jit, jit_word_t _p,
2320     jit_word_t za, jit_word_t xa, jit_word_t zb, jit_word_t xc,
2321     jit_word_t xb ,jit_word_t r3, jit_word_t r2, jit_word_t r1)
2322 {
2323     assert(!(_p & ~0x3fL));
2324     assert(!(za &  ~0x1L));
2325     assert(!(xa &  ~0x3L));
2326     assert(!(zb &  ~0x1L));
2327     assert(!(xc &  ~0x3L));
2328     assert(!(xb &  ~0x3L));
2329     assert(!(r3 & ~0x7fL));
2330     assert(!(r2 & ~0x7fL));
2331     assert(!(r1 & ~0x7fL));
2332     TSTREG2(r2, r3);
2333     TSTPRED(_p);
2334     TSTREG1(r1);
2335     inst((7L<<37)|(za<<36)|(xa<<34)|(zb<<33)|(xc<<30)|
2336          (xb<<28)|(r3<<20)|(r2<<13)|(r1<<6), INST_I);
2337     SETREG(r1);
2338 }
2339
2340 static void
2341 _I3(jit_state_t *_jit, jit_word_t _p,
2342     jit_word_t mb, jit_word_t r2, jit_word_t r1)
2343 {
2344     assert(!(_p & ~0x3fL));
2345     assert(!(mb &  ~0xfL));
2346     assert(!(r2 & ~0x7fL));
2347     assert(!(r1 & ~0x7fL));
2348     TSTREG1(r2);
2349     TSTPRED(_p);
2350     TSTREG1(r1);
2351     inst((7L<<37)|(3L<<34)|(2L<<30)|(2L<<28)|
2352          (mb<<20)|(r2<<13)|(r1<<6)|_p, INST_I);
2353     SETREG(r1);
2354 }
2355
2356 static void
2357 _I4(jit_state_t *_jit, jit_word_t _p,
2358     jit_word_t mh, jit_word_t r2, jit_word_t r1)
2359 {
2360     assert(!(_p & ~0x3fL));
2361     assert(!(mh & ~0xffL));
2362     assert(!(r2 & ~0x7fL));
2363     assert(!(r1 & ~0x7fL));
2364     TSTREG1(r2);
2365     TSTPRED(_p);
2366     TSTREG1(r1);
2367     inst((7L<<37)|(3L<<34)|(1L<<33)|(2L<<30)|
2368          (2L<<28)|(mh<<20)|(r2<<13)|(r1<<6)|_p, INST_I);
2369     SETREG(r1);
2370 }
2371
2372 static void
2373 _I5(jit_state_t *_jit, jit_word_t _p,
2374     jit_word_t za, jit_word_t zb, jit_word_t x2,
2375     jit_word_t r3, jit_word_t r2, jit_word_t r1)
2376 {
2377     assert(!(_p & ~0x3fL));
2378     assert(!(za &  ~0x1L));
2379     assert(!(zb &  ~0x1L));
2380     assert(!(x2 &  ~0x3L));
2381     assert(!(r3 & ~0x7fL));
2382     assert(!(r2 & ~0x7fL));
2383     assert(!(r1 & ~0x7fL));
2384     TSTREG2(r2, r3);
2385     TSTPRED(_p);
2386     TSTREG1(r1);
2387     inst((7L<<37)|(za<<36)|(zb<<33)|(x2<<28)|
2388          (r3<<20)|(r2<<13)|(r1<<6)|_p, INST_I);
2389     SETREG(r1);
2390 }
2391
2392 static void
2393 _I6(jit_state_t *_jit, jit_word_t _p,
2394     jit_word_t za, jit_word_t zb, jit_word_t x2,
2395     jit_word_t r3, jit_word_t ct, jit_word_t r1)
2396 {
2397     assert(!(_p & ~0x3fL));
2398     assert(!(za &  ~0x1L));
2399     assert(!(zb &  ~0x1L));
2400     assert(!(x2 &  ~0x3L));
2401     assert(!(r3 & ~0x7fL));
2402     assert(!(ct & ~0x1fL));
2403     assert(!(r1 & ~0x7fL));
2404     TSTREG1(r3);
2405     TSTPRED(_p);
2406     TSTREG1(r1);
2407     inst((7L<<37)|(za<<36)|(1L<<34)|(zb<<33)|
2408          (x2<<28)|(r3<<20)|(ct<<14)|(r1<<6)|_p, INST_I);
2409     SETREG(r1);
2410 }
2411
2412 static void
2413 _I7(jit_state_t *_jit, jit_word_t _p,
2414     jit_word_t za, jit_word_t zb, jit_word_t r3, jit_word_t r2, jit_word_t r1)
2415 {
2416     assert(!(_p & ~0x3fL));
2417     assert(!(za &  ~0x1L));
2418     assert(!(zb &  ~0x1L));
2419     assert(!(r3 & ~0x7fL));
2420     assert(!(r2 & ~0x7fL));
2421     assert(!(r1 & ~0x7fL));
2422     TSTREG2(r2, r3);
2423     TSTPRED(_p);
2424     TSTREG1(r1);
2425     inst((7L<<37)|(za<<36)|(zb<<33)|(1L<<30)|
2426          (r3<<20)|(r2<<13)|(r1<<6)|_p, INST_I);
2427     SETREG(r1);
2428 }
2429
2430 static void
2431 _I8(jit_state_t *_jit, jit_word_t _p,
2432     jit_word_t za, jit_word_t zb, jit_word_t im, jit_word_t r2, jit_word_t r1)
2433 {
2434     assert(!(_p & ~0x3fL));
2435     assert(!(za &  ~0x1L));
2436     assert(!(zb &  ~0x1L));
2437     assert(!(im & ~0x1fL));
2438     assert(!(r2 & ~0x7fL));
2439     assert(!(r1 & ~0x7fL));
2440     TSTREG1(r2);
2441     TSTPRED(_p);
2442     TSTREG1(r1);
2443     inst((7L<<37)|(za<<36)|(3L<<34)|(zb<<33)|(1L<<30)|(1L<<28)|
2444          (im<<20)|(r2<<13)|(r1<<6), INST_I);
2445     SETREG(r1);
2446 }
2447
2448 static void
2449 _I9(jit_state_t *_jit, jit_word_t _p,
2450     jit_word_t x2, jit_word_t r3, jit_word_t r1)
2451 {
2452     assert(!(_p & ~0x3fL));
2453     assert(!(x2 &  ~0x3L));
2454     assert(!(r3 & ~0x7fL));
2455     assert(!(r1 & ~0x7fL));
2456     TSTREG1(r3);
2457     TSTPRED(_p);
2458     TSTREG1(r1);
2459     inst((7L<<37)|(1L<<34)|(1L<<34)|(1L<<33)|
2460          (x2<<30)|(1L<<28)|(r3<<20)|(r1<<6)|_p, INST_I);
2461     SETREG(r1);
2462 }
2463
2464 static void
2465 _I10(jit_state_t *_jit, jit_word_t _p,
2466      jit_word_t ct, jit_word_t r3, jit_word_t r2, jit_word_t r1)
2467 {
2468     assert(!(_p & ~0x3fL));
2469     assert(!(ct & ~0x3fL));
2470     assert(!(r3 & ~0x7fL));
2471     assert(!(r2 & ~0x7fL));
2472     assert(!(r1 & ~0x7fL));
2473     TSTREG2(r2, r3);
2474     TSTPRED(_p);
2475     TSTREG1(r1);
2476     inst((5L<<37)|(3L<<34)|(ct<<27)|(r3<<20)|(r2<<13)|(r1<<6)|_p, INST_I);
2477     SETREG(r1);
2478 }
2479
2480 static void
2481 _I11(jit_state_t *_jit, jit_word_t _p,
2482      jit_word_t len, jit_word_t r3,
2483      jit_word_t pos, jit_word_t y, jit_word_t r1)
2484 {
2485     assert(!(_p  & ~0x3fL));
2486     assert(!(len & ~0x3fL));
2487     assert(!(r3  & ~0x7fL));
2488     assert(!(pos & ~0x1fL));
2489     assert(!(y   &  ~0x1L));
2490     assert(!(r1  & ~0x7fL));
2491     TSTREG1(r3);
2492     TSTPRED(_p);
2493     TSTREG1(r1);
2494     inst((5L<<37)|(1L<<34)|(len<<27)|(r3<<20)|
2495          (pos<<14)|(y<<13)|(r1<<6)|_p, INST_I);
2496     SETREG(r1);
2497 }
2498
2499 static void
2500 _I12(jit_state_t *_jit, jit_word_t _p,
2501      jit_word_t len, jit_word_t pos, jit_word_t r2, jit_word_t r1)
2502 {
2503     assert(!(_p  & ~0x3fL));
2504     assert(!(len & ~0x3fL));
2505     assert(!(pos & ~0x3fL));
2506     assert(!(r2  & ~0x7fL));
2507     assert(!(r1  & ~0x7fL));
2508     TSTREG1(r2);
2509     TSTPRED(_p);
2510     TSTREG1(r1);
2511     inst((5L<<37)|(1L<<34)|(1L<<33)|(len<<27)|
2512          (pos<<20)|(r2<<13)|(r1<<6)|_p, INST_I);
2513     SETREG(r1);
2514 }
2515
2516 static void
2517 _I13(jit_state_t *_jit, jit_word_t _p,
2518      jit_word_t len, jit_word_t pos, jit_word_t im, jit_word_t r1)
2519 {
2520     assert(!(_p  & ~0x3fL));
2521     assert(!(len & ~0x3fL));
2522     assert(!(pos & ~0x3fL));
2523     assert(!(im  & ~0x7fL));
2524     assert(!(r1  & ~0x7fL));
2525     TSTPRED(_p);
2526     TSTREG1(r1);
2527     inst((5L<<37)|(((im>>7)&1L)<<36)|(1L<<34)|(1L<<33)|(len<<27)|
2528          (1L<<26)|(pos<<20)|((im&0x7fL)<<13)|(r1<<6)|_p, INST_I);
2529     SETREG(r1);
2530 }
2531
2532 static void
2533 _I14(jit_state_t *_jit, jit_word_t _p,
2534      jit_word_t s, jit_word_t len, jit_word_t r3, jit_word_t pos, jit_word_t r1)
2535 {
2536     assert(!(_p  & ~0x3fL));
2537     assert(!(s   &  ~0x1L));
2538     assert(!(len & ~0x3fL));
2539     assert(!(r3  & ~0x7fL));
2540     assert(!(pos & ~0x1fL));
2541     assert(!(r1  & ~0x7fL));
2542     TSTREG1(r3);
2543     TSTPRED(_p);
2544     TSTREG1(r1);
2545     inst((5L<<37)|(s<<36)|(3L<<34)|(1L<<33)|
2546          (len<<27)|(r3<<20)|(pos<<14)|(r1<<6)|_p, INST_I);
2547     SETREG(r1);
2548 }
2549
2550 static void
2551 _I15(jit_state_t *_jit, jit_word_t _p,
2552      jit_word_t pos, jit_word_t len,
2553      jit_word_t r3, jit_word_t r2, jit_word_t r1)
2554 {
2555     assert(!(_p  & ~0x3fL));
2556     assert(!(pos & ~0x3fL));
2557     assert(!(len &  ~0xfL));
2558     assert(!(r3  & ~0x7fL));
2559     assert(!(r2  & ~0x7fL));
2560     assert(!(r1  & ~0x7fL));
2561     TSTREG2(r2, r3);
2562     TSTPRED(_p);
2563     TSTREG1(r1);
2564     inst((4L<<37)|(pos<<31)|(len<<27)|(r3<<20)|(r2<<13)|(r1<<6)|_p, INST_I);
2565     SETREG(r1);
2566 }
2567
2568 static void
2569 _I16(jit_state_t *_jit, jit_word_t _p,
2570      jit_word_t tb, jit_word_t ta, jit_word_t p2,
2571      jit_word_t r3, jit_word_t ps, jit_word_t c, jit_word_t p1)
2572 {
2573     assert(!(_p & ~0x3fL));
2574     assert(!(tb &  ~0x1L));
2575     assert(!(ta &  ~0x1L));
2576     assert(!(p2 & ~0x7fL));
2577     assert(!(r3 & ~0x7fL));
2578     assert(!(ps & ~0x3fL));
2579     assert(!(c  &  ~0x1L));
2580     assert(!(p1 & ~0x3fL));
2581     TSTREG1(r3);
2582     TSTPRED(_p);
2583     inst((5L<<37)|(tb<<36)|(ta<<33)|(p2<<27)|
2584          (r3<<20)|(ps<<14)|(c<<12)|(p1<<6), INST_I);
2585     if (p1)
2586         _jitc->pred |= 1 << p1;
2587     if (p2)
2588         _jitc->pred |= 1 << p2;
2589 }
2590
2591 static void
2592 _I17(jit_state_t *_jit, jit_word_t _p,
2593      jit_word_t tb, jit_word_t ta, jit_word_t p2,
2594      jit_word_t r3, jit_word_t c, jit_word_t p1)
2595 {
2596     assert(!(_p & ~0x3fL));
2597     assert(!(tb &  ~0x1L));
2598     assert(!(ta &  ~0x1L));
2599     assert(!(p2 & ~0x7fL));
2600     assert(!(r3 & ~0x7fL));
2601     assert(!(c  &  ~0x1L));
2602     assert(!(p1 & ~0x3fL));
2603     TSTREG1(r3);
2604     TSTPRED(_p);
2605     inst((5L<<37)|(tb<<36)|(ta<<33)|(p2<<27)|
2606          (r3<<20)|(1L<<13)|(c<<12)|(p1<<6)|_p, INST_I);
2607     if (p1)
2608         _jitc->pred |= 1 << p1;
2609     if (p2)
2610         _jitc->pred |= 1 << p2;
2611 }
2612
2613 static void
2614 _I18(jit_state_t *_jit, jit_word_t _p,
2615      jit_word_t im, jit_word_t y)
2616 {
2617     assert(!(_p &     ~0x3fL));
2618     assert(!(im & ~0x1fffffL));
2619     assert(!(y  &      ~0x1L));
2620     TSTPRED(_p);
2621     inst((((im>>20)&1L)<<26)|(1L<<27)|(y<<26)|((im&0xffffL)<<6)|_p, INST_I);
2622 }
2623
2624 static void
2625 _I19(jit_state_t *_jit, jit_word_t _p,
2626      jit_word_t im)
2627 {
2628     assert(!(_p &     ~0x3fL));
2629     assert(!(im & ~0x1fffffL));
2630     TSTPRED(_p);
2631     inst(((im>>20)&1L)|((im&0xffffL)<<6)|_p, INST_I);
2632 }
2633
2634 static void
2635 _I20(jit_state_t *_jit, jit_word_t _p,
2636      jit_word_t r2, jit_word_t im)
2637 {
2638     assert(!(_p &     ~0x3fL));
2639     assert(!(r2 &     ~0x7fL));
2640     assert(!(im & ~0x1fffffL));
2641     TSTREG1(r2);
2642     TSTPRED(_p);
2643     inst(((im>>20)&1L)|(1L<<33)|(((im>>7)&0x1fffL)<<20)|
2644          (r2<<13)|((im&0x7fL)<<6)|_p, INST_I);
2645 }
2646
2647 static void
2648 _I21(jit_state_t *_jit, jit_word_t _p,
2649      jit_word_t im, jit_word_t ih, jit_word_t x,
2650      jit_word_t wh, jit_word_t r2, jit_word_t b1)
2651 {
2652     assert(!(_p &     ~0x3fL));
2653     assert(!(im &    ~0x1ffL));
2654     assert(!(ih &      ~0x1L));
2655     assert(!(x  &      ~0x1L));
2656     assert(!(wh &      ~0x3L));
2657     assert(!(r2 &     ~0x7fL));
2658     assert(!(b1 &      ~0x7L));
2659     TSTREG1(r2);
2660     TSTPRED(_p);
2661     inst((7L<<33)|(im<<24)|(ih<<23)|(x<<22)|(wh<<20)|
2662          (r2<<13)|(b1<<6), INST_I);
2663 }
2664
2665 static void
2666 _I22(jit_state_t *_jit, jit_word_t _p,
2667      jit_word_t b2, jit_word_t r1)
2668 {
2669     assert(!(_p & ~0x3fL));
2670     assert(!(b2 &  ~0x7L));
2671     assert(!(r1 & ~0x7fL));
2672     TSTPRED(_p);
2673     inst((0x31L<<27)|(b2<<13)|(r1<<6)|_p, INST_I);
2674 }
2675
2676 static void
2677 _I23(jit_state_t *_jit, jit_word_t _p,
2678      jit_word_t r2, jit_word_t im)
2679 {
2680     assert(!(_p &   ~0x3fL));
2681     assert(!(r2 &   ~0x7fL));
2682     assert(!(im & ~0xffffL));
2683     TSTREG1(r2);
2684     TSTPRED(_p);
2685     inst((((im>>15)&1L)<<36)|(3L<<33)|(((im>>7)&0xffL)<<24)|
2686          (r2<<13)|(im&0x7fL)|_p, INST_I);
2687 }
2688
2689 static void
2690 _I24(jit_state_t *_jit, jit_word_t _p,
2691      jit_word_t im)
2692 {
2693     jit_uint8_t         cc = INST_I;
2694     assert(!(_p &      ~0x3fL));
2695     assert(!(im & ~0xfffffffL));
2696     TSTPRED(_p);
2697     inst((((im>>27)&1L)<<36)|(2L<<33)|((im&0x7ffffffL)<<6)|_p, cc);
2698 }
2699
2700 static void
2701 _I25(jit_state_t *_jit, jit_word_t _p,
2702      jit_word_t x6, jit_word_t r1)
2703 {
2704     assert(!(_p & ~0x3fL));
2705     assert(!(x6 & ~0x3fL));
2706     assert(!(r1 & ~0x7fL));
2707     TSTPRED(_p);
2708     TSTREG1(r1);
2709     inst((x6<<27)|(r1<<6)|_p, INST_I);
2710     SETREG(r1);
2711 }
2712
2713 static void
2714 _I26(jit_state_t *_jit, jit_word_t _p,
2715      jit_word_t ar,jit_word_t r2)
2716 {
2717     assert(!(_p & ~0x3fL));
2718     assert(!(ar & ~0x7fL));
2719     assert(!(r2 & ~0x7fL));
2720     TSTREG1(r2);
2721     TSTPRED(_p);
2722     inst((0x2aL<<27)|(ar<<20)|(r2<<13)|_p, INST_I);
2723 }
2724
2725 static void
2726 _I27(jit_state_t *_jit, jit_word_t _p,
2727      jit_word_t ar,jit_word_t im)
2728 {
2729     assert(!(_p & ~0x3fL));
2730     assert(!(ar & ~0x7fL));
2731     assert(!(im & ~0xffL));
2732     TSTPRED(_p);
2733     inst((((im>>7)&1L)<<36)|(0xaL<<27)|(ar<<20)|((im&0x7fL)<<13)|_p, INST_I);
2734 }
2735
2736 static void
2737 _I28(jit_state_t *_jit, jit_word_t _p,
2738      jit_word_t ar, jit_word_t r1)
2739 {
2740     assert(!(_p & ~0x3fL));
2741     assert(!(ar & ~0x7fL));
2742     assert(!(r1 & ~0x7fL));
2743     TSTPRED(_p);
2744     TSTREG1(r1);
2745     inst((0x32L<<27)|(ar<<20)|(r1<<6)|_p, INST_I);
2746     SETREG(r1);
2747 }
2748
2749 static void
2750 _I29(jit_state_t *_jit, jit_word_t _p,
2751      jit_word_t x6, jit_word_t r3,jit_word_t r1)
2752 {
2753     assert(!(_p & ~0x3fL));
2754     assert(!(x6 & ~0x3fL));
2755     assert(!(r3 & ~0x7fL));
2756     assert(!(r1 & ~0x7fL));
2757     TSTREG1(r3);
2758     TSTPRED(_p);
2759     TSTREG1(r1);
2760     inst((x6<<27)|(r3<<20)|(r1<<6)|_p, INST_I);
2761     SETREG(r1);
2762 }
2763
2764 static void
2765 _I30(jit_state_t *_jit, jit_word_t _p,
2766      jit_word_t ta, jit_word_t tb, jit_word_t p2,
2767      jit_word_t im, jit_word_t c, jit_word_t p1)
2768 {
2769     assert(!(_p & ~0x3fL));
2770     assert(!(ta &  ~0x1L));
2771     assert(!(tb &  ~0x1L));
2772     assert(!(p2 & ~0x3fL));
2773     assert(!(im & ~0x1fL));
2774     assert(!(c  &  ~0x1L));
2775     assert(!(p1 & ~0x1fL));
2776     TSTPRED(_p);
2777     inst((5L<<37)|(tb<<36)|(ta<<33)|(1L<<19)|(im<<14)|
2778          (1L<<13)|(c<<12)|(p1<<6)|_p, INST_I);
2779     if (p1)
2780         _jitc->pred |= 1 << p1;
2781     if (p2)
2782         _jitc->pred |= 1 << p2;
2783 }
2784
2785 static void
2786 _M1(jit_state_t *_jit, jit_word_t _p,
2787     jit_word_t x6, jit_word_t ht, jit_word_t x, jit_word_t r3, jit_word_t r1)
2788 {
2789     assert(!(_p & ~0x3fL));
2790     assert(!(x6 & ~0x3fL));
2791     assert(!(ht &  ~0x3L));
2792     assert(!(x  &  ~0x1L));
2793     assert(!(r3 & ~0x7fL));
2794     assert(!(r1 & ~0x7fL));
2795     TSTREG1(r3);
2796     TSTPRED(_p);
2797     TSTREG1(r1);
2798     inst((4L<<37)|(x6<<30)|(ht<<28)|(x<<27)|(r3<<20)|(r1<<6)|_p, INST_M);
2799     SETREG(r1);
2800 }
2801
2802 static void
2803 _M2(jit_state_t *_jit, jit_word_t _p,
2804     jit_word_t x6, jit_word_t ht, jit_word_t r3, jit_word_t r2, jit_word_t r1)
2805 {
2806     assert(!(_p & ~0x3fL));
2807     assert(!(x6 & ~0x3fL));
2808     assert(!(ht &  ~0x3L));
2809     assert(!(r3 & ~0x7fL));
2810     assert(!(r2 & ~0x7fL));
2811     assert(!(r1 & ~0x7fL));
2812     TSTREG2(r2, r3);
2813     TSTPRED(_p);
2814     TSTREG1(r1);
2815     inst((4L<<37)|(1L<<36)|(x6<<30)|(ht<<28)|
2816          (r3<<20)|(r2<<13)|(r1<<6)|_p, INST_M);
2817     SETREG(r1);
2818     SETREG(r3);
2819 }
2820
2821 static void
2822 _M3(jit_state_t *_jit, jit_word_t _p,
2823     jit_word_t x6, jit_word_t ht, jit_word_t r3, jit_word_t im, jit_word_t r1)
2824 {
2825     jit_uint8_t         cc = INST_M;
2826     assert(!(_p &  ~0x3fL));
2827     assert(!(x6 &  ~0x3fL));
2828     assert(!(ht &   ~0x3L));
2829     assert(!(r3 &  ~0x7fL));
2830     assert(im > -256 && im <= 255);
2831     assert(!(r1 &  ~0x7fL));
2832     TSTREG1(r3);
2833     TSTPRED(_p);
2834     TSTREG1(r1);
2835     inst((5L<<37)|(((im>>8)&1L)<<36)|(x6<<30)|(ht<<28)|
2836          (((im>>7)&1L)<<27)|(r3<<20)|((im&0x7fL)<<13)|(r1<<6)|_p, cc);
2837     SETREG(r1);
2838     SETREG(r3);
2839 }
2840
2841 static void
2842 _M5(jit_state_t *_jit, jit_word_t _p,
2843     jit_word_t x6, jit_word_t ht, jit_word_t r3, jit_word_t r2, jit_word_t im)
2844 {
2845     assert(!(_p &  ~0x3fL));
2846     assert(!(x6 &  ~0x3fL));
2847     assert(!(ht &   ~0x3L));
2848     assert(!(r3 &  ~0x7fL));
2849     assert(!(r2 &  ~0x7fL));
2850     assert(im > -256 && im <= 255);
2851     TSTREG2(r2, r3);
2852     TSTPRED(_p);
2853     inst((5L<<37)|(((im>>8)&1L)<<36)|(x6<<30)|(ht<<28)|
2854          (((im>>7)&1L)<<27)|(r3<<20)|(r2<<13)|((im&0x7fL)<<6)|_p, INST_M);
2855     SETREG(r3);
2856 }
2857
2858 static void
2859 _M6(jit_state_t *_jit, jit_word_t _p,
2860     jit_word_t x6, jit_word_t ht, jit_word_t x, jit_word_t r3, jit_word_t r2)
2861 {
2862     assert(!(_p & ~0x3fL));
2863     assert(!(x6 & ~0x3fL));
2864     assert(!(ht &  ~0x3L));
2865     assert(!(x  &  ~0x1L));
2866     assert(!(r3 & ~0x7fL));
2867     assert(!(r2 & ~0x7fL));
2868     TSTREG2(r2, r3);
2869     TSTPRED(_p);
2870     inst((4L<<37)|(x6<<30)|(ht<<28)|(x<<27)|(r3<<20)|(r2<<13)|_p, INST_M);
2871 }
2872
2873 static void
2874 _M13(jit_state_t *_jit, jit_word_t _p,
2875      jit_word_t x6, jit_word_t ht, jit_word_t r3, jit_word_t r2)
2876 {
2877     assert(!(_p & ~0x3fL));
2878     assert(!(x6 & ~0x3fL));
2879     assert(!(ht &  ~0x3L));
2880     assert(!(r3 & ~0x7fL));
2881     assert(!(r2 & ~0x7fL));
2882     TSTREG1(r3);
2883     if (r2)
2884         TSTFREG1(r2);
2885     TSTPRED(_p);
2886     inst((6L<<37)|(x6<<30)|(ht<<28)|(r3<<20)|(r2<<13)|_p, INST_M);
2887 }
2888
2889 static void
2890 _M14(jit_state_t *_jit, jit_word_t _p,
2891      jit_word_t x6, jit_word_t ht, jit_word_t r3, jit_word_t r2)
2892 {
2893     assert(!(_p & ~0x3fL));
2894     assert(!(x6 & ~0x3fL));
2895     assert(!(ht &  ~0x3L));
2896     assert(!(r3 & ~0x7fL));
2897     assert(!(r2 & ~0x7fL));
2898     TSTREG2(r2, r3);
2899     TSTPRED(_p);
2900     inst((6L<<37)|(1L<<36)|(x6<<30)|(ht<<28)|(r3<<20)|(r2<<13)|_p, INST_M);
2901 }
2902
2903 static void
2904 _M15(jit_state_t *_jit, jit_word_t _p,
2905      jit_word_t x6, jit_word_t ht, jit_word_t r3, jit_word_t im)
2906 {
2907     assert(!(_p &  ~0x3fL));
2908     assert(!(x6 &  ~0x3fL));
2909     assert(!(ht &   ~0x3L));
2910     assert(!(r3 &  ~0x7fL));
2911     assert(!(im & ~0x1ffL));
2912     TSTREG1(r3);
2913     TSTPRED(_p);
2914     inst((7L<<37)|(((im>>8)&1L)<<36)|(x6<<30)|(ht<<28)|
2915          (((im>>7)&1L)<<27)|(r3<<20)|((im&0x7fL)<<13)|_p, INST_M);
2916 }
2917
2918 static void
2919 _M16(jit_state_t *_jit, jit_word_t _p,
2920      jit_word_t x6, jit_word_t ht, jit_word_t r3, jit_word_t r2, jit_word_t r1)
2921 {
2922     assert(!(_p & ~0x3fL));
2923     assert(!(x6 & ~0x3fL));
2924     assert(!(ht &  ~0x3L));
2925     assert(!(r3 & ~0x7fL));
2926     assert(!(r2 & ~0x7fL));
2927     assert(!(r1 & ~0x7fL));
2928     TSTREG2(r2, r3);
2929     TSTPRED(_p);
2930     TSTREG1(r1);
2931     inst((4L<<37)|(x6<<30)|(ht<<28)|(1L<<27)|
2932          (r3<<20)|(r2<<13)|(r1<<6)|_p, INST_M);
2933     SETREG(r1);
2934 }
2935
2936 static void
2937 _M17(jit_state_t *_jit, jit_word_t _p,
2938      jit_word_t x6, jit_word_t ht, jit_word_t r3, jit_word_t im, jit_word_t r1)
2939 {
2940     assert(!(_p & ~0x3fL));
2941     assert(!(x6 & ~0x3fL));
2942     assert(!(ht &  ~0x3L));
2943     assert(!(r3 & ~0x7fL));
2944     assert(!(im &  ~0x7L));
2945     assert(!(r1 & ~0x7fL));
2946     TSTREG1(r3);
2947     TSTPRED(_p);
2948     TSTREG1(r1);
2949     inst((4L<<37)|(x6<<30)|(ht<<28)|(1L<<27)|
2950          (r3<<20)|(im<<13)|(r1<<6)|_p, INST_M);
2951     SETREG(r1);
2952 }
2953
2954 static void
2955 _M20x(jit_state_t *_jit, jit_word_t _p,
2956       jit_word_t x3, jit_word_t r2, jit_word_t im)
2957 {
2958     assert(!(_p &     ~0x3fL));
2959     assert(!(x3 &      ~0x7L));
2960     assert(!(r2 &     ~0x7fL));
2961     assert(!(im & ~0x1fffffL));
2962     if (x3 == 1)
2963         TSTREG1(r2);
2964     else
2965         TSTFREG1(r2);
2966     TSTPRED(_p);
2967     inst((1L<<37)|(((im>>20)&1L)<<36)|(x3<<33)|
2968          (((im>>7)&0x1fffL)<<20)|(r2<<13)|((im&0x7fL)<<6)|_p, INST_M);
2969 }
2970
2971 static void
2972 _M22x(jit_state_t *_jit, jit_word_t _p,
2973       jit_word_t x3, jit_word_t im, jit_word_t r1)
2974 {
2975     assert(!(_p &     ~0x3fL));
2976     assert(!(x3 &      ~0x7L));
2977     assert(!(im & ~0x1fffffL));
2978     assert(!(r1 &     ~0x7fL));
2979     TSTPRED(_p);
2980     if (x3 < 6)
2981         TSTREG1(r1);
2982     else
2983         TSTFREG1(r1);
2984     inst((((im>>20)&1L)<<36)|(x3<<33)|((im&0xffffL)<<13)|(r1<<6)|_p, INST_M);
2985     if (x3 < 6)
2986         SETREG(r1);
2987     else
2988         SETFREG(r1);
2989 }
2990
2991 static void
2992 _M24(jit_state_t *_jit, jit_word_t _p,
2993      jit_word_t x2, jit_word_t x4)
2994 {
2995     assert(!(_p & ~0x3fL));
2996     assert(!(x2 &  ~0x3L));
2997     assert(!(x4 &  ~0xfL));
2998     TSTPRED(_p);
2999     inst((x2<<31)|(x4<<27)|_p, INST_M);
3000 }
3001
3002 static void
3003 _M26x(jit_state_t *_jit, jit_word_t _p,
3004       jit_word_t x4, jit_word_t r1)
3005 {
3006     assert(!(_p & ~0x3fL));
3007     assert(!(x4 &  ~0xfL));
3008     assert(!(r1 & ~0x7fL));
3009     if (x4 == 2)
3010         TSTREG1(r1);
3011     else
3012         TSTFREG1(r1);
3013     TSTPRED(_p);
3014     inst((1L<<31)|(x4<<27)|(r1<<6)|_p, INST_M);
3015 }
3016
3017 static void
3018 _M28(jit_state_t *_jit, jit_word_t _p,
3019      jit_word_t x, jit_word_t r3)
3020 {
3021     assert(!(_p & ~0x3fL));
3022     assert(!(x  &  ~0x1L));
3023     assert(!(r3 & ~0x7fL));
3024     TSTREG1(r3);
3025     TSTPRED(_p);
3026     inst((1L<<37)|(x<<36)|(0x30L<<27)|(r3<<20)|_p, INST_M);
3027 }
3028
3029 static void
3030 _M29(jit_state_t *_jit, jit_word_t _p,
3031      jit_word_t ar, jit_word_t r2)
3032 {
3033     assert(!(_p & ~0x3fL));
3034     assert(!(ar & ~0x7fL));
3035     assert(!(r2 & ~0x7fL));
3036     TSTREG1(r2);
3037     TSTPRED(_p);
3038     inst((1L<<37)|(0x2aL<<27)|(ar<<20)|(r2<<13)|_p, INST_M);
3039 }
3040
3041 static void
3042 _M30(jit_state_t *_jit, jit_word_t _p,
3043      jit_word_t ar, jit_word_t im)
3044 {
3045     assert(!(_p & ~0x3fL));
3046     assert(!(ar  & ~0x7L));
3047     assert(!(im & ~0xffL));
3048     TSTPRED(_p);
3049     inst((((im>>7)&1L)<<36)|(2L<<31)|(0x8L<<27)|
3050          (ar<<20)|((im&0x7fL)<<13)|_p, INST_M);
3051 }
3052
3053 static void
3054 _M31(jit_state_t *_jit, jit_word_t _p,
3055      jit_word_t ar, jit_word_t r1)
3056 {
3057     assert(!(_p & ~0x3fL));
3058     assert(!(ar  & ~0x7L));
3059     assert(!(r1 & ~0x7fL));
3060     TSTPRED(_p);
3061     TSTREG1(r1);
3062     inst((1L<<37)|(0x22L<<27)|(ar<<20)|(r1<<6)|_p, INST_M);
3063     SETREG(r1);
3064 }
3065
3066 static void
3067 _M32(jit_state_t *_jit, jit_word_t _p,
3068      jit_word_t cr, jit_word_t r2)
3069 {
3070     assert(!(_p & ~0x3fL));
3071     assert(!(cr  & ~0x7L));
3072     assert(!(r2 & ~0x7fL));
3073     TSTREG1(r2);
3074     TSTPRED(_p);
3075     inst((1L<<37)|(0x2cL<<27)|(cr<<20)|(r2<<13)|_p, INST_M);
3076 }
3077
3078 static void
3079 _M33(jit_state_t *_jit, jit_word_t _p,
3080      jit_word_t cr, jit_word_t r1)
3081 {
3082     assert(!(_p & ~0x3fL));
3083     assert(!(cr  & ~0x7L));
3084     assert(!(r1 & ~0x7fL));
3085     TSTPRED(_p);
3086     TSTREG1(r1);
3087     inst((1L<<37)|(0x24L<<27)|(cr<<20)|(r1<<6)|_p, INST_M);
3088     SETREG(r1);
3089 }
3090
3091 static void
3092 _M34(jit_state_t *_jit, jit_word_t _p,
3093      jit_word_t sor, jit_word_t sol, jit_word_t sof, jit_word_t r1)
3094 {
3095     assert(!(_p  & ~0x3fL));
3096     assert(!(sor &  ~0xfL));
3097     assert(!(sol & ~0x7fL));
3098     assert(!(sof & ~0x7fL));
3099     assert(!(r1  & ~0x7fL));
3100     TSTPRED(_p);
3101     inst((1L<<37)|(6L<<33)|(sor<<27)|(sol<<20)|(sof<<13)|(r1<<6)|_p, INST_M);
3102 }
3103
3104 static void
3105 _M35(jit_state_t *_jit, jit_word_t _p,
3106      jit_word_t x6, jit_word_t r2)
3107 {
3108     assert(!(_p & ~0x3fL));
3109     assert(!(x6 & ~0x3fL));
3110     assert(!(r2 & ~0x7fL));
3111     TSTREG1(r2);
3112     TSTPRED(_p);
3113     inst((1L<<37)|(x6<<27)|(r2<<13)|_p, INST_M);
3114 }
3115
3116 static void
3117 _M36(jit_state_t *_jit, jit_word_t _p,
3118      jit_word_t x6, jit_word_t r1)
3119 {
3120     assert(!(_p & ~0x3fL));
3121     assert(!(x6 & ~0x3fL));
3122     assert(!(r1 & ~0x7fL));
3123     TSTPRED(_p);
3124     TSTREG1(r1);
3125     inst((1L<<37)|(x6<<27)|(r1<<6)|_p, INST_M);
3126     SETREG(r1);
3127 }
3128
3129 static void
3130 _M37(jit_state_t *_jit, jit_word_t _p,
3131      jit_word_t im)
3132 {
3133     assert(!(_p &    ~0x3fL));
3134     assert(!(im & ~0x1ffffL));
3135     TSTPRED(_p);
3136     inst((((im>>20)&1L)<<36)|((im&0xffffL)<<6)|_p, INST_M);
3137 }
3138
3139 static void
3140 _M38(jit_state_t *_jit, jit_word_t _p,
3141      jit_word_t x6, jit_word_t r3, jit_word_t r2, jit_word_t r1)
3142 {
3143     assert(!(_p &  ~0x3fL));
3144     assert(!(x6 &  ~0x3fL));
3145     assert(!(r3 &  ~0x7fL));
3146     assert(!(r2 &  ~0x7fL));
3147     assert(!(r1 &  ~0x7fL));
3148     TSTREG2(r2, r3);
3149     TSTPRED(_p);
3150     TSTREG1(r1);
3151     inst((1L<<37)|(x6<<27)|(r3<<20)|(r2<<13)|(r1<<6)|_p, INST_M);
3152     SETREG(r1);
3153 }
3154
3155 static void
3156 _M39(jit_state_t *_jit, jit_word_t _p,
3157      jit_word_t x6, jit_word_t r3, jit_word_t im, jit_word_t r1)
3158 {
3159     assert(!(_p &  ~0x3fL));
3160     assert(!(x6 &  ~0x3fL));
3161     assert(!(r3 &  ~0x7fL));
3162     assert(!(im &   ~0x7L));
3163     assert(!(r1 &  ~0x7fL));
3164     TSTREG1(r3);
3165     TSTPRED(_p);
3166     TSTREG1(r1);
3167     inst((1L<<37)|(x6<<27)|(r3<<20)|(im<<13)|(r1<<6)|_p, INST_M);
3168     SETREG(r1);
3169 }
3170
3171 static void
3172 _M40(jit_state_t *_jit, jit_word_t _p,
3173      jit_word_t x6, jit_word_t r3, jit_word_t im)
3174 {
3175     assert(!(_p &  ~0x3fL));
3176     assert(!(x6 &  ~0x3fL));
3177     assert(!(r3 &  ~0x7fL));
3178     assert(!(im &   ~0x7L));
3179     TSTREG1(r3);
3180     TSTPRED(_p);
3181     inst((1L<<37)|(x6<<27)|(r3<<20)|(im<<13)|_p, INST_M);
3182 }
3183
3184 static void
3185 _M41(jit_state_t *_jit, jit_word_t _p,
3186      jit_word_t x6, jit_word_t r2)
3187 {
3188     assert(!(_p &  ~0x3fL));
3189     assert(!(x6 &  ~0x3fL));
3190     assert(!(r2 &  ~0x7fL));
3191     TSTREG1(r2);
3192     TSTPRED(_p);
3193     inst((1L<<37)|(x6<<27)|(r2<<13)|_p, INST_M);
3194 }
3195
3196 static void
3197 _M42(jit_state_t *_jit, jit_word_t _p,
3198      jit_word_t x6, jit_word_t r3, jit_word_t r2)
3199 {
3200     assert(!(_p &  ~0x3fL));
3201     assert(!(x6 &  ~0x3fL));
3202     assert(!(r3 &  ~0x7fL));
3203     assert(!(r2 &  ~0x7fL));
3204     TSTREG1(r2);
3205     TSTPRED(_p);
3206     inst((1L<<37)|(x6<<27)|(r3<<20)|(r2<<13)|_p, INST_M);
3207 }
3208
3209 static void
3210 _M43(jit_state_t *_jit, jit_word_t _p,
3211      jit_word_t x6, jit_word_t r3, jit_word_t r1)
3212 {
3213     assert(!(_p &  ~0x3fL));
3214     assert(!(x6 &  ~0x3fL));
3215     assert(!(r3 &  ~0x7fL));
3216     assert(!(r1 &  ~0x7fL));
3217     TSTPRED(_p);
3218     TSTREG1(r1);
3219     inst((1L<<37)|(x6<<27)|(r3<<20)|(r1<<6)|_p, INST_M);
3220     SETREG(r1);
3221 }
3222
3223 static void
3224 _M44(jit_state_t *_jit, jit_word_t _p,
3225      jit_word_t x4, jit_word_t im)
3226 {
3227     assert(!(_p &    ~0x3fL));
3228     assert(!(x4 &     ~0xfL));
3229     assert(!(im & ~0xfffffL));
3230     TSTPRED(_p);
3231     inst((((im>>23)&1L)<<36)|(((im>>21)&3L)<<31)|
3232          (x4<<27)|((im&0x1ffffL)<<6)|_p, INST_M);
3233 }
3234
3235 static void
3236 _M45(jit_state_t *_jit, jit_word_t _p,
3237      jit_word_t x6, jit_word_t r3, jit_word_t r2)
3238 {
3239     assert(!(_p &  ~0x3fL));
3240     assert(!(x6 &  ~0x3fL));
3241     assert(!(r3 &  ~0x7fL));
3242     assert(!(r2 &  ~0x7fL));
3243     TSTREG2(r2, r3);
3244     TSTPRED(_p);
3245     inst((1L<<37)|(x6<<27)|(r3<<20)|(r2<<13)|_p, INST_M);
3246 }
3247
3248 static void
3249 _M46(jit_state_t *_jit, jit_word_t _p,
3250      jit_word_t x6, jit_word_t r3, jit_word_t r1)
3251 {
3252     assert(!(_p & ~0x3fL));
3253     assert(!(x6 & ~0x3fL));
3254     assert(!(r3 & ~0x7fL));
3255     assert(!(r1 & ~0x7fL));
3256     TSTREG1(r3);
3257     TSTPRED(_p);
3258     if (r1)     TSTREG1(r1);
3259     inst((1L<<37)|(x6<<27)|(r3<<20)|(r1<<6)|_p, INST_M);
3260     if (r1)     SETREG(r1);
3261 }
3262
3263 static void
3264 _M48(jit_state_t *_jit, jit_word_t _p,
3265      jit_word_t y, jit_word_t im)
3266 {
3267     assert(!(_p &    ~0x3fL));
3268     assert(!(y  &     ~0x1L));
3269     assert(!(im & ~0x1ffffL));
3270     TSTPRED(_p);
3271     inst((((im>>20)&1L)<<36)|(1L<<27)|(y<<26)|((im&0xffffL)<<6)|_p, INST_M);
3272 }
3273
3274 static void
3275 _B1(jit_state_t *_jit, jit_word_t _p,
3276     jit_word_t d, jit_word_t wh, jit_word_t im, jit_word_t p, jit_word_t tp)
3277 {
3278     assert(!(_p &    ~0x3fL));
3279     assert(!(d  &     ~0x1L));
3280     assert(!(wh &     ~0x3L));
3281     assert(im >= -1048576 && im <= 1048575);
3282     assert(!(p  &     ~0x1L));
3283     assert(!(tp &     ~0x7L));
3284     TSTPRED(_p);
3285     inst((4L<<37)|(((im>>20)&1L)<<36)|(d<<35)|(wh<<33)|
3286          ((im&0xfffffL)<<13)|(p<<12)|(tp<<6)|_p, INST_B);
3287 }
3288
3289 static void
3290 _B3(jit_state_t *_jit, jit_word_t _p,
3291     jit_word_t d, jit_word_t wh, jit_word_t im, jit_word_t p, jit_word_t b)
3292 {
3293     assert(!(_p &    ~0x3fL));
3294     assert(!(d  &     ~0x1L));
3295     assert(!(wh &     ~0x3L));
3296     assert(im >= -1048576 && im <= 1048575);
3297     assert(!(p  &     ~0x1L));
3298     assert(!(b  &     ~0x3L));
3299     TSTPRED(_p);
3300     inst((5L<<37)|(((im>>20)&1L)<<36)|(d<<35)|(wh<<33)|
3301          ((im&0xfffffL)<<13)|(p<<12)|(b<<6)|_p, INST_B);
3302 }
3303
3304 static void
3305 _B4(jit_state_t *_jit, jit_word_t _p,
3306     jit_word_t d, jit_word_t wh, jit_word_t x6,
3307     jit_word_t b, jit_word_t p, jit_word_t tp)
3308 {
3309     assert(!(_p & ~0x3fL));
3310     assert(!(d  &  ~0x1L));
3311     assert(!(wh &  ~0x3L));
3312     assert(!(x6 & ~0x3fL));
3313     assert(!(b  &  ~0x7L));
3314     assert(!(p  &  ~0x1L));
3315     assert(!(tp &  ~0x7L));
3316     TSTPRED(_p);
3317     inst((d<<35)|(wh<<33)|(x6<<27)|(b<<13)|(p<<12)|(tp<<6)|_p, INST_B);
3318 }
3319
3320 static void
3321 _B5(jit_state_t *_jit, jit_word_t _p,
3322     jit_word_t d, jit_word_t wh, jit_word_t b2, jit_word_t p, jit_word_t b1)
3323 {
3324     assert(!(_p & ~0x3fL));
3325     assert(!(d  &  ~0x1L));
3326     assert(!(wh &  ~0x3L));
3327     assert(!(b2 &  ~0x7L));
3328     assert(!(p  &  ~0x1L));
3329     assert(!(b1 &  ~0x7L));
3330     TSTPRED(_p);
3331     inst((1L<<37)|(d<<35)|(wh<<32)|(b2<<13)|(p<<12)|(b1<<6)|_p, INST_B);
3332 }
3333
3334 static void
3335 _B6(jit_state_t *_jit, jit_word_t _p,
3336     jit_word_t ih, jit_word_t im, jit_word_t tag, jit_word_t wh)
3337 {
3338     assert(!(_p &     ~0x3fL));
3339     assert(!(ih  &     ~0x1L));
3340     assert(!(im  & ~0x1ffffL));
3341     assert(!(tag &   ~0x1ffL));
3342     assert(!(wh  &     ~0x3L));
3343     TSTPRED(_p);
3344     inst((7L<<37)|(((im>>20)&1L)<<36)|(ih<<35)|(((tag>>7)&3L)<<33)|
3345          ((im&0xfffffL)<<13)|((tag&0x7fL)<<6)|(wh<<3)|_p, INST_B);
3346 }
3347
3348 static void
3349 _B7(jit_state_t *_jit, jit_word_t _p,
3350     jit_word_t ih, jit_word_t x6, jit_word_t b2, jit_word_t tag, jit_word_t wh)
3351 {
3352     assert(!(_p &   ~0x3fL));
3353     assert(!(ih  &   ~0x1L));
3354     assert(!(x6  &  ~0x3fL));
3355     assert(!(b2  &   ~0x7L));
3356     assert(!(tag & ~0x1ffL));
3357     assert(!(wh  &   ~0x3L));
3358     TSTPRED(_p);
3359     inst((2L<<37)|(ih<<35)|(((tag>>7)&3L)<<33)|(x6<<27)|
3360          (b2<<13)|((tag&0x7fL)<<6)|(wh<<3)|_p, INST_B);
3361 }
3362
3363 static void
3364 _B8(jit_state_t *_jit, jit_word_t _p,
3365     jit_word_t x6)
3366 {
3367     assert(!(_p & ~0x3fL));
3368     assert(!(x6 & ~0x3fL));
3369     TSTPRED(_p);
3370     inst((x6<<27)|_p, INST_B);
3371 }
3372
3373 static void
3374 _B9(jit_state_t *_jit, jit_word_t _p,
3375     jit_word_t op, jit_word_t x6, jit_word_t im)
3376 {
3377     assert(!(_p &    ~0x3fL));
3378     assert(!(op &     ~0xfL));
3379     assert(!(x6 &    ~0x3fL));
3380     assert(!(im & ~0x1ffffL));
3381     TSTPRED(_p);
3382     inst((op<<37)|(((im>>20)&1L)<<36)|(x6<<27)|((im&0xffffL)<<6)|_p, INST_B);
3383 }
3384
3385 static void
3386 _X1(jit_state_t *_jit, jit_word_t _p,
3387     jit_word_t im)
3388 {
3389     jit_word_t          i41, i1, i20;
3390     assert(!(_p &               ~0x3fL));
3391     assert(im > -0x2000000000000000 && im <= 0x1fffffffffffffff);
3392     i41 = (im >> 22) & 0x1ffffffffffL;
3393     i1  = (im >> 21) &           0x1L;
3394     i20 =  im        &       0xfffffL;
3395     TSTPRED(_p);
3396     inst(i41, INST_L);
3397     inst((i1<<36)|(i20<<6)|_p, INST_X);
3398 }
3399
3400 static void
3401 _X2(jit_state_t *_jit, jit_word_t _p,
3402     jit_word_t r1, jit_word_t im)
3403 {
3404     jit_word_t          i1, i41, ic, i5, i9, i7;
3405     assert(!(_p & ~0x3fL));
3406     assert(!(r1 & ~0x7fL));
3407     i1  = (im >> 63) &           0x1L;
3408     i41 = (im >> 22) & 0x1ffffffffffL;
3409     ic  = (im >> 21) &           0x1L;
3410     i5  = (im >> 16) &          0x1fL;
3411     i9  = (im >>  7) &         0x1ffL;
3412     i7  =  im        &          0x7fL;
3413     TSTPRED(_p);
3414     TSTREG1(r1);
3415     inst(i41, INST_L);
3416     inst((6L<<37)|(i1<<36)|(i9<<27)|(i5<<22)|
3417          (ic<<21)|(i7<<13)|(r1<<6)|_p, INST_X);
3418     SETREG(r1);
3419 }
3420
3421 static void
3422 _X3x(jit_state_t *_jit, jit_word_t _p,
3423      jit_word_t op, jit_word_t d, jit_word_t wh,
3424      jit_word_t p, jit_word_t tp, jit_word_t im)
3425 {
3426     /* target64 = IP + ((i1 << 59 | imm39 << 20 | imm20b) << 4) */
3427     jit_word_t          i1, i41, i20;
3428     assert(!(_p &               ~0x3fL));
3429     assert(!(op &                ~0xfL));
3430     assert(!(d  &                ~0x1L));
3431     assert(!(wh &                ~0x3L));
3432     assert(!(p  &                ~0x1L));
3433     assert(!(tp &                ~0x7L));
3434     i1  = (im >> 61) &           0x1L;
3435     i41 = (im >> 22) & 0x1ffffffffffL;
3436     i20 =  im        &       0xfffffL;
3437     TSTPRED(_p);
3438     inst(i41, INST_L);
3439     inst((op<<37)|(i1<<36)|(d<<35)|(wh<<33)|
3440          (i20<<13)|(p<<12)|(tp<<6)|_p, INST_X);
3441 }
3442
3443 static void
3444 _X5(jit_state_t *_jit, jit_word_t _p,
3445     jit_word_t y, jit_word_t im)
3446 {
3447     jit_word_t          i41, i1, i20;
3448     assert(!(_p &               ~0x3fL));
3449     assert(im > -0x2000000000000000 && im <= 0x1fffffffffffffff);
3450     i41 = (im >> 22) & 0x1ffffffffffL;
3451     i1  = (im >> 21) &           0x1L;
3452     i20 =  im        &       0xfffffL;
3453     TSTPRED(_p);
3454     inst(i41, INST_L);
3455     inst((i1<<36)|(1L<<27)|(y<<26)|(i20<<6)|_p, INST_X);
3456 }
3457
3458 static void
3459 _nop(jit_state_t *_jit, jit_int32_t i0)
3460 {
3461     for (; i0 > 0; i0 -= 8) {
3462         NOP_M(0);
3463         sync();
3464     }
3465     assert(i0 == 0);
3466 }
3467
3468 static void
3469 _movr(jit_state_t *_jit, jit_int32_t r0, jit_int32_t r1)
3470 {
3471     /* patch pushargr */
3472     if (r0 >= 120)
3473         r0 = _jitc->rout + (r0 - 120);
3474
3475     MOV(r0, r1);
3476 }
3477
3478 static void
3479 _movi(jit_state_t *_jit, jit_int32_t r0, jit_word_t i0)
3480 {
3481     /* patch pushargi */
3482     if (r0 >= 120)
3483         r0 = _jitc->rout + (r0 - 120);
3484
3485     if (i0 >= -2097152 && i0 <= 2097151)
3486         MOVI(r0, i0);
3487     else
3488         MOVL(r0, i0);
3489 }
3490
3491 static jit_word_t
3492 _movi_p(jit_state_t *_jit, jit_int32_t r0, jit_word_t i0)
3493 {
3494     jit_word_t          w;
3495     sync();
3496     w = _jit->pc.w;
3497     MOVL(r0, i0);
3498     return (w);
3499 }
3500
3501 static void
3502 _movnr(jit_state_t *_jit, jit_int32_t r0, jit_int32_t r1, jit_int32_t r2)
3503 {
3504     CMP_EQ(PR_6, PR_7, r2, GR_0);
3505     MOV_p(r0, r1, PR_7);
3506 }
3507
3508 static void
3509 _movzr(jit_state_t *_jit, jit_int32_t r0, jit_int32_t r1, jit_int32_t r2)
3510 {
3511     CMP_EQ(PR_6, PR_7, r2, GR_0);
3512     MOV_p(r0, r1, PR_6);
3513 }
3514
3515 static void
3516 _casx(jit_state_t *_jit, jit_int32_t r0, jit_int32_t r1,
3517       jit_int32_t r2, jit_int32_t r3, jit_word_t i0)
3518 {
3519     jit_int32_t         r1_reg, iscasi;
3520     if ((iscasi = (r1 == _NOREG))) {
3521         r1_reg = jit_get_reg(jit_class_gpr);
3522         r1 = rn(r1_reg);
3523         movi(r1, i0);
3524     }
3525     sync();
3526     MOV_M_ar_rn(AR_CCV, r2);
3527     CMPXCHG8_ACQ(r0, r1, r3);
3528     eqr(r0, r0, r2);
3529     if (iscasi)
3530         jit_unget_reg(r1_reg);
3531 }
3532
3533
3534 static void
3535 _bswapr_us(jit_state_t *_jit, jit_int32_t r0, jit_int32_t r1)
3536 {
3537         bswapr_ul(r0, r1);
3538         rshi_u(r0, r0, 48);
3539 }
3540
3541 static void
3542 _bswapr_ui(jit_state_t *_jit, jit_int32_t r0, jit_int32_t r1)
3543 {
3544         bswapr_ul(r0, r1);
3545         rshi_u(r0, r0, 32);
3546 }
3547
3548 static void
3549 _addi(jit_state_t *_jit, jit_int32_t r0, jit_int32_t r1, jit_word_t i0)
3550 {
3551     jit_int32_t         reg;
3552     if (i0 >= -8192 && i0 <= 8191)
3553         ADDS(r0, i0, r1);
3554     else if (!(r1 & ~3) && i0 >= -2097152 && i0 <= 2097151)
3555         ADDL(r1, i0, r0);
3556     else {
3557         reg = jit_get_reg(jit_class_gpr);
3558         movi(rn(reg), i0);
3559         addr(r0, r1, rn(reg));
3560         jit_unget_reg(reg);
3561     }
3562 }
3563
3564 static void
3565 _addcr(jit_state_t *_jit, jit_int32_t r0, jit_int32_t r1, jit_int32_t r2)
3566 {
3567     jit_int32_t         reg;
3568     if (jit_carry == _NOREG)
3569         jit_carry = jit_get_reg(jit_class_gpr);
3570     if (r0 == r1) {
3571         reg = jit_get_reg(jit_class_gpr);
3572         addr(rn(reg), r1, r2);
3573         ltr_u(rn(jit_carry), rn(reg), r1);
3574         movr(r0, rn(reg));
3575         jit_unget_reg(reg);
3576     }
3577     else {
3578         addr(r0, r1, r2);
3579         ltr_u(rn(jit_carry), r0, r1);
3580     }
3581 }
3582
3583 static void
3584 _addci(jit_state_t *_jit, jit_int32_t r0, jit_int32_t r1, jit_word_t i0)
3585 {
3586     jit_int32_t         reg;
3587     if (jit_carry == _NOREG)
3588         jit_carry = jit_get_reg(jit_class_gpr);
3589     if (r0 == r1) {
3590         reg = jit_get_reg(jit_class_gpr);
3591         addi(rn(reg), r1, i0);
3592         ltr_u(rn(jit_carry), rn(reg), r1);
3593         movr(r0, rn(reg));
3594         jit_unget_reg(reg);
3595     }
3596     else {
3597         addi(r0, r1, i0);
3598         ltr_u(rn(jit_carry), r0, r1);
3599     }
3600 }
3601
3602 static void
3603 _addxr(jit_state_t *_jit, jit_int32_t r0, jit_int32_t r1, jit_int32_t r2)
3604 {
3605     jit_int32_t         reg;
3606     assert(jit_carry != _NOREG);
3607     reg = jit_get_reg(jit_class_gpr);
3608     movr(rn(reg), rn(jit_carry));
3609     addcr(r0, r1, r2);
3610     addcr(r0, r0, rn(reg));
3611     jit_unget_reg(reg);
3612 }
3613
3614 static void
3615 _addxi(jit_state_t *_jit, jit_int32_t r0, jit_int32_t r1, jit_word_t i0)
3616 {
3617     jit_int32_t         reg;
3618     assert(jit_carry != _NOREG);
3619     reg = jit_get_reg(jit_class_gpr);
3620     movr(rn(reg), rn(jit_carry));
3621     addci(r0, r1, i0);
3622     addcr(r0, r0, rn(reg));
3623     jit_unget_reg(reg);
3624 }
3625
3626 static void
3627 _subi(jit_state_t *_jit, jit_int32_t r0, jit_int32_t r1, jit_word_t i0)
3628 {
3629     jit_int32_t         reg;
3630     reg = jit_get_reg(jit_class_gpr);
3631     movi(rn(reg), i0);
3632     subr(r0, r1, rn(reg));
3633     jit_unget_reg(reg);
3634 }
3635
3636 static void
3637 _subcr(jit_state_t *_jit, jit_int32_t r0, jit_int32_t r1, jit_int32_t r2)
3638 {
3639     jit_int32_t         reg;
3640     if (jit_carry == _NOREG)
3641         jit_carry = jit_get_reg(jit_class_gpr);
3642     if (r0 == r1) {
3643         reg = jit_get_reg(jit_class_gpr);
3644         subr(rn(reg), r1, r2);
3645         ltr_u(rn(jit_carry), r1, rn(reg));
3646         movr(r0, rn(reg));
3647         jit_unget_reg(reg);
3648     }
3649     else {
3650         subr(r0, r1, r2);
3651         ltr_u(rn(jit_carry), r1, r0);
3652     }
3653 }
3654
3655 static void
3656 _subci(jit_state_t *_jit, jit_int32_t r0, jit_int32_t r1, jit_word_t i0)
3657 {
3658     jit_int32_t         reg;
3659     if (jit_carry == _NOREG)
3660         jit_carry = jit_get_reg(jit_class_gpr);
3661     if (r0 == r1) {
3662         reg = jit_get_reg(jit_class_gpr);
3663         addi(rn(reg), r1, -i0);
3664         ltr_u(rn(jit_carry), r1, rn(reg));
3665         movr(r0, rn(reg));
3666         jit_unget_reg(reg);
3667     }
3668     else {
3669         addi(r0, r1, -i0);
3670         ltr_u(rn(jit_carry), r1, r0);
3671     }
3672 }
3673
3674 static void
3675 _subxr(jit_state_t *_jit, jit_int32_t r0, jit_int32_t r1, jit_int32_t r2)
3676 {
3677     jit_int32_t         reg;
3678     assert(jit_carry != _NOREG);
3679     reg = jit_get_reg(jit_class_gpr);
3680     movr(rn(reg), rn(jit_carry));
3681     subcr(r0, r1, r2);
3682     subcr(r0, r0, rn(reg));
3683     jit_unget_reg(reg);
3684 }
3685
3686 static void
3687 _subxi(jit_state_t *_jit, jit_int32_t r0, jit_int32_t r1, jit_word_t i0)
3688 {
3689     jit_int32_t         reg;
3690     assert(jit_carry != _NOREG);
3691     reg = jit_get_reg(jit_class_gpr);
3692     movr(rn(reg), rn(jit_carry));
3693     subci(r0, r1, i0);
3694     subcr(r0, r0, rn(reg));
3695     jit_unget_reg(reg);
3696 }
3697
3698 static void
3699 _rsbi(jit_state_t *_jit, jit_int32_t r0, jit_int32_t r1, jit_word_t i0)
3700 {
3701     jit_int32_t         reg;
3702     if (i0 >= -128 && i0 <= 127)
3703         SUBI(r0, i0, r1);
3704     else if (!(r1 & ~3) && i0 >= -2097151 && i0 <= 2097152)
3705         ADDL(r1, -i0, r0);
3706     else {
3707         reg = jit_get_reg(jit_class_gpr);
3708         movi(rn(reg), i0);
3709         subr(r0, rn(reg), r1);
3710         jit_unget_reg(reg);
3711     }
3712 }
3713
3714 static void
3715 _mulr(jit_state_t *_jit, jit_int32_t r0, jit_int32_t r1, jit_int32_t r2)
3716 {
3717     jit_int32_t         f0, f1;
3718     f0 = jit_get_reg(jit_class_fpr);
3719     f1 = jit_get_reg(jit_class_fpr);
3720     SETF_SIG(rn(f0), r1);
3721     SETF_SIG(rn(f1), r2);
3722     XMPY_L(rn(f0), rn(f0), rn(f1));
3723     GETF_SIG(r0, rn(f0));
3724     jit_unget_reg(f0);
3725     jit_unget_reg(f1);
3726 }
3727
3728 static void
3729 _muli(jit_state_t *_jit, jit_int32_t r0, jit_int32_t r1, jit_word_t i0)
3730 {
3731     jit_int32_t         reg;
3732     reg = jit_get_reg(jit_class_gpr);
3733     movi(rn(reg), i0);
3734     mulr(r0, r1, rn(reg));
3735     jit_unget_reg(reg);
3736 }
3737
3738 #if !defined(__GNUC__)
3739 static long
3740 __divdi3(long u, long v)
3741 {
3742     return (u / v);
3743 }
3744
3745 static unsigned long
3746 __udivdi3(unsigned long u, unsigned long v)
3747 {
3748     return (u / v);
3749 }
3750
3751 static long
3752 __moddi3(long u, long v)
3753 {
3754     return (u % v);
3755 }
3756
3757 static unsigned long
3758 __umoddi3(unsigned long u, unsigned long v)
3759 {
3760     return (u % v);
3761 }
3762 #endif
3763
3764 static void
3765 _divr(jit_state_t *_jit, jit_int32_t r0, jit_int32_t r1, jit_int32_t r2)
3766 {
3767     MOV(_jitc->rout, r1);
3768     MOV(_jitc->rout + 1, r2);
3769     calli((jit_word_t)__divdi3);
3770     MOV(r0, GR_8);
3771 }
3772
3773 static void
3774 _divi(jit_state_t *_jit, jit_int32_t r0, jit_int32_t r1, jit_word_t i0)
3775 {
3776     jit_int32_t         reg;
3777     switch (i0) {
3778         case 1:
3779             movr(r0, r1);
3780             return;
3781         case -1:
3782             negr(r0, r1);
3783             return;
3784         default:
3785             if (i0 > 0 && !(i0 & (i0 - 1))) {
3786                 movr(r0, r1);
3787                 rshi(r0, r0, ffsl(i0) - 1);
3788                 return;
3789             }
3790             break;
3791     }
3792     reg = jit_get_reg(jit_class_gpr);
3793     movi(rn(reg), i0);
3794     divr(r0, r1, rn(reg));
3795     jit_unget_reg(reg);
3796 }
3797
3798 static void
3799 _divr_u(jit_state_t *_jit, jit_int32_t r0, jit_int32_t r1, jit_int32_t r2)
3800 {
3801     MOV(_jitc->rout, r1);
3802     MOV(_jitc->rout + 1, r2);
3803     calli((jit_word_t)__udivdi3);
3804     MOV(r0, GR_8);
3805 }
3806
3807 static void
3808 _divi_u(jit_state_t *_jit, jit_int32_t r0, jit_int32_t r1, jit_word_t i0)
3809 {
3810     jit_int32_t         reg;
3811     if (i0 == 1) {
3812         movr(r0, r1);
3813         return;
3814     }
3815     else if (i0 > 0 && !(i0 & (i0 - 1))) {
3816         movr(r0, r1);
3817         rshi_u(r0, r0, ffsl(i0) - 1);
3818         return;
3819     }
3820     reg = jit_get_reg(jit_class_gpr);
3821     movi(rn(reg), i0);
3822     divr_u(r0, r1, rn(reg));
3823     jit_unget_reg(reg);
3824 }
3825
3826 static void
3827 _remr(jit_state_t *_jit, jit_int32_t r0, jit_int32_t r1, jit_int32_t r2)
3828 {
3829     MOV(_jitc->rout, r1);
3830     MOV(_jitc->rout + 1, r2);
3831     calli((jit_word_t)__moddi3);
3832     MOV(r0, GR_8);
3833 }
3834
3835 static void
3836 _remi(jit_state_t *_jit, jit_int32_t r0, jit_int32_t r1, jit_word_t i0)
3837 {
3838     jit_int32_t         reg;
3839     if (i0 == 1 || i0 == -1) {
3840         MOV(r0, GR_0);
3841         return;
3842     }
3843     reg = jit_get_reg(jit_class_gpr);
3844     movi(rn(reg), i0);
3845     remr(r0, r1, rn(reg));
3846     jit_unget_reg(reg);
3847 }
3848
3849 static void
3850 _remr_u(jit_state_t *_jit, jit_int32_t r0, jit_int32_t r1, jit_int32_t r2)
3851 {
3852     MOV(_jitc->rout, r1);
3853     MOV(_jitc->rout + 1, r2);
3854     calli((jit_word_t)__umoddi3);
3855     MOV(r0, GR_8);
3856 }
3857
3858 static void
3859 _remi_u(jit_state_t *_jit, jit_int32_t r0, jit_int32_t r1, jit_word_t i0)
3860 {
3861     jit_int32_t         reg;
3862     if (i0 == 1) {
3863         MOV(r0, GR_0);
3864         return;
3865     }
3866     else if (i0 > 0 && !(i0 & (i0 - 1))) {
3867         andi(r0, r1, i0 - 1);
3868         return;
3869     }
3870     reg = jit_get_reg(jit_class_gpr);
3871     movi(rn(reg), i0);
3872     remr_u(r0, r1, rn(reg));
3873     jit_unget_reg(reg);
3874 }
3875
3876 static void
3877 _mulh(jit_state_t *_jit,
3878       jit_int32_t r0, jit_int32_t r1, jit_int32_t r2, jit_bool_t sign)
3879 {
3880     jit_int32_t         f0, f1;
3881     f0 = jit_get_reg(jit_class_fpr);
3882     f1 = jit_get_reg(jit_class_fpr);
3883     SETF_SIG(rn(f0), r1);
3884     SETF_SIG(rn(f1), r2);
3885     if (sign)
3886         XMPY_H(rn(f0), rn(f0), rn(f1));
3887     else
3888         XMPY_HU(rn(f0), rn(f0), rn(f1));
3889     GETF_SIG(r0, rn(f0));
3890     jit_unget_reg(f0);
3891     jit_unget_reg(f1);
3892 }
3893
3894 static void
3895 _iqmulr(jit_state_t *_jit, jit_int32_t r0, jit_int32_t r1,
3896         jit_int32_t r2, jit_int32_t r3, jit_bool_t sign)
3897 {
3898     jit_int32_t         reg;
3899     if (r0 == r2 || r0 == r3) {
3900         reg = jit_get_reg(jit_class_gpr);
3901         mulr(rn(reg), r2, r3);
3902     }
3903     else
3904         mulr(r0, r2, r3);
3905     mulh(r1, r2, r3, sign);
3906     if (r0 == r2 || r0 == r3) {
3907         movr(r0, rn(reg));
3908         jit_unget_reg(reg);
3909     }
3910 }
3911
3912 static void
3913 _iqmuli(jit_state_t *_jit, jit_int32_t r0, jit_int32_t r1,
3914         jit_int32_t r2, jit_word_t i0, jit_bool_t sign)
3915 {
3916     jit_int32_t         reg;
3917     reg = jit_get_reg(jit_class_gpr);
3918     movi(rn(reg), i0);
3919     iqmulr(r0, r1, r2, rn(reg), sign);
3920     jit_unget_reg(reg);
3921 }
3922
3923 static void
3924 _iqdivr(jit_state_t *_jit, jit_int32_t r0, jit_int32_t r1,
3925         jit_int32_t r2, jit_int32_t r3, jit_bool_t sign)
3926 {
3927     jit_int32_t         sv0, rg0;
3928     jit_int32_t         sv1, rg1;
3929
3930     if (r0 == r2 || r0 == r3) {
3931         sv0 = jit_get_reg(jit_class_gpr);
3932         rg0 = rn(sv0);
3933     }
3934     else
3935         rg0 = r0;
3936     if (r1 == r2 || r1 == r3) {
3937         sv1 = jit_get_reg(jit_class_gpr);
3938         rg1 = rn(sv1);
3939     }
3940     else
3941         rg1 = r1;
3942
3943     if (sign)
3944         divr(rg0, r2, r3);
3945     else
3946         divr_u(rg0, r2, r3);
3947     mulr(rg1, r3, rg0);
3948     subr(rg1, r2, rg1);
3949     if (rg0 != r0) {
3950         movr(r0, rg0);
3951         jit_unget_reg(sv0);
3952     }
3953     if (rg1 != r1) {
3954         movr(r1, rg1);
3955         jit_unget_reg(sv1);
3956     }
3957 }
3958
3959 static void
3960 _iqdivi(jit_state_t *_jit, jit_int32_t r0, jit_int32_t r1,
3961         jit_int32_t r2, jit_word_t i0, jit_bool_t sign)
3962 {
3963     jit_int32_t         reg;
3964     reg = jit_get_reg(jit_class_gpr);
3965     movi(rn(reg), i0);
3966     iqdivr(r0, r1, r2, rn(reg), sign);
3967     jit_unget_reg(reg);
3968 }
3969
3970 static void
3971 _andi(jit_state_t *_jit, jit_int32_t r0, jit_int32_t r1, jit_word_t i0)
3972 {
3973     jit_int32_t         reg;
3974     if (i0 >= -128 && i0 <= 127)
3975         ANDI(r0, i0, r1);
3976     else if (~i0 >= -128 && ~i0 <= 127)
3977         ANDCMI(r0, ~i0, r1);
3978     else {
3979         reg = jit_get_reg(jit_class_gpr);
3980         movi(rn(reg), i0);
3981         andr(r0, r1, rn(reg));
3982         jit_unget_reg(reg);
3983     }
3984 }
3985
3986 static void
3987 _ori(jit_state_t *_jit, jit_int32_t r0, jit_int32_t r1, jit_word_t i0)
3988 {
3989     jit_int32_t         reg;
3990     if (i0 >= -128 && i0 <= 127)
3991         ORI(r0, i0, r1);
3992     else {
3993         reg = jit_get_reg(jit_class_gpr);
3994         movi(rn(reg), i0);
3995         orr(r0, r1, rn(reg));
3996         jit_unget_reg(reg);
3997     }
3998 }
3999
4000 static void
4001 _xori(jit_state_t *_jit, jit_int32_t r0, jit_int32_t r1, jit_word_t i0)
4002 {
4003     jit_int32_t         reg;
4004     if (i0 >= -128 && i0 <= 127)
4005         XORI(r0, i0, r1);
4006     else {
4007         reg = jit_get_reg(jit_class_gpr);
4008         movi(rn(reg), i0);
4009         xorr(r0, r1, rn(reg));
4010         jit_unget_reg(reg);
4011     }
4012 }
4013
4014 static void
4015 _lshi(jit_state_t *_jit, jit_int32_t r0, jit_int32_t r1, jit_word_t i0)
4016 {
4017     jit_int32_t         reg;
4018     reg = jit_get_reg(jit_class_gpr);
4019     movi(rn(reg), i0);
4020     lshr(r0, r1, rn(reg));
4021     jit_unget_reg(reg);
4022 }
4023
4024 static void
4025 _rshi(jit_state_t *_jit, jit_int32_t r0, jit_int32_t r1, jit_word_t i0)
4026 {
4027     jit_int32_t         reg;
4028     reg = jit_get_reg(jit_class_gpr);
4029     movi(rn(reg), i0);
4030     rshr(r0, r1, rn(reg));
4031     jit_unget_reg(reg);
4032 }
4033
4034 static void
4035 _rshi_u(jit_state_t *_jit, jit_int32_t r0, jit_int32_t r1, jit_word_t i0)
4036 {
4037     jit_int32_t         reg;
4038     reg = jit_get_reg(jit_class_gpr);
4039     movi(rn(reg), i0);
4040     rshr_u(r0, r1, rn(reg));
4041     jit_unget_reg(reg);
4042 }
4043
4044 static void
4045 _ltr(jit_state_t *_jit, jit_int32_t r0, jit_int32_t r1, jit_int32_t r2)
4046 {
4047     CMP_LT(PR_6, PR_7, r1, r2);
4048     MOVI_p(r0, 1, PR_6);
4049     MOV_p(r0, GR_0, PR_7);
4050 }
4051
4052 static void
4053 _lti(jit_state_t *_jit, jit_int32_t r0, jit_int32_t r1, jit_word_t i0)
4054 {
4055     jit_int32_t         reg;
4056     if (i0 >= -127 && i0 <= 128)
4057         CMPI_LT(PR_7, PR_6, i0 - 1, r1);
4058     else {
4059         reg = jit_get_reg(jit_class_gpr);
4060         movi(rn(reg), i0);
4061         CMP_LT(PR_6, PR_7, r1, rn(reg));
4062         jit_unget_reg(reg);
4063     }
4064     MOVI_p(r0, 1, PR_6);
4065     MOV_p(r0, GR_0, PR_7);
4066 }
4067
4068 static void
4069 _ltr_u(jit_state_t *_jit, jit_int32_t r0, jit_int32_t r1, jit_int32_t r2)
4070 {
4071     CMP_LTU(PR_6, PR_7, r1, r2);
4072     MOVI_p(r0, 1, PR_6);
4073     MOV_p(r0, GR_0, PR_7);
4074 }
4075
4076 static void
4077 _lti_u(jit_state_t *_jit, jit_int32_t r0, jit_int32_t r1, jit_word_t i0)
4078 {
4079     jit_int32_t         reg;
4080     if (i0 >= -127 && i0 <= 128)
4081         CMPI_LTU(PR_7, PR_6, i0 - 1, r1);
4082     else {
4083         reg = jit_get_reg(jit_class_gpr);
4084         movi(rn(reg), i0);
4085         CMP_LTU(PR_6, PR_7, r1, rn(reg));
4086         jit_unget_reg(reg);
4087     }
4088     MOVI_p(r0, 1, PR_6);
4089     MOV_p(r0, GR_0, PR_7);
4090 }
4091
4092 static void
4093 _ler(jit_state_t *_jit, jit_int32_t r0, jit_int32_t r1, jit_int32_t r2)
4094 {
4095     CMP_LT(PR_6, PR_7, r2, r1);
4096     MOV_p(r0, GR_0, PR_6);
4097     MOVI_p(r0, 1, PR_7);
4098 }
4099
4100 static void
4101 _lei(jit_state_t *_jit, jit_int32_t r0, jit_int32_t r1, jit_word_t i0)
4102 {
4103     jit_int32_t         reg;
4104     reg = jit_get_reg(jit_class_gpr);
4105     movi(rn(reg), i0);
4106     ler(r0, r1, rn(reg));
4107     jit_unget_reg(reg);
4108 }
4109
4110 static void
4111 _ler_u(jit_state_t *_jit, jit_int32_t r0, jit_int32_t r1, jit_int32_t r2)
4112 {
4113     CMP_LTU(PR_6, PR_7, r2, r1);
4114     MOV_p(r0, GR_0, PR_6);
4115     MOVI_p(r0, 1, PR_7);
4116 }
4117
4118 static void
4119 _lei_u(jit_state_t *_jit, jit_int32_t r0, jit_int32_t r1, jit_word_t i0)
4120 {
4121     jit_int32_t         reg;
4122     reg = jit_get_reg(jit_class_gpr);
4123     movi(rn(reg), i0);
4124     ler_u(r0, r1, rn(reg));
4125     jit_unget_reg(reg);
4126 }
4127
4128 static void
4129 _eqr(jit_state_t *_jit, jit_int32_t r0, jit_int32_t r1, jit_int32_t r2)
4130 {
4131     CMP_EQ(PR_6, PR_7, r1, r2);
4132     MOVI_p(r0, 1, PR_6);
4133     MOV_p(r0, GR_0, PR_7);
4134 }
4135
4136 static void
4137 _eqi(jit_state_t *_jit, jit_int32_t r0, jit_int32_t r1, jit_word_t i0)
4138 {
4139     jit_int32_t         reg;
4140     if (i0 >= -128 && i0 <= 127)
4141         CMPI_EQ(PR_6, PR_7, i0, r1);
4142     else {
4143         reg = jit_get_reg(jit_class_gpr);
4144         movi(rn(reg), i0);
4145         CMP_EQ(PR_6, PR_7, r1, rn(reg));
4146         jit_unget_reg(reg);
4147     }
4148     MOVI_p(r0, 1, PR_6);
4149     MOV_p(r0, GR_0, PR_7);
4150 }
4151
4152 static void
4153 _ger(jit_state_t *_jit, jit_int32_t r0, jit_int32_t r1, jit_int32_t r2)
4154 {
4155     CMP_LT(PR_6, PR_7, r1, r2);
4156     MOV_p(r0, GR_0, PR_6);
4157     MOVI_p(r0, 1, PR_7);
4158 }
4159
4160 static void
4161 _gei(jit_state_t *_jit, jit_int32_t r0, jit_int32_t r1, jit_word_t i0)
4162 {
4163     jit_int32_t         reg;
4164     if (i0 >= -127 && i0 <= 128)
4165         CMPI_LT(PR_7, PR_6, i0 - 1, r1);
4166     else {
4167         reg = jit_get_reg(jit_class_gpr);
4168         movi(rn(reg), i0);
4169         CMP_LT(PR_6, PR_7, r1, rn(reg));
4170         jit_unget_reg(reg);
4171     }
4172     MOV_p(r0, GR_0, PR_6);
4173     MOVI_p(r0, 1, PR_7);
4174 }
4175
4176 static void
4177 _ger_u(jit_state_t *_jit, jit_int32_t r0, jit_int32_t r1, jit_int32_t r2)
4178 {
4179     CMP_LTU(PR_6, PR_7, r1, r2);
4180     MOV_p(r0, GR_0, PR_6);
4181     MOVI_p(r0, 1, PR_7);
4182 }
4183
4184 static void
4185 _gei_u(jit_state_t *_jit, jit_int32_t r0, jit_int32_t r1, jit_word_t i0)
4186 {
4187     jit_int32_t         reg;
4188     if (i0 >= -127 && i0 <= 128)
4189         CMPI_LTU(PR_7, PR_6, i0 - 1, r1);
4190     else {
4191         reg = jit_get_reg(jit_class_gpr);
4192         movi(rn(reg), i0);
4193         CMP_LTU(PR_6, PR_7, r1, rn(reg));
4194         jit_unget_reg(reg);
4195     }
4196     MOV_p(r0, GR_0, PR_6);
4197     MOVI_p(r0, 1, PR_7);
4198 }
4199
4200 static void
4201 _gtr(jit_state_t *_jit, jit_int32_t r0, jit_int32_t r1, jit_int32_t r2)
4202 {
4203     CMP_LT(PR_6, PR_7, r2, r1);
4204     MOVI_p(r0, 1, PR_6);
4205     MOV_p(r0, GR_0, PR_7);
4206 }
4207
4208 static void
4209 _gti(jit_state_t *_jit, jit_int32_t r0, jit_int32_t r1, jit_word_t i0)
4210 {
4211     jit_int32_t         reg;
4212     reg = jit_get_reg(jit_class_gpr);
4213     movi(rn(reg), i0);
4214     gtr(r0, r1, rn(reg));
4215     jit_unget_reg(reg);
4216 }
4217
4218 static void
4219 _gtr_u(jit_state_t *_jit, jit_int32_t r0, jit_int32_t r1, jit_int32_t r2)
4220 {
4221     CMP_LTU(PR_6, PR_7, r2, r1);
4222     MOVI_p(r0, 1, PR_6);
4223     MOV_p(r0, GR_0, PR_7);
4224 }
4225
4226 static void
4227 _gti_u(jit_state_t *_jit, jit_int32_t r0, jit_int32_t r1, jit_word_t i0)
4228 {
4229     jit_int32_t         reg;
4230     reg = jit_get_reg(jit_class_gpr);
4231     movi(rn(reg), i0);
4232     gtr_u(r0, r1, rn(reg));
4233     jit_unget_reg(reg);
4234 }
4235
4236 static void
4237 _ner(jit_state_t *_jit, jit_int32_t r0, jit_int32_t r1, jit_int32_t r2)
4238 {
4239     CMP_EQ(PR_6, PR_7, r1, r2);
4240     MOV_p(r0, GR_0, PR_6);
4241     MOVI_p(r0, 1, PR_7);
4242 }
4243
4244 static void
4245 _nei(jit_state_t *_jit, jit_int32_t r0, jit_int32_t r1, jit_word_t i0)
4246 {
4247     jit_int32_t         reg;
4248     if (i0 >= -128 && i0 <= 127)
4249         CMPI_EQ(PR_6, PR_7, i0, r1);
4250     else {
4251         reg = jit_get_reg(jit_class_gpr);
4252         movi(rn(reg), i0);
4253         CMP_EQ(PR_6, PR_7, r1, rn(reg));
4254         jit_unget_reg(reg);
4255     }
4256     MOV_p(r0, GR_0, PR_6);
4257     MOVI_p(r0, 1, PR_7);
4258 }
4259
4260 static void
4261 _ldr_c(jit_state_t *_jit, jit_int32_t r0, jit_int32_t r1)
4262 {
4263     ldr_uc(r0, r1);
4264     extr_c(r0, r0);
4265 }
4266
4267 static void
4268 _ldi_c(jit_state_t *_jit, jit_int32_t r0, jit_word_t i0)
4269 {
4270     jit_int32_t         reg;
4271     reg = jit_get_reg(jit_class_gpr);
4272     movi(rn(reg), i0);
4273     ldr_c(r0, rn(reg));
4274     jit_unget_reg(reg);
4275 }
4276
4277 static void
4278 _ldi_uc(jit_state_t *_jit, jit_int32_t r0, jit_word_t i0)
4279 {
4280     jit_int32_t         reg;
4281     reg = jit_get_reg(jit_class_gpr);
4282     movi(rn(reg), i0);
4283     ldr_uc(r0, rn(reg));
4284     jit_unget_reg(reg);
4285 }
4286
4287 static void
4288 _ldr_s(jit_state_t *_jit, jit_int32_t r0, jit_int32_t r1)
4289 {
4290     ldr_us(r0, r1);
4291     extr_s(r0, r0);
4292 }
4293
4294 static void
4295 _ldi_s(jit_state_t *_jit, jit_int32_t r0, jit_word_t i0)
4296 {
4297     jit_int32_t         reg;
4298     reg = jit_get_reg(jit_class_gpr);
4299     movi(rn(reg), i0);
4300     ldr_s(r0, rn(reg));
4301     jit_unget_reg(reg);
4302 }
4303
4304 static void
4305 _ldi_us(jit_state_t *_jit, jit_int32_t r0, jit_word_t i0)
4306 {
4307     jit_int32_t         reg;
4308     reg = jit_get_reg(jit_class_gpr);
4309     movi(rn(reg), i0);
4310     ldr_us(r0, rn(reg));
4311     jit_unget_reg(reg);
4312 }
4313
4314 static void
4315 _ldr_i(jit_state_t *_jit, jit_int32_t r0, jit_int32_t r1)
4316 {
4317     ldr_ui(r0, r1);
4318     extr_i(r0, r0);
4319 }
4320
4321 static void
4322 _ldi_i(jit_state_t *_jit, jit_int32_t r0, jit_word_t i0)
4323 {
4324     jit_int32_t         reg;
4325     reg = jit_get_reg(jit_class_gpr);
4326     movi(rn(reg), i0);
4327     ldr_i(r0, rn(reg));
4328     jit_unget_reg(reg);
4329 }
4330
4331 static void
4332 _ldi_ui(jit_state_t *_jit, jit_int32_t r0, jit_word_t i0)
4333 {
4334     jit_int32_t         reg;
4335     reg = jit_get_reg(jit_class_gpr);
4336     movi(rn(reg), i0);
4337     ldr_ui(r0, rn(reg));
4338     jit_unget_reg(reg);
4339 }
4340
4341 static void
4342 _ldi_l(jit_state_t *_jit, jit_int32_t r0, jit_word_t i0)
4343 {
4344     jit_int32_t         reg;
4345     reg = jit_get_reg(jit_class_gpr);
4346     movi(rn(reg), i0);
4347     ldr_l(r0, rn(reg));
4348     jit_unget_reg(reg);
4349 }
4350
4351 static void
4352 _ldxr_c(jit_state_t *_jit, jit_int32_t r0, jit_int32_t r1, jit_int32_t r2)
4353 {
4354     jit_int32_t         reg;
4355     reg = jit_get_reg(jit_class_gpr);
4356     addr(rn(reg), r1, r2);
4357     ldr_c(r0, rn(reg));
4358     jit_unget_reg(reg);
4359 }
4360
4361 static void
4362 _ldxi_c(jit_state_t *_jit, jit_int32_t r0, jit_int32_t r1, jit_word_t i0)
4363 {
4364     jit_int32_t         reg;
4365     if (i0) {
4366         reg = jit_get_reg(jit_class_gpr);
4367         movi(rn(reg), i0);
4368         ldxr_c(r0, r1, rn(reg));
4369         jit_unget_reg(reg);
4370     }
4371     else
4372         ldr_c(r0, r1);
4373 }
4374
4375 static void
4376 _ldxr_uc(jit_state_t *_jit, jit_int32_t r0, jit_int32_t r1, jit_int32_t r2)
4377 {
4378     jit_int32_t         reg;
4379     reg = jit_get_reg(jit_class_gpr);
4380     addr(rn(reg), r1, r2);
4381     ldr_uc(r0, rn(reg));
4382     jit_unget_reg(reg);
4383 }
4384
4385 static void
4386 _ldxi_uc(jit_state_t *_jit, jit_int32_t r0, jit_int32_t r1, jit_word_t i0)
4387 {
4388     jit_int32_t         reg;
4389     if (i0) {
4390         reg = jit_get_reg(jit_class_gpr);
4391         movi(rn(reg), i0);
4392         ldxr_uc(r0, r1, rn(reg));
4393         jit_unget_reg(reg);
4394     }
4395     else
4396         ldr_uc(r0, r1);
4397 }
4398
4399 static void
4400 _ldxr_s(jit_state_t *_jit, jit_int32_t r0, jit_int32_t r1, jit_int32_t r2)
4401 {
4402     jit_int32_t         reg;
4403     reg = jit_get_reg(jit_class_gpr);
4404     addr(rn(reg), r1, r2);
4405     ldr_s(r0, rn(reg));
4406     jit_unget_reg(reg);
4407 }
4408
4409 static void
4410 _ldxi_s(jit_state_t *_jit, jit_int32_t r0, jit_int32_t r1, jit_word_t i0)
4411 {
4412     jit_int32_t         reg;
4413     if (i0) {
4414         reg = jit_get_reg(jit_class_gpr);
4415         movi(rn(reg), i0);
4416         ldxr_s(r0, r1, rn(reg));
4417         jit_unget_reg(reg);
4418     }
4419     else
4420         ldr_s(r0, r1);
4421 }
4422
4423 static void
4424 _ldxr_us(jit_state_t *_jit, jit_int32_t r0, jit_int32_t r1, jit_int32_t r2)
4425 {
4426     jit_int32_t         reg;
4427     reg = jit_get_reg(jit_class_gpr);
4428     addr(rn(reg), r1, r2);
4429     ldr_us(r0, rn(reg));
4430     jit_unget_reg(reg);
4431 }
4432
4433 static void
4434 _ldxi_us(jit_state_t *_jit, jit_int32_t r0, jit_int32_t r1, jit_word_t i0)
4435 {
4436     jit_int32_t         reg;
4437     if (i0) {
4438         reg = jit_get_reg(jit_class_gpr);
4439         movi(rn(reg), i0);
4440         ldxr_us(r0, r1, rn(reg));
4441         jit_unget_reg(reg);
4442     }
4443     else
4444         ldr_us(r0, r1);
4445 }
4446
4447 static void
4448 _ldxr_i(jit_state_t *_jit, jit_int32_t r0, jit_int32_t r1, jit_int32_t r2)
4449 {
4450     jit_int32_t         reg;
4451     reg = jit_get_reg(jit_class_gpr);
4452     addr(rn(reg), r1, r2);
4453     ldr_i(r0, rn(reg));
4454     jit_unget_reg(reg);
4455 }
4456
4457 static void
4458 _ldxi_i(jit_state_t *_jit, jit_int32_t r0, jit_int32_t r1, jit_word_t i0)
4459 {
4460     jit_int32_t         reg;
4461     if (i0) {
4462         reg = jit_get_reg(jit_class_gpr);
4463         movi(rn(reg), i0);
4464         ldxr_i(r0, r1, rn(reg));
4465         jit_unget_reg(reg);
4466     }
4467     else
4468         ldr_i(r0, r1);
4469 }
4470
4471 static void
4472 _ldxr_ui(jit_state_t *_jit, jit_int32_t r0, jit_int32_t r1, jit_int32_t r2)
4473 {
4474     jit_int32_t         reg;
4475     reg = jit_get_reg(jit_class_gpr);
4476     addr(rn(reg), r1, r2);
4477     ldr_ui(r0, rn(reg));
4478     jit_unget_reg(reg);
4479 }
4480
4481 static void
4482 _ldxi_ui(jit_state_t *_jit, jit_int32_t r0, jit_int32_t r1, jit_word_t i0)
4483 {
4484     jit_int32_t         reg;
4485     if (i0) {
4486         reg = jit_get_reg(jit_class_gpr);
4487         movi(rn(reg), i0);
4488         ldxr_ui(r0, r1, rn(reg));
4489         jit_unget_reg(reg);
4490     }
4491     else
4492         ldr_ui(r0, r1);
4493 }
4494
4495 static void
4496 _ldxr_l(jit_state_t *_jit, jit_int32_t r0, jit_int32_t r1, jit_int32_t r2)
4497 {
4498     jit_int32_t         reg;
4499     reg = jit_get_reg(jit_class_gpr);
4500     addr(rn(reg), r1, r2);
4501     ldr_l(r0, rn(reg));
4502     jit_unget_reg(reg);
4503 }
4504
4505 static void
4506 _ldxi_l(jit_state_t *_jit, jit_int32_t r0, jit_int32_t r1, jit_word_t i0)
4507 {
4508     jit_int32_t         reg;
4509     if (i0) {
4510         reg = jit_get_reg(jit_class_gpr);
4511         movi(rn(reg), i0);
4512         ldxr_l(r0, r1, rn(reg));
4513         jit_unget_reg(reg);
4514     }
4515     else
4516         ldr_l(r0, r1);
4517 }
4518
4519 static void
4520 _sti_c(jit_state_t *_jit, jit_word_t i0, jit_int32_t r0)
4521 {
4522     jit_int32_t         reg;
4523     reg = jit_get_reg(jit_class_gpr);
4524     movi(rn(reg), i0);
4525     str_c(rn(reg), r0);
4526     jit_unget_reg(reg);
4527 }
4528
4529 static void
4530 _sti_s(jit_state_t *_jit, jit_word_t i0, jit_int32_t r0)
4531 {
4532     jit_int32_t         reg;
4533     reg = jit_get_reg(jit_class_gpr);
4534     movi(rn(reg), i0);
4535     str_s(rn(reg), r0);
4536     jit_unget_reg(reg);
4537 }
4538
4539 static void
4540 _sti_i(jit_state_t *_jit, jit_word_t i0, jit_int32_t r0)
4541 {
4542     jit_int32_t         reg;
4543     reg = jit_get_reg(jit_class_gpr);
4544     movi(rn(reg), i0);
4545     str_i(rn(reg), r0);
4546     jit_unget_reg(reg);
4547 }
4548
4549 static void
4550 _sti_l(jit_state_t *_jit, jit_word_t i0, jit_int32_t r0)
4551 {
4552     jit_int32_t         reg;
4553     reg = jit_get_reg(jit_class_gpr);
4554     movi(rn(reg), i0);
4555     str_l(rn(reg), r0);
4556     jit_unget_reg(reg);
4557 }
4558
4559 static void
4560 _stxr_c(jit_state_t *_jit, jit_int32_t r0, jit_int32_t r1, jit_int32_t r2)
4561 {
4562     jit_int32_t         reg;
4563     reg = jit_get_reg(jit_class_gpr);
4564     addr(rn(reg), r0, r1);
4565     str_c(rn(reg), r2);
4566     jit_unget_reg(reg);
4567 }
4568
4569 static void
4570 _stxi_c(jit_state_t *_jit, jit_word_t i0, jit_int32_t r0, jit_int32_t r1)
4571 {
4572     jit_int32_t         reg;
4573     if (i0) {
4574         reg = jit_get_reg(jit_class_gpr);
4575         addi(rn(reg), r0, i0);
4576         str_c(rn(reg), r1);
4577         jit_unget_reg(reg);
4578     }
4579     else
4580         str_c(r0, r1);
4581 }
4582
4583 static void
4584 _stxr_s(jit_state_t *_jit, jit_int32_t r0, jit_int32_t r1, jit_int32_t r2)
4585 {
4586     jit_int32_t         reg;
4587     reg = jit_get_reg(jit_class_gpr);
4588     addr(rn(reg), r0, r1);
4589     str_s(rn(reg), r2);
4590     jit_unget_reg(reg);
4591 }
4592
4593 static void
4594 _stxi_s(jit_state_t *_jit, jit_word_t i0, jit_int32_t r0, jit_int32_t r1)
4595 {
4596     jit_int32_t         reg;
4597     if (i0) {
4598         reg = jit_get_reg(jit_class_gpr);
4599         addi(rn(reg), r0, i0);
4600         str_s(rn(reg), r1);
4601         jit_unget_reg(reg);
4602     }
4603     else
4604         str_s(r0, r1);
4605 }
4606
4607 static void
4608 _stxr_i(jit_state_t *_jit, jit_int32_t r0, jit_int32_t r1, jit_int32_t r2)
4609 {
4610     jit_int32_t         reg;
4611     reg = jit_get_reg(jit_class_gpr);
4612     addr(rn(reg), r0, r1);
4613     str_i(rn(reg), r2);
4614     jit_unget_reg(reg);
4615 }
4616
4617 static void
4618 _stxi_i(jit_state_t *_jit, jit_word_t i0, jit_int32_t r0, jit_int32_t r1)
4619 {
4620     jit_int32_t         reg;
4621     if (i0) {
4622         reg = jit_get_reg(jit_class_gpr);
4623         addi(rn(reg), r0, i0);
4624         str_i(rn(reg), r1);
4625         jit_unget_reg(reg);
4626     }
4627     else
4628         str_i(r0, r1);
4629 }
4630
4631 static void
4632 _stxr_l(jit_state_t *_jit, jit_int32_t r0, jit_int32_t r1, jit_int32_t r2)
4633 {
4634     jit_int32_t         reg;
4635     reg = jit_get_reg(jit_class_gpr);
4636     addr(rn(reg), r0, r1);
4637     str_l(rn(reg), r2);
4638     jit_unget_reg(reg);
4639 }
4640
4641 static void
4642 _stxi_l(jit_state_t *_jit, jit_word_t i0, jit_int32_t r0, jit_int32_t r1)
4643 {
4644     jit_int32_t         reg;
4645     if (i0) {
4646         reg = jit_get_reg(jit_class_gpr);
4647         addi(rn(reg), r0, i0);
4648         str_l(rn(reg), r1);
4649         jit_unget_reg(reg);
4650     }
4651     else
4652         str_l(r0, r1);
4653 }
4654
4655 static jit_word_t
4656 _bltr(jit_state_t *_jit, jit_word_t i0, jit_int32_t r0, jit_int32_t r1)
4657 {
4658     jit_word_t          w;
4659     CMP_LT(PR_6, PR_7, r0, r1);
4660     sync();
4661     w = _jit->pc.w;
4662     BRI_COND((i0 - w) >> 4, PR_6);
4663     return (w);
4664 }
4665
4666 static jit_word_t
4667 _blti(jit_state_t *_jit, jit_word_t i0, jit_int32_t r0, jit_word_t i1)
4668 {
4669     jit_word_t          w;
4670     jit_int32_t         reg;
4671     if (i1 >= -127 && i1 <= 128)
4672         CMPI_LT(PR_7, PR_6, i1 - 1, r0);
4673     else {
4674         reg = jit_get_reg(jit_class_gpr);
4675         movi(rn(reg), i1);
4676         CMP_LT(PR_6, PR_7, r0, rn(reg));
4677         jit_unget_reg(reg);
4678     }
4679     sync();
4680     w = _jit->pc.w;
4681     BRI_COND((i0 - w) >> 4, PR_6);
4682     return (w);
4683 }
4684
4685 static jit_word_t
4686 _bltr_u(jit_state_t *_jit, jit_word_t i0, jit_int32_t r0, jit_int32_t r1)
4687 {
4688     jit_word_t          w;
4689     CMP_LTU(PR_6, PR_7, r0, r1);
4690     sync();
4691     w = _jit->pc.w;
4692     BRI_COND((i0 - w) >> 4, PR_6);
4693     return (w);
4694 }
4695
4696 static jit_word_t
4697 _blti_u(jit_state_t *_jit, jit_word_t i0, jit_int32_t r0, jit_word_t i1)
4698 {
4699     jit_word_t          w;
4700     jit_int32_t         reg;
4701     if (i1 >= -127 && i1 <= 128)
4702         CMPI_LTU(PR_7, PR_6, i1 - 1, r0);
4703     else {
4704         reg = jit_get_reg(jit_class_gpr);
4705         movi(rn(reg), i1);
4706         CMP_LTU(PR_6, PR_7, r0, rn(reg));
4707         jit_unget_reg(reg);
4708     }
4709     sync();
4710     w = _jit->pc.w;
4711     BRI_COND((i0 - w) >> 4, PR_6);
4712     return (w);
4713 }
4714
4715 static jit_word_t
4716 _bler(jit_state_t *_jit, jit_word_t i0, jit_int32_t r0, jit_int32_t r1)
4717 {
4718     jit_word_t          w;
4719     CMP_LT(PR_6, PR_7, r1, r0);
4720     sync();
4721     w = _jit->pc.w;
4722     BRI_COND((i0 - w) >> 4, PR_7);
4723     return (w);
4724 }
4725
4726 static jit_word_t
4727 _blei(jit_state_t *_jit, jit_word_t i0, jit_int32_t r0, jit_word_t i1)
4728 {
4729     jit_word_t          w;
4730     jit_int32_t         reg;
4731     reg = jit_get_reg(jit_class_gpr|jit_class_nospill);
4732     movi(rn(reg), i1);
4733     w = bler(i0, r0, rn(reg));
4734     jit_unget_reg(reg);
4735     return (w);
4736 }
4737
4738 static jit_word_t
4739 _bler_u(jit_state_t *_jit, jit_word_t i0, jit_int32_t r0, jit_int32_t r1)
4740 {
4741     jit_word_t          w;
4742     CMP_LTU(PR_6, PR_7, r1, r0);
4743     sync();
4744     w = _jit->pc.w;
4745     BRI_COND((i0 - w) >> 4, PR_7);
4746     return (w);
4747 }
4748
4749 static jit_word_t
4750 _blei_u(jit_state_t *_jit, jit_word_t i0, jit_int32_t r0, jit_word_t i1)
4751 {
4752     jit_word_t          w;
4753     jit_int32_t         reg;
4754     reg = jit_get_reg(jit_class_gpr|jit_class_nospill);
4755     movi(rn(reg), i1);
4756     w = bler_u(i0, r0, rn(reg));
4757     jit_unget_reg(reg);
4758     return (w);
4759 }
4760
4761 static jit_word_t
4762 _beqr(jit_state_t *_jit, jit_word_t i0, jit_int32_t r0, jit_int32_t r1)
4763 {
4764     jit_word_t          w;
4765     CMP_EQ(PR_6, PR_7, r0, r1);
4766     sync();
4767     w = _jit->pc.w;
4768     BRI_COND((i0 - w) >> 4, PR_6);
4769     return (w);
4770 }
4771
4772 static jit_word_t
4773 _beqi(jit_state_t *_jit, jit_word_t i0, jit_int32_t r0, jit_word_t i1)
4774 {
4775     jit_word_t          w;
4776     jit_int32_t         reg;
4777     if (i1 >= -128 && i1 <= 127)
4778         CMPI_EQ(PR_6, PR_7, i1, r0);
4779     else {
4780         reg = jit_get_reg(jit_class_gpr);
4781         movi(rn(reg), i1);
4782         CMP_EQ(PR_6, PR_7, r0, rn(reg));
4783         jit_unget_reg(reg);
4784     }
4785     sync();
4786     w = _jit->pc.w;
4787     BRI_COND((i0 - w) >> 4, PR_6);
4788     return (w);
4789 }
4790
4791 static jit_word_t
4792 _bger(jit_state_t *_jit, jit_word_t i0, jit_int32_t r0, jit_int32_t r1)
4793 {
4794     jit_word_t          w;
4795     CMP_LT(PR_6, PR_7, r0, r1);
4796     sync();
4797     w = _jit->pc.w;
4798     BRI_COND((i0 - w) >> 4, PR_7);
4799     return (w);
4800 }
4801
4802 static jit_word_t
4803 _bgei(jit_state_t *_jit, jit_word_t i0, jit_int32_t r0, jit_word_t i1)
4804 {
4805     jit_word_t          w;
4806     jit_int32_t         reg;
4807     if (i1 >= -127 && i1 <= 128)
4808         CMPI_LT(PR_7, PR_6, i1 - 1, r0);
4809     else {
4810         reg = jit_get_reg(jit_class_gpr);
4811         movi(rn(reg), i1);
4812         CMP_LT(PR_6, PR_7, r0, rn(reg));
4813         jit_unget_reg(reg);
4814     }
4815     sync();
4816     w = _jit->pc.w;
4817     BRI_COND((i0 - w) >> 4, PR_7);
4818     return (w);
4819 }
4820
4821 static jit_word_t
4822 _bger_u(jit_state_t *_jit, jit_word_t i0, jit_int32_t r0, jit_int32_t r1)
4823 {
4824     jit_word_t          w;
4825     CMP_LTU(PR_6, PR_7, r0, r1);
4826     sync();
4827     w = _jit->pc.w;
4828     BRI_COND((i0 - w) >> 4, PR_7);
4829     return (w);
4830 }
4831
4832 static jit_word_t
4833 _bgei_u(jit_state_t *_jit, jit_word_t i0, jit_int32_t r0, jit_word_t i1)
4834 {
4835     jit_word_t          w;
4836     jit_int32_t         reg;
4837     if (i1 >= -127 && i1 <= 128)
4838         CMPI_LTU(PR_7, PR_6, i1 - 1, r0);
4839     else {
4840         reg = jit_get_reg(jit_class_gpr);
4841         movi(rn(reg), i1);
4842         CMP_LTU(PR_6, PR_7, r0, rn(reg));
4843         jit_unget_reg(reg);
4844     }
4845     sync();
4846     w = _jit->pc.w;
4847     BRI_COND((i0 - w) >> 4, PR_7);
4848     return (w);
4849 }
4850
4851 static jit_word_t
4852 _bgtr(jit_state_t *_jit, jit_word_t i0, jit_int32_t r0, jit_int32_t r1)
4853 {
4854     jit_word_t          w;
4855     CMP_LT(PR_6, PR_7, r1, r0);
4856     sync();
4857     w = _jit->pc.w;
4858     BRI_COND((i0 - w) >> 4, PR_6);
4859     return (w);
4860 }
4861
4862 static jit_word_t
4863 _bgti(jit_state_t *_jit, jit_word_t i0, jit_int32_t r0, jit_word_t i1)
4864 {
4865     jit_word_t          w;
4866     jit_int32_t         reg;
4867     reg = jit_get_reg(jit_class_gpr|jit_class_nospill);
4868     movi(rn(reg), i1);
4869     w = bgtr(i0, r0, rn(reg));
4870     jit_unget_reg(reg);
4871     return (w);
4872 }
4873
4874 static jit_word_t
4875 _bgtr_u(jit_state_t *_jit, jit_word_t i0, jit_int32_t r0, jit_int32_t r1)
4876 {
4877     jit_word_t          w;
4878     CMP_LTU(PR_6, PR_7, r1, r0);
4879     sync();
4880     w = _jit->pc.w;
4881     BRI_COND((i0 - w) >> 4, PR_6);
4882     return (w);
4883 }
4884
4885 static jit_word_t
4886 _bgti_u(jit_state_t *_jit, jit_word_t i0, jit_int32_t r0, jit_word_t i1)
4887 {
4888     jit_word_t          w;
4889     jit_int32_t         reg;
4890     reg = jit_get_reg(jit_class_gpr|jit_class_nospill);
4891     movi(rn(reg), i1);
4892     w = bgtr_u(i0, r0, rn(reg));
4893     jit_unget_reg(reg);
4894     return (w);
4895 }
4896
4897 static jit_word_t
4898 _bner(jit_state_t *_jit, jit_word_t i0, jit_int32_t r0, jit_int32_t r1)
4899 {
4900     jit_word_t          w;
4901     CMP_EQ(PR_6, PR_7, r0, r1);
4902     sync();
4903     w = _jit->pc.w;
4904     BRI_COND((i0 - w) >> 4, PR_7);
4905     return (w);
4906 }
4907
4908 static jit_word_t
4909 _bnei(jit_state_t *_jit, jit_word_t i0, jit_int32_t r0, jit_word_t i1)
4910 {
4911     jit_word_t          w;
4912     jit_int32_t         reg;
4913     if (i1 >= -128 && i1 <= 127)
4914         CMPI_EQ(PR_6, PR_7, i1, r0);
4915     else {
4916         reg = jit_get_reg(jit_class_gpr);
4917         movi(rn(reg), i1);
4918         CMP_EQ(PR_6, PR_7, r0, rn(reg));
4919         jit_unget_reg(reg);
4920     }
4921     sync();
4922     w = _jit->pc.w;
4923     BRI_COND((i0 - w) >> 4, PR_7);
4924     return (w);
4925 }
4926
4927 static jit_word_t
4928 _bmsr(jit_state_t *_jit, jit_word_t i0, jit_int32_t r0, jit_int32_t r1)
4929 {
4930     jit_word_t          w;
4931     jit_int32_t         reg;
4932     reg = jit_get_reg(jit_class_gpr);
4933     andr(rn(reg), r0, r1);
4934     CMPI_EQ(PR_6, PR_7, 0, rn(reg));
4935     jit_unget_reg(reg);
4936     sync();
4937     w = _jit->pc.w;
4938     BRI_COND((i0 - w) >> 4, PR_7);
4939     return (w);
4940 }
4941
4942 static jit_word_t
4943 _bmsi(jit_state_t *_jit, jit_word_t i0, jit_int32_t r0, jit_word_t i1)
4944 {
4945     jit_word_t          w;
4946     jit_int32_t         reg;
4947     reg = jit_get_reg(jit_class_gpr);
4948     movi(rn(reg), i1);
4949     andr(rn(reg), r0, rn(reg));
4950     CMPI_EQ(PR_6, PR_7, 0, rn(reg));
4951     jit_unget_reg(reg);
4952     sync();
4953     w = _jit->pc.w;
4954     BRI_COND((i0 - w) >> 4, PR_7);
4955     return (w);
4956 }
4957
4958 static jit_word_t
4959 _bmcr(jit_state_t *_jit, jit_word_t i0, jit_int32_t r0, jit_int32_t r1)
4960 {
4961     jit_word_t          w;
4962     jit_int32_t         reg;
4963     reg = jit_get_reg(jit_class_gpr);
4964     andr(rn(reg), r0, r1);
4965     CMPI_EQ(PR_6, PR_7, 0, rn(reg));
4966     jit_unget_reg(reg);
4967     sync();
4968     w = _jit->pc.w;
4969     BRI_COND((i0 - w) >> 4, PR_6);
4970     return (w);
4971 }
4972
4973 static jit_word_t
4974 _bmci(jit_state_t *_jit, jit_word_t i0, jit_int32_t r0, jit_word_t i1)
4975 {
4976     jit_word_t          w;
4977     jit_int32_t         reg;
4978     reg = jit_get_reg(jit_class_gpr);
4979     movi(rn(reg), i1);
4980     andr(rn(reg), r0, rn(reg));
4981     CMPI_EQ(PR_6, PR_7, 0, rn(reg));
4982     jit_unget_reg(reg);
4983     sync();
4984     w = _jit->pc.w;
4985     BRI_COND((i0 - w) >> 4, PR_6);
4986     return (w);
4987 }
4988
4989 static jit_word_t
4990 _baddr(jit_state_t *_jit, jit_word_t i0, jit_int32_t r0, jit_int32_t r1,
4991        jit_bool_t carry)
4992 {
4993     jit_word_t          w;
4994     jit_int32_t         t0;
4995     jit_int32_t         t1;
4996     jit_int32_t         t2;
4997     /* t1 = r0 + r1;    overflow = r1 < 0 ? r0 < t1 : t1 < r0 */
4998     t0 = jit_get_reg(jit_class_gpr|jit_class_nospill);
4999     t1 = jit_get_reg(jit_class_gpr|jit_class_nospill);
5000     t2 = jit_get_reg(jit_class_gpr|jit_class_nospill);
5001     lti(rn(t0), r1, 0);                 /* t0 = r1 < 0 */
5002     addr(rn(t1), r0, r1);               /* t1 = r0 + r1 */
5003     ltr(rn(t2), rn(t1), r0);            /* t2 = t1 < r0 */
5004     ltr(rn(t1), r0, rn(t1));            /* t1 = r0 < t1 */
5005     CMPI_EQ(PR_6, PR_7, 0, rn(t0));
5006     CMPI_EQ_p(PR_8, PR_9, 0, rn(t2), PR_6);/* if (t0==0) p8=t2==0,p9=t2!=0; */
5007     CMPI_EQ_p(PR_8, PR_9, 0, rn(t1), PR_7);/* if (t0!=0) p8=t1==0,p9=t1!=0; */
5008     addr(r0, r0, r1);
5009     sync();
5010     w = _jit->pc.w;
5011     BRI_COND((i0 - w) >> 4, carry ? PR_9 : PR_8);
5012     jit_unget_reg(t2);
5013     jit_unget_reg(t1);
5014     jit_unget_reg(t0);
5015     return (w);
5016 }
5017
5018 static jit_word_t
5019 _baddi(jit_state_t *_jit, jit_word_t i0, jit_int32_t r0, jit_word_t i1,
5020        jit_bool_t carry)
5021 {
5022     jit_word_t          w;
5023     jit_int32_t         reg;
5024     reg = jit_get_reg(jit_class_gpr|jit_class_nospill);
5025     movi(rn(reg), i1);
5026     w = baddr(i0, r0, rn(reg), carry);
5027     jit_unget_reg(reg);
5028     return (w);
5029 }
5030
5031 static jit_word_t
5032 _baddr_u(jit_state_t *_jit, jit_word_t i0, jit_int32_t r0, jit_int32_t r1,
5033          jit_bool_t carry)
5034 {
5035     jit_word_t          w;
5036     jit_int32_t         t0;
5037     jit_int32_t         t1;
5038     t0 = jit_get_reg(jit_class_gpr|jit_class_nospill);
5039     t1 = jit_get_reg(jit_class_gpr|jit_class_nospill);
5040     addr(rn(t0), r0, r1);
5041     ltr_u(rn(t1), rn(t0), r0);
5042     CMPI_EQ(PR_6, PR_7, 0, rn(t1));
5043     MOV(r0, rn(t0));
5044     sync();
5045     w = _jit->pc.w;
5046     BRI_COND((i0 - w) >> 4, carry ? PR_7 : PR_6);
5047     jit_unget_reg(t1);
5048     jit_unget_reg(t0);
5049     return (w);
5050 }
5051
5052 static jit_word_t
5053 _baddi_u(jit_state_t *_jit, jit_word_t i0, jit_int32_t r0, jit_word_t i1,
5054          jit_bool_t carry)
5055 {
5056     jit_word_t          w;
5057     jit_int32_t         reg;
5058     reg = jit_get_reg(jit_class_gpr|jit_class_nospill);
5059     movi(rn(reg), i1);
5060     w = baddr_u(i0, r0, rn(reg), carry);
5061     jit_unget_reg(reg);
5062     return (w);
5063 }
5064
5065 static jit_word_t
5066 _bsubr(jit_state_t *_jit, jit_word_t i0, jit_int32_t r0, jit_int32_t r1,
5067        jit_bool_t carry)
5068 {
5069     jit_word_t          w;
5070     jit_int32_t         t0;
5071     jit_int32_t         t1;
5072     jit_int32_t         t2;
5073     /* t1 = r0 - r1;    overflow = 0 < r1 ? r0 < t1 : t1 < r0 */
5074     t0 = jit_get_reg(jit_class_gpr|jit_class_nospill);
5075     t1 = jit_get_reg(jit_class_gpr|jit_class_nospill);
5076     t2 = jit_get_reg(jit_class_gpr|jit_class_nospill);
5077     gti(rn(t0), r1, 0);                 /* t0 = r1 > 0 */
5078     subr(rn(t1), r0, r1);               /* t1 = r0 - r1 */
5079     ltr(rn(t2), rn(t1), r0);            /* t2 = t1 < r0 */
5080     ltr(rn(t1), r0, rn(t1));            /* t1 = r0 < t1 */
5081     CMPI_EQ(PR_6, PR_7, 0, rn(t0));
5082     CMPI_EQ_p(PR_8, PR_9, 0, rn(t2), PR_6);/* if (t0==0) p4=t2==0,p5=t2!=0; */
5083     CMPI_EQ_p(PR_8, PR_9, 0, rn(t1), PR_7);/* if (t0!=0) p4=t1==0,p5=t1!=0; */
5084     subr(r0, r0, r1);
5085     sync();
5086     w = _jit->pc.w;
5087     BRI_COND((i0 - w) >> 4, carry ? PR_9 : PR_8);
5088     jit_unget_reg(t2);
5089     jit_unget_reg(t1);
5090     jit_unget_reg(t0);
5091     return (w);
5092 }
5093
5094 static jit_word_t
5095 _bsubi(jit_state_t *_jit, jit_word_t i0, jit_int32_t r0, jit_word_t i1,
5096        jit_bool_t carry)
5097 {
5098     jit_word_t          w;
5099     jit_int32_t         reg;
5100     reg = jit_get_reg(jit_class_gpr|jit_class_nospill);
5101     movi(rn(reg), i1);
5102     w = bsubr(i0, r0, rn(reg), carry);
5103     jit_unget_reg(reg);
5104     return (w);
5105 }
5106
5107 static jit_word_t
5108 _bsubr_u(jit_state_t *_jit, jit_word_t i0, jit_int32_t r0, jit_int32_t r1,
5109          jit_bool_t carry)
5110 {
5111     jit_word_t          w;
5112     jit_int32_t         t0;
5113     jit_int32_t         t1;
5114     t0 = jit_get_reg(jit_class_gpr|jit_class_nospill);
5115     t1 = jit_get_reg(jit_class_gpr|jit_class_nospill);
5116     subr(rn(t0), r0, r1);
5117     ltr_u(rn(t1), r0, rn(t0));
5118     CMPI_EQ(PR_6, PR_7, 0, rn(t1));
5119     MOV(r0, rn(t0));
5120     sync();
5121     w = _jit->pc.w;
5122     BRI_COND((i0 - w) >> 4, carry ? PR_7 : PR_6);
5123     jit_unget_reg(t1);
5124     jit_unget_reg(t0);
5125     return (w);
5126 }
5127
5128 static jit_word_t
5129 _bsubi_u(jit_state_t *_jit, jit_word_t i0, jit_int32_t r0, jit_word_t i1,
5130          jit_bool_t carry)
5131 {
5132     jit_word_t          w;
5133     jit_int32_t         reg;
5134     reg = jit_get_reg(jit_class_gpr|jit_class_nospill);
5135     movi(rn(reg), i1);
5136     w = bsubr_u(i0, r0, rn(reg), carry);
5137     jit_unget_reg(reg);
5138     return (w);
5139 }
5140
5141 static void
5142 _jmpr(jit_state_t *_jit, jit_int32_t r0)
5143 {
5144     MOV_br_rn(BR_6, r0);
5145     BR(BR_6);
5146 }
5147
5148 static void
5149 _jmpi(jit_state_t *_jit, jit_word_t i0)
5150 {
5151     jit_word_t          d;
5152     sync();
5153     d = ((jit_word_t)i0 - _jit->pc.w) >> 4;
5154     if (d >= -16777216 && d <= 16777215)
5155         BRI(d);
5156     else
5157         BRL(d);
5158 }
5159
5160 static jit_word_t
5161 _jmpi_p(jit_state_t *_jit, jit_word_t i0)
5162 {
5163     jit_word_t          d, w;
5164     sync();
5165     w = _jit->pc.w;
5166     d = ((jit_word_t)i0 - w) >> 4;
5167     BRL(d);
5168     return (w);
5169 }
5170
5171 static void
5172 _callr(jit_state_t *_jit, jit_int32_t r0)
5173 {
5174     jit_int32_t         reg;
5175     reg = jit_get_reg(jit_class_gpr);
5176     LD8_inc(rn(reg), r0, 8);
5177     MOV_br_rn(BR_6, rn(reg));
5178     jit_unget_reg(reg);
5179     LD8(GR_1, r0);
5180     BR_CALL(BR_0, BR_6);
5181 }
5182
5183 static void
5184 _calli(jit_state_t *_jit, jit_word_t i0)
5185 {
5186     jit_int32_t         reg;
5187     reg = jit_get_reg(jit_class_gpr);
5188     movi(rn(reg), i0);
5189     callr(rn(reg));
5190     jit_unget_reg(reg);
5191 }
5192
5193 static jit_word_t
5194 _calli_p(jit_state_t *_jit, jit_word_t i0)
5195 {
5196     jit_word_t          w;
5197     jit_int32_t         reg;
5198     reg = jit_get_reg(jit_class_gpr);
5199     w = movi_p(rn(reg), i0);
5200     callr(rn(reg));
5201     jit_unget_reg(reg);
5202     return (w);
5203 }
5204
5205 static void
5206 _prolog(jit_state_t *_jit, jit_node_t *node)
5207 {
5208     jit_int32_t         reg, ruse, rout;
5209
5210     if (_jitc->function->define_frame || _jitc->function->assume_frame) {
5211         jit_int32_t     frame = -_jitc->function->frame;
5212         assert(_jitc->function->self.aoff >= frame);
5213         if (_jitc->function->assume_frame)
5214             return;
5215         _jitc->function->self.aoff = frame;
5216     }
5217     if (_jitc->function->allocar)
5218         _jitc->function->self.aoff &= -16;
5219     _jitc->function->stack = ((_jitc->function->self.alen -
5220                                _jitc->function->self.aoff) + 15) & -16;
5221
5222     /* First lowest unused register is first output register */
5223     for (reg = _R115; reg >= _R40; reg--) {
5224         if (jit_regset_tstbit(&_jitc->function->regset, reg))
5225             break;
5226     }
5227     _jitc->breg = rn(reg) + 1;
5228     _jitc->rout = _jitc->breg + 5;
5229     ruse = _jitc->rout - GR_32;
5230
5231     /* How many out argument registers required? */
5232     if (!_jitc->function->define_frame) {
5233         for (reg = _OUT7; reg >= _OUT0; --reg) {
5234             if (jit_regset_tstbit(&_jitc->function->regset, reg))
5235                 break;
5236         }
5237         rout = (reg + 1) - _OUT0;
5238     }
5239     else
5240         rout = 8;
5241
5242     /* Do not know if will call div/mod functions (sqrt* needs one) */
5243     if (rout < 2)
5244         rout = 2;
5245
5246     /* Match gcc prolog */
5247     ALLOC(_jitc->breg + 1, ruse, rout);
5248     MOV(_jitc->breg + 2, GR_12);
5249     MOV_rn_br(_jitc->breg, BR_0);
5250     MOV(_jitc->breg + 3, GR_1);
5251
5252     /* lightning specific, use r4 as frame pointer */
5253     MOV(_jitc->breg + 4, GR_4);
5254     addi(GR_4, GR_12, -(stack_framesize + params_offset));
5255
5256     /* adjust stack pointer */
5257     addi(GR_12, GR_12, -(stack_framesize +
5258                          (params_offset << 1) + _jitc->function->stack));
5259
5260     if (jit_regset_tstbit(&_jitc->function->regset, JIT_F0))
5261         STF_SPILL(GR_4, rn(JIT_F0));
5262     if (jit_regset_tstbit(&_jitc->function->regset, JIT_F1)) {
5263         addi(GR_2, GR_4, 16);
5264         STF_SPILL(GR_2, rn(JIT_F1));
5265     }
5266     if (jit_regset_tstbit(&_jitc->function->regset, JIT_F2)) {
5267         addi(GR_2, GR_4, 32);
5268         STF_SPILL(GR_2, rn(JIT_F2));
5269     }
5270     if (jit_regset_tstbit(&_jitc->function->regset, JIT_F3)) {
5271         addi(GR_2, GR_4, 48);
5272         STF_SPILL(GR_2, rn(JIT_F3));
5273     }
5274     if (jit_regset_tstbit(&_jitc->function->regset, JIT_F4)) {
5275         addi(GR_2, GR_4, 64);
5276         STF_SPILL(GR_2, rn(JIT_F4));
5277     }
5278     if (jit_regset_tstbit(&_jitc->function->regset, JIT_F5)) {
5279         addi(GR_2, GR_4, 80);
5280         STF_SPILL(GR_2, rn(JIT_F5));
5281     }
5282
5283     if (_jitc->function->allocar) {
5284         reg = jit_get_reg(jit_class_gpr);
5285         movi(rn(reg), _jitc->function->self.aoff);
5286         stxi_i(_jitc->function->aoffoff, GR_4, rn(reg));
5287         jit_unget_reg(reg);
5288     }
5289
5290     if (_jitc->function->self.call & jit_call_varargs) {
5291         for (reg = _jitc->function->vagp; reg < 8; ++reg)
5292             stxi(112 + reg * 8, GR_4, GR_32 + reg);
5293     }
5294 }
5295
5296 static void
5297 _epilog(jit_state_t *_jit, jit_node_t *node)
5298 {
5299     if (_jitc->function->assume_frame)
5300         return;
5301     if (jit_regset_tstbit(&_jitc->function->regset, JIT_F0))
5302         LDF_FILL(rn(JIT_F0), GR_4);
5303     if (jit_regset_tstbit(&_jitc->function->regset, JIT_F1)) {
5304         addi(GR_2, GR_4, 16);
5305         LDF_FILL(rn(JIT_F1), GR_2);
5306     }
5307     if (jit_regset_tstbit(&_jitc->function->regset, JIT_F2)) {
5308         addi(GR_2, GR_4, 32);
5309         LDF_FILL(rn(JIT_F2), GR_2);
5310     }
5311     if (jit_regset_tstbit(&_jitc->function->regset, JIT_F3)) {
5312         addi(GR_2, GR_4, 48);
5313         LDF_FILL(rn(JIT_F3), GR_2);
5314     }
5315     if (jit_regset_tstbit(&_jitc->function->regset, JIT_F4)) {
5316         addi(GR_2, GR_4, 64);
5317         LDF_FILL(rn(JIT_F4), GR_2);
5318     }
5319     if (jit_regset_tstbit(&_jitc->function->regset, JIT_F5)) {
5320         addi(GR_2, GR_4, 80);
5321         LDF_FILL(rn(JIT_F5), GR_2);
5322     }
5323     /* Match gcc epilog */
5324     MOV(GR_1, _jitc->breg + 3);
5325     MOV_I_ar_rn(AR_PFS, _jitc->breg + 1);
5326     MOV_br_rn(BR_0, _jitc->breg);
5327     MOV(GR_12, _jitc->breg + 2);
5328     /* Restore lightning specific r4 as frame pointer */
5329     MOV(GR_4, _jitc->breg + 4);
5330     BR_RET(BR_0);
5331     flush();
5332 }
5333
5334 static void
5335 _vastart(jit_state_t *_jit, jit_int32_t r0)
5336 {
5337     assert(_jitc->function->self.call & jit_call_varargs);
5338     /* Initialize va_list to the first stack argument. */
5339     if (_jitc->function->vagp < 8)
5340         addi(r0, GR_4, 112 + _jitc->function->vagp * 8);
5341     else
5342         addi(r0, GR_4, _jitc->function->self.size);
5343 }
5344
5345 static void
5346 _vaarg(jit_state_t *_jit, jit_int32_t r0, jit_int32_t r1)
5347 {
5348     assert(_jitc->function->self.call & jit_call_varargs);
5349     /* Load argument. */
5350     ldr(r0, r1);
5351     /* Update va_list. */
5352     addi(r1, r1, 8);
5353 }
5354
5355 static void
5356 _patch_at(jit_state_t *_jit, jit_code_t code,
5357           jit_word_t instr, jit_word_t label)
5358 {
5359     jit_word_t           l, h, tm, s0, s1, s2;
5360     union {
5361         jit_word_t       w;
5362         jit_word_t      *p;
5363     } c;
5364     jit_word_t           i1, i41, i20, ic, i5, i9, i7;
5365     c.w = instr;
5366     get_bundle(c.p, l, h, tm, s0, s1, s2);
5367     switch (code) {
5368         case jit_code_movi:
5369             /* Handle jit functions as C function, so that jit function
5370              * pointers can be passed to C code, and jit code does not
5371              * need to try to differentiate them. */
5372         case jit_code_calli:
5373             i1  = (label >> 63) &           0x1L;
5374             i41 = (label >> 22) & 0x1ffffffffffL;
5375             ic  = (label >> 21) &           0x1L;
5376             i5  = (label >> 16) &          0x1fL;
5377             i9  = (label >>  7) &         0x1ffL;
5378             i7  =  label        &          0x7fL;
5379             s1 = i41;
5380             assert((tm & ~1) == TM_M_L_X_ &&
5381                    (s2 & 0xfL<<37) == (6L<<37) &&
5382                    s0 == nop_m);
5383             s2 &= (6L<<37)|(0x7fL<<6);
5384             s2 |= (i1<<36)|(i9<<27)|(i5<<22)|(ic<<21)|(i7<<13);
5385             break;
5386         case jit_code_jmpi:
5387             if (_jitc->jump) {
5388                 /* kludge to hide function descriptors; check that gp
5389                  * is zero, what is done for all jit functions */
5390                 if (((long *)label)[1] == 0) {
5391                     for (ic = 0; ic < _jitc->prolog.offset; ic++) {
5392                         if (_jitc->prolog.ptr[ic] == label) {
5393                             label += 16;
5394                             break;
5395                         }
5396                     }
5397                 }
5398             }
5399             ic = (label - instr) >> 4;
5400             i1  = (ic >> 61) &           0x1L;
5401             i41 = (ic >> 22) & 0x1ffffffffffL;
5402             i20 =  ic        &       0xfffffL;
5403             assert((tm & ~1) == TM_M_L_X_ &&
5404                    (s2 & 0xfL<<37) == (0xcL<<37) &&
5405                    s0 == nop_m);
5406             s1 = i41;
5407             s2 &= (0xcL<<37)|(0x7L<<33)|(1L<<12);
5408             s2 |= (i1<<36)|(i20<<13);
5409             break;
5410         default:
5411             /* Only B1 in slot 0 expected due to need to either
5412              * a stop to update predicates, or a sync before
5413              * unconditional short branch */
5414             ic = (label - instr) >> 4;
5415             assert((s0 >> 37) == 4 && (s0 & (7 << 6)) == 0);
5416             s0 &= (4L<<37)|(7L<<33)|(1L<<12)|0x1f;
5417             s0 |= (((ic>>20)&1L)<<36)|((ic&0xfffffL)<<13);
5418             break;
5419     }
5420     set_bundle(c.p, l, h, tm, s0, s1, s2);
5421 }
5422 #endif