deps/lightning/lib/jit_ia64-cpu.c

   1 /*
   2  * Copyright (C) 2013-2019  Free Software Foundation, Inc.
   3  *
   4  * This file is part of GNU lightning.
   5  *
   6  * GNU lightning is free software; you can redistribute it and/or modify it
   7  * under the terms of the GNU Lesser General Public License as published
   8  * by the Free Software Foundation; either version 3, or (at your option)
   9  * any later version.
  10  *
  11  * GNU lightning is distributed in the hope that it will be useful, but
  12  * WITHOUT ANY WARRANTY; without even the implied warranty of MERCHANTABILITY
  13  * or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU Lesser General Public
  14  * License for more details.
  15  *
  16  * Authors:
  17  *      Paulo Cesar Pereira de Andrade
  18  */
  19
  20 #if PROTO
  21 #define stack_framesize                 144
  22 #define params_offset                   16
  23 #define INST_NONE                       0       /* should never be generated */
  24 #define INST_STOP                       1       /* or'ed if stop is required */
  25 #define INST_A                          2       /* M- or I- unit */
  26 #define INST_As                         3
  27 #define INST_I                          4
  28 #define INST_Is                         5
  29 #define INST_M                          6
  30 #define INST_Ms                         7
  31 #define INST_F                          8
  32 #define INST_Fs                         9
  33 #define INST_B                          10
  34 #define INST_Bs                         11
  35 #define INST_L                          12
  36 #define INST_Ls                         13
  37 #define INST_X                          14
  38 #define INST_Xs                         15
  39
  40 /* Data and instructions are referenced by 64-bit addresses. Instructions
  41  * are stored in memory in little endian byte order, in which the least
  42  * significant byte appears in the lowest addressed byte of a memory
  43  * location. For data, modes for both big and little endian byte order are
  44  * supported and can be controlled by a bit in the User Mask Register.
  45  */
  46 #define il(ii)                          *_jit->pc.ul++ = ii
  47 #define set_bundle(p, l, h, tm, s0, s1, s2)                             \
  48     do {                                                                \
  49         l = tm | ((s0 & 0x1ffffffffffL) << 5L) | (s1 << 46L);           \
  50         h = ((s1 >> 18L) & 0x7fffffLL) | (s2 << 23L);                   \
  51         p[0] = byte_swap_if_big_endian(l);                              \
  52         p[1] = byte_swap_if_big_endian(h);                              \
  53     } while (0)
  54 #define get_bundle(p, l, h, tm, s0, s1, s2)                             \
  55     do {                                                                \
  56         l = byte_swap_if_big_endian(p[0]);                              \
  57         h = byte_swap_if_big_endian(p[1]);                              \
  58         tm = l & 0x1f;                                                  \
  59         s0 = (l >> 5L) & 0x1ffffffffffL;                                \
  60         s1 = ((l >> 46L) & 0x3ffffL) | ((h & 0x7fffffL) << 18L);        \
  61         s2 = (h >> 23L) & 0x1ffffffffffL;                               \
  62     } while (0)
  63
  64 /*  Need to insert a stop if a modified register would (or could)
  65  *  be read in the same cycle.
  66  */
  67 #define TSTREG1(r0)                                                     \
  68     do {                                                                \
  69         if (jit_regset_tstbit(&_jitc->regs, r0))                        \
  70             stop();                                                     \
  71     } while (0)
  72 #define TSTREG2(r0, r1)                                                 \
  73     do {                                                                \
  74         if (jit_regset_tstbit(&_jitc->regs, r0) ||                      \
  75             jit_regset_tstbit(&_jitc->regs, r1))                        \
  76             stop();                                                     \
  77     } while (0)
  78 #define TSTPRED(p0)                                                     \
  79     do {                                                                \
  80         if (p0 && (_jitc->pred & (1 << p0)))                            \
  81             stop();                                                     \
  82     } while (0)
  83 /* Record register was modified */
  84 #define SETREG(r0)              jit_regset_setbit(&_jitc->regs, r0)
  85
  86 /* Avoid using constants in macros and code */
  87 typedef enum {
  88     GR_0,               GR_1,           GR_2,           GR_3,
  89     GR_4,               GR_5,           GR_6,           GR_7,
  90     GR_8,               GR_9,           GR_10,          GR_11,
  91     GR_12,              GR_13,          GR_14,          GR_15,
  92     GR_16,              GR_17,          GR_18,          GR_19,
  93     GR_20,              GR_21,          GR_22,          GR_23,
  94     GR_24,              GR_25,          GR_26,          GR_27,
  95     GR_28,              GR_29,          GR_30,          GR_31,
  96     GR_32,              GR_33,          GR_34,          GR_35,
  97     GR_36,              GR_37,          GR_38,          GR_39,
  98     GR_40,              GR_41,          GR_42,          GR_43,
  99     /* GR_44...GR_127 */
 100 } gr_reg_t;
 101
 102 typedef enum {
 103     PR_0,               /* constant - always 1 */
 104     /* p0-p5             -  preserved */
 105     PR_1,               PR_2,           PR_3,           PR_4,
 106     PR_5,
 107     /* p6-p15            - scratch */
 108     PR_6,               PR_7,           PR_8,           PR_9,
 109     PR_10,              PR_11,          PR_12,          PR_13,
 110     PR_14,              PR_15,
 111     /* p16-...           - preserved - rotating */
 112 } pr_reg_t;
 113
 114 typedef enum {
 115     BR_0,               /* scratch - Return link */
 116     /* b1-b5             - preserved */
 117     BR_1,               BR_2,           BR_3,           BR_4,
 118     BR_5,
 119     /* b6-b7            - scratch */
 120     BR_6,               BR_7
 121 } br_reg_t;
 122
 123 typedef enum {
 124     AR_KR0,             AR_KR1,         AR_KR2,         AR_KR3,
 125     AR_KR4,             AR_KR5,         AR_KR6,         AR_KR7,
 126     AR_8,               AR_9,           AR_10,          AR_11,
 127     AR_12,              AR_13,          AR_14,          AR_15,
 128     AR_RSC,             AR_BSP,         AR_BSPSTORE,    AR_RNAT,
 129     AR_20,              AR_FCR,         AR_22,          AR_23,
 130     AR_EFLAG,           AR_CSD,         AR_SSD,         AR_CFLG,
 131     AR_FSR,             AR_FIR,         AR_FDR,         AR_31,
 132     AR_CCV,             AR_33,          AR_34,          AR_35,
 133     AR_UNAT,            AR_37,          AR_38,          AR_39,
 134     AR_FPSR,            AR_41,          AR_42,          AR_43,
 135     AR_ITC,             AR_RUC,         AR_46,          AR_47,
 136     AR_48,              AR_49,          AR_50,          AR_51,
 137     AR_52,              AR_53,          AR_54,          AR_55,
 138     AR_56,              AR_57,          AR_58,          AR_59,
 139     AR_60,              AR_61,          AR_62,          AR_63,
 140     AR_PFS,             AR_LC,          AR_EC,
 141     /* AR_67 ... AR_127 */
 142 } ar_reg_t;
 143
 144 typedef enum {
 145     TM_M_I_I_,  TM_M_I_Is,      TM_M_IsI_,      TM_M_IsIs,
 146     TM_M_L_X_,  TM_M_L_Xs,      TM_ILL_06,      TM_ILL_07,
 147     TM_M_M_I_,  TM_M_M_Is,      TM_MsM_I_,      TM_MsM_Is,
 148     TM_M_F_I_,  TM_M_F_Is,      TM_M_M_F_,      TM_M_M_Fs,
 149     TM_M_I_B_,  TM_M_I_Bs,      TM_M_B_B_,      TM_M_B_Bs,
 150     TM_ILL_14,  TM_ILL_15,      TM_B_B_B_,      TM_B_B_Bs,
 151     TM_M_M_B_,  TM_M_M_Bs,      TM_ILL_1A,      TM_ILL_1B,
 152     TM_M_F_B_,  TM_M_F_Bs,      TM_ILL_1E,      TM_ILL_1F,
 153 } template_t;
 154
 155 #define MWH_SPTK                        0
 156 #define MWH_NONE                        1
 157 #define MWH_DPTK                        2
 158
 159 #define IH_NONE                         0
 160 #define IH_IMP                          1
 161
 162 #define LD_NONE                         0
 163 #define LD_NT1                          1
 164 #define LD_NTA                          3
 165
 166 #define ST_NONE                         0
 167 #define ST_NTA                          3
 168
 169 #define LF_NONE                         0
 170 #define LF_NT1                          1
 171 #define LF_NT2                          2
 172 #define LF_NTA                          3
 173
 174 #define BR_PH_FEW                       0
 175 #define BR_PH_MANY                      1
 176
 177 #define BR_BWH_SPTK                     0
 178 #define BR_BWH_SPNT                     1
 179 #define BR_BWH_DPTK                     2
 180 #define BR_BWH_DPNT                     3
 181
 182 #define BRI_BWH_SPTK                    1
 183 #define BRI_BWH_SPNT                    3
 184 #define BRI_BWH_DPTK                    5
 185 #define BRI_BWH_DPNT                    7
 186
 187 #define BR_DH_NONE                      0
 188 #define BR_DH_CLR                       1
 189
 190 #define BR_IH_NONE                      0
 191 #define BR_IH_IMP                       1
 192
 193 #define BR_IPWH_SPTK                    0
 194 #define BR_IPWH_LOOP                    1
 195 #define BR_IPWH_DPTK                    2
 196 #define BR_IPWH_EXIT                    3
 197
 198 #define BR_INDWH_SPTK                   0
 199 #define BR_INDWH_DPTK                   2
 200
 201 #define MUX_BRCST                       0
 202 #define MUX_REV                         11
 203 #define MUX_MIX                         8
 204 #define MUX_SHUF                        9
 205 #define MUX_ALT                         10
 206
 207 #define ldr(r0,r1)                      ldr_l(r0,r1)
 208 #define ldi(r0,i0)                      ldi_l(r0,i0)
 209 #define str(r0,r1)                      str_l(r0,r1)
 210 #define sti(i0,r0)                      str_l(i0,r0)
 211 #define ldxr(r0,r1,r2)                  ldxr_l(r0,r1,r2)
 212 #define ldxi(r0,r1,i0)                  ldxi_l(r0,r1,i0)
 213 #define stxr(r0,r1,r2)                  stxr_l(r0,r1,r2)
 214 #define stxi(i0,r0,r1)                  stxi_l(i0,r0,r1)
 215
 216 #if !HAVE_FFSL
 217 #  define ffsl(l)                       __builtin_ffsl(l)
 218 #endif
 219
 220 /* libgcc */
 221 #if defined(__GNUC__)
 222 extern long __divdi3(long,long);
 223 extern unsigned long __udivdi3(unsigned long,unsigned long);
 224 extern long __moddi3(long,long);
 225 extern unsigned long __umoddi3(unsigned long,unsigned long);
 226 #else
 227 static long __divdi3(long,long);
 228 static unsigned long __udivdi3(unsigned long,unsigned long);
 229 static long __moddi3(long,long);
 230 static unsigned long __umoddi3(unsigned long,unsigned long);
 231 #endif
 232 #define out(n,tm,s0,s1,s2)              _out(_jit,n,tm,s0,s1,s2)
 233 static void _out(jit_state_t*,int,int,jit_word_t,jit_word_t,jit_word_t);
 234 #define stop()                          _stop(_jit)
 235 static void _stop(jit_state_t*);
 236 #define sync()                          _sync(_jit)
 237 static void _sync(jit_state_t*);
 238 #define flush()                         _flush(_jit)
 239 static void _flush(jit_state_t*);
 240 #define inst(w, t)                      _inst(_jit, w, t)
 241 static void _inst(jit_state_t*, jit_word_t, jit_uint8_t);
 242 #define A1(x4,x2,r3,r2,r1)              _A1(_jit,0,x4,x2,r3,r2,r1)
 243 static void _A1(jit_state_t*, jit_word_t,
 244                 jit_word_t,jit_word_t,jit_word_t,jit_word_t,jit_word_t);
 245 #define A2(x4,ct,r3,r2,r1)              A1(x4,ct,r3,r2,r1)
 246 #define A3(x4,x2,r3,im,r1)              _A3(_jit,0,x4,x2,r3,im,r1)
 247 static void _A3(jit_state_t*, jit_word_t,
 248                 jit_word_t,jit_word_t,jit_word_t,jit_word_t,jit_word_t);
 249 #define A4(x2a,r3,im,r1)                _A4(_jit,0,x2a,r3,im,r1)
 250 static void _A4(jit_state_t*,jit_word_t,
 251                 jit_word_t,jit_word_t,jit_word_t,jit_word_t);
 252 #define A5(r3,im,r1)                    _A5(_jit,0,r3,im,r1)
 253 static void _A5(jit_state_t*,jit_word_t,
 254                 jit_word_t,jit_word_t,jit_word_t);
 255 #define A6(o,x2,ta,p2,r3,r2,c,p1)       _A6(_jit,0,o,x2,ta,p2,r3,r2,c,p1)
 256 static void _A6(jit_state_t*,jit_word_t,
 257                 jit_word_t,jit_word_t,jit_word_t,jit_word_t,
 258                 jit_word_t,jit_word_t,jit_word_t,jit_word_t);
 259 #define A7(o,x2,ta,p2,r3,c,p1)          _A7(_jit,0,o,x2,ta,p2,r3,c,p1)
 260 static void _A7(jit_state_t*,jit_word_t,
 261                 jit_word_t,jit_word_t,jit_word_t,
 262                 jit_word_t,jit_word_t,jit_word_t,jit_word_t)
 263     maybe_unused;
 264 #define A8(o,x2,ta,p2,r3,im,c,p1)       _A8(_jit,0,o,x2,ta,p2,r3,im,c,p1)
 265 static void _A8(jit_state_t*,jit_word_t,
 266                 jit_word_t,jit_word_t,jit_word_t,jit_word_t,
 267                 jit_word_t,jit_word_t,jit_word_t,jit_word_t);
 268 #define A9(za,zb,x4,x2,r3,r2,r1)        _A9(_jit,0,za,zb,x4,x2,r3,r2,r1)
 269 static void _A9(jit_state_t*,jit_word_t,
 270                 jit_word_t,jit_word_t,jit_word_t,
 271                 jit_word_t,jit_word_t,jit_word_t,jit_word_t)
 272     maybe_unused;
 273 #define A10(x4,ct,r3,r2,r1)             A9(0,1,x4,ct,r3,r2,r1)
 274 #define I1(ct,x2,r3,r2,r1)              _I1(_jit,0,ct,x2,r3,r2,r1)
 275 static void _I1(jit_state_t*,jit_word_t,
 276                 jit_word_t,jit_word_t,jit_word_t,jit_word_t,jit_word_t)
 277     maybe_unused;
 278 #define I2(za,x2a,zb,x2c,x2b,r3,r2,r1)  _I2(_jit,0,za,x2a,zb,x2c,x2b,r3,r2,r1)
 279 static void _I2(jit_state_t*,jit_word_t,
 280                 jit_word_t,jit_word_t,jit_word_t,jit_word_t,
 281                 jit_word_t,jit_word_t,jit_word_t,jit_word_t)
 282     maybe_unused;
 283 #define I3(mbt,r2,r1)                   _I3(_jit,0,mbt,r2,r1)
 284 static void _I3(jit_state_t*,jit_word_t,
 285                 jit_word_t,jit_word_t,jit_word_t)
 286 #if __BYTE_ORDER == __BIG_ENDIAN
 287     maybe_unused
 288 #endif
 289     ;
 290 #define I4(mht,r2,r1)                   _I4(_jit,0,mht,r2,r1)
 291 static void _I4(jit_state_t*,jit_word_t,
 292                 jit_word_t,jit_word_t,jit_word_t)
 293     maybe_unused;
 294 #define I5(za,zb,x2b,r3,r2,r1)          _I5(_jit,0,za,zb,x2b,r3,r2,r1)
 295 static void _I5(jit_state_t*,jit_word_t,
 296                 jit_word_t,jit_word_t,jit_word_t,
 297                 jit_word_t,jit_word_t,jit_word_t);
 298 #define I6(za,zb,x2b,r3,ct,r1)          _I6(_jit,0,za,zb,x2b,r3,ct,r1)
 299 static void _I6(jit_state_t*,jit_word_t,
 300                 jit_word_t,jit_word_t,jit_word_t,
 301                 jit_word_t,jit_word_t,jit_word_t)
 302     maybe_unused;
 303 #define I7(za,zb,r3,r2,r1)              _I7(_jit,0,za,zb,r3,r2,r1)
 304 static void _I7(jit_state_t*,jit_word_t,
 305                 jit_word_t,jit_word_t,jit_word_t,jit_word_t,jit_word_t);
 306 #define I8(za,zb,ct,r2,r1)              _I8(_jit,0,za,zb,ct,r2,r1)
 307 static void _I8(jit_state_t*,jit_word_t,
 308                 jit_word_t,jit_word_t,jit_word_t,jit_word_t,jit_word_t)
 309     maybe_unused;
 310 #define I9(x2c,r3,r1)                   _I9(_jit,0,x2c,r3,r1)
 311 static void _I9(jit_state_t*,jit_word_t,
 312                 jit_word_t,jit_word_t,jit_word_t)
 313     maybe_unused;
 314 #define I10(ct,r3,r2,r1)                _I10(_jit,0,ct,r3,r2,r1)
 315 static void _I10(jit_state_t*,jit_word_t,
 316                  jit_word_t,jit_word_t,jit_word_t,jit_word_t)
 317     maybe_unused;
 318 #define I11(len,r3,pos,y,r1)            _I11(_jit,0,len,r3,pos,y,r1)
 319 static void _I11(jit_state_t*,jit_word_t,
 320                  jit_word_t,jit_word_t,jit_word_t,jit_word_t,jit_word_t)
 321     maybe_unused;
 322 #define I12(len,pos,r2,r1)              _I12(_jit,0,len,pos,r2,r1)
 323 static void _I12(jit_state_t*,jit_word_t,
 324                  jit_word_t,jit_word_t,jit_word_t,jit_word_t)
 325     maybe_unused;
 326 #define I13(len,pos,im,r1)              _I13(_jit,0,len,pos,im,r1)
 327 static void _I13(jit_state_t*,jit_word_t,
 328                  jit_word_t,jit_word_t,jit_word_t,jit_word_t)
 329     maybe_unused;
 330 #define I14(s,len,r3,pos,r1)            _I14(_jit,0,s,len,r3,pos,r1)
 331 static void _I14(jit_state_t*,jit_word_t,
 332                  jit_word_t,jit_word_t,jit_word_t,jit_word_t,jit_word_t)
 333     maybe_unused;
 334 #define I15(pos,len,r3,r2,r1)           _I15(_jit,0,pos,len,r3,r2,r1)
 335 static void _I15(jit_state_t*,jit_word_t,
 336                  jit_word_t,jit_word_t,jit_word_t,jit_word_t,jit_word_t)
 337     maybe_unused;
 338 #define I16(tb,ta,p2,r3,pos,c,p1)       _I16(_jit,0,tb,ta,p2,r3,pos,c,p1)
 339 static void _I16(jit_state_t*,jit_word_t,
 340                  jit_word_t,jit_word_t,jit_word_t,
 341                  jit_word_t,jit_word_t,jit_word_t,jit_word_t)
 342     maybe_unused;
 343 #define I17(tb,ta,p2,r3,c,p1)           _I17(_jit,0,tb,ta,p2,r3,c,p1)
 344 static void _I17(jit_state_t*,jit_word_t,
 345                  jit_word_t,jit_word_t,jit_word_t,
 346                  jit_word_t,jit_word_t,jit_word_t)
 347     maybe_unused;
 348 #define I18(im,y)                       _I18(_jit,0,im,y)
 349 static void _I18(jit_state_t*,jit_word_t,
 350                  jit_word_t,jit_word_t)
 351     maybe_unused;
 352 #define I19(im)                         _I19(_jit,0,im)
 353 static void _I19(jit_state_t*,jit_word_t,
 354                  jit_word_t)
 355     maybe_unused;
 356 #define I20(r2,im)                      _I20(_jit,0,r2,im)
 357 static void _I20(jit_state_t*,jit_word_t,
 358                  jit_word_t,jit_word_t)
 359     maybe_unused;
 360 #define I21(im,ih,x,wh,r2,b1)           _I21(_jit,0,im,ih,x,wh,r2,b1)
 361 static void _I21(jit_state_t*,jit_word_t,
 362                  jit_word_t,jit_word_t,
 363                  jit_word_t,jit_word_t,jit_word_t,jit_word_t);
 364 #define I22(b2,r1)                      _I22(_jit,0,b2,r1)
 365 static void _I22(jit_state_t*,jit_word_t,
 366                  jit_word_t,jit_word_t);
 367 #define I23(r2,im)                      _I23(_jit,0,r2,im)
 368 static void _I23(jit_state_t*,jit_word_t,
 369                  jit_word_t,jit_word_t)
 370     maybe_unused;
 371 #define I24(im)                         _I24(_jit,0,im)
 372 static void _I24(jit_state_t*,jit_word_t,
 373                  jit_word_t)
 374     maybe_unused;
 375 #define I25(x6,r1)                      _I25(_jit,0,x6,r1)
 376 static void _I25(jit_state_t*,jit_word_t,
 377                  jit_word_t,jit_word_t)
 378     maybe_unused;
 379 #define I26(ar,r2)                      _I26(_jit,0,ar,r2)
 380 static void _I26(jit_state_t*,jit_word_t,
 381                  jit_word_t,jit_word_t);
 382 #define I27(ar,im)                      _I27(_jit,0,ar,im)
 383 static void _I27(jit_state_t*,jit_word_t,
 384                  jit_word_t,jit_word_t)
 385     maybe_unused;
 386 #define I28(ar,r1)                      _I28(_jit,0,ar,r1)
 387 static void _I28(jit_state_t*,jit_word_t,
 388                  jit_word_t,jit_word_t)
 389     maybe_unused;
 390 #define I29(x6,r3,r1)                   _I29(_jit,0,x6,r3,r1)
 391 static void _I29(jit_state_t*,jit_word_t,
 392                  jit_word_t,jit_word_t,jit_word_t);
 393 #define I30(tb,ta,p2,im,c,p1)           _I30(_jit,0,ta,tb,p2,im,c,p1)
 394 static void _I30(jit_state_t*,jit_word_t,
 395                  jit_word_t,jit_word_t,jit_word_t,
 396                  jit_word_t,jit_word_t,jit_word_t)
 397     maybe_unused;
 398 #define M1(x6,ht,x,r3,r1)               _M1(_jit,0,x6,ht,x,r3,r1)
 399 static void _M1(jit_state_t*,jit_word_t,
 400                 jit_word_t,jit_word_t,jit_word_t,jit_word_t,jit_word_t);
 401 #define M2(x6,ht,r3,r2,r1)              _M2(_jit,0,x6,ht,r3,r2,r1)
 402 static void _M2(jit_state_t*,jit_word_t,
 403                 jit_word_t,jit_word_t,jit_word_t,jit_word_t,jit_word_t)
 404     maybe_unused;
 405 #define M3(x6,ht,r3,im,r1)              _M3(_jit,0,x6,ht,r3,im,r1)
 406 static void _M3(jit_state_t*,jit_word_t,
 407                 jit_word_t,jit_word_t,jit_word_t,jit_word_t,jit_word_t);
 408 #define M5(x6,ht,r3,r2,im)              _M5(_jit,0,x6,ht,r3,r2,im)
 409 static void _M5(jit_state_t*,jit_word_t,
 410                 jit_word_t,jit_word_t,jit_word_t,jit_word_t,jit_word_t)
 411     maybe_unused;
 412 #define M6(x6,ht,x,r3,r2)               _M6(_jit,0,x6,ht,x,r3,r2)
 413 static void _M6(jit_state_t*,jit_word_t,
 414                 jit_word_t,jit_word_t,jit_word_t,jit_word_t,jit_word_t);
 415 #define M13(x6,ht,r3,f2)                _M13(_jit,0,x6,ht,r3,f2)
 416 static void _M13(jit_state_t*,jit_word_t,
 417                  jit_word_t,jit_word_t,jit_word_t,jit_word_t);
 418 #define M14(x6,ht,r3,r2)                _M14(_jit,0,x6,ht,r3,r2)
 419 static void _M14(jit_state_t*,jit_word_t,
 420                  jit_word_t,jit_word_t,jit_word_t,jit_word_t)
 421     maybe_unused;
 422 #define M15(x6,ht,r3,im)                _M15(_jit,0,x6,ht,r3,im)
 423 static void _M15(jit_state_t*,jit_word_t,
 424                  jit_word_t,jit_word_t,jit_word_t,jit_word_t)
 425     maybe_unused;
 426 #define M16(x6,ht,r3,r2,r1)             _M16(_jit,0,x6,ht,r3,r2,r1)
 427 static void _M16(jit_state_t*,jit_word_t,
 428                  jit_word_t,jit_word_t,jit_word_t,jit_word_t,jit_word_t)
 429     maybe_unused;
 430 #define M17(x6,ht,r3,im,r1)             _M17(_jit,0,x6,ht,r3,im,r1)
 431 static void _M17(jit_state_t*,jit_word_t,
 432                  jit_word_t,jit_word_t,jit_word_t,jit_word_t,jit_word_t)
 433     maybe_unused;
 434 #define M20(r2,im)                      M20x(0x1,r2,im)
 435 #define M20x(x3,r2,im)                  _M20x(_jit,0,x3,r2,im)
 436 static void _M20x(jit_state_t*,jit_word_t,
 437                   jit_word_t,jit_word_t,jit_word_t)
 438     maybe_unused;
 439 #define M22(x3,im,r1)                   M22x(x3,im,r1)
 440 #define M22x(x3,im,r1)                  _M22x(_jit,0,x3,im,r1)
 441 static void _M22x(jit_state_t*,jit_word_t,
 442                   jit_word_t,jit_word_t,jit_word_t)
 443     maybe_unused;
 444 #define M24(x2,x4)                      _M24(_jit,0,x2,x4)
 445 #define M25(x4)                         M24(0,x4)
 446 static void _M24(jit_state_t*,jit_word_t,
 447                  jit_word_t,jit_word_t)
 448     maybe_unused;
 449 #define M26(r1)                         M26x(2,r1)
 450 #define M26x(x4,r1)                     _M26x(_jit,0,x4,r1)
 451 static void _M26x(jit_state_t*,jit_word_t,
 452                   jit_word_t,jit_word_t)
 453     maybe_unused;
 454 #define M28(x,r3)                       _M28(_jit,0,x,r3)
 455 static void _M28(jit_state_t*,jit_word_t,
 456                  jit_word_t,jit_word_t)
 457     maybe_unused;
 458 #define M29(ar,r2)                      _M29(_jit,0,ar,r2)
 459 static void _M29(jit_state_t*,jit_word_t,
 460                  jit_word_t,jit_word_t)
 461     maybe_unused;
 462 #define M30(ar,im)                      _M30(_jit,0,ar,im)
 463 static void _M30(jit_state_t*,jit_word_t,jit_word_t,jit_word_t)
 464     maybe_unused;
 465 #define M31(ar,r1)                      _M31(_jit,0,ar,r1)
 466 static void _M31(jit_state_t*,jit_word_t,
 467                  jit_word_t,jit_word_t)
 468     maybe_unused;
 469 #define M32(cr,r2)                      _M32(_jit,0,cr,r2)
 470 static void _M32(jit_state_t*,jit_word_t,
 471                  jit_word_t,jit_word_t)
 472     maybe_unused;
 473 #define M33(cr,r1)                      _M33(_jit,0,cr,r1)
 474 static void _M33(jit_state_t*,jit_word_t,
 475                  jit_word_t,jit_word_t)
 476     maybe_unused;
 477 #define M34(sor,sol,sof,r1)             _M34(_jit,0,sor,sol,sof,r1)
 478 static void _M34(jit_state_t*,jit_word_t,
 479                  jit_word_t,jit_word_t,jit_word_t,jit_word_t);
 480 #define M35(x6,r2)                      _M35(_jit,0,x6,r2)
 481 static void _M35(jit_state_t*,jit_word_t,
 482                  jit_word_t,jit_word_t)
 483     maybe_unused;
 484 #define M36(x6,r1)                      _M36(_jit,0,x6,r1)
 485 static void _M36(jit_state_t*,jit_word_t,
 486                  jit_word_t,jit_word_t)
 487     maybe_unused;
 488 #define M37(im)                         _M37(_jit,0,im)
 489 static void _M37(jit_state_t*,jit_word_t,
 490                  jit_word_t)
 491     maybe_unused;
 492 #define M38(x6,r3,r2,r1)                _M38(_jit,0,x6,r3,r2,r1)
 493 static void _M38(jit_state_t*,jit_word_t,
 494                  jit_word_t,jit_word_t,jit_word_t,jit_word_t)
 495     maybe_unused;
 496 #define M39(x6,r3,im,r1)                _M39(_jit,0,x6,r3,im,r1)
 497 static void _M39(jit_state_t*,jit_word_t,
 498                  jit_word_t,jit_word_t,jit_word_t,jit_word_t)
 499     maybe_unused;
 500 #define M40(x6,r3,im)                   _M40(_jit,0,x6,r3,im)
 501 static void _M40(jit_state_t*,jit_word_t,
 502                  jit_word_t,jit_word_t,jit_word_t)
 503     maybe_unused;
 504 #define M41(x6,r2)                      _M41(_jit,0,x6,r2)
 505 static void _M41(jit_state_t*,jit_word_t,
 506                  jit_word_t,jit_word_t)
 507     maybe_unused;
 508 #define M42(x6,r3,r2)                   _M42(_jit,0,x6,r3,r2)
 509 static void _M42(jit_state_t*,jit_word_t,
 510                  jit_word_t,jit_word_t,jit_word_t)
 511     maybe_unused;
 512 #define M43(x6,r3,r1)                   _M43(_jit,0,x6,r3,r1)
 513 static void _M43(jit_state_t*,jit_word_t,
 514                  jit_word_t,jit_word_t,jit_word_t)
 515     maybe_unused;
 516 #define M44(x4,im)                      _M44(_jit,0,x4,im)
 517 static void _M44(jit_state_t*,jit_word_t,
 518                  jit_word_t,jit_word_t)
 519     maybe_unused;
 520 #define M45(x6,r3,r2)                   _M45(_jit,0,x6,r3,r2)
 521 static void _M45(jit_state_t*,jit_word_t,
 522                  jit_word_t,jit_word_t,jit_word_t)
 523     maybe_unused;
 524 #define M46(x6,r3,r1)                   _M46(_jit,0,x6,r3,r1)
 525 #define M47(x6,r3)                      M46(x6,r3,0)
 526 static void _M46(jit_state_t*,jit_word_t,
 527                  jit_word_t,jit_word_t,jit_word_t)
 528     maybe_unused;
 529 #define M48(y,im)                       _M48(_jit,0,y,im)
 530 static void _M48(jit_state_t*,jit_word_t,
 531                  jit_word_t,jit_word_t)
 532     maybe_unused;
 533 #define B1(d,wh,im,p,tp)                _B1(_jit,0,d,wh,im,p,tp)
 534 #define B2(d,wh,im,p,tp)                B1(d,wh,im,p,tp)
 535 static void _B1(jit_state_t*,jit_word_t,
 536                 jit_word_t,jit_word_t,jit_word_t,jit_word_t,jit_word_t);
 537 #define B3(d,wh,im,p,b)                 _B3(_jit,0,d,wh,im,p,b)
 538 static void _B3(jit_state_t*,jit_word_t,
 539                 jit_word_t,jit_word_t,jit_word_t,jit_word_t,jit_word_t)
 540     maybe_unused;
 541 #define B4(d,wh,x6,b,p,tp)              _B4(_jit,0,d,wh,x6,b,p,tp)
 542 static void _B4(jit_state_t*,jit_word_t,
 543                 jit_word_t,jit_word_t,jit_word_t,
 544                 jit_word_t,jit_word_t,jit_word_t);
 545 #define B5(d,wh,b2,p,b1)                _B5(_jit,0,d,wh,b2,p,b1)
 546 static void _B5(jit_state_t*,jit_word_t,
 547                 jit_word_t,jit_word_t,jit_word_t,jit_word_t,jit_word_t);
 548 #define B6(ih,im,tag,wh)                _B6(_jit,0,ih,im,tag,wh)
 549 static void _B6(jit_state_t*,jit_word_t,
 550                 jit_word_t,jit_word_t,jit_word_t,jit_word_t)
 551     maybe_unused;
 552 #define B7(ih,x6,b2,tag,wh)             _B7(_jit,0,ih,x6,b2,tag,wh)
 553 static void _B7(jit_state_t*,jit_word_t,
 554                 jit_word_t,jit_word_t,jit_word_t,jit_word_t,jit_word_t)
 555     maybe_unused;
 556 #define B8(x6)                          _B8(_jit,0,x6)
 557 static void _B8(jit_state_t*,jit_word_t,
 558                 jit_word_t)
 559     maybe_unused;
 560 #define B9(op,x6,im)                    _B9(_jit,0,op,x6,im)
 561 static void _B9(jit_state_t*,jit_word_t,
 562                 jit_word_t,jit_word_t,jit_word_t)
 563     maybe_unused;
 564 #define X1(im)                          _X1(_jit,0,im)
 565 static void _X1(jit_state_t*,jit_word_t,
 566                 jit_word_t)
 567     maybe_unused;
 568 #define X2(r1,im)                       _X2(_jit,0,r1,im)
 569 static void _X2(jit_state_t*,jit_word_t,
 570                 jit_word_t,jit_word_t);
 571 #define X3x(o,d,wh,p,tp,im)             _X3x(_jit,0,o,d,wh,p,tp,im)
 572 #define X3(d,wh,p,tp,im)                X3x(0xc,d,wh,p,tp,im)
 573 #define X4(d,wh,p,tp,im)                X3x(0xd,d,wh,p,tp,im)
 574 static void _X3x(jit_state_t*,jit_word_t,
 575                  jit_word_t,jit_word_t,jit_word_t,
 576                  jit_word_t,jit_word_t,jit_word_t);
 577 #define X5(y,im)                        _X5(_jit,0,y,im)
 578 static void _X5(jit_state_t*,jit_word_t,
 579                 jit_word_t,jit_word_t)
 580     maybe_unused;
 581
 582 /* add */
 583 #define ADD(r1,r2,r3)                   A1(0,0,r3,r2,r1)
 584 #define ADD1(r1,r2,r3)                  A1(0,1,r3,r2,r1)
 585 #define ADDS(r1,im,r3)                  A4(2,r3,im,r1)
 586 #define ADDS_p(r1,im,r3,_p)             _A4(_jit,_p,2,r3,im,r1)
 587 #define ADDL(r1,im,r3)                  A5(r3,im,r1)
 588 #define ADDL_p(r1,im,r3,_p)             _A5(_jit,_p,r3,im,r1)
 589 /* addp4 */
 590 #define ADDP4(r1,r2,r3)                 A1(2,0,r3,r2,r1)
 591 #define ADDIP4(r1,im,r3)                A4(3,r3,im,r1)
 592 /* alloc */
 593 #define ALLOCR(r1,i,l,o,r)              M34((r)>>3,(i)+(l),(i)+(l)+(o),r1)
 594 #define ALLOC(r1,i,o)                   ALLOCR(r1,i,0,o,0)
 595 /* and */
 596 #define AND(r1,r2,r3)                   A1(3,0,r3,r2,r1)
 597 #define ANDI(r1,im,r3)                  A3(0xb,0,r3,im,r1)
 598 /* andcm */
 599 #define ANDCM(r1,r2,r3)                 A1(3,1,r3,r2,r1)
 600 #define ANDCMI(r1,im,r3)                A3(0xb,1,r3,im,r1)
 601 /* br */
 602 #define BRI(im)                         B1(BR_DH_NONE,BR_BWH_SPTK,im,BR_PH_FEW,0)
 603 #define BRI_COND(im,_p)                 _B1(_jit,_p,BR_DH_NONE,BR_BWH_SPTK,im,BR_PH_FEW,0)
 604 #define BRI_WEXIT(im)                   B1(BR_DH_NONE,BR_BWH_SPTK,im,BR_PH_MANY,2)
 605 #define BRI_WTOP(im)                    B1(BR_DH_NONE,BR_BWH_SPTK,im,BR_PH_MANY,3)
 606 #define BRI_CALL(b,im)                  B3(BR_DH_NONE,BR_BWH_SPTK,im,BR_PH_MANY,b)
 607 #define BRI_CLOOP(im)                   B2(BR_DH_NONE,BR_BWH_SPTK,im,BR_PH_MANY,5)
 608 #define BRI_CEXIT(im)                   B2(BR_DH_NONE,BR_BWH_SPTK,im,BR_PH_MANY,6)
 609 #define BRI_CTOP(im)                    B2(BR_DH_NONE,BR_BWH_SPTK,im,BR_PH_MANY,7)
 610 #define BR_COND(b,_p)                   _B4(_jit,_p,BR_DH_NONE,BR_BWH_SPTK,0x20,b,BR_PH_FEW,0)
 611 #define BR(b)                           B4(BR_DH_NONE,BR_BWH_SPTK,0x20,b,BR_PH_FEW,0)
 612 #define BR_IA(b)                        B4(BR_DH_NONE,BR_BWH_SPTK,0x20,b,BR_PH_MANY,1)
 613 #define BR_RET(b)                       B4(BR_DH_NONE,BR_BWH_SPTK,0x21,b,BR_PH_MANY,4)
 614 #define BR_CALL(b1,b2)                  B5(BR_DH_NONE,BRI_BWH_SPTK,b2,BR_PH_MANY,b1)
 615 /* break */
 616 #define BREAK_I(im)                     I19(im)
 617 #define BREAK_M(im)                     M37(im)
 618 #define BREAK_B(im)                     B9(0,0,im)
 619 #define BREAK_X(im)                     X1(im)
 620 /* brl */
 621 #define BRL(im)                         X3(BR_DH_NONE,BR_BWH_SPTK,BR_PH_MANY,0,im)
 622 #define BRL_COND(im,_p)                 _X3(_jit,_p,BR_DH_NONE,BR_BWH_SPTK,BR_PH_MANY,0,im)
 623 #define BRL_CALL(b1,im)                 X4(BR_DH_NONE,BR_BWH_SPTK,BR_PH_MANY,b1,im)
 624 /* brp */
 625 #define BRP(im,tag)                     B6(BR_IH_NONE,im,tag,BR_IPWH_SPTK)
 626 #define BRPI(b2,tag)                    B7(BR_IH_NONE,0x10,b2,tag,BR_INDWH_SPTK)
 627 #define BRPI_RET(b2,tag)                B7(BR_IH_NONE,0x11,b2,tag,BR_INDWH_SPTK)
 628 /* bsw */
 629 #define BSW_0()                         B8(0x0c)
 630 #define BSW_1()                         B8(0x0d)
 631 /* chk */
 632 #define CHK_S_I(r2,im)                  I20(r2,im)
 633 #define CHK_S_M(r2,im)                  M20(r2,im)
 634 #define CHK_A_NC(r1,im)                 M22(0x4,im,r1)
 635 #define CHK_A_CLR(r1,im)                M22(0x5,im,r1)
 636 /* clrrrb */
 637 #define CLRRRB()                        B8(0x04)
 638 #define CLRRRB_PR()                     B8(0x05)
 639 /* clz */
 640 #define CLZ(r1,r3)                      I9(3,r3,r1)
 641 /* cmp */
 642 #define CMP_LT(p1,p2,r2,r3)             A6(0xc,0,0,p2,r3,r2,0,p1)
 643 #define CMP_LT_p(p1,p2,r2,r3,_p)        A6(_jit,_p,0xc,0,0,p2,r3,r2,0,p1)
 644 #define CMP_LTU(p1,p2,r2,r3)            A6(0xd,0,0,p2,r3,r2,0,p1)
 645 #define CMP_EQ(p1,p2,r2,r3)             A6(0xe,0,0,p2,r3,r2,0,p1)
 646 #define CMP_LT_UNC(p1,p2,r2,r3)         A6(0xc,0,0,p2,r3,r2,1,p1)
 647 #define CMP_LTU_UNC(p1,p2,r2,r3)        A6(0xd,0,0,p2,r3,r2,1,p1)
 648 #define CMP_EQ_UNC(p1,p2,r2,r3)         A6(0xe,0,0,p2,r3,r2,1,p1)
 649 #define CMP_EQ_AND(p1,p2,r2,r3)         A6(0xc,0,1,p2,r3,r2,0,p1)
 650 #define CMP_EQ_OR(p1,p2,r2,r3)          A6(0xd,0,1,p2,r3,r2,0,p1)
 651 #define CMP_EQ_OR_ANDCM(p1,p2,r2,r3)    A6(0xe,0,1,p2,r3,r2,0,p1)
 652 #define CMP_NE_AND(p1,p2,r2,r3)         A6(0xc,0,1,p2,r3,r2,1,p1)
 653 #define CMP_NE_OR(p1,p2,r2,r3)          A6(0xd,0,1,p2,r3,r2,1,p1)
 654 #define CMP_NE_OR_ANDCM(p1,p2,r2,r3)    A6(0xe,0,1,p2,r3,r2,1,p1)
 655 #define CMPI_LT(p1,p2,im,r3)            A8(0xc,2,0,p2,r3,im,0,p1)
 656 #define CMPI_LTU(p1,p2,im,r3)           A8(0xd,2,0,p2,r3,im,0,p1)
 657 #define CMPI_EQ(p1,p2,im,r3)            A8(0xe,2,0,p2,r3,im,0,p1)
 658 #define CMPI_EQ_p(p1,p2,im,r3,_p)       _A8(_jit,_p,0xe,2,0,p2,r3,im,0,p1)
 659 #define CMPI_LT_UNC(p1,p2,im,r3)        A8(0xc,2,0,p2,r3,im,1,p1)
 660 #define CMPI_LTU_UNC(p1,p2,im,r3)       A8(0xd,2,0,p2,r3,im,1,p1)
 661 #define CMPI_EQ_UNC(p1,p2,im,r3)        A8(0xe,2,0,p2,r3,im,1,p1)
 662 #define CMPI_EQ_AND(p1,p2,im,r3)        A8(0xc,2,1,p2,r3,im,0,p1)
 663 #define CMPI_EQ_OR(p1,p2,im,r3)         A8(0xd,2,1,p2,r3,im,0,p1)
 664 #define CMPI_EQ_ANDCM(p1,p2,im,r3)      A8(0xe,2,1,p2,r3,im,0,p1)
 665 #define CMPI_NE_AND(p1,p2,im,r3)        A8(0xc,2,1,p2,r3,im,1,p1)
 666 #define CMPI_NE_OR(p1,p2,im,r3)         A8(0xd,2,1,p2,r3,im,1,p1)
 667 #define CMPI_NE_ANDCM(p1,p2,im,r3)      A8(0xe,2,1,p2,r3,im,1,p1)
 668 #define ZCMP_GT_AND(p1,p2,r3)           A7(0xc,0,0,p2,r3,0,p1)
 669 #define ZCMP_GT_OR(p1,p2,r3)            A7(0xd,0,0,p2,r3,0,p1)
 670 #define ZCMP_GT_ANDCM(p1,p2,r3)         A7(0xe,0,0,p2,r3,0,p1)
 671 #define ZCMP_LE_AND(p1,p2,r3)           A7(0xc,0,0,p2,r3,1,p1)
 672 #define ZCMP_LE_OR(p1,p2,r3)            A7(0xd,0,0,p2,r3,1,p1)
 673 #define ZCMP_LE_ANDCM(p1,p2,r3)         A7(0xe,0,0,p2,r3,1,p1)
 674 #define ZCMP_GE_AND(p1,p2,r3)           A7(0xc,0,1,p2,r3,0,p1)
 675 #define ZCMP_GE_OR(p1,p2,r3)            A7(0xd,0,1,p2,r3,0,p1)
 676 #define ZCMP_GE_ANDCM(p1,p2,r3)         A7(0xe,0,1,p2,r3,0,p1)
 677 #define ZCMP_LT_AND(p1,p2,r3)           A7(0xc,0,1,p2,r3,1,p1)
 678 #define ZCMP_LT_OR(p1,p2,r3)            A7(0xd,0,1,p2,r3,1,p1)
 679 #define ZCMP_LT_ANDCM(p1,p2,r3)         A7(0xe,0,1,p2,r3,1,p1)
 680 /* cmp4 */
 681 #define CMP4_LT(p1,p2,r2,r3)            A6(0xc,1,0,p2,r3,r2,0,p1)
 682 #define CMP4_LTU(p1,p2,r2,r3)           A6(0xd,1,0,p2,r3,r2,0,p1)
 683 #define CMP4_EQ(p1,p2,r2,r3)            A6(0xe,1,0,p2,r3,r2,0,p1)
 684 #define CMP4_LT_UNC(p1,p2,r2,r3)        A6(0xc,1,0,p2,r3,r2,1,p1)
 685 #define CMP4_LTU_UNC(p1,p2,r2,r3)       A6(0xd,1,0,p2,r3,r2,1,p1)
 686 #define CMP4_EQ_UNC(p1,p2,r2,r3)        A6(0xe,1,0,p2,r3,r2,1,p1)
 687 #define CMP4_EQ_AND(p1,p2,r2,r3)        A6(0xc,1,1,p2,r3,r2,0,p1)
 688 #define CMP4_EQ_OR(p1,p2,r2,r3)         A6(0xd,1,1,p2,r3,r2,0,p1)
 689 #define CMP4_EQ_XOR(p1,p2,r2,r3)        A6(0xe,1,1,p2,r3,r2,0,p1)
 690 #define CMP4_NE_AND(p1,p2,r2,r3)        A6(0xc,1,1,p2,r3,r2,1,p1)
 691 #define CMP4_NE_OR(p1,p2,r2,r3)         A6(0xd,1,1,p2,r3,r2,1,p1)
 692 #define CMP4_NE_XOR(p1,p2,r2,r3)        A6(0xe,1,1,p2,r3,r2,1,p1)
 693 #define CMP4I_LT(p1,p2,im,r3)           A8(0xc,3,0,p2,r3,im,0,p1)
 694 #define CMP4I_LTU(p1,p2,im,r3)          A8(0xd,3,0,p2,r3,im,0,p1)
 695 #define CMP4I_EQ(p1,p2,im,r3)           A8(0xe,3,0,p2,r3,im,0,p1)
 696 #define CMP4I_LT_UNC(p1,p2,im,r3)       A8(0xc,3,0,p2,r3,im,1,p1)
 697 #define CMP4I_LTU_UNC(p1,p2,im,r3)      A8(0xd,3,0,p2,r3,im,1,p1)
 698 #define CMP4I_EQ_UNC(p1,p2,im,r3)       A8(0xe,3,0,p2,r3,im,1,p1)
 699 #define CMP4I_EQ_AND(p1,p2,im,r3)       A8(0xc,3,1,p2,r3,im,0,p1)
 700 #define CMP4I_EQ_OR(p1,p2,im,r3)        A8(0xd,3,1,p2,r3,im,0,p1)
 701 #define CMP4I_EQ_ANDCM(p1,p2,im,r3)     A8(0xe,3,1,p2,r3,im,0,p1)
 702 #define CMP4I_NE_AND(p1,p2,im,r3)       A8(0xc,3,1,p2,r3,im,1,p1)
 703 #define CMP4I_NE_OR(p1,p2,im,r3)        A8(0xd,3,1,p2,r3,im,1,p1)
 704 #define CMP4I_NE_ANDCM(p1,p2,im,r3)     A8(0xe,3,1,p2,r3,im,1,p1)
 705 #define ZCMP4_GT_AND(p1,p2,r3)          A7(0xc,1,0,p2,r3,0,p1)
 706 #define ZCMP4_GT_OR(p1,p2,r3)           A7(0xd,1,0,p2,r3,0,p1)
 707 #define ZCMP4_GT_ANDCM(p1,p2,r3)        A7(0xe,1,0,p2,r3,0,p1)
 708 #define ZCMP4_LE_AND(p1,p2,r3)          A7(0xc,1,0,p2,r3,1,p1)
 709 #define ZCMP4_LE_OR(p1,p2,r3)           A7(0xd,1,0,p2,r3,1,p1)
 710 #define ZCMP4_LE_ANDCM(p1,p2,r3)        A7(0xe,1,0,p2,r3,1,p1)
 711 #define ZCMP4_GE_AND(p1,p2,r3)          A7(0xc,1,1,p2,r3,0,p1)
 712 #define ZCMP4_GE_OR(p1,p2,r3)           A7(0xd,1,1,p2,r3,0,p1)
 713 #define ZCMP4_GE_ANDCM(p1,p2,r3)        A7(0xe,1,1,p2,r3,0,p1)
 714 #define ZCMP4_LT_AND(p1,p2,r3)          A7(0xc,1,1,p2,r3,1,p1)
 715 #define ZCMP4_LT_OR(p1,p2,r3)           A7(0xd,1,1,p2,r3,1,p1)
 716 #define ZCMP4_LT_ANDCM(p1,p2,r3)        A7(0xe,1,1,p2,r3,1,p1)
 717 /* cmpxchg */
 718 #define CMPXCHG1_ACQ(r1,r3,r2)          M16(0x00,LD_NONE,r3,r2,r1)
 719 #define CMPXCHG2_ACQ(r1,r3,r2)          M16(0x01,LD_NONE,r3,r2,r1)
 720 #define CMPXCHG4_ACQ(r1,r3,r2)          M16(0x02,LD_NONE,r3,r2,r1)
 721 #define CMPXCHG8_ACQ(r1,r3,r2)          M16(0x03,LD_NONE,r3,r2,r1)
 722 #define CMPXCHG1_REL(r1,r3,r2)          M16(0x04,LD_NONE,r3,r2,r1)
 723 #define CMPXCHG2_REL(r1,r3,r2)          M16(0x05,LD_NONE,r3,r2,r1)
 724 #define CMPXCHG4_REL(r1,r3,r2)          M16(0x06,LD_NONE,r3,r2,r1)
 725 #define CMPXCHG8_REL(r1,r3,r2)          M16(0x07,LD_NONE,r3,r2,r1)
 726 #define CMP8XCHG16_ACQ(r1,r3,r2)        M16(0x20,LD_NONE,r3,r2,r1)
 727 #define CMP8XCHG16_REL(r1,r3,r2)        M16(0x24,LD_NONE,r3,r2,r1)
 728 /* cover */
 729 #define COVER()                         B8(0x02)
 730 /* cxz */
 731 #define CZX1_L(r1,r3)                   I29(0x18,r3,r1)
 732 #define CZX2_L(r1,r3)                   I29(0x19,r3,r1)
 733 #define CZX1_R(r1,r3)                   I29(0x1c,r3,r1)
 734 #define CZX2_R(r1,r3)                   I29(0x1d,r3,r1)
 735 /* dep */
 736 #define DEP_Z(r1,r2,pos,len)            I12(len,pos,r2,r1)
 737 #define DEPI_Z(r1,im,pos,len)           I13(len,pos,im,r1)
 738 #define DEPs(r1,r2,r3,pos,len)          I14(1,len,r3,pos,r1)
 739 #define DEPu(r1,r2,r3,pos,len)          I14(0,len,r3,pos,r1)
 740 #define DEP(r1,r2,r3,pos,len)           I15(pos,len,r3,r2,r1)
 741 /* epc */
 742 #define EPC()                           B8(0x10)
 743 /* extr */
 744 #define EXTR(r1,r3,pos,len)             I11(len,r3,pos,1,r1)
 745 #define EXTR_U(r1,r3,pos,len)           I11(len,r3,pos,0,r1)
 746 /* fc */
 747 #define FC(r3)                          M28(0,r3)
 748 #define FC_I(r3)                        M28(1,r3)
 749 /* fetchadd */
 750 #define FETCHADD4_ACQ(r1,r3,im)         M17(0x12,LD_NONE,r3,im,r1)
 751 #define FETCHADD8_ACQ(r1,r3,im)         M17(0x13,LD_NONE,r3,im,r1)
 752 #define FETCHADD4_REL(r1,r3,im)         M17(0x16,LD_NONE,r3,im,r1)
 753 #define FETCHADD8_REL(r1,r3,im)         M17(0x17,LD_NONE,r3,im,r1)
 754 /* flushrs */
 755 #define FLUSHRS()                       M25(0xc)
 756 /* fwb */
 757 #define FWB()                           M24(2,0)
 758 /* hint */
 759 #define HINT_I(im)                      I18(im,1)
 760 #define HINT_M(im)                      M48(1,im)
 761 #define HINT_B(im)                      B9(2,1,im)
 762 #define HINT_X(im)                      X5(1,im)
 763 /* invala */
 764 #define INVALA()                        M24(1,0)
 765 #define INVALA_E(r1)                    M26(r1)
 766 /* itc */
 767 #define ITC_I(r2)                       M41(0x2f,r2)
 768 #define ITC_D(r2)                       M41(0x2e,r2)
 769 /* itr */
 770 #define ITR_I(r3,r2)                    M42(0x0f,r3,r2)
 771 #define ITR_D(r3,r2)                    M42(0x0e,r3,r2)
 772 /* ld */
 773 #define LD1(r1,r3)                      M1(0x00,LD_NONE,0,r3,r1)
 774 #define LD2(r1,r3)                      M1(0x01,LD_NONE,0,r3,r1)
 775 #define LD4(r1,r3)                      M1(0x02,LD_NONE,0,r3,r1)
 776 #define LD8(r1,r3)                      M1(0x03,LD_NONE,0,r3,r1)
 777 #define LD1_S(r1,r3)                    M1(0x04,LD_NONE,0,r3,r1)
 778 #define LD2_S(r1,r3)                    M1(0x05,LD_NONE,0,r3,r1)
 779 #define LD4_S(r1,r3)                    M1(0x06,LD_NONE,0,r3,r1)
 780 #define LD8_S(r1,r3)                    M1(0x07,LD_NONE,0,r3,r1)
 781 #define LD1_A(r1,r3)                    M1(0x08,LD_NONE,0,r3,r1)
 782 #define LD2_A(r1,r3)                    M1(0x09,LD_NONE,0,r3,r1)
 783 #define LD4_A(r1,r3)                    M1(0x0a,LD_NONE,0,r3,r1)
 784 #define LD8_A(r1,r3)                    M1(0x0b,LD_NONE,0,r3,r1)
 785 #define LD1_SA(r1,r3)                   M1(0x0c,LD_NONE,0,r3,r1)
 786 #define LD2_SA(r1,r3)                   M1(0x0d,LD_NONE,0,r3,r1)
 787 #define LD4_SA(r1,r3)                   M1(0x0e,LD_NONE,0,r3,r1)
 788 #define LD8_SA(r1,r3)                   M1(0x0f,LD_NONE,0,r3,r1)
 789 #define LD1_BIAS(r1,r3)                 M1(0x10,LD_NONE,0,r3,r1)
 790 #define LD2_BIAS(r1,r3)                 M1(0x11,LD_NONE,0,r3,r1)
 791 #define LD4_BIAS(r1,r3)                 M1(0x12,LD_NONE,0,r3,r1)
 792 #define LD8_BIAS(r1,r3)                 M1(0x13,LD_NONE,0,r3,r1)
 793 #define LD1_ACQ(r1,r3)                  M1(0x14,LD_NONE,0,r3,r1)
 794 #define LD2_ACQ(r1,r3)                  M1(0x15,LD_NONE,0,r3,r1)
 795 #define LD4_ACQ(r1,r3)                  M1(0x16,LD_NONE,0,r3,r1)
 796 #define LD8_ACQ(r1,r3)                  M1(0x17,LD_NONE,0,r3,r1)
 797 #define LD8_FILL(r1,r3)                 M1(0x1b,LD_NONE,0,r3,r1)
 798 #define LD1_C_CLR(r1,r3)                M1(0x20,LD_NONE,0,r3,r1)
 799 #define LD2_C_CLR(r1,r3)                M1(0x21,LD_NONE,0,r3,r1)
 800 #define LD4_C_CLR(r1,r3)                M1(0x22,LD_NONE,0,r3,r1)
 801 #define LD8_C_CLR(r1,r3)                M1(0x23,LD_NONE,0,r3,r1)
 802 #define LD1_C_NC(r1,r3)                 M1(0x24,LD_NONE,0,r3,r1)
 803 #define LD2_C_NC(r1,r3)                 M1(0x25,LD_NONE,0,r3,r1)
 804 #define LD4_C_NC(r1,r3)                 M1(0x26,LD_NONE,0,r3,r1)
 805 #define LD8_C_NC(r1,r3)                 M1(0x27,LD_NONE,0,r3,r1)
 806 #define LD1_C_CLR_ACQ(r1,r3)            M1(0x28,LD_NONE,0,r3,r1)
 807 #define LD2_C_CLR_ACQ(r1,r3)            M1(0x29,LD_NONE,0,r3,r1)
 808 #define LD4_C_CLR_ACQ(r1,r3)            M1(0x2a,LD_NONE,0,r3,r1)
 809 #define LD8_C_CLR_ACQ(r1,r3)            M1(0x2b,LD_NONE,0,r3,r1)
 810 #define LD16(r1,r3)                     M1(0x28,LD_NONE,1,r3,r1)
 811 #define LD16_ACQ(r1,r3)                 M1(0x2c,LD_NONE,1,r3,r1)
 812 #define LD1_inc(r1,r3,im)               M3(0x00,LD_NONE,r3,im,r1)
 813 #define LD2_inc(r1,r3,im)               M3(0x01,LD_NONE,r3,im,r1)
 814 #define LD4_inc(r1,r3,im)               M3(0x02,LD_NONE,r3,im,r1)
 815 #define LD8_inc(r1,r3,im)               M3(0x03,LD_NONE,r3,im,r1)
 816 #define LD1_S_inc(r1,r3,im)             M3(0x04,LD_NONE,r3,im,r1)
 817 #define LD2_S_inc(r1,r3,im)             M3(0x05,LD_NONE,r3,im,r1)
 818 #define LD4_S_inc(r1,r3,im)             M3(0x06,LD_NONE,r3,im,r1)
 819 #define LD8_S_inc(r1,r3,im)             M3(0x07,LD_NONE,r3,im,r1)
 820 #define LD1_A_inc(r1,r3,im)             M3(0x08,LD_NONE,r3,im,r1)
 821 #define LD2_A_inc(r1,r3,im)             M3(0x09,LD_NONE,r3,im,r1)
 822 #define LD4_A_inc(r1,r3,im)             M3(0x0a,LD_NONE,r3,im,r1)
 823 #define LD8_A_inc(r1,r3,im)             M3(0x0b,LD_NONE,r3,im,r1)
 824 #define LD1_SA_inc(r1,r3,im)            M3(0x0c,LD_NONE,r3,im,r1)
 825 #define LD2_SA_inc(r1,r3,im)            M3(0x0d,LD_NONE,r3,im,r1)
 826 #define LD4_SA_inc(r1,r3,im)            M3(0x0e,LD_NONE,r3,im,r1)
 827 #define LD8_SA_inc(r1,r3,im)            M3(0x0f,LD_NONE,r3,im,r1)
 828 #define LD1_BIAS_inc(r1,r3,im)          M3(0x10,LD_NONE,r3,im,r1)
 829 #define LD2_BIAS_inc(r1,r3,im)          M3(0x11,LD_NONE,r3,im,r1)
 830 #define LD4_BIAS_inc(r1,r3,im)          M3(0x12,LD_NONE,r3,im,r1)
 831 #define LD8_BIAS_inc(r1,r3,im)          M3(0x13,LD_NONE,r3,im,r1)
 832 #define LD1_ACQ_inc(r1,r3,im)           M3(0x14,LD_NONE,r3,im,r1)
 833 #define LD2_ACQ_inc(r1,r3,im)           M3(0x15,LD_NONE,r3,im,r1)
 834 #define LD4_ACQ_inc(r1,r3,im)           M3(0x16,LD_NONE,r3,im,r1)
 835 #define LD8_AVQ_inc(r1,r3,im)           M3(0x17,LD_NONE,r3,im,r1)
 836 #define LD8_FILL_inc(r1,r3,im)          M3(0x1b,LD_NONE,r3,im,r1)
 837 #define LD1_C_CLR_inc(r1,r3,im)         M3(0x20,LD_NONE,r3,im,r1)
 838 #define LD2_C_CLR_inc(r1,r3,im)         M3(0x21,LD_NONE,r3,im,r1)
 839 #define LD4_C_CLR_inc(r1,r3,im)         M3(0x22,LD_NONE,r3,im,r1)
 840 #define LD8_C_CLR_inc(r1,r3,im)         M3(0x23,LD_NONE,r3,im,r1)
 841 #define LD1_C_NC_inc(r1,r3,im)          M3(0x24,LD_NONE,r3,im,r1)
 842 #define LD2_C_NC_inc(r1,r3,im)          M3(0x25,LD_NONE,r3,im,r1)
 843 #define LD4_C_NC_inc(r1,r3,im)          M3(0x26,LD_NONE,r3,im,r1)
 844 #define LD8_C_NC_inc(r1,r3,im)          M3(0x27,LD_NONE,r3,im,r1)
 845 #define LD1_C_CLR_ACQ_inc(r1,r3,im)     M3(0x28,LD_NONE,r3,im,r1)
 846 #define LD2_C_CLR_ACQ_inc(r1,r3,im)     M3(0x29,LD_NONE,r3,im,r1)
 847 #define LD4_C_CLR_ACQ_inc(r1,r3,im)     M3(0x2a,LD_NONE,r3,im,r1)
 848 #define LD8_C_CLR_ACQ_inc(r1,r3,im)     M3(0x2b,LD_NONE,r3,im,r1)
 849 #define LDX1(r1,r3,r2)                  M2(0x00,LD_NONE,r3,r2,r1)
 850 #define LDX2(r1,r3,r2)                  M2(0x01,LD_NONE,r3,r2,r1)
 851 #define LDX4(r1,r3,r2)                  M2(0x02,LD_NONE,r3,r2,r1)
 852 #define LDX8(r1,r3,r2)                  M2(0x03,LD_NONE,r3,r2,r1)
 853 #define LDX1_S(r1,r3,r2)                M2(0x04,LD_NONE,r3,r2,r1)
 854 #define LDX2_S(r1,r3,r2)                M2(0x05,LD_NONE,r3,r2,r1)
 855 #define LDX4_S(r1,r3,r2)                M2(0x06,LD_NONE,r3,r2,r1)
 856 #define LDX8_S(r1,r3,r2)                M2(0x07,LD_NONE,r3,r2,r1)
 857 #define LDX1_A(r1,r3,r2)                M2(0x08,LD_NONE,r3,r2,r1)
 858 #define LDX2_A(r1,r3,r2)                M2(0x09,LD_NONE,r3,r2,r1)
 859 #define LDX4_A(r1,r3,r2)                M2(0x0a,LD_NONE,r3,r2,r1)
 860 #define LDX8_A(r1,r3,r2)                M2(0x0b,LD_NONE,r3,r2,r1)
 861 #define LDX1_SA(r1,r3,r2)               M2(0x0c,LD_NONE,r3,r2,r1)
 862 #define LDX2_SA(r1,r3,r2)               M2(0x0d,LD_NONE,r3,r2,r1)
 863 #define LDX4_SA(r1,r3,r2)               M2(0x0e,LD_NONE,r3,r2,r1)
 864 #define LDX8_SA(r1,r3,r2)               M2(0x0f,LD_NONE,r3,r2,r1)
 865 #define LDX1_BIAS(r1,r3,r2)             M2(0x10,LD_NONE,r3,r2,r1)
 866 #define LDX2_BIAS(r1,r3,r2)             M2(0x11,LD_NONE,r3,r2,r1)
 867 #define LDX4_BIAS(r1,r3,r2)             M2(0x12,LD_NONE,r3,r2,r1)
 868 #define LDX8_BIAS(r1,r3,r2)             M2(0x13,LD_NONE,r3,r2,r1)
 869 #define LDX1_ACQ(r1,r3,r2)              M2(0x14,LD_NONE,r3,r2,r1)
 870 #define LDX2_ACQ(r1,r3,r2)              M2(0x15,LD_NONE,r3,r2,r1)
 871 #define LDX4_ACQ(r1,r3,r2)              M2(0x16,LD_NONE,r3,r2,r1)
 872 #define LDX8_ACQ(r1,r3,r2)              M2(0x17,LD_NONE,r3,r2,r1)
 873 #define LDX8_FILL(r1,r3,r2)             M2(0x1b,LD_NONE,r3,r2,r1)
 874 #define LDX1_C_CLR(r1,r3,r2)            M2(0x20,LD_NONE,r3,r2,r1)
 875 #define LDX2_C_CLR(r1,r3,r2)            M2(0x21,LD_NONE,r3,r2,r1)
 876 #define LDX4_C_CLR(r1,r3,r2)            M2(0x22,LD_NONE,r3,r2,r1)
 877 #define LDX8_C_CLR(r1,r3,r2)            M2(0x23,LD_NONE,r3,r2,r1)
 878 #define LDX1_C_NC(r1,r3,r2)             M2(0x24,LD_NONE,r3,r2,r1)
 879 #define LDX2_C_NC(r1,r3,r2)             M2(0x25,LD_NONE,r3,r2,r1)
 880 #define LDX4_C_NC(r1,r3,r2)             M2(0x26,LD_NONE,r3,r2,r1)
 881 #define LDX8_C_NC(r1,r3,r2)             M2(0x27,LD_NONE,r3,r2,r1)
 882 #define LDX1_C_CLR_ACQ(r1,r3,r2)        M2(0x28,LD_NONE,r3,r2,r1)
 883 #define LDX2_C_CLR_ACQ(r1,r3,r2)        M2(0x29,LD_NONE,r3,r2,r1)
 884 #define LDX4_C_CLR_ACQ(r1,r3,r2)        M2(0x2a,LD_NONE,r3,r2,r1)
 885 #define LDX8_C_CLR_ACQ(r1,r3,r2)        M2(0x2b,LD_NONE,r3,r2,r1)
 886 /* lfetch */
 887 #define LFETCH_EXCL(r3)                 M13(0x2d,LF_NONE,r3,GR_0)
 888 #define LFETCH_FAULT(r3)                M13(0x2e,LF_NONE,r3,GR_0)
 889 #define LFETCH_FAULT_EXCL(r3)           M13(0x2f,LF_NONE,r3,GR_0)
 890 #define LXFETCH(r3,r2)                  M14(0x2c,LF_NONE,r3,r2)
 891 #define LXFETCH_EXCL(r3,r2)             M14(0x2d,LF_NONE,r3,r2)
 892 #define LXFETCH_FAULT(r3,r2)            M14(0x2e,LF_NONE,r3,r2)
 893 #define LXFETCH_FAULT_EXCL(r3,r2)       M14(0x2f,LF_NONE,r3,r2)
 894 #define LFETCHI(r3,im)                  M15(0x2c,LF_NONE,r3,im)
 895 #define LFETCHI_EXCL(r3,im)             M15(0x2d,LF_NONE,r3,im)
 896 #define LFETCHI_FAULT(r3,im)            M15(0x2e,LF_NONE,r3,im)
 897 #define LFETCHI_FAULT_EXCL(r3,im)       M15(0x2f,LF_NONE,r3,im)
 898 /* loadrs */
 899 #define LOADRS()                        M25(0xa)
 900 /* mf */
 901 #define MF()                            M24(2,2)
 902 #define MF_A()                          M24(2,3)
 903 /* mix */
 904 #define MIX1_R(r1,r2,r3)                I2(0,2,0,2,0,r3,r2,r1)
 905 #define MIX2_R(r1,r2,r3)                I2(0,2,1,2,0,r3,r2,r1)
 906 #define MIX4_R(r1,r2,r3)                I2(1,2,0,2,0,r3,r2,r1)
 907 #define MIX1_L(r1,r2,r3)                I2(0,2,0,2,2,r3,r2,r1)
 908 #define MIX2_L(r1,r2,r3)                I2(0,2,1,2,2,r3,r2,r1)
 909 #define MIX4_L(r1,r2,r3)                I2(1,2,0,2,2,r3,r2,r1)
 910 /* mov - Move Application Register */
 911 #define MOV_I_rn_ar(r1,ar)              I28(ar,r1)
 912 #define MOV_I_ar_rn(ar,r2)              I26(ar,r2)
 913 #define MOV_I_ar_im(ar,im)              I27(ar,im)
 914 #define MOV_M_rn_a(r1,ar)               M31(r1,ar)
 915 #define MOV_M_ar_rn(ar,r2)              M29(ar,r2)
 916 #define MOV_M_ar_im(ar,im)              M30(ar,im)
 917 /* mov - Move Branch Register */
 918 #define MOV_rn_br(r1,b2)                I22(b2,r1)
 919 #define MOV_br_rn_tg(b1,r2,tag)         I21(tag,IH_NONE,0,MWH_NONE,r2,b1)
 920 #define MOV_br_rn(b1,r2)                MOV_br_rn_tg(b1,r2,0)
 921 #define MOV_RET_br_rn_tg(b1,r2,tag)     I21(tag,IH_NONE,1,MWH_NONE,r2,b1)
 922 /* mov - Move Control Register */
 923 #define MOV_rn_cr(cr,r1)                M33(cr,r1)
 924 #define MOV_cr_rr(cr,r2)                M32(cr,r2)
 925 /* mov - Move General Register */
 926 #define MOV(r0,r1)                      ADDS(r0,0,r1)
 927 #define MOV_p(r0,r1,_p)                 ADDS_p(r0,0,r1,_p)
 928 /* mov - Move Immediate */
 929 #define MOVI(r1,im)                     ADDL(r1,im,GR_0)
 930 #define MOVI_p(r1,im,_p)                ADDL_p(r1,im,GR_0,_p)
 931 /* mov - Move Indirect Register */
 932 #define MOV_rn_RR(r1,r3)                M43(0x10,r3,r1)
 933 #define MOV_rn_DBR(r1,r3)               M43(0x11,r3,r1)
 934 #define MOV_rn_IBR(r1,r3)               M43(0x012,r3,r1)
 935 #define MOV_rn_PKR(r1,r3)               M43(0x13,r3,r1)
 936 #define MOV_rn_PMC(r1,r3)               M43(0x14,r3,r1)
 937 #define MOV_rn_PMD(r1,r3)               M43(0x15,r3,r1)
 938 #define MOV_rn_CPUID(r1,r3)             M43(0x17,r3,r1)
 939 #define MOV_RR_rn(r3,r2)                M42(0x00,r3,r2)
 940 #define MOV_DBR_rn(r3,r2)               M42(0x01,r3,r2)
 941 #define MOV_IBR_rn(r3,r2)               M42(0x02,r3,r2)
 942 #define MOV_PKR_rn(r3,r2)               M42(0x03,r3,r2)
 943 #define MOV_PMC_rn(r3,r2)               M42(0x04,r3,r2)
 944 #define MOV_PMD_rn(r3,r2)               M42(0x05,r3,r2)
 945 /* mov - Move Instruction Pointer */
 946 #define MOV_rn_ip(r1)                   I25(0x30,r1)
 947 /* mov - Move Predicates */
 948 #define MOV_rn_pr(r1)                   I25(0x33,r1)
 949 #define MOV_pr_rn(r2,im)                I23(r2,im)
 950 #define MOVI_pr(im)                     I24(im)
 951 /* mov - Move Processor Status Register */
 952 #define MOV_rn_psr(r1)                  M36(0x25,r1)
 953 #define MOV_psr_l_rn(r2)                M35(0x2d,r2)
 954 /* mov - Move User Mask */
 955 #define MOV_rn_psr_um(r1)               M36(0x21,r1)
 956 #define MOV_psr_um_rn(r2)               M35(0x29,r2)
 957 /* movl */
 958 #define MOVL(r1,im)                     X2(r1,im)
 959 /* mpy4 */
 960 #define MPY4(r1,r2,r3)                  I2(1,0,0,3,1,r3,r2,r1)
 961 /* mpyshl4 */
 962 #define MPYSHL4(r1,r2,r3)               I2(1,0,0,3,3,r3,r2,r1)
 963 /* mux */
 964 #define MUX1(r1,r2,mbt)                 I3(mbt,r2,r1)
 965 #define MUX2(r1,r2,mht)                 I4(mht,r2,r1)
 966 /* nop */
 967 #define NOP_I(im)                       I18(im,0)
 968 #define NOP_M(im)                       M48(0,im)
 969 #define NOP_B(im)                       B9(2,0,im)
 970 #define NOP_X(im)                       X5(0,im)
 971 /* or */
 972 #define OR(r1,r2,r3)                    A1(3,2,r3,r2,r1)
 973 #define ORI(r1,im,r3)                   A3(0xb,2,r3,im,r1)
 974 /* pack */
 975 #define PACK2_USS(r1,r2,r3)             I2(0,2,1,0,0,r3,r2,r1)
 976 #define PACK2_SSS(r1,r2,r3)             I2(0,2,1,0,2,r3,r2,r1)
 977 #define PACK4_SSS(r1,r2,r3)             I2(1,2,0,0,2,r3,r2,r1)
 978 /* padd */
 979 #define PADD1(r1,r2,r3)                 A9(0,0,0,0,r3,r2,r1)
 980 #define PADD1_SSS(r1,r2,r3)             A9(0,0,0,1,r3,r2,r1)
 981 #define PADD1_UUU(r1,r2,r3)             A9(0,0,0,2,r3,r2,r1)
 982 #define PADD1_UUS(r1,r2,r3)             A9(0,0,0,3,r3,r2,r1)
 983 #define PADD2(r1,r2,r3)                 A9(0,1,0,0,r3,r2,r1)
 984 #define PADD2_SSS(r1,r2,r3)             A9(0,1,0,1,r3,r2,r1)
 985 #define PADD2_UUU(r1,r2,r3)             A9(0,1,0,2,r3,r2,r1)
 986 #define PADD2_UUS(r1,r2,r3)             A9(0,1,0,3,r3,r2,r1)
 987 #define PADD4(r1,r2,r3)                 A9(1,0,0,0,r3,r2,r1)
 988 /* pavg */
 989 #define PAVG1(r1,r2,r3)                 A9(0,0,2,2,r3,r2,r1)
 990 #define PAVG2(r1,r2,r3)                 A9(0,1,2,2,r3,r2,r1)
 991 #define PAVG1_RAZ(r1,r2,r3)             A9(0,0,2,3,r3,r2,r1)
 992 #define PAVG2_RAZ(r1,r2,r3)             A9(0,1,2,3,r3,r2,r1)
 993 /* pavgsub */
 994 #define PAVGSUB1(r1,r2,r3)              A9(0,0,3,2,r3,r2,r1)
 995 #define PAVGSUB2(r1,r2,r3)              A9(0,1,3,2,r3,r2,r1)
 996 /* pcmp */
 997 #define PCMP1_EQ(r1,r2,r3)              A9(0,0,9,0,r3,r2,r1)
 998 #define PCMP2_EQ(r1,r2,r3)              A9(0,1,9,0,r3,r2,r1)
 999 #define PCMP4_EQ(r1,r2,r3)              A9(1,0,9,0,r3,r2,r1)
1000 #define PCMP1_GT(r1,r2,r3)              A9(0,0,9,1,r3,r2,r1)
1001 #define PCMP2_GT(r1,r2,r3)              A9(0,1,9,1,r3,r2,r1)
1002 #define PCMP4_GT(r1,r2,r3)              A9(1,0,9,1,r3,r2,r1)
1003 /* pmax */
1004 #define PMAX1_U(r1,r2,r3)               I2(0,2,0,1,1,r3,r2,r1)
1005 #define PMAX2(r1,r2,r3)                 I2(0,2,1,1,3,r3,r2,r1)
1006 /* pmin */
1007 #define PMIN1_U(r1,r2,r3)               I2(0,2,0,0,1,r3,r2,r1)
1008 #define PMIN2(r1,r2,r3)                 I2(0,2,1,0,3,r3,r2,r1)
1009 /* pmpy */
1010 #define PMPY2_R(r1,r2,r3)               I2(0,2,1,3,1,r3,r2,r1)
1011 #define PMPY2_L(r1,r2,r3)               I2(0,2,1,3,3,r3,r2,r1)
1012 /* pmpyshr */
1013 #define PMPYSHR2(r1,r2,r3,im)           I1(im,3,r3,r2,r1)
1014 #define PMPYSHR2_U(r1,r2,r3,im)         I1(im,1,r3,r2,r1)
1015 /* popcnt */
1016 #define POPCNT(r1,r3)                   I9(2,r3,r1)
1017 /* probe */
1018 #define PROBE_R(r1,r3,r2)               M38(0x38,r3,r2,r1)
1019 #define PROBE_W(r1,r3,r2)               M38(0x39,r3,r2,r1)
1020 #define PROBEI_R(r1,r3,im)              M39(0x18,r3,im,r1)
1021 #define PROBEI_W(r1,r3,im)              M39(0x19,r3,im,r1)
1022 #define PROBE_RW_FAULT(r3,im)           M40(0x31,r3,im)
1023 #define PROBE_R_FAULT(r3,im)            M40(0x32,r3,im)
1024 #define PROBE_W_FAULT(r3,im)            M40(0x33,r3,im)
1025 /* psad */
1026 #define PSAD1(r1,r2,r3)                 I2(0,2,0,2,3,r3,r2,r1)
1027 /* pshl */
1028 #define PSHL2(r1,r2,r3)                 I7(0,1,r3,r2,r1)
1029 #define PSHL4(r1,r2,r3)                 I7(1,0,r3,r2,r1)
1030 #define PSHL2I(r1,r2,im)                I8(0,1,im,r2,r1)
1031 #define PSHL4I(r1,r2,im)                I8(1,0,im,r2,r1)
1032 /* pshladd */
1033 #define PSHLADD2(r1,r2,im,r3)           A10(4,im,r3,r2,r1)
1034 /* pshr */
1035 #define PSHR2(r1,r3,r2)                 I5(0,1,2,r3,r2,r1)
1036 #define PSHR2I(r1,r3,im)                I6(0,1,3,r3,im,r1)
1037 #define PSHR2_U(r1,r3,r2)               I5(0,1,0,r3,r2,r1)
1038 #define PSHR2I_U(r1,r3,im)              I6(0,1,1,r3,im,r1)
1039 #define PSHR4(r1,r3,r2)                 I5(1,0,2,r3,r2,r1)
1040 #define PSHR4I(r1,r3,im)                I6(1,0,3,r3,im,r1)
1041 #define PSHR4_U(r1,r3,r2)               I5(1,0,0,r3,r2,r1)
1042 #define PSHR4I_U(r1,r3,im)              I6(1,0,1,r3,im,r1)
1043 /* pshradd */
1044 #define PSHRADD2(r1,r2,im,r3)           A10(6,im,r3,r2,r1)
1045 /* psub */
1046 #define PSUB1(r1,r2,r3)                 A9(0,0,1,0,r3,r2,r1)
1047 #define PSUB1_SSS(r1,r2,r3)             A9(0,0,1,1,r3,r2,r1)
1048 #define PSUB1_UUU(r1,r2,r3)             A9(0,0,1,2,r3,r2,r1)
1049 #define PSUB1_UUS(r1,r2,r3)             A9(0,0,1,3,r3,r2,r1)
1050 #define PSUB2(r1,r2,r3)                 A9(0,1,1,0,r3,r2,r1)
1051 #define PSUB2_SSS(r1,r2,r3)             A9(0,1,1,1,r3,r2,r1)
1052 #define PSUB2_UUU(r1,r2,r3)             A9(0,1,1,2,r3,r2,r1)
1053 #define PSUB2_UUS(r1,r2,r3)             A9(0,1,1,3,r3,r2,r1)
1054 #define PSUB4(r1,r2,r3)                 A9(1,0,1,0,r3,r2,r1)
1055 /* ptc.e */
1056 #define PTC_E(r3)                       M47(0x34,r3)
1057 /* ptc.g, ptc.ga */
1058 #define PTC_G(r3,r2)                    M45(0xa,r3,r2)
1059 #define PTC_GA(r3,r2)                   M45(0xb,r3,r2)
1060 /* ptc.l */
1061 #define PTC_L(r3,r2)                    M45(0x9,r3,r2)
1062 /* ptr */
1063 #define PTR_D(r3,r2)                    M45(0xc,r3,r2)
1064 #define PTR_I(r3,r2)                    M45(0xd,r3,r2)
1065 /* rfi */
1066 #define RFI()                           B8(0x08)
1067 /* rsm */
1068 #define RSM(im)                         M44(7,im)
1069 /* rum */
1070 #define RUM(im)                         M44(5,im)
1071 /* shl */
1072 #define SHL(r1,r2,r3)                   I7(1,1,r3,r2,r1)
1073 /* shladd */
1074 #define SHLADD(r1,r2,im,r3)             A2(4,im,r3,r2,r1)
1075 /* shladdp4 */
1076 #define SHLADDP4(r1,r2,im,r3)           A2(6,im,r3,r2,r1)
1077 /* shr */
1078 #define SHR(r1,r3,r2)                   I5(1,1,2,r3,r2,r1)
1079 #define SHR_U(r1,r3,r2)                 I5(1,1,0,r3,r2,r1)
1080 /* shrp */
1081 #define SHRP(r1,r2,r3,im)               I10(im,r3,r2,r1)
1082 /* srlz */
1083 #define SRLZ_I()                        M24(3,1)
1084 #define SRLZ_D()                        M24(3,0)
1085 /* ssm */
1086 #define SSM(im)                         M44(6,im)
1087 /* st */
1088 #define ST1(r3,r2)                      M6(0x30,ST_NONE,0,r3,r2)
1089 #define ST2(r3,r2)                      M6(0x31,ST_NONE,0,r3,r2)
1090 #define ST4(r3,r2)                      M6(0x32,ST_NONE,0,r3,r2)
1091 #define ST8(r3,r2)                      M6(0x33,ST_NONE,0,r3,r2)
1092 #define ST1_REL(r3,r2)                  M6(0x34,ST_NONE,0,r3,r2)
1093 #define ST2_REL(r3,r2)                  M6(0x35,ST_NONE,0,r3,r2)
1094 #define ST4_REL(r3,r2)                  M6(0x36,ST_NONE,0,r3,r2)
1095 #define ST8_REL(r3,r2)                  M6(0x37,ST_NONE,0,r3,r2)
1096 #define ST8_SPILL(r3,r2)                M6(0x3b,ST_NONE,0,r3,r2)
1097 #define ST16(r3,r2)                     M6(0x30,ST_NONE,1,r3,r2)
1098 #define ST16_REL(r3,r2)                 M6(0x34,ST_NONE,1,r3,r2)
1099 #define ST1_inc(r3,r2,im)               M5(0x30,ST_NONE,r3,r2,im)
1100 #define ST2_inc(r3,r2,im)               M5(0x31,ST_NONE,r3,r2,im)
1101 #define ST4_inc(r3,r2,im)               M5(0x32,ST_NONE,r3,r2,im)
1102 #define ST8_inc(r3,r2,im)               M5(0x33,ST_NONE,r3,r2,im)
1103 #define ST1_REL_inc(r3,r2,im)           M5(0x34,ST_NONE,r3,r2,im)
1104 #define ST2_REL_inc(r3,r2,im)           M5(0x35,ST_NONE,r3,r2,im)
1105 #define ST4_REL_inc(r3,r2,im)           M5(0x36,ST_NONE,r3,r2,im)
1106 #define ST8_REL_inc(r3,r2,im)           M5(0x37,ST_NONE,r3,r2,im)
1107 #define ST8_SPILL_inc(r3,r2,im)         M5(0x3b,ST_NONE,r3,r2,im)
1108 /* sub */
1109 #define SUB(r1,r2,r3)                   A1(1,1,r3,r2,r1)
1110 #define SUB1(r1,r2,r3)                  A1(1,0,r3,r2,r1)
1111 #define SUBI(r1,im,r3)                  A3(9,1,r3,im,r1)
1112 /* sum */
1113 #define SUM(im)                         M44(4,im)
1114 /* sxt */
1115 #define SXT1(r1,r3)                     I29(0x14,r3,r1)
1116 #define SXT2(r1,r3)                     I29(0x15,r3,r1)
1117 #define SXT4(r1,r3)                     I29(0x16,r3,r1)
1118 /* sync */
1119 #define SYNC_I()                        M24(3,3)
1120 /* tak */
1121 #define TAK(r1,r3)                      M46(0x1f,r3,r1)
1122 /* tbit */
1123 #define TBIT_Z(p1,p2,r3,pos)            I16(0,0,p2,r3,pos,0,p1)
1124 #define TBIT_Z_UNC(p1,p2,r3,pos)        I16(0,0,p2,r3,pos,1,p1)
1125 #define TBIT_Z_AND(p1,p2,r3,pos)        I16(1,0,p2,r3,pos,0,p1)
1126 #define TBIT_NZ_AND(p1,p2,r3,pos)       I16(1,0,p2,r3,pos,1,p1)
1127 #define TBIT_Z_OR(p1,p2,r3,pos)         I16(0,1,p2,r3,pos,0,p1)
1128 #define TBIT_NZ_OR(p1,p2,r3,pos)        I16(0,1,p2,r3,pos,1,p1)
1129 #define TBIT_Z_ANDCM(p1,p2,r3,pos)      I16(1,1,p2,r3,pos,0,p1)
1130 #define TBIT_NZ_ANDCM(p1,p2,r3,pos)     I16(1,1,p2,r3,pos,1,p1)
1131 /* tf */
1132 #define TF_Z(p1,p2,im)                  I30(0,0,p2,im,0,p1)
1133 #define TF_Z_UNC(p1,p2,im)              I30(0,0,p2,im,1,p1)
1134 #define TF_Z_AND(p1,p2,im)              I30(1,0,p2,im,0,p1)
1135 #define TF_NZ_AND(p1,p2,im)             I30(1,0,p2,im,1,p1)
1136 #define TF_Z_OR(p1,p2,im)               I30(0,1,p2,im,0,p1)
1137 #define TF_NZ_OR(p1,p2,im)              I30(0,1,p2,im,1,p1)
1138 #define TF_Z_ANDCM(p1,p2,im)            I30(1,1,p2,im,0,p1)
1139 #define TF_NZ_ANDCM(p1,p2,im)           I30(1,1,p2,im,1,p1)
1140 /* thash */
1141 #define THASH(r1,r3)                    M46(0x1a,r3,r1)
1142 /* tnat */
1143 #define TNAT_Z(p1,p2,r3)                I17(0,0,p2,r3,0,p1)
1144 #define TNAT_Z_UNC(p1,p2,r3)            I17(0,0,p2,r3,1,p1)
1145 #define TNAT_Z_AND(p1,p2,r3)            I17(1,0,p2,r3,0,p1)
1146 #define TNAT_NZ_AND(p1,p2,r3)           I17(1,0,p2,r3,1,p1)
1147 #define TNAT_Z_OR(p1,p2,r3)             I17(0,1,p2,r3,0,p1)
1148 #define TNAT_NZ_OR(p1,p2,r3)            I17(0,1,p2,r3,1,p1)
1149 #define TNAT_Z_ANDCM(p1,p2,r3)          I17(1,1,p2,r3,0,p1)
1150 #define TNAT_NZ_ANDCM(p1,p2,r3)         I17(1,1,p2,r3,1,p1)
1151 /* tpa */
1152 #define TPA(r1,r3)                      M46(0x1e,r3,r1)
1153 /* ttag */
1154 #define TTAG(r1,r3)                     M46(0x1b,r3,r1)
1155 /* unpack */
1156 #define UNPACK1_H(r1,r2,r3)             I2(0,2,0,1,0,r3,r2,r1)
1157 #define UNPACK2_H(r1,r2,r3)             I2(0,2,1,1,0,r3,r2,r1)
1158 #define UNPACK4_H(r1,r2,r3)             I2(1,2,0,1,0,r3,r2,r1)
1159 #define UNPACK1_L(r1,r2,r3)             I2(0,2,0,1,2,r3,r2,r1)
1160 #define UNPACK2_L(r1,r2,r3)             I2(0,2,1,1,2,r3,r2,r1)
1161 #define UNPACK4_L(r1,r2,r3)             I2(1,2,0,1,2,r3,r2,r1)
1162 /* vmsw */
1163 #define VMSW_0()                        B8(0x18)
1164 #define VMSW_1()                        B8(0x19)
1165 /* xchg */
1166 #define XCHG1_ACQ(r1,r3,r2)             M16(0x08,LD_NONE,r3,r2,r1)
1167 #define XCHG2_ACQ(r1,r3,r2)             M16(0x09,LD_NONE,r3,r2,r1)
1168 #define XCHG4_ACQ(r1,r3,r2)             M16(0x0a,LD_NONE,r3,r2,r1)
1169 #define XCHG8_ACQ(r1,r3,r2)             M16(0x0b,LD_NONE,r3,r2,r1)
1170 /* xor */
1171 #define XOR(r1,r2,r3)                   A1(3,3,r3,r2,r1)
1172 #define XORI(r1,im,r3)                  A3(0xb,3,r3,im,r1)
1173 /* zxt */
1174 #define ZXT1(r1,r3)                     I29(0x10,r3,r1)
1175 #define ZXT2(r1,r3)                     I29(0x11,r3,r1)
1176 #define ZXT4(r1,r3)                     I29(0x12,r3,r1)
1177
1178 #define addr(r0,r1,r2)                  ADD(r0,r1,r2)
1179 #define addi(r0,r1,i0)                  _addi(_jit,r0,r1,i0)
1180 static void _addi(jit_state_t*,jit_int32_t,jit_int32_t,jit_word_t);
1181 #define addcr(r0, r1, r2)               _addcr(_jit, r0, r1, r2)
1182 static void _addcr(jit_state_t*, jit_int32_t, jit_int32_t, jit_int32_t);
1183 #define addci(r0, r1, i0)               _addci(_jit, r0, r1, i0)
1184 static void _addci(jit_state_t*, jit_int32_t, jit_int32_t, jit_word_t);
1185 #  define addxr(r0, r1, r2)             _addxr(_jit, r0, r1, r2)
1186 static void _addxr(jit_state_t*, jit_int32_t, jit_int32_t, jit_int32_t);
1187 #  define addxi(r0, r1, i0)             _addxi(_jit, r0, r1, i0)
1188 static void _addxi(jit_state_t*, jit_int32_t, jit_int32_t, jit_word_t);
1189 #define subr(r0,r1,r2)                  SUB(r0,r1,r2)
1190 #define subi(r0,r1,i0)                  _subi(_jit,r0,r1,i0)
1191 static void _subi(jit_state_t*,jit_int32_t,jit_int32_t,jit_word_t);
1192 #  define subcr(r0, r1, r2)             _subcr(_jit, r0, r1, r2)
1193 static void _subcr(jit_state_t*,jit_int32_t,jit_int32_t,jit_int32_t);
1194 #  define subci(r0, r1, i0)             _subci(_jit, r0, r1, i0)
1195 static void _subci(jit_state_t*,jit_int32_t,jit_int32_t,jit_word_t);
1196 #  define subxr(r0, r1, r2)             _subxr(_jit, r0, r1, r2)
1197 static void _subxr(jit_state_t*,jit_int32_t,jit_int32_t,jit_int32_t);
1198 #  define subxi(r0, r1, i0)             _subxi(_jit, r0, r1, i0)
1199 static void _subxi(jit_state_t*,jit_int32_t,jit_int32_t,jit_word_t);
1200 #  define rsbi(r0, r1, i0)              _rsbi(_jit, r0, r1, i0)
1201 static void _rsbi(jit_state_t*,jit_int32_t,jit_int32_t,jit_word_t);
1202 #define mulr(r0,r1,r2)                  _mulr(_jit,r0,r1,r2)
1203 static void _mulr(jit_state_t*,jit_int32_t,jit_int32_t,jit_int32_t);
1204 #define muli(r0,r1,i0)                  _muli(_jit,r0,r1,i0)
1205 static void _muli(jit_state_t*,jit_int32_t,jit_int32_t,jit_word_t);
1206 #define divr(r0,r1,r2)                  _divr(_jit,r0,r1,r2)
1207 static void _divr(jit_state_t*,jit_int32_t,jit_int32_t,jit_int32_t);
1208 #define divi(r0,r1,i0)                  _divi(_jit,r0,r1,i0)
1209 static void _divi(jit_state_t*,jit_int32_t,jit_int32_t,jit_word_t);
1210 #define divr_u(r0,r1,r2)                _divr_u(_jit,r0,r1,r2)
1211 static void _divr_u(jit_state_t*,jit_int32_t,jit_int32_t,jit_int32_t);
1212 #define divi_u(r0,r1,i0)                _divi_u(_jit,r0,r1,i0)
1213 static void _divi_u(jit_state_t*,jit_int32_t,jit_int32_t,jit_word_t);
1214 #define remr(r0,r1,r2)                  _remr(_jit,r0,r1,r2)
1215 static void _remr(jit_state_t*,jit_int32_t,jit_int32_t,jit_int32_t);
1216 #define remi(r0,r1,i0)                  _remi(_jit,r0,r1,i0)
1217 static void _remi(jit_state_t*,jit_int32_t,jit_int32_t,jit_word_t);
1218 #define remr_u(r0,r1,r2)                _remr_u(_jit,r0,r1,r2)
1219 static void _remr_u(jit_state_t*,jit_int32_t,jit_int32_t,jit_int32_t);
1220 #define remi_u(r0,r1,i0)                _remi_u(_jit,r0,r1,i0)
1221 static void _remi_u(jit_state_t*,jit_int32_t,jit_int32_t,jit_word_t);
1222 #define mulh(r0,r1,r2,sign)             _mulh(_jit,r0,r1,r2,sign)
1223 static void _mulh(jit_state_t*,jit_bool_t,jit_int32_t,jit_int32_t,jit_int32_t);
1224 #define qmulr(r0,r1,r2,r3)              iqmulr(r0,r1,r2,r3,1)
1225 #define qmulr_u(r0,r1,r2,r3)            iqmulr(r0,r1,r2,r3,0)
1226 #define iqmulr(r0,r1,r2,r3,sign)        _iqmulr(_jit,r0,r1,r2,r3,sign)
1227 static void _iqmulr(jit_state_t*,jit_int32_t,jit_int32_t,
1228                     jit_int32_t,jit_int32_t,jit_bool_t);
1229 #define qmuli(r0,r1,r2,i0)              iqmuli(r0,r1,r2,i0,1)
1230 #define qmuli_u(r0,r1,r2,i0)            iqmuli(r0,r1,r2,i0,0)
1231 #define iqmuli(r0,r1,r2,i0,sign)        _iqmuli(_jit,r0,r1,r2,i0,sign)
1232 static void _iqmuli(jit_state_t*,jit_int32_t,jit_int32_t,
1233                     jit_int32_t,jit_word_t,jit_bool_t);
1234 #define qdivr(r0,r1,r2,r3)              iqdivr(r0,r1,r2,r3,1)
1235 #define qdivr_u(r0,r1,r2,r3)            iqdivr(r0,r1,r2,r3,0)
1236 #define iqdivr(r0,r1,r2,r3,sign)        _iqdivr(_jit,r0,r1,r2,r3,sign)
1237 static void _iqdivr(jit_state_t*,jit_int32_t,jit_int32_t,
1238                     jit_int32_t,jit_int32_t,jit_bool_t);
1239 #define qdivi(r0,r1,r2,i0)              iqdivi(r0,r1,r2,i0,1)
1240 #define qdivi_u(r0,r1,r2,i0)            iqdivi(r0,r1,r2,i0,0)
1241 #define iqdivi(r0,r1,r2,i0,sign)        _iqdivi(_jit,r0,r1,r2,i0,sign)
1242 static void _iqdivi(jit_state_t*,jit_int32_t,jit_int32_t,
1243                     jit_int32_t,jit_word_t,jit_bool_t);
1244 #define andr(r0,r1,r2)                  AND(r0,r1,r2)
1245 #define andi(r0,r1,i0)                  _andi(_jit,r0,r1,i0)
1246 static void _andi(jit_state_t*,jit_int32_t,jit_int32_t,jit_word_t);
1247 #define orr(r0,r1,r2)                   OR(r0,r1,r2)
1248 #define ori(r0,r1,i0)                   _ori(_jit,r0,r1,i0)
1249 static void _ori(jit_state_t*,jit_int32_t,jit_int32_t,jit_word_t);
1250 #define xorr(r0,r1,r2)                  XOR(r0,r1,r2)
1251 #define xori(r0,r1,i0)                  _xori(_jit,r0,r1,i0)
1252 static void _xori(jit_state_t*,jit_int32_t,jit_int32_t,jit_word_t);
1253 #define lshr(r0,r1,r2)                  SHL(r0,r1,r2)
1254 #define lshi(r0,r1,i0)                  _lshi(_jit,r0,r1,i0)
1255 static void _lshi(jit_state_t*,jit_int32_t,jit_int32_t,jit_word_t);
1256 #define rshr(r0,r1,r2)                  SHR(r0,r1,r2)
1257 #define rshi(r0,r1,i0)                  _rshi(_jit,r0,r1,i0)
1258 static void _rshi(jit_state_t*,jit_int32_t,jit_int32_t,jit_word_t);
1259 #define rshr_u(r0,r1,r2)                SHR_U(r0,r1,r2)
1260 #define rshi_u(r0,r1,i0)                _rshi_u(_jit,r0,r1,i0)
1261 static void _rshi_u(jit_state_t*,jit_int32_t,jit_int32_t,jit_word_t);
1262 #define ltr(r0,r1,r2)                   _ltr(_jit,r0,r1,r2)
1263 static void _ltr(jit_state_t*,jit_int32_t,jit_int32_t,jit_int32_t);
1264 #define lti(r0,r1,i0)                   _lti(_jit,r0,r1,i0)
1265 static void _lti(jit_state_t*,jit_int32_t,jit_int32_t,jit_word_t);
1266 #define ltr_u(r0,r1,r2)                 _ltr_u(_jit,r0,r1,r2)
1267 static void _ltr_u(jit_state_t*,jit_int32_t,jit_int32_t,jit_int32_t);
1268 #define lti_u(r0,r1,i0)                 _lti_u(_jit,r0,r1,i0)
1269 static void _lti_u(jit_state_t*,jit_int32_t,jit_int32_t,jit_word_t);
1270 #define ler(r0,r1,r2)                   _ler(_jit,r0,r1,r2)
1271 static void _ler(jit_state_t*,jit_int32_t,jit_int32_t,jit_int32_t);
1272 #define lei(r0,r1,i0)                   _lei(_jit,r0,r1,i0)
1273 static void _lei(jit_state_t*,jit_int32_t,jit_int32_t,jit_word_t);
1274 #define ler_u(r0,r1,r2)                 _ler_u(_jit,r0,r1,r2)
1275 static void _ler_u(jit_state_t*,jit_int32_t,jit_int32_t,jit_int32_t);
1276 #define lei_u(r0,r1,i0)                 _lei_u(_jit,r0,r1,i0)
1277 static void _lei_u(jit_state_t*,jit_int32_t,jit_int32_t,jit_word_t);
1278 #define eqr(r0,r1,r2)                   _eqr(_jit,r0,r1,r2)
1279 static void _eqr(jit_state_t*,jit_int32_t,jit_int32_t,jit_int32_t);
1280 #define eqi(r0,r1,i0)                   _eqi(_jit,r0,r1,i0)
1281 static void _eqi(jit_state_t*,jit_int32_t,jit_int32_t,jit_word_t);
1282 #define ger(r0,r1,r2)                   _ger(_jit,r0,r1,r2)
1283 static void _ger(jit_state_t*,jit_int32_t,jit_int32_t,jit_int32_t);
1284 #define gei(r0,r1,i0)                   _gei(_jit,r0,r1,i0)
1285 static void _gei(jit_state_t*,jit_int32_t,jit_int32_t,jit_word_t);
1286 #define ger_u(r0,r1,r2)                 _ger_u(_jit,r0,r1,r2)
1287 static void _ger_u(jit_state_t*,jit_int32_t,jit_int32_t,jit_int32_t);
1288 #define gei_u(r0,r1,i0)                 _gei_u(_jit,r0,r1,i0)
1289 static void _gei_u(jit_state_t*,jit_int32_t,jit_int32_t,jit_word_t);
1290 #define gtr(r0,r1,r2)                   _gtr(_jit,r0,r1,r2)
1291 static void _gtr(jit_state_t*,jit_int32_t,jit_int32_t,jit_int32_t);
1292 #define gti(r0,r1,i0)                   _gti(_jit,r0,r1,i0)
1293 static void _gti(jit_state_t*,jit_int32_t,jit_int32_t,jit_word_t);
1294 #define gtr_u(r0,r1,r2)                 _gtr_u(_jit,r0,r1,r2)
1295 static void _gtr_u(jit_state_t*,jit_int32_t,jit_int32_t,jit_int32_t);
1296 #define gti_u(r0,r1,i0)                 _gti_u(_jit,r0,r1,i0)
1297 static void _gti_u(jit_state_t*,jit_int32_t,jit_int32_t,jit_word_t);
1298 #define ner(r0,r1,r2)                   _ner(_jit,r0,r1,r2)
1299 static void _ner(jit_state_t*,jit_int32_t,jit_int32_t,jit_int32_t);
1300 #define nei(r0,r1,i0)                   _nei(_jit,r0,r1,i0)
1301 static void _nei(jit_state_t*,jit_int32_t,jit_int32_t,jit_word_t);
1302 #define negr(r0,r1)                     subr(r0,0,r1)
1303 #define comr(r0,r1)                     ANDCMI(r0,-1,r1)
1304 #define movr(r0,r1)                     _movr(_jit,r0,r1)
1305 static void _movr(jit_state_t*,jit_int32_t,jit_int32_t);
1306 #define movi(r0,i0)                     _movi(_jit,r0,i0)
1307 static void _movi(jit_state_t*,jit_int32_t,jit_word_t);
1308 #define movi_p(r0,i0)                   _movi_p(_jit,r0,i0)
1309 static jit_word_t _movi_p(jit_state_t*,jit_int32_t,jit_word_t);
1310 #if __BYTE_ORDER == __LITTLE_ENDIAN
1311 #  define htonr_us(r0,r1)               _htonr_us(_jit,r0,r1)
1312 static void _htonr_us(jit_state_t*,jit_int32_t,jit_int32_t);
1313 #  define htonr_ui(r0,r1)               _htonr_ui(_jit,r0,r1)
1314 static void _htonr_ui(jit_state_t*,jit_int32_t,jit_int32_t);
1315 #  define htonr_ul(r0,r1)               MUX1(r0,r1,MUX_REV)
1316 #else
1317 #  define htonr_us(r0,r1)               extr_us(r0,r1)
1318 #  define htonr_ui(r0,r1)               extr_ui(r0,r1)
1319 #  define htonr_ul(r0,r1)               movr(r0,r1)
1320 #endif
1321 #define extr_c(r0,r1)                   SXT1(r0,r1)
1322 #define extr_uc(r0,r1)                  ZXT1(r0,r1)
1323 #define extr_s(r0,r1)                   SXT2(r0,r1)
1324 #define extr_us(r0,r1)                  ZXT2(r0,r1)
1325 #define extr_i(r0,r1)                   SXT4(r0,r1)
1326 #define extr_ui(r0,r1)                  ZXT4(r0,r1)
1327 #define bltr(i0,r0,r1)                  _bltr(_jit,i0,r0,r1)
1328 static jit_word_t _bltr(jit_state_t*,jit_word_t,jit_int32_t,jit_int32_t);
1329 #define blti(i0,r0,i1)                  _blti(_jit,i0,r0,i1)
1330 static jit_word_t _blti(jit_state_t*,jit_word_t,jit_int32_t,jit_word_t);
1331 #define bltr_u(i0,r0,r1)                _bltr_u(_jit,i0,r0,r1)
1332 static jit_word_t _bltr_u(jit_state_t*,jit_word_t,jit_int32_t,jit_int32_t);
1333 #define blti_u(i0,r0,i1)                _blti_u(_jit,i0,r0,i1)
1334 static jit_word_t _blti_u(jit_state_t*,jit_word_t,jit_int32_t,jit_word_t);
1335 #define bler(i0,r0,r1)                  _bler(_jit,i0,r0,r1)
1336 static jit_word_t _bler(jit_state_t*,jit_word_t,jit_int32_t,jit_int32_t);
1337 #define blei(i0,r0,i1)                  _blei(_jit,i0,r0,i1)
1338 static jit_word_t _blei(jit_state_t*,jit_word_t,jit_int32_t,jit_word_t);
1339 #define bler_u(i0,r0,r1)                _bler_u(_jit,i0,r0,r1)
1340 static jit_word_t _bler_u(jit_state_t*,jit_word_t,jit_int32_t,jit_int32_t);
1341 #define blei_u(i0,r0,i1)                _blei_u(_jit,i0,r0,i1)
1342 static jit_word_t _blei_u(jit_state_t*,jit_word_t,jit_int32_t,jit_word_t);
1343 #define beqr(i0,r0,r1)                  _beqr(_jit,i0,r0,r1)
1344 static jit_word_t _beqr(jit_state_t*,jit_word_t,jit_int32_t,jit_int32_t);
1345 #define beqi(i0,r0,i1)                  _beqi(_jit,i0,r0,i1)
1346 static jit_word_t _beqi(jit_state_t*,jit_word_t,jit_int32_t,jit_word_t);
1347 #define bger(i0,r0,r1)                  _bger(_jit,i0,r0,r1)
1348 static jit_word_t _bger(jit_state_t*,jit_word_t,jit_int32_t,jit_int32_t);
1349 #define bgei(i0,r0,i1)                  _bgei(_jit,i0,r0,i1)
1350 static jit_word_t _bgei(jit_state_t*,jit_word_t,jit_int32_t,jit_word_t);
1351 #define bger_u(i0,r0,r1)                _bger_u(_jit,i0,r0,r1)
1352 static jit_word_t _bger_u(jit_state_t*,jit_word_t,jit_int32_t,jit_int32_t);
1353 #define bgei_u(i0,r0,i1)                _bgei_u(_jit,i0,r0,i1)
1354 static jit_word_t _bgei_u(jit_state_t*,jit_word_t,jit_int32_t,jit_word_t);
1355 #define bgtr(i0,r0,r1)                  _bgtr(_jit,i0,r0,r1)
1356 static jit_word_t _bgtr(jit_state_t*,jit_word_t,jit_int32_t,jit_int32_t);
1357 #define bgti(i0,r0,i1)                  _bgti(_jit,i0,r0,i1)
1358 static jit_word_t _bgti(jit_state_t*,jit_word_t,jit_int32_t,jit_word_t);
1359 #define bgtr_u(i0,r0,r1)                _bgtr_u(_jit,i0,r0,r1)
1360 static jit_word_t _bgtr_u(jit_state_t*,jit_word_t,jit_int32_t,jit_int32_t);
1361 #define bgti_u(i0,r0,i1)                _bgti_u(_jit,i0,r0,i1)
1362 static jit_word_t _bgti_u(jit_state_t*,jit_word_t,jit_int32_t,jit_word_t);
1363 #define bner(i0,r0,r1)                  _bner(_jit,i0,r0,r1)
1364 static jit_word_t _bner(jit_state_t*,jit_word_t,jit_int32_t,jit_int32_t);
1365 #define bnei(i0,r0,i1)                  _bnei(_jit,i0,r0,i1)
1366 static jit_word_t _bnei(jit_state_t*,jit_word_t,jit_int32_t,jit_word_t);
1367 #define bmsr(i0,r0,r1)                  _bmsr(_jit,i0,r0,r1)
1368 static jit_word_t _bmsr(jit_state_t*,jit_word_t,jit_int32_t,jit_int32_t);
1369 #define bmsi(i0,r0,i1)                  _bmsi(_jit,i0,r0,i1)
1370 static jit_word_t _bmsi(jit_state_t*,jit_word_t,jit_int32_t,jit_word_t);
1371 #define bmcr(i0,r0,r1)                  _bmcr(_jit,i0,r0,r1)
1372 static jit_word_t _bmcr(jit_state_t*,jit_word_t,jit_int32_t,jit_int32_t);
1373 #define bmci(i0,r0,i1)                  _bmci(_jit,i0,r0,i1)
1374 static jit_word_t _bmci(jit_state_t*,jit_word_t,jit_int32_t,jit_word_t);
1375 #define baddr(i0,r0,r1,cc)              _baddr(_jit,i0,r0,r1,cc)
1376 static jit_word_t _baddr(jit_state_t*,jit_word_t,
1377                          jit_int32_t,jit_int32_t,jit_bool_t);
1378 #define baddi(i0,r0,i1,cc)              _baddi(_jit,i0,r0,i1,cc)
1379 static jit_word_t _baddi(jit_state_t*,jit_word_t,
1380                          jit_int32_t,jit_word_t,jit_bool_t);
1381 #define baddr_u(i0,r0,r1,cc)            _baddr_u(_jit,i0,r0,r1,cc)
1382 static jit_word_t _baddr_u(jit_state_t*,jit_word_t,
1383                            jit_int32_t,jit_int32_t,jit_bool_t);
1384 #define baddi_u(i0,r0,i1,cc)            _baddi_u(_jit,i0,r0,i1,cc)
1385 static jit_word_t _baddi_u(jit_state_t*,jit_word_t,
1386                            jit_int32_t,jit_word_t,jit_bool_t);
1387 #define bsubr(i0,r0,r1,cc)              _bsubr(_jit,i0,r0,r1,cc)
1388 static jit_word_t _bsubr(jit_state_t*,jit_word_t,
1389                          jit_int32_t,jit_int32_t,jit_bool_t);
1390 #define bsubi(i0,r0,i1,cc)              _bsubi(_jit,i0,r0,i1,cc)
1391 static jit_word_t _bsubi(jit_state_t*,jit_word_t,
1392                          jit_int32_t,jit_word_t,jit_bool_t);
1393 #define bsubr_u(i0,r0,r1,cc)            _bsubr_u(_jit,i0,r0,r1,cc)
1394 static jit_word_t _bsubr_u(jit_state_t*,jit_word_t,
1395                            jit_int32_t,jit_int32_t,jit_bool_t);
1396 #define bsubi_u(i0,r0,i1,cc)            _bsubi_u(_jit,i0,r0,i1,cc)
1397 static jit_word_t _bsubi_u(jit_state_t*,jit_word_t,
1398                            jit_int32_t,jit_word_t,jit_bool_t);
1399 #define boaddr(i0,r0,r1)                baddr(i0,r0,r1,1)
1400 #define boaddi(i0,r0,i1)                baddi(i0,r0,i1,1)
1401 #define boaddr_u(i0,r0,r1)              baddr_u(i0,r0,r1,1)
1402 #define boaddi_u(i0,r0,i1)              baddi_u(i0,r0,i1,1)
1403 #define bxaddr(i0,r0,r1)                baddr(i0,r0,r1,0)
1404 #define bxaddi(i0,r0,i1)                baddi(i0,r0,i1,0)
1405 #define bxaddr_u(i0,r0,r1)              baddr_u(i0,r0,r1,0)
1406 #define bxaddi_u(i0,r0,i1)              baddi_u(i0,r0,i1,0)
1407 #define bosubr(i0,r0,r1)                bsubr(i0,r0,r1,1)
1408 #define bosubi(i0,r0,i1)                bsubi(i0,r0,i1,1)
1409 #define bosubr_u(i0,r0,r1)              bsubr_u(i0,r0,r1,1)
1410 #define bosubi_u(i0,r0,i1)              bsubi_u(i0,r0,i1,1)
1411 #define bxsubr(i0,r0,r1)                bsubr(i0,r0,r1,0)
1412 #define bxsubi(i0,r0,i1)                bsubi(i0,r0,i1,0)
1413 #define bxsubr_u(i0,r0,r1)              bsubr_u(i0,r0,r1,0)
1414 #define bxsubi_u(i0,r0,i1)              bsubi_u(i0,r0,i1,0)
1415 #define ldr_c(r0,r1)                    _ldr_c(_jit,r0,r1)
1416 static void _ldr_c(jit_state_t*,jit_int32_t,jit_int32_t);
1417 #define ldi_c(r0,i0)                    _ldi_c(_jit,r0,i0)
1418 static void _ldi_c(jit_state_t*,jit_int32_t,jit_word_t);
1419 #define ldr_uc(r0,r1)                   LD1(r0,r1)
1420 #define ldi_uc(r0,i0)                   _ldi_uc(_jit,r0,i0)
1421 static void _ldi_uc(jit_state_t*,jit_int32_t,jit_word_t);
1422 #define ldr_s(r0,r1)                    _ldr_s(_jit,r0,r1)
1423 static void _ldr_s(jit_state_t*,jit_int32_t,jit_int32_t);
1424 #define ldi_s(r0,i0)                    _ldi_s(_jit,r0,i0)
1425 static void _ldi_s(jit_state_t*,jit_int32_t,jit_word_t);
1426 #define ldr_us(r0,r1)                   LD2(r0,r1)
1427 #define ldi_us(r0,i0)                   _ldi_us(_jit,r0,i0)
1428 static void _ldi_us(jit_state_t*,jit_int32_t,jit_word_t);
1429 #define ldr_i(r0,r1)                    _ldr_i(_jit,r0,r1)
1430 static void _ldr_i(jit_state_t*,jit_int32_t,jit_int32_t);
1431 #define ldi_i(r0,i0)                    _ldi_i(_jit,r0,i0)
1432 static void _ldi_i(jit_state_t*,jit_int32_t,jit_word_t);
1433 #define ldr_ui(r0,r1)                   LD4(r0,r1)
1434 #define ldi_ui(r0,i0)                   _ldi_ui(_jit,r0,i0)
1435 static void _ldi_ui(jit_state_t*,jit_int32_t,jit_word_t);
1436 #define ldr_l(r0,r1)                    LD8(r0,r1)
1437 #define ldi_l(r0,i0)                    _ldi_l(_jit,r0,i0)
1438 static void _ldi_l(jit_state_t*,jit_int32_t,jit_word_t);
1439 #define ldxr_c(r0,r1,r2)                _ldxr_c(_jit,r0,r1,r2)
1440 static void _ldxr_c(jit_state_t*,jit_int32_t,jit_int32_t,jit_int32_t);
1441 #define ldxi_c(r0,r1,i0)                _ldxi_c(_jit,r0,r1,i0)
1442 static void _ldxi_c(jit_state_t*,jit_int32_t,jit_int32_t,jit_word_t);
1443 #define ldxr_uc(r0,r1,r2)               _ldxr_uc(_jit,r0,r1,r2)
1444 static void _ldxr_uc(jit_state_t*,jit_int32_t,jit_int32_t,jit_int32_t);
1445 #define ldxi_uc(r0,r1,i0)               _ldxi_uc(_jit,r0,r1,i0)
1446 static void _ldxi_uc(jit_state_t*,jit_int32_t,jit_int32_t,jit_word_t);
1447 #define ldxr_s(r0,r1,r2)                _ldxr_s(_jit,r0,r1,r2)
1448 static void _ldxr_s(jit_state_t*,jit_int32_t,jit_int32_t,jit_int32_t);
1449 #define ldxi_s(r0,r1,i0)                _ldxi_s(_jit,r0,r1,i0)
1450 static void _ldxi_s(jit_state_t*,jit_int32_t,jit_int32_t,jit_word_t);
1451 #define ldxr_us(r0,r1,r2)               _ldxr_us(_jit,r0,r1,r2)
1452 static void _ldxr_us(jit_state_t*,jit_int32_t,jit_int32_t,jit_int32_t);
1453 #define ldxi_us(r0,r1,i0)               _ldxi_us(_jit,r0,r1,i0)
1454 static void _ldxi_us(jit_state_t*,jit_int32_t,jit_int32_t,jit_word_t);
1455 #define ldxr_i(r0,r1,r2)                _ldxr_i(_jit,r0,r1,r2)
1456 static void _ldxr_i(jit_state_t*,jit_int32_t,jit_int32_t,jit_int32_t);
1457 #define ldxi_i(r0,r1,i0)                _ldxi_i(_jit,r0,r1,i0)
1458 static void _ldxi_i(jit_state_t*,jit_int32_t,jit_int32_t,jit_word_t);
1459 #define ldxr_ui(r0,r1,r2)               _ldxr_ui(_jit,r0,r1,r2)
1460 static void _ldxr_ui(jit_state_t*,jit_int32_t,jit_int32_t,jit_int32_t);
1461 #define ldxi_ui(r0,r1,i0)               _ldxi_ui(_jit,r0,r1,i0)
1462 static void _ldxi_ui(jit_state_t*,jit_int32_t,jit_int32_t,jit_word_t);
1463 #define ldxr_l(r0,r1,r2)                _ldxr_l(_jit,r0,r1,r2)
1464 static void _ldxr_l(jit_state_t*,jit_int32_t,jit_int32_t,jit_int32_t);
1465 #define ldxi_l(r0,r1,i0)                _ldxi_l(_jit,r0,r1,i0)
1466 static void _ldxi_l(jit_state_t*,jit_int32_t,jit_int32_t,jit_word_t);
1467 #define str_c(r0,r1)                    ST1(r0,r1)
1468 #define sti_c(i0,r0)                    _sti_c(_jit,i0,r0)
1469 static void _sti_c(jit_state_t*,jit_word_t,jit_int32_t);
1470 #define str_s(r0,r1)                    ST2(r0,r1)
1471 #define sti_s(i0,r0)                    _sti_s(_jit,i0,r0)
1472 static void _sti_s(jit_state_t*,jit_word_t,jit_int32_t);
1473 #define str_i(r0,r1)                    ST4(r0,r1)
1474 #define sti_i(i0,r0)                    _sti_i(_jit,i0,r0)
1475 static void _sti_i(jit_state_t*,jit_word_t,jit_int32_t);
1476 #define str_l(r0,r1)                    ST8(r0,r1)
1477 #define sti_l(i0,r0)                    _sti_l(_jit,i0,r0)
1478 static void _sti_l(jit_state_t*,jit_word_t,jit_int32_t);
1479 #define stxr_c(r0,r1,r2)                _stxr_c(_jit,r0,r1,r2)
1480 static void _stxr_c(jit_state_t*,jit_int32_t,jit_int32_t,jit_int32_t);
1481 #define stxi_c(i0,r0,r1)                _stxi_c(_jit,i0,r0,r1)
1482 static void _stxi_c(jit_state_t*,jit_word_t,jit_int32_t,jit_int32_t);
1483 #define stxr_s(r0,r1,r2)                _stxr_s(_jit,r0,r1,r2)
1484 static void _stxr_s(jit_state_t*,jit_int32_t,jit_int32_t,jit_int32_t);
1485 #define stxi_s(i0,r0,r1)                _stxi_s(_jit,i0,r0,r1)
1486 static void _stxi_s(jit_state_t*,jit_word_t,jit_int32_t,jit_int32_t);
1487 #define stxr_i(r0,r1,r2)                _stxr_i(_jit,r0,r1,r2)
1488 static void _stxr_i(jit_state_t*,jit_int32_t,jit_int32_t,jit_int32_t);
1489 #define stxi_i(i0,r0,r1)                _stxi_i(_jit,i0,r0,r1)
1490 static void _stxi_i(jit_state_t*,jit_word_t,jit_int32_t,jit_int32_t);
1491 #define stxr_l(r0,r1,r2)                _stxr_l(_jit,r0,r1,r2)
1492 static void _stxr_l(jit_state_t*,jit_int32_t,jit_int32_t,jit_int32_t);
1493 #define stxi_l(i0,r0,r1)                _stxi_l(_jit,i0,r0,r1)
1494 static void _stxi_l(jit_state_t*,jit_word_t,jit_int32_t,jit_int32_t);
1495 #define jmpr(r0)                        _jmpr(_jit,r0)
1496 static void _jmpr(jit_state_t*,jit_int32_t);
1497 #define jmpi(i0)                        _jmpi(_jit,i0)
1498 static void _jmpi(jit_state_t*,jit_word_t);
1499 #define jmpi_p(i0)                      _jmpi_p(_jit,i0)
1500 static jit_word_t _jmpi_p(jit_state_t*,jit_word_t);
1501 #define callr(r0)                       _callr(_jit,r0)
1502 static void _callr(jit_state_t*,jit_int32_t);
1503 #define calli(i0)                       _calli(_jit,i0)
1504 static void _calli(jit_state_t*,jit_word_t);
1505 #define calli_p(i0)                     _calli_p(_jit,i0)
1506 static jit_word_t _calli_p(jit_state_t*,jit_word_t);
1507 #define prolog(node)                    _prolog(_jit,node)
1508 static void _prolog(jit_state_t*,jit_node_t*);
1509 #define epilog(node)                    _epilog(_jit,node)
1510 static void _epilog(jit_state_t*,jit_node_t*);
1511 #  define vastart(r0)                   _vastart(_jit, r0)
1512 static void _vastart(jit_state_t*, jit_int32_t);
1513 #  define vaarg(r0, r1)                 _vaarg(_jit, r0, r1)
1514 static void _vaarg(jit_state_t*, jit_int32_t, jit_int32_t);
1515 #define patch_at(node,instr,label)      _patch_at(_jit,node,instr,label)
1516 static void _patch_at(jit_state_t*,jit_code_t,jit_word_t,jit_word_t);
1517 #endif
1518
1519 #if CODE
1520 #if __BYTE_ORDER == __BIG_ENDIAN
1521 static jit_word_t
1522 byte_swap_if_big_endian(jit_word_t w)
1523 {
1524     union {
1525         char    c[8];
1526         long    w;
1527     } l, h;
1528     l.w = w;
1529     h.c[0] = l.c[7];
1530     h.c[1] = l.c[6];
1531     h.c[2] = l.c[5];
1532     h.c[3] = l.c[4];
1533     h.c[4] = l.c[3];
1534     h.c[5] = l.c[2];
1535     h.c[6] = l.c[1];
1536     h.c[7] = l.c[0];
1537     return (h.w);
1538 }
1539 #else
1540 #define byte_swap_if_big_endian(w)              (w)
1541 #endif
1542
1543 static void
1544 _out(jit_state_t *_jit, int n, int tm,
1545      jit_word_t s0, jit_word_t s1, jit_word_t s2)
1546 {
1547     int                  i;
1548     jit_word_t           l, h, *w;
1549     set_bundle(_jit->pc.ul, l, h, tm, s0, s1, s2);
1550     _jit->pc.ul += 2;
1551     w = (jit_word_t *)_jitc->inst;
1552     for (i = n; i < _jitc->ioff; i++)
1553         w[i - n] = w[i];
1554     _jitc->ioff -= n;
1555 }
1556
1557 #define nop_m           0x0008000000L
1558 #define nop_i           0x0008000000L
1559 #define nop_b           0x4000000000L
1560 static void
1561 _stop(jit_state_t *_jit)
1562 {
1563     /* Clear set of live registers */
1564     jit_regset_set_ui(&_jitc->regs, 0);
1565     _jitc->pred = 0;
1566     /* Flag a stop is required */
1567     if (_jitc->ioff)
1568         _jitc->inst[_jitc->ioff - 1].t |= INST_STOP;
1569     else
1570         inst(nop_m, INST_Ms);
1571 }
1572
1573 static void
1574 _sync(jit_state_t *_jit)
1575 {
1576     /* Taken branches are supposed to not need a stop, so, it
1577      * should not be required to stop if no registers live in
1578      * sequential code */
1579     if (jit_regset_cmp_ui(&_jitc->regs, 0) != 0 || _jitc->pred)
1580         stop();
1581     do
1582         flush();
1583     while (_jitc->ioff);
1584 }
1585
1586 #define A_0             INST_A
1587 #define As0             INST_As
1588 #define I_0             INST_I
1589 #define Is0             INST_Is
1590 #define M_0             INST_M
1591 #define Ms0             INST_Ms
1592 #define F_0             INST_F
1593 #define Fs0             INST_Fs
1594 #define B_0             INST_B
1595 #define Bs0             INST_Bs
1596 #define L_0             INST_L
1597 #define Ls0             INST_Ls
1598 #define X_0             INST_X
1599 #define Xs0             INST_Xs
1600 #define A_1             (INST_A<<4)
1601 #define As1             (INST_As<<4)
1602 #define I_1             (INST_I<<4)
1603 #define Is1             (INST_Is<<4)
1604 #define M_1             (INST_M<<4)
1605 #define Ms1             (INST_Ms<<4)
1606 #define F_1             (INST_F<<4)
1607 #define Fs1             (INST_Fs<<4)
1608 #define B_1             (INST_B<<4)
1609 #define Bs1             (INST_Bs<<4)
1610 #define L_1             (INST_L<<4)
1611 #define Ls1             (INST_Ls<<4)
1612 #define X_1             (INST_X<<4)
1613 #define Xs1             (INST_Xs<<4)
1614 #define A_2             (INST_A<<8)
1615 #define As2             (INST_As<<8)
1616 #define I_2             (INST_I<<8)
1617 #define Is2             (INST_Is<<8)
1618 #define M_2             (INST_M<<8)
1619 #define Ms2             (INST_Ms<<8)
1620 #define F_2             (INST_F<<8)
1621 #define Fs2             (INST_Fs<<8)
1622 #define B_2             (INST_B<<8)
1623 #define Bs2             (INST_Bs<<8)
1624 #define L_2             (INST_L<<8)
1625 #define Ls2             (INST_Ls<<8)
1626 #define X_2             (INST_X<<8)
1627 #define Xs2             (INST_Xs<<8)
1628
1629 #define I_              I_0
1630 #define I_I_            I_0|I_1
1631 #define I_Is            I_0|Is1
1632 #define I_B_            I_0|B_1
1633 #define I_Bs            I_0|Bs1
1634 #define Is              Is0
1635 #define IsI_            Is0|I_1
1636 #define IsIs            Is0|Is1
1637 #define M_              M_0
1638 #define M_I_            M_0|I_1
1639 #define M_Is            M_0|Is1
1640 #define M_M_            M_0|M_1
1641 #define M_Ms            M_0|Ms1
1642 #define M_F_            M_0|F_1
1643 #define M_Fs            M_0|Fs1
1644 #define M_B_            M_0|B_1
1645 #define M_Bs            M_0|Bs1
1646 #define M_I_I_          M_0|I_1|I_2
1647 #define M_I_Is          M_0|I_1|Is2
1648 #define M_I_B_          M_0|I_1|B_2
1649 #define M_I_Bs          M_0|I_1|Bs2
1650 #define M_IsI_          M_0|Is1|I_2
1651 #define M_IsIs          M_0|Is1|Is2
1652 #define M_M_I_          M_0|M_1|I_2
1653 #define M_M_Is          M_0|M_1|Is2
1654 #define M_M_F_          M_0|M_1|F_2
1655 #define M_M_Fs          M_0|M_1|Fs2
1656 #define M_M_B_          M_0|M_1|B_2
1657 #define M_M_Bs          M_0|M_1|Bs2
1658 #define M_F_I_          M_0|F_1|I_2
1659 #define M_F_Is          M_0|F_1|Is2
1660 #define M_F_B_          M_0|F_1|B_2
1661 #define M_F_Bs          M_0|F_1|Bs2
1662 #define M_B_B_          M_0|B_1|B_2
1663 #define M_B_Bs          M_0|B_1|Bs2
1664 #define M_L_X_          M_0|L_1|X_2
1665 #define M_L_Xs          M_0|L_1|Xs2
1666 #define Ms              Ms0
1667 #define MsI_            Ms0|I_1
1668 #define MsIs            Ms0|Is1
1669 #define MsM_            Ms0|M_1
1670 #define MsMs            Ms0|Ms1
1671 #define MsM_I_          Ms0|M_1|I_2
1672 #define MsM_Is          Ms0|M_1|Is2
1673 #define F_              F_0
1674 #define F_I_            F_0|I_1
1675 #define F_Is            F_0|Is1
1676 #define F_B_            F_0|B_1
1677 #define F_Bs            F_0|Bs1
1678 #define Fs              Fs0
1679 #define B_              B_0
1680 #define B_B_            B_0|B_1
1681 #define B_Bs            B_0|Bs1
1682 #define B_B_B_          B_0|B_1|B_2
1683 #define B_B_Bs          B_0|B_1|Bs2
1684 #define Bs              Bs0
1685 #define L_X_            L_0|X_1
1686 #define L_Xs            L_0|Xs1
1687
1688 static jit_word_t
1689 templat(jit_word_t cc)
1690 {
1691     switch (cc) {
1692         case I_:
1693         case I_I_:      case I_Is:
1694         case I_B_:      case I_Bs:
1695         case Is:
1696         case IsI_:      case IsIs:
1697         case M_:
1698         case M_I_:      case M_Is:
1699         case M_M_:      case M_Ms:
1700         case M_F_:      case M_Fs:
1701         case M_B_:      case M_Bs:
1702         case M_I_I_:    case M_I_Is:
1703         case M_I_B_:    case M_I_Bs:
1704         case M_IsI_:    case M_IsIs:
1705         case M_M_I_:    case M_M_Is:
1706         case M_M_F_:    case M_M_Fs:
1707         case M_M_B_:    case M_M_Bs:
1708         case M_F_I_:    case M_F_Is:
1709         case M_F_B_:    case M_F_Bs:
1710         case M_B_B_:    case M_B_Bs:
1711         case M_L_X_:    case M_L_Xs:
1712         case Ms:
1713         case MsI_:      case MsIs:
1714         case MsM_:      case MsMs:
1715         case MsM_I_:    case MsM_Is:
1716         case F_:
1717         case F_I_:      case F_Is:
1718         case F_B_:      case F_Bs:
1719         case Fs:
1720         case B_:
1721         case B_B_:      case B_Bs:
1722         case B_B_B_:    case B_B_Bs:
1723         case Bs:
1724         case L_X_:      case L_Xs:
1725             return (cc);
1726         default:
1727             return (0);
1728     }
1729 }
1730
1731 /* match* functions recurse attempting to find a template for A-
1732  * instructions, that may be executed in M- or I- unit.
1733  * It also uses an heuristic of trying first M- for slot 0 and 2,
1734  * and I- for slot 1, but tries all possible matches.
1735  */
1736 static jit_word_t
1737 match_2(jit_word_t cc)
1738 {
1739     jit_word_t          t;
1740     if ((t = templat(cc | M_2)))
1741         return (t);
1742     if ((t = templat(cc | I_2)))
1743         return (t);
1744     return (0);
1745 }
1746
1747 static jit_word_t
1748 matchs2(jit_word_t cc)
1749 {
1750     jit_word_t          t;
1751     if ((t = templat(cc | Ms2)))
1752         return (t);
1753     if ((t = templat(cc | Is2)))
1754         return (t);
1755     return (0);
1756 }
1757
1758 static jit_word_t
1759 match2(jit_word_t cc)
1760 {
1761     if ((cc & 0xf00) == A_2)
1762         return (match_2(cc & ~0xf00));
1763     if ((cc & 0xf00) == As2)
1764         return (matchs2(cc & ~0xf00));
1765     return (0);
1766 }
1767
1768 static jit_word_t
1769 match_1(jit_word_t cc)
1770 {
1771     jit_word_t          t;
1772     if ((t = templat(cc | I_1)))
1773         return (t);
1774     if ((t = templat(cc | M_1)))
1775         return (t);
1776     if ((t = match2(cc | I_1)))
1777         return (t);
1778     if ((t = match2(cc | M_1)))
1779         return (t);
1780     return (0);
1781 }
1782
1783 static jit_word_t
1784 matchs1(jit_word_t cc)
1785 {
1786     jit_word_t          t;
1787     if ((t = templat(cc | Is1)))
1788         return (t);
1789     if ((t = templat(cc | Ms1)))
1790         return (t);
1791     if ((t = match2(cc | Is1)))
1792         return (t);
1793     if ((t = match2(cc | Ms1)))
1794         return (t);
1795     return (0);
1796 }
1797
1798 static jit_word_t
1799 match1(jit_word_t cc)
1800 {
1801     if ((cc & 0x0f0) == A_1)
1802         return (match_1(cc & ~0x0f0));
1803     if ((cc & 0x0f0) == As1)
1804         return (matchs1(cc & ~0x0f0));
1805     return (0);
1806 }
1807
1808 static jit_word_t
1809 match_0(jit_word_t cc)
1810 {
1811     jit_word_t          t;
1812     if ((t = templat(cc | M_0)))
1813         return (t);
1814     if ((t = templat(cc | I_0)))
1815         return (t);
1816     if ((t = match1(cc | M_0)))
1817         return (t);
1818     if ((t = match1(cc | I_0)))
1819         return (t);
1820     return (0);
1821 }
1822
1823 static jit_word_t
1824 matchs0(jit_word_t cc)
1825 {
1826     jit_word_t          t;
1827     if ((t = templat(cc | Ms0)))
1828         return (t);
1829     if ((t = templat(cc | Is0)))
1830         return (t);
1831     if ((t = match1(cc | Ms0)))
1832         return (t);
1833     if ((t = match1(cc | Is0)))
1834         return (t);
1835     return (0);
1836 }
1837
1838 static jit_word_t
1839 match0(jit_word_t cc)
1840 {
1841     if ((cc & 0x00f) == A_0)
1842         return (match_0(cc & ~0x00f));
1843     if ((cc & 0x00f) == As0)
1844         return (matchs0(cc & ~0x00f));
1845     return (0);
1846 }
1847
1848 static void
1849 _flush(jit_state_t *_jit)
1850 {
1851     int                 n, soff;
1852     jit_word_t          t, cc, tm, s0, s1, s2;
1853
1854     if (!_jitc->ioff)
1855         return;
1856     for (cc = 0, n = soff = 0; n < _jitc->ioff; n++, soff += 4)
1857         cc |= (jit_uword_t)(_jitc->inst[n].t) << soff;
1858
1859     soff = 0xf00;
1860     while (soff) {
1861         /* Try to find a template, or reduce down
1862          * to one instruction if no template match */
1863         if ((t = templat(cc))) {
1864             cc = t;
1865             break;
1866         }
1867         /* A- instructions may be execute in M- or I- unit */
1868         if ((t = match0(cc))) {
1869             cc = t;
1870             break;
1871         }
1872         cc &= ~soff;
1873         soff >>= 4;
1874     }
1875     assert(soff);
1876
1877     /* Prefer tail nop if need to add some nop, so that patching is easier */
1878 #define ii(n)           _jitc->inst[n].i
1879     switch (cc) {
1880         case I_:
1881             n = 1;              tm = TM_M_I_I_;
1882             s0 = nop_m;         s1 = ii(0);             s2 = nop_i;
1883             break;
1884         case I_I_:
1885             n = 2;              tm = TM_M_I_I_;
1886             s0 = nop_m;         s1 = ii(0);             s2 = ii(1);
1887             break;
1888         case I_Is:
1889             n = 2;              tm = TM_M_I_Is;
1890             s0 = nop_m;         s1 = ii(0);             s2 = ii(1);
1891             break;
1892         case I_B_:
1893             n = 2;              tm = TM_M_I_B_;
1894             s0 = nop_m;         s1 = ii(0);             s2 = ii(1);
1895             break;
1896         case I_Bs:
1897             n = 2;              tm = TM_M_I_Bs;
1898             s0 = nop_m;         s1 = ii(0);             s2 = ii(1);
1899             break;
1900         case Is:
1901             n = 1;              tm = TM_M_IsI_;
1902             s0 = nop_m;         s1 = ii(0);             s2 = nop_i;
1903             break;
1904         case IsI_:
1905             n = 2;              tm = TM_M_IsI_;
1906             s0 = nop_m;         s1 = ii(0);             s2 = ii(1);
1907             break;
1908         case IsIs:
1909             n = 2;              tm = TM_M_IsIs;
1910             s0 = nop_m;         s1 = ii(0);             s2 = ii(1);
1911             break;
1912         case M_:
1913             n = 1;              tm = TM_M_I_I_;
1914             s0 = ii(0);         s1 = nop_i;             s2 = nop_i;
1915             break;
1916         case M_I_:
1917             n = 2;              tm = TM_M_I_I_;
1918             s0 = ii(0);         s1 = ii(1);             s2 = nop_i;
1919             break;
1920         case M_Is:
1921             n = 2;              tm = TM_M_IsI_;
1922             s0 = ii(0);         s1 = ii(1);             s2 = nop_i;
1923             break;
1924         case M_M_:
1925             n = 2;              tm = TM_M_M_I_;
1926             s0 = ii(0);         s1 = ii(1);             s2 = nop_i;
1927             break;
1928         case M_Ms:
1929             n = 2;              tm = TM_M_M_Is;
1930             s0 = ii(0);         s1 = ii(1);             s2 = nop_i;
1931             break;
1932         case M_F_:
1933             n = 2;              tm = TM_M_F_I_;
1934             s0 = ii(0);         s1 = ii(1);             s2 = nop_i;
1935             break;
1936         case M_Fs:
1937             n = 2;              tm = TM_M_F_Is;
1938             s0 = ii(0);         s1 = ii(1);             s2 = nop_i;
1939             break;
1940         case M_B_:
1941             n = 2;              tm = TM_M_B_B_;
1942             s0 = ii(0);         s1 = ii(1);             s2 = nop_b;
1943             break;
1944         case M_Bs:
1945             n = 2;              tm = TM_M_B_Bs;
1946             s0 = ii(0);         s1 = ii(1);             s2 = nop_b;
1947             break;
1948         case M_I_I_:
1949             n = 3;              tm = TM_M_I_I_;
1950             s0 = ii(0);         s1 = ii(1);             s2 = ii(2);
1951             break;
1952         case M_I_Is:
1953             n = 3;              tm = TM_M_I_Is;
1954             s0 = ii(0);         s1 = ii(1);             s2 = ii(2);
1955             break;
1956         case M_I_B_:
1957             n = 3;              tm = TM_M_I_B_;
1958             s0 = ii(0);         s1 = ii(1);             s2 = ii(2);
1959             break;
1960         case M_I_Bs:
1961             n = 3;              tm = TM_M_I_Bs;
1962             s0 = ii(0);         s1 = ii(1);             s2 = ii(2);
1963             break;
1964         case M_IsI_:
1965             n = 3;              tm = TM_M_IsI_;
1966             s0 = ii(0);         s1 = ii(1);             s2 = ii(2);
1967             break;
1968         case M_IsIs:
1969             n = 3;              tm = TM_M_IsIs;
1970             s0 = ii(0);         s1 = ii(1);             s2 = ii(2);
1971             break;
1972         case M_M_I_:
1973             n = 3;              tm = TM_M_M_I_;
1974             s0 = ii(0);         s1 = ii(1);             s2 = ii(2);
1975             break;
1976         case M_M_Is:
1977             n = 3;              tm = TM_M_M_Is;
1978             s0 = ii(0);         s1 = ii(1);             s2 = ii(2);
1979             break;
1980         case M_M_F_:
1981             n = 3;              tm = TM_M_M_F_;
1982             s0 = ii(0);         s1 = ii(1);             s2 = ii(2);
1983             break;
1984         case M_M_Fs:
1985             n = 3;              tm = TM_M_M_Fs;
1986             s0 = ii(0);         s1 = ii(1);             s2 = ii(2);
1987             break;
1988         case M_M_B_:
1989             n = 3;              tm = TM_M_M_B_;
1990             s0 = ii(0);         s1 = ii(1);             s2 = ii(2);
1991             break;
1992         case M_M_Bs:
1993             n = 3;              tm = TM_M_M_Bs;
1994             s0 = ii(0);         s1 = ii(1);             s2 = ii(2);
1995             break;
1996         case M_F_I_:
1997             n = 3;              tm = TM_M_F_I_;
1998             s0 = ii(0);         s1 = ii(1);             s2 = ii(2);
1999             break;
2000         case M_F_Is:
2001             n = 3;              tm = TM_M_F_Is;
2002             s0 = ii(0);         s1 = ii(1);             s2 = ii(2);
2003             break;
2004         case M_F_B_:
2005             n = 3;              tm = TM_M_F_B_;
2006             s0 = ii(0);         s1 = ii(1);             s2 = ii(2);
2007             break;
2008         case M_F_Bs:
2009             n = 3;              tm = TM_M_F_Bs;
2010             s0 = ii(0);         s1 = ii(1);             s2 = ii(2);
2011             break;
2012         case M_B_B_:
2013             n = 3;              tm = TM_M_B_B_;
2014             s0 = ii(0);         s1 = ii(1);             s2 = ii(2);
2015             break;
2016         case M_B_Bs:
2017             n = 3;              tm = TM_M_B_Bs;
2018             s0 = ii(0);         s1 = ii(1);             s2 = ii(2);
2019             break;
2020         case M_L_X_:
2021             n = 3;              tm = TM_M_L_X_;
2022             s0 = ii(0);         s1 = ii(1);             s2 = ii(2);
2023             break;
2024         case M_L_Xs:
2025             n = 3;              tm = TM_M_L_Xs;
2026             s0 = ii(0);         s1 = ii(1);             s2 = ii(2);
2027             break;
2028         case Ms:
2029             n = 1;              tm = TM_MsM_I_;
2030             s0 = ii(0);         s1 = nop_m;             s2 = nop_i;
2031             break;
2032         case MsI_:
2033             n = 2;              tm = TM_MsM_I_;
2034             s0 = ii(0);         s1 = nop_m;             s2 = ii(1);
2035             break;
2036         case MsIs:
2037             n = 2;              tm = TM_MsM_Is;
2038             s0 = ii(0);         s1 = nop_m;             s2 = ii(1);
2039             break;
2040         case MsM_:
2041             n = 2;              tm = TM_MsM_I_;
2042             s0 = ii(0);         s1 = ii(1);             s2 = nop_i;
2043             break;
2044         case MsMs:
2045             n = 2;              tm = TM_MsM_Is;
2046             s0 = ii(0);         s1 = ii(1);             s2 = nop_i;
2047             break;
2048         case MsM_I_:
2049             n = 3;              tm = TM_MsM_I_;
2050             s0 = ii(0);         s1 = ii(1);             s2 = ii(2);
2051             break;
2052         case MsM_Is:
2053             n = 3;              tm = TM_MsM_Is;
2054             s0 = ii(0);         s1 = ii(1);             s2 = ii(2);
2055             break;
2056         case F_:
2057             n = 1;              tm = TM_M_F_I_;
2058             s0 = nop_m;         s1 = ii(0);             s2 = nop_i;
2059             break;
2060         case F_I_:
2061             n = 2;              tm = TM_M_F_I_;
2062             s0 = nop_m;         s1 = ii(0);             s2 = ii(1);
2063             break;
2064         case F_Is:
2065             n = 2;              tm = TM_M_F_Is;
2066             s0 = nop_m;         s1 = ii(0);             s2 = ii(1);
2067             break;
2068         case F_B_:
2069             n = 2;              tm = TM_M_F_B_;
2070             s0 = nop_m;         s1 = ii(0);             s2 = ii(1);
2071             break;
2072         case F_Bs:
2073             n = 2;              tm = TM_M_F_Bs;
2074             s0 = nop_m;         s1 = ii(0);             s2 = ii(1);
2075             break;
2076         case Fs:
2077             n = 1;              tm = TM_M_F_Is;
2078             s0 = nop_m;         s1 = ii(0);             s2 = nop_i;
2079             break;
2080         case B_:
2081             n = 1;              tm = TM_B_B_B_;
2082             s0 = ii(0);         s1 = nop_b;             s2 = nop_b;
2083             break;
2084         case B_B_:
2085             n = 2;              tm = TM_B_B_B_;
2086             s0 = ii(0);         s1 = ii(1);             s2 = nop_b;
2087             break;
2088         case B_Bs:
2089             n = 2;              tm = TM_B_B_Bs;
2090             s0 = ii(0);         s1 = ii(1);             s2 = nop_b;
2091             break;
2092         case B_B_B_:
2093             n = 3;              tm = TM_B_B_B_;
2094             s0 = ii(0);         s1 = ii(1);             s2 = ii(2);
2095             break;
2096         case B_B_Bs:
2097             n = 3;              tm = TM_B_B_Bs;
2098             s0 = ii(0);         s1 = ii(1);             s2 = ii(2);
2099             break;
2100         case Bs:
2101             n = 1;              tm = TM_B_B_Bs;
2102             s0 = ii(0);         s1 = nop_b;             s2 = nop_b;
2103             break;
2104         case L_X_:
2105             n = 2;              tm = TM_M_L_X_;
2106             s0 = nop_m;         s1 = ii(0);             s2 = ii(1);
2107             break;
2108         case L_Xs:
2109             n = 2;              tm = TM_M_L_Xs;
2110             s0 = nop_m;         s1 = ii(0);             s2 = ii(1);
2111             break;
2112         default:
2113             abort();
2114     }
2115     out(n, tm, s0, s1, s2);
2116 }
2117
2118 static void
2119 _inst(jit_state_t *_jit, jit_word_t i, jit_uint8_t t)
2120 {
2121     if (_jitc->ioff > 2)
2122         flush();
2123     assert(!(i & 0x11111e0000000000L));
2124     _jitc->inst[_jitc->ioff].i = i;
2125     _jitc->inst[_jitc->ioff].t = t;
2126     ++_jitc->ioff;
2127 }
2128
2129 static void
2130 _A1(jit_state_t *_jit, jit_word_t _p,
2131     jit_word_t x4, jit_word_t x2, jit_word_t r3, jit_word_t r2, jit_word_t r1)
2132 {
2133     assert(!(_p & ~0x3fL));
2134     assert(!(x4 &  ~0xfL));
2135     assert(!(x2 &  ~0x3L));
2136     assert(!(r3 & ~0x7fL));
2137     assert(!(r2 & ~0x7fL));
2138     assert(!(r1 & ~0x7fL));
2139     TSTREG2(r2, r3);
2140     TSTPRED(_p);
2141     TSTREG1(r1);
2142     inst((8L<<37)|(x4<<29)|(x2<<27)|(r3<<20)|(r2<<13)|(r1<<6)|_p, INST_A);
2143     SETREG(r1);
2144 }
2145
2146 static void
2147 _A3(jit_state_t *_jit, jit_word_t _p,
2148     jit_word_t x4, jit_word_t x2, jit_word_t r3, jit_word_t im, jit_word_t r1)
2149 {
2150     assert(!(_p & ~0x3fL));
2151     assert(!(x4 &  ~0xfL));
2152     assert(!(x2 &  ~0x3L));
2153     assert(!(r3 & ~0x7fL));
2154     assert(im >= -128 && im <= 127);
2155     assert(!(r1 & ~0x7f));
2156     TSTREG1(r3);
2157     TSTPRED(_p);
2158     TSTREG1(r1);
2159     inst((8L<<37)|(((im>>7)&1L)<<36)|(x4<<29)|(x2<<27)|
2160          (r3<<20)|((im&0x7fL)<<13)|(r1<<6)|_p, INST_A);
2161     SETREG(r1);
2162 }
2163
2164 static void
2165 _A4(jit_state_t *_jit, jit_word_t _p,
2166     jit_word_t x2, jit_word_t r3, jit_word_t im, jit_word_t r1)
2167 {
2168     assert(!(_p & ~0x3fL));
2169     assert(!(x2 &  ~0x3L));
2170     assert(!(r3 & ~0x7fL));
2171     assert(im >= -8192 && im <= 8191);
2172     assert(!(r1  & ~0x7f));
2173     TSTREG1(r3);
2174     TSTPRED(_p);
2175     TSTREG1(r1);
2176     inst((8L<<37)|(((im>>13)&1L)<<36)|(x2<<34)|(((im>>7)&0x3fL)<<27)|
2177          (r3<<20)|((im&0x7fL)<<13)|(r1<<6)|_p, INST_A);
2178     SETREG(r1);
2179 }
2180
2181 static void
2182 _A5(jit_state_t *_jit, jit_word_t _p,
2183     jit_word_t r3, jit_word_t im, jit_word_t r1)
2184 {
2185     jit_word_t          s, i5, i9, i7;
2186     assert(!(_p & ~0x3fL));
2187     assert(!(r3  & ~0x3L));
2188     assert(im >= -2097152 && im <= 2097151);
2189     assert(!(r1  & ~0x7fL));
2190     /* imm22 = sign_ext(s << 21 | imm5c << 16 | imm9d << 7 | imm7b, 22) */
2191     s  = (im & 0x200000) >> 21;
2192     i5 = (im & 0x1f0000) >> 16;
2193     i9 = (im &   0xff80) >>  7;
2194     i7 =  im &     0x7f;
2195     TSTREG1(r3);
2196     TSTPRED(_p);
2197     TSTREG1(r1);
2198     inst((9L<<37)|(s<<36)|(i9<<27)|(i5<<22)|
2199          (r3<<20)|(i7<<13)|(r1<<6)|_p, INST_A);
2200     SETREG(r1);
2201 }
2202
2203 static void
2204 _A6(jit_state_t *_jit, jit_word_t _p,
2205     jit_word_t o, jit_word_t x2, jit_word_t ta, jit_word_t p2,
2206     jit_word_t r3, jit_word_t r2, jit_word_t c, jit_word_t p1)
2207 {
2208     assert(!(_p & ~0x3fL));
2209     assert(!(o  &  ~0xfL));
2210     assert(!(x2 &  ~0x3L));
2211     assert(!(ta &  ~0x1L));
2212     assert(!(p2 & ~0x7fL));
2213     assert(!(r3 & ~0x7fL));
2214     assert(!(r2 & ~0x7fL));
2215     assert(!(c  &  ~0x1L));
2216     assert(!(p1 & ~0x3fL));
2217     TSTREG2(r2, r3);
2218     TSTPRED(_p);
2219     inst((o<<37)|(x2<<34)|(ta<<33)|(p2<<27)|(r3<<20)|
2220          (r2<<13)|(c<<12)|(p1<<6)|_p, INST_A);
2221     if (p1)
2222         _jitc->pred |= 1 << p1;
2223     if (p2)
2224         _jitc->pred |= 1 << p2;
2225 }
2226
2227 static void
2228 _A7(jit_state_t *_jit, jit_word_t _p,
2229     jit_word_t o, jit_word_t x2, jit_word_t ta,
2230     jit_word_t p2, jit_word_t r3, jit_word_t c, jit_word_t p1)
2231 {
2232     assert(!(_p & ~0x3fL));
2233     assert(!(o  &  ~0xfL));
2234     assert(!(x2 &  ~0x3L));
2235     assert(!(ta &  ~0x1L));
2236     assert(!(p2 & ~0x7fL));
2237     assert(!(r3 & ~0x7fL));
2238     assert(!(c  &  ~0x1L));
2239     assert(!(p1 & ~0x3fL));
2240     TSTREG1(r3);
2241     TSTPRED(_p);
2242     inst((o<<37)|(1L<<36)|(x2<<34)|(ta<<33)|
2243          (p2<<27)|(r3<<20)|(c<<12)|(p1<<6)|_p, INST_A);
2244     if (p1)
2245         _jitc->pred |= 1 << p1;
2246     if (p2)
2247         _jitc->pred |= 1 << p2;
2248 }
2249
2250 static void
2251 _A8(jit_state_t *_jit, jit_word_t _p,
2252     jit_word_t o, jit_word_t x2, jit_word_t ta, jit_word_t p2,
2253     jit_word_t r3, jit_word_t im, jit_word_t c, jit_word_t p1)
2254 {
2255     assert(!(_p & ~0x3fL));
2256     assert(!(o  &  ~0xfL));
2257     assert(!(x2 &  ~0x3L));
2258     assert(!(ta &  ~0x1L));
2259     assert(!(p2 & ~0x7fL));
2260     assert(!(r3 & ~0x7fL));
2261     assert(im >= -128 && im <= 127);
2262     assert(!(c  &  ~0x1L));
2263     assert(!(p1 & ~0x3fL));
2264     TSTREG1(r3);
2265     TSTPRED(_p);
2266     inst((o<<37)|(((im>>7)&1L)<<36)|(x2<<34)|(ta<<33)|(p2<<27)|(r3<<20)|
2267          ((im&0x7fL)<<13)|(c<<12)|(p1<<6)|_p, INST_A);
2268     if (p1)
2269         _jitc->pred |= 1 << p1;
2270     if (p2)
2271         _jitc->pred |= 1 << p2;
2272 }
2273
2274 static void
2275 _A9(jit_state_t *_jit, jit_word_t _p,
2276     jit_word_t za, jit_word_t zb, jit_word_t x4,
2277     jit_word_t x2, jit_word_t r3, jit_word_t r2, jit_word_t r1)
2278 {
2279     assert(!(_p & ~0x3fL));
2280     assert(!(za &  ~0x1L));
2281     assert(!(zb &  ~0x1L));
2282     assert(!(x4 &  ~0xfL));
2283     assert(!(x2 &  ~0x3L));
2284     assert(!(r3 & ~0x7fL));
2285     assert(!(r2 & ~0x7fL));
2286     assert(!(r1 & ~0x7fL));
2287     TSTREG2(r2, r3);
2288     TSTPRED(_p);
2289     TSTREG1(r1);
2290     inst((8L<<37)|(za<<36)|(1L<<34)|(zb<<33)|(x4<<29)|(x2<<27)|
2291          (r3<<20)|(r2<<13)|(r1<<6)|_p, INST_A);
2292     SETREG(r1);
2293 }
2294
2295 static void
2296 _I1(jit_state_t *_jit, jit_word_t _p,
2297     jit_word_t ct, jit_word_t x2, jit_word_t r3, jit_word_t r2, jit_word_t r1)
2298 {
2299     assert(!(_p & ~0x3fL));
2300     assert(!(ct &  ~0x3L));
2301     assert(!(x2 &  ~0x3L));
2302     assert(!(r3 & ~0x7fL));
2303     assert(!(r2 & ~0x7fL));
2304     assert(!(r1 & ~0x7fL));
2305     TSTREG2(r2, r3);
2306     TSTPRED(_p);
2307     TSTREG1(r1);
2308     inst((7L<<37)|(1L<<33)|(ct<<30)|(x2<<28)|
2309          (r3<<20)|(r2<<13)|(r1<<6)|_p, INST_I);
2310     SETREG(r1);
2311 }
2312
2313 static void
2314 _I2(jit_state_t *_jit, jit_word_t _p,
2315     jit_word_t za, jit_word_t xa, jit_word_t zb, jit_word_t xc,
2316     jit_word_t xb ,jit_word_t r3, jit_word_t r2, jit_word_t r1)
2317 {
2318     assert(!(_p & ~0x3fL));
2319     assert(!(za &  ~0x1L));
2320     assert(!(xa &  ~0x3L));
2321     assert(!(zb &  ~0x1L));
2322     assert(!(xc &  ~0x3L));
2323     assert(!(xb &  ~0x3L));
2324     assert(!(r3 & ~0x7fL));
2325     assert(!(r2 & ~0x7fL));
2326     assert(!(r1 & ~0x7fL));
2327     TSTREG2(r2, r3);
2328     TSTPRED(_p);
2329     TSTREG1(r1);
2330     inst((7L<<37)|(za<<36)|(xa<<34)|(zb<<33)|(xc<<30)|
2331          (xb<<28)|(r3<<20)|(r2<<13)|(r1<<6), INST_I);
2332     SETREG(r1);
2333 }
2334
2335 static void
2336 _I3(jit_state_t *_jit, jit_word_t _p,
2337     jit_word_t mb, jit_word_t r2, jit_word_t r1)
2338 {
2339     assert(!(_p & ~0x3fL));
2340     assert(!(mb &  ~0xfL));
2341     assert(!(r2 & ~0x7fL));
2342     assert(!(r1 & ~0x7fL));
2343     TSTREG1(r2);
2344     TSTPRED(_p);
2345     TSTREG1(r1);
2346     inst((7L<<37)|(3L<<34)|(2L<<30)|(2L<<28)|
2347          (mb<<20)|(r2<<13)|(r1<<6)|_p, INST_I);
2348     SETREG(r1);
2349 }
2350
2351 static void
2352 _I4(jit_state_t *_jit, jit_word_t _p,
2353     jit_word_t mh, jit_word_t r2, jit_word_t r1)
2354 {
2355     assert(!(_p & ~0x3fL));
2356     assert(!(mh & ~0xffL));
2357     assert(!(r2 & ~0x7fL));
2358     assert(!(r1 & ~0x7fL));
2359     TSTREG1(r2);
2360     TSTPRED(_p);
2361     TSTREG1(r1);
2362     inst((7L<<37)|(3L<<34)|(1L<<33)|(2L<<30)|
2363          (2L<<28)|(mh<<20)|(r2<<13)|(r1<<6)|_p, INST_I);
2364     SETREG(r1);
2365 }
2366
2367 static void
2368 _I5(jit_state_t *_jit, jit_word_t _p,
2369     jit_word_t za, jit_word_t zb, jit_word_t x2,
2370     jit_word_t r3, jit_word_t r2, jit_word_t r1)
2371 {
2372     assert(!(_p & ~0x3fL));
2373     assert(!(za &  ~0x1L));
2374     assert(!(zb &  ~0x1L));
2375     assert(!(x2 &  ~0x3L));
2376     assert(!(r3 & ~0x7fL));
2377     assert(!(r2 & ~0x7fL));
2378     assert(!(r1 & ~0x7fL));
2379     TSTREG2(r2, r3);
2380     TSTPRED(_p);
2381     TSTREG1(r1);
2382     inst((7L<<37)|(za<<36)|(zb<<33)|(x2<<28)|
2383          (r3<<20)|(r2<<13)|(r1<<6)|_p, INST_I);
2384     SETREG(r1);
2385 }
2386
2387 static void
2388 _I6(jit_state_t *_jit, jit_word_t _p,
2389     jit_word_t za, jit_word_t zb, jit_word_t x2,
2390     jit_word_t r3, jit_word_t ct, jit_word_t r1)
2391 {
2392     assert(!(_p & ~0x3fL));
2393     assert(!(za &  ~0x1L));
2394     assert(!(zb &  ~0x1L));
2395     assert(!(x2 &  ~0x3L));
2396     assert(!(r3 & ~0x7fL));
2397     assert(!(ct & ~0x1fL));
2398     assert(!(r1 & ~0x7fL));
2399     TSTREG1(r3);
2400     TSTPRED(_p);
2401     TSTREG1(r1);
2402     inst((7L<<37)|(za<<36)|(1L<<34)|(zb<<33)|
2403          (x2<<28)|(r3<<20)|(ct<<14)|(r1<<6)|_p, INST_I);
2404     SETREG(r1);
2405 }
2406
2407 static void
2408 _I7(jit_state_t *_jit, jit_word_t _p,
2409     jit_word_t za, jit_word_t zb, jit_word_t r3, jit_word_t r2, jit_word_t r1)
2410 {
2411     assert(!(_p & ~0x3fL));
2412     assert(!(za &  ~0x1L));
2413     assert(!(zb &  ~0x1L));
2414     assert(!(r3 & ~0x7fL));
2415     assert(!(r2 & ~0x7fL));
2416     assert(!(r1 & ~0x7fL));
2417     TSTREG2(r2, r3);
2418     TSTPRED(_p);
2419     TSTREG1(r1);
2420     inst((7L<<37)|(za<<36)|(zb<<33)|(1L<<30)|
2421          (r3<<20)|(r2<<13)|(r1<<6)|_p, INST_I);
2422     SETREG(r1);
2423 }
2424
2425 static void
2426 _I8(jit_state_t *_jit, jit_word_t _p,
2427     jit_word_t za, jit_word_t zb, jit_word_t im, jit_word_t r2, jit_word_t r1)
2428 {
2429     assert(!(_p & ~0x3fL));
2430     assert(!(za &  ~0x1L));
2431     assert(!(zb &  ~0x1L));
2432     assert(!(im & ~0x1fL));
2433     assert(!(r2 & ~0x7fL));
2434     assert(!(r1 & ~0x7fL));
2435     TSTREG1(r2);
2436     TSTPRED(_p);
2437     TSTREG1(r1);
2438     inst((7L<<37)|(za<<36)|(3L<<34)|(zb<<33)|(1L<<30)|(1L<<28)|
2439          (im<<20)|(r2<<13)|(r1<<6), INST_I);
2440     SETREG(r1);
2441 }
2442
2443 static void
2444 _I9(jit_state_t *_jit, jit_word_t _p,
2445     jit_word_t x2, jit_word_t r3, jit_word_t r1)
2446 {
2447     assert(!(_p & ~0x3fL));
2448     assert(!(x2 &  ~0x3L));
2449     assert(!(r3 & ~0x7fL));
2450     assert(!(r1 & ~0x7fL));
2451     TSTREG1(r3);
2452     TSTPRED(_p);
2453     TSTREG1(r1);
2454     inst((7L<<37)|(1L<<34)|(1L<<34)|(1L<<33)|
2455          (x2<<30)|(1L<<28)|(r3<<20)|(r1<<6)|_p, INST_I);
2456     SETREG(r1);
2457 }
2458
2459 static void
2460 _I10(jit_state_t *_jit, jit_word_t _p,
2461      jit_word_t ct, jit_word_t r3, jit_word_t r2, jit_word_t r1)
2462 {
2463     assert(!(_p & ~0x3fL));
2464     assert(!(ct & ~0x3fL));
2465     assert(!(r3 & ~0x7fL));
2466     assert(!(r2 & ~0x7fL));
2467     assert(!(r1 & ~0x7fL));
2468     TSTREG2(r2, r3);
2469     TSTPRED(_p);
2470     TSTREG1(r1);
2471     inst((5L<<37)|(3L<<34)|(ct<<27)|(r3<<20)|(r2<<13)|(r1<<6)|_p, INST_I);
2472     SETREG(r1);
2473 }
2474
2475 static void
2476 _I11(jit_state_t *_jit, jit_word_t _p,
2477      jit_word_t len, jit_word_t r3,
2478      jit_word_t pos, jit_word_t y, jit_word_t r1)
2479 {
2480     assert(!(_p  & ~0x3fL));
2481     assert(!(len & ~0x3fL));
2482     assert(!(r3  & ~0x7fL));
2483     assert(!(pos & ~0x1fL));
2484     assert(!(y   &  ~0x1L));
2485     assert(!(r1  & ~0x7fL));
2486     TSTREG1(r3);
2487     TSTPRED(_p);
2488     TSTREG1(r1);
2489     inst((5L<<37)|(1L<<34)|(len<<27)|(r3<<20)|
2490          (pos<<14)|(y<<13)|(r1<<6)|_p, INST_I);
2491     SETREG(r1);
2492 }
2493
2494 static void
2495 _I12(jit_state_t *_jit, jit_word_t _p,
2496      jit_word_t len, jit_word_t pos, jit_word_t r2, jit_word_t r1)
2497 {
2498     assert(!(_p  & ~0x3fL));
2499     assert(!(len & ~0x3fL));
2500     assert(!(pos & ~0x3fL));
2501     assert(!(r2  & ~0x7fL));
2502     assert(!(r1  & ~0x7fL));
2503     TSTREG1(r2);
2504     TSTPRED(_p);
2505     TSTREG1(r1);
2506     inst((5L<<37)|(1L<<34)|(1L<<33)|(len<<27)|
2507          (pos<<20)|(r2<<13)|(r1<<6)|_p, INST_I);
2508     SETREG(r1);
2509 }
2510
2511 static void
2512 _I13(jit_state_t *_jit, jit_word_t _p,
2513      jit_word_t len, jit_word_t pos, jit_word_t im, jit_word_t r1)
2514 {
2515     assert(!(_p  & ~0x3fL));
2516     assert(!(len & ~0x3fL));
2517     assert(!(pos & ~0x3fL));
2518     assert(!(im  & ~0x7fL));
2519     assert(!(r1  & ~0x7fL));
2520     TSTPRED(_p);
2521     TSTREG1(r1);
2522     inst((5L<<37)|(((im>>7)&1L)<<36)|(1L<<34)|(1L<<33)|(len<<27)|
2523          (1L<<26)|(pos<<20)|((im&0x7fL)<<13)|(r1<<6)|_p, INST_I);
2524     SETREG(r1);
2525 }
2526
2527 static void
2528 _I14(jit_state_t *_jit, jit_word_t _p,
2529      jit_word_t s, jit_word_t len, jit_word_t r3, jit_word_t pos, jit_word_t r1)
2530 {
2531     assert(!(_p  & ~0x3fL));
2532     assert(!(s   &  ~0x1L));
2533     assert(!(len & ~0x3fL));
2534     assert(!(r3  & ~0x7fL));
2535     assert(!(pos & ~0x1fL));
2536     assert(!(r1  & ~0x7fL));
2537     TSTREG1(r3);
2538     TSTPRED(_p);
2539     TSTREG1(r1);
2540     inst((5L<<37)|(s<<36)|(3L<<34)|(1L<<33)|
2541          (len<<27)|(r3<<20)|(pos<<14)|(r1<<6)|_p, INST_I);
2542     SETREG(r1);
2543 }
2544
2545 static void
2546 _I15(jit_state_t *_jit, jit_word_t _p,
2547      jit_word_t pos, jit_word_t len,
2548      jit_word_t r3, jit_word_t r2, jit_word_t r1)
2549 {
2550     assert(!(_p  & ~0x3fL));
2551     assert(!(pos & ~0x3fL));
2552     assert(!(len &  ~0xfL));
2553     assert(!(r3  & ~0x7fL));
2554     assert(!(r2  & ~0x7fL));
2555     assert(!(r1  & ~0x7fL));
2556     TSTREG2(r2, r3);
2557     TSTPRED(_p);
2558     TSTREG1(r1);
2559     inst((4L<<37)|(pos<<31)|(len<<27)|(r3<<20)|(r2<<13)|(r1<<6)|_p, INST_I);
2560     SETREG(r1);
2561 }
2562
2563 static void
2564 _I16(jit_state_t *_jit, jit_word_t _p,
2565      jit_word_t tb, jit_word_t ta, jit_word_t p2,
2566      jit_word_t r3, jit_word_t ps, jit_word_t c, jit_word_t p1)
2567 {
2568     assert(!(_p & ~0x3fL));
2569     assert(!(tb &  ~0x1L));
2570     assert(!(ta &  ~0x1L));
2571     assert(!(p2 & ~0x7fL));
2572     assert(!(r3 & ~0x7fL));
2573     assert(!(ps & ~0x3fL));
2574     assert(!(c  &  ~0x1L));
2575     assert(!(p1 & ~0x3fL));
2576     TSTREG1(r3);
2577     TSTPRED(_p);
2578     inst((5L<<37)|(tb<<36)|(ta<<33)|(p2<<27)|
2579          (r3<<20)|(ps<<14)|(c<<12)|(p1<<6), INST_I);
2580     if (p1)
2581         _jitc->pred |= 1 << p1;
2582     if (p2)
2583         _jitc->pred |= 1 << p2;
2584 }
2585
2586 static void
2587 _I17(jit_state_t *_jit, jit_word_t _p,
2588      jit_word_t tb, jit_word_t ta, jit_word_t p2,
2589      jit_word_t r3, jit_word_t c, jit_word_t p1)
2590 {
2591     assert(!(_p & ~0x3fL));
2592     assert(!(tb &  ~0x1L));
2593     assert(!(ta &  ~0x1L));
2594     assert(!(p2 & ~0x7fL));
2595     assert(!(r3 & ~0x7fL));
2596     assert(!(c  &  ~0x1L));
2597     assert(!(p1 & ~0x3fL));
2598     TSTREG1(r3);
2599     TSTPRED(_p);
2600     inst((5L<<37)|(tb<<36)|(ta<<33)|(p2<<27)|
2601          (r3<<20)|(1L<<13)|(c<<12)|(p1<<6)|_p, INST_I);
2602     if (p1)
2603         _jitc->pred |= 1 << p1;
2604     if (p2)
2605         _jitc->pred |= 1 << p2;
2606 }
2607
2608 static void
2609 _I18(jit_state_t *_jit, jit_word_t _p,
2610      jit_word_t im, jit_word_t y)
2611 {
2612     assert(!(_p &     ~0x3fL));
2613     assert(!(im & ~0x1fffffL));
2614     assert(!(y  &      ~0x1L));
2615     TSTPRED(_p);
2616     inst((((im>>20)&1L)<<26)|(1L<<27)|(y<<26)|((im&0xffffL)<<6)|_p, INST_I);
2617 }
2618
2619 static void
2620 _I19(jit_state_t *_jit, jit_word_t _p,
2621      jit_word_t im)
2622 {
2623     assert(!(_p &     ~0x3fL));
2624     assert(!(im & ~0x1fffffL));
2625     TSTPRED(_p);
2626     inst(((im>>20)&1L)|((im&0xffffL)<<6)|_p, INST_I);
2627 }
2628
2629 static void
2630 _I20(jit_state_t *_jit, jit_word_t _p,
2631      jit_word_t r2, jit_word_t im)
2632 {
2633     assert(!(_p &     ~0x3fL));
2634     assert(!(r2 &     ~0x7fL));
2635     assert(!(im & ~0x1fffffL));
2636     TSTREG1(r2);
2637     TSTPRED(_p);
2638     inst(((im>>20)&1L)|(1L<<33)|(((im>>7)&0x1fffL)<<20)|
2639          (r2<<13)|((im&0x7fL)<<6)|_p, INST_I);
2640 }
2641
2642 static void
2643 _I21(jit_state_t *_jit, jit_word_t _p,
2644      jit_word_t im, jit_word_t ih, jit_word_t x,
2645      jit_word_t wh, jit_word_t r2, jit_word_t b1)
2646 {
2647     assert(!(_p &     ~0x3fL));
2648     assert(!(im &    ~0x1ffL));
2649     assert(!(ih &      ~0x1L));
2650     assert(!(x  &      ~0x1L));
2651     assert(!(wh &      ~0x3L));
2652     assert(!(r2 &     ~0x7fL));
2653     assert(!(b1 &      ~0x7L));
2654     TSTREG1(r2);
2655     TSTPRED(_p);
2656     inst((7L<<33)|(im<<24)|(ih<<23)|(x<<22)|(wh<<20)|
2657          (r2<<13)|(b1<<6), INST_I);
2658 }
2659
2660 static void
2661 _I22(jit_state_t *_jit, jit_word_t _p,
2662      jit_word_t b2, jit_word_t r1)
2663 {
2664     assert(!(_p & ~0x3fL));
2665     assert(!(b2 &  ~0x7L));
2666     assert(!(r1 & ~0x7fL));
2667     TSTPRED(_p);
2668     inst((0x31L<<27)|(b2<<13)|(r1<<6)|_p, INST_I);
2669 }
2670
2671 static void
2672 _I23(jit_state_t *_jit, jit_word_t _p,
2673      jit_word_t r2, jit_word_t im)
2674 {
2675     assert(!(_p &   ~0x3fL));
2676     assert(!(r2 &   ~0x7fL));
2677     assert(!(im & ~0xffffL));
2678     TSTREG1(r2);
2679     TSTPRED(_p);
2680     inst((((im>>15)&1L)<<36)|(3L<<33)|(((im>>7)&0xffL)<<24)|
2681          (r2<<13)|(im&0x7fL)|_p, INST_I);
2682 }
2683
2684 static void
2685 _I24(jit_state_t *_jit, jit_word_t _p,
2686      jit_word_t im)
2687 {
2688     jit_uint8_t         cc = INST_I;
2689     assert(!(_p &      ~0x3fL));
2690     assert(!(im & ~0xfffffffL));
2691     TSTPRED(_p);
2692     inst((((im>>27)&1L)<<36)|(2L<<33)|((im&0x7ffffffL)<<6)|_p, cc);
2693 }
2694
2695 static void
2696 _I25(jit_state_t *_jit, jit_word_t _p,
2697      jit_word_t x6, jit_word_t r1)
2698 {
2699     assert(!(_p & ~0x3fL));
2700     assert(!(x6 & ~0x3fL));
2701     assert(!(r1 & ~0x7fL));
2702     TSTPRED(_p);
2703     TSTREG1(r1);
2704     inst((x6<<27)|(r1<<6)|_p, INST_I);
2705     SETREG(r1);
2706 }
2707
2708 static void
2709 _I26(jit_state_t *_jit, jit_word_t _p,
2710      jit_word_t ar,jit_word_t r2)
2711 {
2712     assert(!(_p & ~0x3fL));
2713     assert(!(ar & ~0x7fL));
2714     assert(!(r2 & ~0x7fL));
2715     TSTREG1(r2);
2716     TSTPRED(_p);
2717     inst((0x2aL<<27)|(ar<<20)|(r2<<13)|_p, INST_I);
2718 }
2719
2720 static void
2721 _I27(jit_state_t *_jit, jit_word_t _p,
2722      jit_word_t ar,jit_word_t im)
2723 {
2724     assert(!(_p & ~0x3fL));
2725     assert(!(ar & ~0x7fL));
2726     assert(!(im & ~0xffL));
2727     TSTPRED(_p);
2728     inst((((im>>7)&1L)<<36)|(0xaL<<27)|(ar<<20)|((im&0x7fL)<<13)|_p, INST_I);
2729 }
2730
2731 static void
2732 _I28(jit_state_t *_jit, jit_word_t _p,
2733      jit_word_t ar, jit_word_t r1)
2734 {
2735     assert(!(_p & ~0x3fL));
2736     assert(!(ar & ~0x7fL));
2737     assert(!(r1 & ~0x7fL));
2738     TSTPRED(_p);
2739     TSTREG1(r1);
2740     inst((0x32L<<27)|(ar<<20)|(r1<<6)|_p, INST_I);
2741     SETREG(r1);
2742 }
2743
2744 static void
2745 _I29(jit_state_t *_jit, jit_word_t _p,
2746      jit_word_t x6, jit_word_t r3,jit_word_t r1)
2747 {
2748     assert(!(_p & ~0x3fL));
2749     assert(!(x6 & ~0x3fL));
2750     assert(!(r3 & ~0x7fL));
2751     assert(!(r1 & ~0x7fL));
2752     TSTREG1(r3);
2753     TSTPRED(_p);
2754     TSTREG1(r1);
2755     inst((x6<<27)|(r3<<20)|(r1<<6)|_p, INST_I);
2756     SETREG(r1);
2757 }
2758
2759 static void
2760 _I30(jit_state_t *_jit, jit_word_t _p,
2761      jit_word_t ta, jit_word_t tb, jit_word_t p2,
2762      jit_word_t im, jit_word_t c, jit_word_t p1)
2763 {
2764     assert(!(_p & ~0x3fL));
2765     assert(!(ta &  ~0x1L));
2766     assert(!(tb &  ~0x1L));
2767     assert(!(p2 & ~0x3fL));
2768     assert(!(im & ~0x1fL));
2769     assert(!(c  &  ~0x1L));
2770     assert(!(p1 & ~0x1fL));
2771     TSTPRED(_p);
2772     inst((5L<<37)|(tb<<36)|(ta<<33)|(1L<<19)|(im<<14)|
2773          (1L<<13)|(c<<12)|(p1<<6)|_p, INST_I);
2774     if (p1)
2775         _jitc->pred |= 1 << p1;
2776     if (p2)
2777         _jitc->pred |= 1 << p2;
2778 }
2779
2780 static void
2781 _M1(jit_state_t *_jit, jit_word_t _p,
2782     jit_word_t x6, jit_word_t ht, jit_word_t x, jit_word_t r3, jit_word_t r1)
2783 {
2784     assert(!(_p & ~0x3fL));
2785     assert(!(x6 & ~0x3fL));
2786     assert(!(ht &  ~0x3L));
2787     assert(!(x  &  ~0x1L));
2788     assert(!(r3 & ~0x7fL));
2789     assert(!(r1 & ~0x7fL));
2790     TSTREG1(r3);
2791     TSTPRED(_p);
2792     TSTREG1(r1);
2793     inst((4L<<37)|(x6<<30)|(ht<<28)|(x<<27)|(r3<<20)|(r1<<6)|_p, INST_M);
2794     SETREG(r1);
2795 }
2796
2797 static void
2798 _M2(jit_state_t *_jit, jit_word_t _p,
2799     jit_word_t x6, jit_word_t ht, jit_word_t r3, jit_word_t r2, jit_word_t r1)
2800 {
2801     assert(!(_p & ~0x3fL));
2802     assert(!(x6 & ~0x3fL));
2803     assert(!(ht &  ~0x3L));
2804     assert(!(r3 & ~0x7fL));
2805     assert(!(r2 & ~0x7fL));
2806     assert(!(r1 & ~0x7fL));
2807     TSTREG2(r2, r3);
2808     TSTPRED(_p);
2809     TSTREG1(r1);
2810     inst((4L<<37)|(1L<<36)|(x6<<30)|(ht<<28)|
2811          (r3<<20)|(r2<<13)|(r1<<6)|_p, INST_M);
2812     SETREG(r1);
2813     SETREG(r3);
2814 }
2815
2816 static void
2817 _M3(jit_state_t *_jit, jit_word_t _p,
2818     jit_word_t x6, jit_word_t ht, jit_word_t r3, jit_word_t im, jit_word_t r1)
2819 {
2820     jit_uint8_t         cc = INST_M;
2821     assert(!(_p &  ~0x3fL));
2822     assert(!(x6 &  ~0x3fL));
2823     assert(!(ht &   ~0x3L));
2824     assert(!(r3 &  ~0x7fL));
2825     assert(im > -256 && im <= 255);
2826     assert(!(r1 &  ~0x7fL));
2827     TSTREG1(r3);
2828     TSTPRED(_p);
2829     TSTREG1(r1);
2830     inst((5L<<37)|(((im>>8)&1L)<<36)|(x6<<30)|(ht<<28)|
2831          (((im>>7)&1L)<<27)|(r3<<20)|((im&0x7fL)<<13)|(r1<<6)|_p, cc);
2832     SETREG(r1);
2833     SETREG(r3);
2834 }
2835
2836 static void
2837 _M5(jit_state_t *_jit, jit_word_t _p,
2838     jit_word_t x6, jit_word_t ht, jit_word_t r3, jit_word_t r2, jit_word_t im)
2839 {
2840     assert(!(_p &  ~0x3fL));
2841     assert(!(x6 &  ~0x3fL));
2842     assert(!(ht &   ~0x3L));
2843     assert(!(r3 &  ~0x7fL));
2844     assert(!(r2 &  ~0x7fL));
2845     assert(im > -256 && im <= 255);
2846     TSTREG2(r2, r3);
2847     TSTPRED(_p);
2848     inst((5L<<37)|(((im>>8)&1L)<<36)|(x6<<30)|(ht<<28)|
2849          (((im>>7)&1L)<<27)|(r3<<20)|(r2<<13)|((im&0x7fL)<<6)|_p, INST_M);
2850     SETREG(r3);
2851 }
2852
2853 static void
2854 _M6(jit_state_t *_jit, jit_word_t _p,
2855     jit_word_t x6, jit_word_t ht, jit_word_t x, jit_word_t r3, jit_word_t r2)
2856 {
2857     assert(!(_p & ~0x3fL));
2858     assert(!(x6 & ~0x3fL));
2859     assert(!(ht &  ~0x3L));
2860     assert(!(x  &  ~0x1L));
2861     assert(!(r3 & ~0x7fL));
2862     assert(!(r2 & ~0x7fL));
2863     TSTREG2(r2, r3);
2864     TSTPRED(_p);
2865     inst((4L<<37)|(x6<<30)|(ht<<28)|(x<<27)|(r3<<20)|(r2<<13)|_p, INST_M);
2866 }
2867
2868 static void
2869 _M13(jit_state_t *_jit, jit_word_t _p,
2870      jit_word_t x6, jit_word_t ht, jit_word_t r3, jit_word_t r2)
2871 {
2872     assert(!(_p & ~0x3fL));
2873     assert(!(x6 & ~0x3fL));
2874     assert(!(ht &  ~0x3L));
2875     assert(!(r3 & ~0x7fL));
2876     assert(!(r2 & ~0x7fL));
2877     TSTREG1(r3);
2878     if (r2)
2879         TSTFREG1(r2);
2880     TSTPRED(_p);
2881     inst((6L<<37)|(x6<<30)|(ht<<28)|(r3<<20)|(r2<<13)|_p, INST_M);
2882 }
2883
2884 static void
2885 _M14(jit_state_t *_jit, jit_word_t _p,
2886      jit_word_t x6, jit_word_t ht, jit_word_t r3, jit_word_t r2)
2887 {
2888     assert(!(_p & ~0x3fL));
2889     assert(!(x6 & ~0x3fL));
2890     assert(!(ht &  ~0x3L));
2891     assert(!(r3 & ~0x7fL));
2892     assert(!(r2 & ~0x7fL));
2893     TSTREG2(r2, r3);
2894     TSTPRED(_p);
2895     inst((6L<<37)|(1L<<36)|(x6<<30)|(ht<<28)|(r3<<20)|(r2<<13)|_p, INST_M);
2896 }
2897
2898 static void
2899 _M15(jit_state_t *_jit, jit_word_t _p,
2900      jit_word_t x6, jit_word_t ht, jit_word_t r3, jit_word_t im)
2901 {
2902     assert(!(_p &  ~0x3fL));
2903     assert(!(x6 &  ~0x3fL));
2904     assert(!(ht &   ~0x3L));
2905     assert(!(r3 &  ~0x7fL));
2906     assert(!(im & ~0x1ffL));
2907     TSTREG1(r3);
2908     TSTPRED(_p);
2909     inst((7L<<37)|(((im>>8)&1L)<<36)|(x6<<30)|(ht<<28)|
2910          (((im>>7)&1L)<<27)|(r3<<20)|((im&0x7fL)<<13)|_p, INST_M);
2911 }
2912
2913 static void
2914 _M16(jit_state_t *_jit, jit_word_t _p,
2915      jit_word_t x6, jit_word_t ht, jit_word_t r3, jit_word_t r2, jit_word_t r1)
2916 {
2917     assert(!(_p & ~0x3fL));
2918     assert(!(x6 & ~0x3fL));
2919     assert(!(ht &  ~0x3L));
2920     assert(!(r3 & ~0x7fL));
2921     assert(!(r2 & ~0x7fL));
2922     assert(!(r1 & ~0x7fL));
2923     TSTREG2(r2, r3);
2924     TSTPRED(_p);
2925     TSTREG1(r1);
2926     inst((4L<<37)|(x6<<30)|(ht<<28)|(1L<<27)|
2927          (r3<<20)|(r2<<13)|(r1<<6)|_p, INST_M);
2928     SETREG(r1);
2929 }
2930
2931 static void
2932 _M17(jit_state_t *_jit, jit_word_t _p,
2933      jit_word_t x6, jit_word_t ht, jit_word_t r3, jit_word_t im, jit_word_t r1)
2934 {
2935     assert(!(_p & ~0x3fL));
2936     assert(!(x6 & ~0x3fL));
2937     assert(!(ht &  ~0x3L));
2938     assert(!(r3 & ~0x7fL));
2939     assert(!(im &  ~0x7L));
2940     assert(!(r1 & ~0x7fL));
2941     TSTREG1(r3);
2942     TSTPRED(_p);
2943     TSTREG1(r1);
2944     inst((4L<<37)|(x6<<30)|(ht<<28)|(1L<<27)|
2945          (r3<<20)|(im<<13)|(r1<<6)|_p, INST_M);
2946     SETREG(r1);
2947 }
2948
2949 static void
2950 _M20x(jit_state_t *_jit, jit_word_t _p,
2951       jit_word_t x3, jit_word_t r2, jit_word_t im)
2952 {
2953     assert(!(_p &     ~0x3fL));
2954     assert(!(x3 &      ~0x7L));
2955     assert(!(r2 &     ~0x7fL));
2956     assert(!(im & ~0x1fffffL));
2957     if (x3 == 1)
2958         TSTREG1(r2);
2959     else
2960         TSTFREG1(r2);
2961     TSTPRED(_p);
2962     inst((1L<<37)|(((im>>20)&1L)<<36)|(x3<<33)|
2963          (((im>>7)&0x1fffL)<<20)|(r2<<13)|((im&0x7fL)<<6)|_p, INST_M);
2964 }
2965
2966 static void
2967 _M22x(jit_state_t *_jit, jit_word_t _p,
2968       jit_word_t x3, jit_word_t im, jit_word_t r1)
2969 {
2970     assert(!(_p &     ~0x3fL));
2971     assert(!(x3 &      ~0x7L));
2972     assert(!(im & ~0x1fffffL));
2973     assert(!(r1 &     ~0x7fL));
2974     TSTPRED(_p);
2975     if (x3 < 6)
2976         TSTREG1(r1);
2977     else
2978         TSTFREG1(r1);
2979     inst((((im>>20)&1L)<<36)|(x3<<33)|((im&0xffffL)<<13)|(r1<<6)|_p, INST_M);
2980     if (x3 < 6)
2981         SETREG(r1);
2982     else
2983         SETFREG(r1);
2984 }
2985
2986 static void
2987 _M24(jit_state_t *_jit, jit_word_t _p,
2988      jit_word_t x2, jit_word_t x4)
2989 {
2990     assert(!(_p & ~0x3fL));
2991     assert(!(x2 &  ~0x3L));
2992     assert(!(x4 &  ~0xfL));
2993     TSTPRED(_p);
2994     inst((x2<<31)|(x4<<27)|_p, INST_M);
2995 }
2996
2997 static void
2998 _M26x(jit_state_t *_jit, jit_word_t _p,
2999       jit_word_t x4, jit_word_t r1)
3000 {
3001     assert(!(_p & ~0x3fL));
3002     assert(!(x4 &  ~0xfL));
3003     assert(!(r1 & ~0x7fL));
3004     if (x4 == 2)
3005         TSTREG1(r1);
3006     else
3007         TSTFREG1(r1);
3008     TSTPRED(_p);
3009     inst((1L<<31)|(x4<<27)|(r1<<6)|_p, INST_M);
3010 }
3011
3012 static void
3013 _M28(jit_state_t *_jit, jit_word_t _p,
3014      jit_word_t x, jit_word_t r3)
3015 {
3016     assert(!(_p & ~0x3fL));
3017     assert(!(x  &  ~0x1L));
3018     assert(!(r3 & ~0x7fL));
3019     TSTREG1(r3);
3020     TSTPRED(_p);
3021     inst((1L<<37)|(x<<36)|(0x30L<<27)|(r3<<20)|_p, INST_M);
3022 }
3023
3024 static void
3025 _M29(jit_state_t *_jit, jit_word_t _p,
3026      jit_word_t ar, jit_word_t r2)
3027 {
3028     assert(!(_p & ~0x3fL));
3029     assert(!(ar  & ~0x7L));
3030     assert(!(r2 & ~0x7fL));
3031     TSTREG1(r2);
3032     TSTPRED(_p);
3033     inst((1L<<37)|(0x2aL<<27)|(ar<<20)|(r2<<13)|_p, INST_M);
3034 }
3035
3036 static void
3037 _M30(jit_state_t *_jit, jit_word_t _p,
3038      jit_word_t ar, jit_word_t im)
3039 {
3040     assert(!(_p & ~0x3fL));
3041     assert(!(ar  & ~0x7L));
3042     assert(!(im & ~0xffL));
3043     TSTPRED(_p);
3044     inst((((im>>7)&1L)<<36)|(2L<<31)|(0x8L<<27)|
3045          (ar<<20)|((im&0x7fL)<<13)|_p, INST_M);
3046 }
3047
3048 static void
3049 _M31(jit_state_t *_jit, jit_word_t _p,
3050      jit_word_t ar, jit_word_t r1)
3051 {
3052     assert(!(_p & ~0x3fL));
3053     assert(!(ar  & ~0x7L));
3054     assert(!(r1 & ~0x7fL));
3055     TSTPRED(_p);
3056     TSTREG1(r1);
3057     inst((1L<<37)|(0x22L<<27)|(ar<<20)|(r1<<6)|_p, INST_M);
3058     SETREG(r1);
3059 }
3060
3061 static void
3062 _M32(jit_state_t *_jit, jit_word_t _p,
3063      jit_word_t cr, jit_word_t r2)
3064 {
3065     assert(!(_p & ~0x3fL));
3066     assert(!(cr  & ~0x7L));
3067     assert(!(r2 & ~0x7fL));
3068     TSTREG1(r2);
3069     TSTPRED(_p);
3070     inst((1L<<37)|(0x2cL<<27)|(cr<<20)|(r2<<13)|_p, INST_M);
3071 }
3072
3073 static void
3074 _M33(jit_state_t *_jit, jit_word_t _p,
3075      jit_word_t cr, jit_word_t r1)
3076 {
3077     assert(!(_p & ~0x3fL));
3078     assert(!(cr  & ~0x7L));
3079     assert(!(r1 & ~0x7fL));
3080     TSTPRED(_p);
3081     TSTREG1(r1);
3082     inst((1L<<37)|(0x24L<<27)|(cr<<20)|(r1<<6)|_p, INST_M);
3083     SETREG(r1);
3084 }
3085
3086 static void
3087 _M34(jit_state_t *_jit, jit_word_t _p,
3088      jit_word_t sor, jit_word_t sol, jit_word_t sof, jit_word_t r1)
3089 {
3090     assert(!(_p  & ~0x3fL));
3091     assert(!(sor &  ~0xfL));
3092     assert(!(sol & ~0x7fL));
3093     assert(!(sof & ~0x7fL));
3094     assert(!(r1  & ~0x7fL));
3095     TSTPRED(_p);
3096     inst((1L<<37)|(6L<<33)|(sor<<27)|(sol<<20)|(sof<<13)|(r1<<6)|_p, INST_M);
3097 }
3098
3099 static void
3100 _M35(jit_state_t *_jit, jit_word_t _p,
3101      jit_word_t x6, jit_word_t r2)
3102 {
3103     assert(!(_p & ~0x3fL));
3104     assert(!(x6 & ~0x3fL));
3105     assert(!(r2 & ~0x7fL));
3106     TSTREG1(r2);
3107     TSTPRED(_p);
3108     inst((1L<<37)|(x6<<27)|(r2<<13)|_p, INST_M);
3109 }
3110
3111 static void
3112 _M36(jit_state_t *_jit, jit_word_t _p,
3113      jit_word_t x6, jit_word_t r1)
3114 {
3115     assert(!(_p & ~0x3fL));
3116     assert(!(x6 & ~0x3fL));
3117     assert(!(r1 & ~0x7fL));
3118     TSTPRED(_p);
3119     TSTREG1(r1);
3120     inst((1L<<37)|(x6<<27)|(r1<<6)|_p, INST_M);
3121     SETREG(r1);
3122 }
3123
3124 static void
3125 _M37(jit_state_t *_jit, jit_word_t _p,
3126      jit_word_t im)
3127 {
3128     assert(!(_p &    ~0x3fL));
3129     assert(!(im & ~0x1ffffL));
3130     TSTPRED(_p);
3131     inst((((im>>20)&1L)<<36)|((im&0xffffL)<<6)|_p, INST_M);
3132 }
3133
3134 static void
3135 _M38(jit_state_t *_jit, jit_word_t _p,
3136      jit_word_t x6, jit_word_t r3, jit_word_t r2, jit_word_t r1)
3137 {
3138     assert(!(_p &  ~0x3fL));
3139     assert(!(x6 &  ~0x3fL));
3140     assert(!(r3 &  ~0x7fL));
3141     assert(!(r2 &  ~0x7fL));
3142     assert(!(r1 &  ~0x7fL));
3143     TSTREG2(r2, r3);
3144     TSTPRED(_p);
3145     TSTREG1(r1);
3146     inst((1L<<37)|(x6<<27)|(r3<<20)|(r2<<13)|(r1<<6)|_p, INST_M);
3147     SETREG(r1);
3148 }
3149
3150 static void
3151 _M39(jit_state_t *_jit, jit_word_t _p,
3152      jit_word_t x6, jit_word_t r3, jit_word_t im, jit_word_t r1)
3153 {
3154     assert(!(_p &  ~0x3fL));
3155     assert(!(x6 &  ~0x3fL));
3156     assert(!(r3 &  ~0x7fL));
3157     assert(!(im &   ~0x7L));
3158     assert(!(r1 &  ~0x7fL));
3159     TSTREG1(r3);
3160     TSTPRED(_p);
3161     TSTREG1(r1);
3162     inst((1L<<37)|(x6<<27)|(r3<<20)|(im<<13)|(r1<<6)|_p, INST_M);
3163     SETREG(r1);
3164 }
3165
3166 static void
3167 _M40(jit_state_t *_jit, jit_word_t _p,
3168      jit_word_t x6, jit_word_t r3, jit_word_t im)
3169 {
3170     assert(!(_p &  ~0x3fL));
3171     assert(!(x6 &  ~0x3fL));
3172     assert(!(r3 &  ~0x7fL));
3173     assert(!(im &   ~0x7L));
3174     TSTREG1(r3);
3175     TSTPRED(_p);
3176     inst((1L<<37)|(x6<<27)|(r3<<20)|(im<<13)|_p, INST_M);
3177 }
3178
3179 static void
3180 _M41(jit_state_t *_jit, jit_word_t _p,
3181      jit_word_t x6, jit_word_t r2)
3182 {
3183     assert(!(_p &  ~0x3fL));
3184     assert(!(x6 &  ~0x3fL));
3185     assert(!(r2 &  ~0x7fL));
3186     TSTREG1(r2);
3187     TSTPRED(_p);
3188     inst((1L<<37)|(x6<<27)|(r2<<13)|_p, INST_M);
3189 }
3190
3191 static void
3192 _M42(jit_state_t *_jit, jit_word_t _p,
3193      jit_word_t x6, jit_word_t r3, jit_word_t r2)
3194 {
3195     assert(!(_p &  ~0x3fL));
3196     assert(!(x6 &  ~0x3fL));
3197     assert(!(r3 &  ~0x7fL));
3198     assert(!(r2 &  ~0x7fL));
3199     TSTREG1(r2);
3200     TSTPRED(_p);
3201     inst((1L<<37)|(x6<<27)|(r3<<20)|(r2<<13)|_p, INST_M);
3202 }
3203
3204 static void
3205 _M43(jit_state_t *_jit, jit_word_t _p,
3206      jit_word_t x6, jit_word_t r3, jit_word_t r1)
3207 {
3208     assert(!(_p &  ~0x3fL));
3209     assert(!(x6 &  ~0x3fL));
3210     assert(!(r3 &  ~0x7fL));
3211     assert(!(r1 &  ~0x7fL));
3212     TSTPRED(_p);
3213     TSTREG1(r1);
3214     inst((1L<<37)|(x6<<27)|(r3<<20)|(r1<<6)|_p, INST_M);
3215     SETREG(r1);
3216 }
3217
3218 static void
3219 _M44(jit_state_t *_jit, jit_word_t _p,
3220      jit_word_t x4, jit_word_t im)
3221 {
3222     assert(!(_p &    ~0x3fL));
3223     assert(!(x4 &     ~0xfL));
3224     assert(!(im & ~0xfffffL));
3225     TSTPRED(_p);
3226     inst((((im>>23)&1L)<<36)|(((im>>21)&3L)<<31)|
3227          (x4<<27)|((im&0x1ffffL)<<6)|_p, INST_M);
3228 }
3229
3230 static void
3231 _M45(jit_state_t *_jit, jit_word_t _p,
3232      jit_word_t x6, jit_word_t r3, jit_word_t r2)
3233 {
3234     assert(!(_p &  ~0x3fL));
3235     assert(!(x6 &  ~0x3fL));
3236     assert(!(r3 &  ~0x7fL));
3237     assert(!(r2 &  ~0x7fL));
3238     TSTREG2(r2, r3);
3239     TSTPRED(_p);
3240     inst((1L<<37)|(x6<<27)|(r3<<20)|(r2<<13)|_p, INST_M);
3241 }
3242
3243 static void
3244 _M46(jit_state_t *_jit, jit_word_t _p,
3245      jit_word_t x6, jit_word_t r3, jit_word_t r1)
3246 {
3247     assert(!(_p & ~0x3fL));
3248     assert(!(x6 & ~0x3fL));
3249     assert(!(r3 & ~0x7fL));
3250     assert(!(r1 & ~0x7fL));
3251     TSTREG1(r3);
3252     TSTPRED(_p);
3253     if (r1)     TSTREG1(r1);
3254     inst((1L<<37)|(x6<<27)|(r3<<20)|(r1<<6)|_p, INST_M);
3255     if (r1)     SETREG(r1);
3256 }
3257
3258 static void
3259 _M48(jit_state_t *_jit, jit_word_t _p,
3260      jit_word_t y, jit_word_t im)
3261 {
3262     assert(!(_p &    ~0x3fL));
3263     assert(!(y  &     ~0x1L));
3264     assert(!(im & ~0x1ffffL));
3265     TSTPRED(_p);
3266     inst((((im>>20)&1L)<<36)|(1L<<27)|(y<<26)|((im&0xffffL)<<6)|_p, INST_M);
3267 }
3268
3269 static void
3270 _B1(jit_state_t *_jit, jit_word_t _p,
3271     jit_word_t d, jit_word_t wh, jit_word_t im, jit_word_t p, jit_word_t tp)
3272 {
3273     assert(!(_p &    ~0x3fL));
3274     assert(!(d  &     ~0x1L));
3275     assert(!(wh &     ~0x3L));
3276     assert(im >= -1048576 && im <= 1048575);
3277     assert(!(p  &     ~0x1L));
3278     assert(!(tp &     ~0x7L));
3279     TSTPRED(_p);
3280     inst((4L<<37)|(((im>>20)&1L)<<36)|(d<<35)|(wh<<33)|
3281          ((im&0xfffffL)<<13)|(p<<12)|(tp<<6)|_p, INST_B);
3282 }
3283
3284 static void
3285 _B3(jit_state_t *_jit, jit_word_t _p,
3286     jit_word_t d, jit_word_t wh, jit_word_t im, jit_word_t p, jit_word_t b)
3287 {
3288     assert(!(_p &    ~0x3fL));
3289     assert(!(d  &     ~0x1L));
3290     assert(!(wh &     ~0x3L));
3291     assert(im >= -1048576 && im <= 1048575);
3292     assert(!(p  &     ~0x1L));
3293     assert(!(b  &     ~0x3L));
3294     TSTPRED(_p);
3295     inst((5L<<37)|(((im>>20)&1L)<<36)|(d<<35)|(wh<<33)|
3296          ((im&0xfffffL)<<13)|(p<<12)|(b<<6)|_p, INST_B);
3297 }
3298
3299 static void
3300 _B4(jit_state_t *_jit, jit_word_t _p,
3301     jit_word_t d, jit_word_t wh, jit_word_t x6,
3302     jit_word_t b, jit_word_t p, jit_word_t tp)
3303 {
3304     assert(!(_p & ~0x3fL));
3305     assert(!(d  &  ~0x1L));
3306     assert(!(wh &  ~0x3L));
3307     assert(!(x6 & ~0x3fL));
3308     assert(!(b  &  ~0x7L));
3309     assert(!(p  &  ~0x1L));
3310     assert(!(tp &  ~0x7L));
3311     TSTPRED(_p);
3312     inst((d<<35)|(wh<<33)|(x6<<27)|(b<<13)|(p<<12)|(tp<<6)|_p, INST_B);
3313 }
3314
3315 static void
3316 _B5(jit_state_t *_jit, jit_word_t _p,
3317     jit_word_t d, jit_word_t wh, jit_word_t b2, jit_word_t p, jit_word_t b1)
3318 {
3319     assert(!(_p & ~0x3fL));
3320     assert(!(d  &  ~0x1L));
3321     assert(!(wh &  ~0x3L));
3322     assert(!(b2 &  ~0x7L));
3323     assert(!(p  &  ~0x1L));
3324     assert(!(b1 &  ~0x7L));
3325     TSTPRED(_p);
3326     inst((1L<<37)|(d<<35)|(wh<<32)|(b2<<13)|(p<<12)|(b1<<6)|_p, INST_B);
3327 }
3328
3329 static void
3330 _B6(jit_state_t *_jit, jit_word_t _p,
3331     jit_word_t ih, jit_word_t im, jit_word_t tag, jit_word_t wh)
3332 {
3333     assert(!(_p &     ~0x3fL));
3334     assert(!(ih  &     ~0x1L));
3335     assert(!(im  & ~0x1ffffL));
3336     assert(!(tag &   ~0x1ffL));
3337     assert(!(wh  &     ~0x3L));
3338     TSTPRED(_p);
3339     inst((7L<<37)|(((im>>20)&1L)<<36)|(ih<<35)|(((tag>>7)&3L)<<33)|
3340          ((im&0xfffffL)<<13)|((tag&0x7fL)<<6)|(wh<<3)|_p, INST_B);
3341 }
3342
3343 static void
3344 _B7(jit_state_t *_jit, jit_word_t _p,
3345     jit_word_t ih, jit_word_t x6, jit_word_t b2, jit_word_t tag, jit_word_t wh)
3346 {
3347     assert(!(_p &   ~0x3fL));
3348     assert(!(ih  &   ~0x1L));
3349     assert(!(x6  &  ~0x3fL));
3350     assert(!(b2  &   ~0x7L));
3351     assert(!(tag & ~0x1ffL));
3352     assert(!(wh  &   ~0x3L));
3353     TSTPRED(_p);
3354     inst((2L<<37)|(ih<<35)|(((tag>>7)&3L)<<33)|(x6<<27)|
3355          (b2<<13)|((tag&0x7fL)<<6)|(wh<<3)|_p, INST_B);
3356 }
3357
3358 static void
3359 _B8(jit_state_t *_jit, jit_word_t _p,
3360     jit_word_t x6)
3361 {
3362     assert(!(_p & ~0x3fL));
3363     assert(!(x6 & ~0x3fL));
3364     TSTPRED(_p);
3365     inst((x6<<27)|_p, INST_B);
3366 }
3367
3368 static void
3369 _B9(jit_state_t *_jit, jit_word_t _p,
3370     jit_word_t op, jit_word_t x6, jit_word_t im)
3371 {
3372     assert(!(_p &    ~0x3fL));
3373     assert(!(op &     ~0xfL));
3374     assert(!(x6 &    ~0x3fL));
3375     assert(!(im & ~0x1ffffL));
3376     TSTPRED(_p);
3377     inst((op<<37)|(((im>>20)&1L)<<36)|(x6<<27)|((im&0xffffL)<<6)|_p, INST_B);
3378 }
3379
3380 static void
3381 _X1(jit_state_t *_jit, jit_word_t _p,
3382     jit_word_t im)
3383 {
3384     jit_word_t          i41, i1, i20;
3385     assert(!(_p &               ~0x3fL));
3386     assert(im > -0x2000000000000000 && im <= 0x1fffffffffffffff);
3387     i41 = (im >> 22) & 0x1ffffffffffL;
3388     i1  = (im >> 21) &           0x1L;
3389     i20 =  im        &       0xfffffL;
3390     TSTPRED(_p);
3391     inst(i41, INST_L);
3392     inst((i1<<36)|(i20<<6)|_p, INST_X);
3393 }
3394
3395 static void
3396 _X2(jit_state_t *_jit, jit_word_t _p,
3397     jit_word_t r1, jit_word_t im)
3398 {
3399     jit_word_t          i1, i41, ic, i5, i9, i7;
3400     assert(!(_p & ~0x3fL));
3401     assert(!(r1 & ~0x7fL));
3402     i1  = (im >> 63) &           0x1L;
3403     i41 = (im >> 22) & 0x1ffffffffffL;
3404     ic  = (im >> 21) &           0x1L;
3405     i5  = (im >> 16) &          0x1fL;
3406     i9  = (im >>  7) &         0x1ffL;
3407     i7  =  im        &          0x7fL;
3408     TSTPRED(_p);
3409     TSTREG1(r1);
3410     inst(i41, INST_L);
3411     inst((6L<<37)|(i1<<36)|(i9<<27)|(i5<<22)|
3412          (ic<<21)|(i7<<13)|(r1<<6)|_p, INST_X);
3413     SETREG(r1);
3414 }
3415
3416 static void
3417 _X3x(jit_state_t *_jit, jit_word_t _p,
3418      jit_word_t op, jit_word_t d, jit_word_t wh,
3419      jit_word_t p, jit_word_t tp, jit_word_t im)
3420 {
3421     /* target64 = IP + ((i1 << 59 | imm39 << 20 | imm20b) << 4) */
3422     jit_word_t          i1, i41, i20;
3423     assert(!(_p &               ~0x3fL));
3424     assert(!(op &                ~0xfL));
3425     assert(!(d  &                ~0x1L));
3426     assert(!(wh &                ~0x3L));
3427     assert(!(p  &                ~0x1L));
3428     assert(!(tp &                ~0x7L));
3429     i1  = (im >> 61) &           0x1L;
3430     i41 = (im >> 22) & 0x1ffffffffffL;
3431     i20 =  im        &       0xfffffL;
3432     TSTPRED(_p);
3433     inst(i41, INST_L);
3434     inst((op<<37)|(i1<<36)|(d<<35)|(wh<<33)|
3435          (i20<<13)|(p<<12)|(tp<<6)|_p, INST_X);
3436 }
3437
3438 static void
3439 _X5(jit_state_t *_jit, jit_word_t _p,
3440     jit_word_t y, jit_word_t im)
3441 {
3442     jit_word_t          i41, i1, i20;
3443     assert(!(_p &               ~0x3fL));
3444     assert(im > -0x2000000000000000 && im <= 0x1fffffffffffffff);
3445     i41 = (im >> 22) & 0x1ffffffffffL;
3446     i1  = (im >> 21) &           0x1L;
3447     i20 =  im        &       0xfffffL;
3448     TSTPRED(_p);
3449     inst(i41, INST_L);
3450     inst((i1<<36)|(1L<<27)|(y<<26)|(i20<<6)|_p, INST_X);
3451 }
3452
3453 static void
3454 _movr(jit_state_t *_jit, jit_int32_t r0, jit_int32_t r1)
3455 {
3456     /* patch pushargr */
3457     if (r0 >= 120)
3458         r0 = _jitc->rout + (r0 - 120);
3459
3460     MOV(r0, r1);
3461 }
3462
3463 static void
3464 _movi(jit_state_t *_jit, jit_int32_t r0, jit_word_t i0)
3465 {
3466     /* patch pushargi */
3467     if (r0 >= 120)
3468         r0 = _jitc->rout + (r0 - 120);
3469
3470     if (i0 >= -2097152 && i0 <= 2097151)
3471         MOVI(r0, i0);
3472     else
3473         MOVL(r0, i0);
3474 }
3475
3476 static jit_word_t
3477 _movi_p(jit_state_t *_jit, jit_int32_t r0, jit_word_t i0)
3478 {
3479     jit_word_t          w;
3480     sync();
3481     w = _jit->pc.w;
3482     MOVL(r0, i0);
3483     return (w);
3484 }
3485
3486 static void
3487 _addi(jit_state_t *_jit, jit_int32_t r0, jit_int32_t r1, jit_word_t i0)
3488 {
3489     jit_int32_t         reg;
3490     if (i0 >= -8192 && i0 <= 8191)
3491         ADDS(r0, i0, r1);
3492     else if (!(r1 & ~3) && i0 >= -2097152 && i0 <= 2097151)
3493         ADDL(r1, i0, r0);
3494     else {
3495         reg = jit_get_reg(jit_class_gpr);
3496         movi(rn(reg), i0);
3497         addr(r0, r1, rn(reg));
3498         jit_unget_reg(reg);
3499     }
3500 }
3501
3502 static void
3503 _addcr(jit_state_t *_jit, jit_int32_t r0, jit_int32_t r1, jit_int32_t r2)
3504 {
3505     jit_int32_t         reg;
3506     if (jit_carry == _NOREG)
3507         jit_carry = jit_get_reg(jit_class_gpr);
3508     if (r0 == r1) {
3509         reg = jit_get_reg(jit_class_gpr);
3510         addr(rn(reg), r1, r2);
3511         ltr_u(rn(jit_carry), rn(reg), r1);
3512         movr(r0, rn(reg));
3513         jit_unget_reg(reg);
3514     }
3515     else {
3516         addr(r0, r1, r2);
3517         ltr_u(rn(jit_carry), r0, r1);
3518     }
3519 }
3520
3521 static void
3522 _addci(jit_state_t *_jit, jit_int32_t r0, jit_int32_t r1, jit_word_t i0)
3523 {
3524     jit_int32_t         reg;
3525     if (jit_carry == _NOREG)
3526         jit_carry = jit_get_reg(jit_class_gpr);
3527     if (r0 == r1) {
3528         reg = jit_get_reg(jit_class_gpr);
3529         addi(rn(reg), r1, i0);
3530         ltr_u(rn(jit_carry), rn(reg), r1);
3531         movr(r0, rn(reg));
3532         jit_unget_reg(reg);
3533     }
3534     else {
3535         addi(r0, r1, i0);
3536         ltr_u(rn(jit_carry), r0, r1);
3537     }
3538 }
3539
3540 static void
3541 _addxr(jit_state_t *_jit, jit_int32_t r0, jit_int32_t r1, jit_int32_t r2)
3542 {
3543     jit_int32_t         reg;
3544     assert(jit_carry != _NOREG);
3545     reg = jit_get_reg(jit_class_gpr);
3546     movr(rn(reg), rn(jit_carry));
3547     addcr(r0, r1, r2);
3548     addcr(r0, r0, rn(reg));
3549     jit_unget_reg(reg);
3550 }
3551
3552 static void
3553 _addxi(jit_state_t *_jit, jit_int32_t r0, jit_int32_t r1, jit_word_t i0)
3554 {
3555     jit_int32_t         reg;
3556     assert(jit_carry != _NOREG);
3557     reg = jit_get_reg(jit_class_gpr);
3558     movr(rn(reg), rn(jit_carry));
3559     addci(r0, r1, i0);
3560     addcr(r0, r0, rn(reg));
3561     jit_unget_reg(reg);
3562 }
3563
3564 static void
3565 _subi(jit_state_t *_jit, jit_int32_t r0, jit_int32_t r1, jit_word_t i0)
3566 {
3567     jit_int32_t         reg;
3568     reg = jit_get_reg(jit_class_gpr);
3569     movi(rn(reg), i0);
3570     subr(r0, r1, rn(reg));
3571     jit_unget_reg(reg);
3572 }
3573
3574 static void
3575 _subcr(jit_state_t *_jit, jit_int32_t r0, jit_int32_t r1, jit_int32_t r2)
3576 {
3577     jit_int32_t         reg;
3578     if (jit_carry == _NOREG)
3579         jit_carry = jit_get_reg(jit_class_gpr);
3580     if (r0 == r1) {
3581         reg = jit_get_reg(jit_class_gpr);
3582         subr(rn(reg), r1, r2);
3583         ltr_u(rn(jit_carry), r1, rn(reg));
3584         movr(r0, rn(reg));
3585         jit_unget_reg(reg);
3586     }
3587     else {
3588         subr(r0, r1, r2);
3589         ltr_u(rn(jit_carry), r1, r0);
3590     }
3591 }
3592
3593 static void
3594 _subci(jit_state_t *_jit, jit_int32_t r0, jit_int32_t r1, jit_word_t i0)
3595 {
3596     jit_int32_t         reg;
3597     if (jit_carry == _NOREG)
3598         jit_carry = jit_get_reg(jit_class_gpr);
3599     if (r0 == r1) {
3600         reg = jit_get_reg(jit_class_gpr);
3601         addi(rn(reg), r1, -i0);
3602         ltr_u(rn(jit_carry), r1, rn(reg));
3603         movr(r0, rn(reg));
3604         jit_unget_reg(reg);
3605     }
3606     else {
3607         addi(r0, r1, -i0);
3608         ltr_u(rn(jit_carry), r1, r0);
3609     }
3610 }
3611
3612 static void
3613 _subxr(jit_state_t *_jit, jit_int32_t r0, jit_int32_t r1, jit_int32_t r2)
3614 {
3615     jit_int32_t         reg;
3616     assert(jit_carry != _NOREG);
3617     reg = jit_get_reg(jit_class_gpr);
3618     movr(rn(reg), rn(jit_carry));
3619     subcr(r0, r1, r2);
3620     subcr(r0, r0, rn(reg));
3621     jit_unget_reg(reg);
3622 }
3623
3624 static void
3625 _subxi(jit_state_t *_jit, jit_int32_t r0, jit_int32_t r1, jit_word_t i0)
3626 {
3627     jit_int32_t         reg;
3628     assert(jit_carry != _NOREG);
3629     reg = jit_get_reg(jit_class_gpr);
3630     movr(rn(reg), rn(jit_carry));
3631     subci(r0, r1, i0);
3632     subcr(r0, r0, rn(reg));
3633     jit_unget_reg(reg);
3634 }
3635
3636 static void
3637 _rsbi(jit_state_t *_jit, jit_int32_t r0, jit_int32_t r1, jit_word_t i0)
3638 {
3639     jit_int32_t         reg;
3640     if (i0 >= -128 && i0 <= 127)
3641         SUBI(r0, i0, r1);
3642     else if (!(r1 & ~3) && i0 >= -2097151 && i0 <= 2097152)
3643         ADDL(r1, -i0, r0);
3644     else {
3645         reg = jit_get_reg(jit_class_gpr);
3646         movi(rn(reg), i0);
3647         subr(r0, rn(reg), r1);
3648         jit_unget_reg(reg);
3649     }
3650 }
3651
3652 static void
3653 _mulr(jit_state_t *_jit, jit_int32_t r0, jit_int32_t r1, jit_int32_t r2)
3654 {
3655     jit_int32_t         f0, f1;
3656     f0 = jit_get_reg(jit_class_fpr);
3657     f1 = jit_get_reg(jit_class_fpr);
3658     SETF_SIG(rn(f0), r1);
3659     SETF_SIG(rn(f1), r2);
3660     XMPY_L(rn(f0), rn(f0), rn(f1));
3661     GETF_SIG(r0, rn(f0));
3662     jit_unget_reg(f0);
3663     jit_unget_reg(f1);
3664 }
3665
3666 static void
3667 _muli(jit_state_t *_jit, jit_int32_t r0, jit_int32_t r1, jit_word_t i0)
3668 {
3669     jit_int32_t         reg;
3670     reg = jit_get_reg(jit_class_gpr);
3671     movi(rn(reg), i0);
3672     mulr(r0, r1, rn(reg));
3673     jit_unget_reg(reg);
3674 }
3675
3676 #if !defined(__GNUC__)
3677 static long
3678 __divdi3(long u, long v)
3679 {
3680     return (u / v);
3681 }
3682
3683 static unsigned long
3684 __udivdi3(unsigned long u, unsigned long v)
3685 {
3686     return (u / v);
3687 }
3688
3689 static long
3690 __moddi3(long u, long v)
3691 {
3692     return (u % v);
3693 }
3694
3695 static unsigned long
3696 __umoddi3(unsigned long u, unsigned long v)
3697 {
3698     return (u % v);
3699 }
3700 #endif
3701
3702 static void
3703 _divr(jit_state_t *_jit, jit_int32_t r0, jit_int32_t r1, jit_int32_t r2)
3704 {
3705     MOV(_jitc->rout, r1);
3706     MOV(_jitc->rout + 1, r2);
3707     calli((jit_word_t)__divdi3);
3708     MOV(r0, GR_8);
3709 }
3710
3711 static void
3712 _divi(jit_state_t *_jit, jit_int32_t r0, jit_int32_t r1, jit_word_t i0)
3713 {
3714     jit_int32_t         reg;
3715     switch (i0) {
3716         case 1:
3717             movr(r0, r1);
3718             return;
3719         case -1:
3720             negr(r0, r1);
3721             return;
3722         default:
3723             if (i0 > 0 && !(i0 & (i0 - 1))) {
3724                 movr(r0, r1);
3725                 rshi(r0, r0, ffsl(i0) - 1);
3726                 return;
3727             }
3728             break;
3729     }
3730     reg = jit_get_reg(jit_class_gpr);
3731     movi(rn(reg), i0);
3732     divr(r0, r1, rn(reg));
3733     jit_unget_reg(reg);
3734 }
3735
3736 static void
3737 _divr_u(jit_state_t *_jit, jit_int32_t r0, jit_int32_t r1, jit_int32_t r2)
3738 {
3739     MOV(_jitc->rout, r1);
3740     MOV(_jitc->rout + 1, r2);
3741     calli((jit_word_t)__udivdi3);
3742     MOV(r0, GR_8);
3743 }
3744
3745 static void
3746 _divi_u(jit_state_t *_jit, jit_int32_t r0, jit_int32_t r1, jit_word_t i0)
3747 {
3748     jit_int32_t         reg;
3749     if (i0 == 1) {
3750         movr(r0, r1);
3751         return;
3752     }
3753     else if (i0 > 0 && !(i0 & (i0 - 1))) {
3754         movr(r0, r1);
3755         rshi_u(r0, r0, ffsl(i0) - 1);
3756         return;
3757     }
3758     reg = jit_get_reg(jit_class_gpr);
3759     movi(rn(reg), i0);
3760     divr_u(r0, r1, rn(reg));
3761     jit_unget_reg(reg);
3762 }
3763
3764 static void
3765 _remr(jit_state_t *_jit, jit_int32_t r0, jit_int32_t r1, jit_int32_t r2)
3766 {
3767     MOV(_jitc->rout, r1);
3768     MOV(_jitc->rout + 1, r2);
3769     calli((jit_word_t)__moddi3);
3770     MOV(r0, GR_8);
3771 }
3772
3773 static void
3774 _remi(jit_state_t *_jit, jit_int32_t r0, jit_int32_t r1, jit_word_t i0)
3775 {
3776     jit_int32_t         reg;
3777     if (i0 == 1 || i0 == -1) {
3778         MOV(r0, GR_0);
3779         return;
3780     }
3781     reg = jit_get_reg(jit_class_gpr);
3782     movi(rn(reg), i0);
3783     remr(r0, r1, rn(reg));
3784     jit_unget_reg(reg);
3785 }
3786
3787 static void
3788 _remr_u(jit_state_t *_jit, jit_int32_t r0, jit_int32_t r1, jit_int32_t r2)
3789 {
3790     MOV(_jitc->rout, r1);
3791     MOV(_jitc->rout + 1, r2);
3792     calli((jit_word_t)__umoddi3);
3793     MOV(r0, GR_8);
3794 }
3795
3796 static void
3797 _remi_u(jit_state_t *_jit, jit_int32_t r0, jit_int32_t r1, jit_word_t i0)
3798 {
3799     jit_int32_t         reg;
3800     if (i0 == 1) {
3801         MOV(r0, GR_0);
3802         return;
3803     }
3804     else if (i0 > 0 && !(i0 & (i0 - 1))) {
3805         andi(r0, r1, i0 - 1);
3806         return;
3807     }
3808     reg = jit_get_reg(jit_class_gpr);
3809     movi(rn(reg), i0);
3810     remr_u(r0, r1, rn(reg));
3811     jit_unget_reg(reg);
3812 }
3813
3814 static void
3815 _mulh(jit_state_t *_jit,
3816       jit_int32_t r0, jit_int32_t r1, jit_int32_t r2, jit_bool_t sign)
3817 {
3818     jit_int32_t         f0, f1;
3819     f0 = jit_get_reg(jit_class_fpr);
3820     f1 = jit_get_reg(jit_class_fpr);
3821     SETF_SIG(rn(f0), r1);
3822     SETF_SIG(rn(f1), r2);
3823     if (sign)
3824         XMPY_H(rn(f0), rn(f0), rn(f1));
3825     else
3826         XMPY_HU(rn(f0), rn(f0), rn(f1));
3827     GETF_SIG(r0, rn(f0));
3828     jit_unget_reg(f0);
3829     jit_unget_reg(f1);
3830 }
3831
3832 static void
3833 _iqmulr(jit_state_t *_jit, jit_int32_t r0, jit_int32_t r1,
3834         jit_int32_t r2, jit_int32_t r3, jit_bool_t sign)
3835 {
3836     jit_int32_t         reg;
3837     if (r0 == r2 || r0 == r3) {
3838         reg = jit_get_reg(jit_class_gpr);
3839         mulr(rn(reg), r2, r3);
3840     }
3841     else
3842         mulr(r0, r2, r3);
3843     mulh(r1, r2, r3, sign);
3844     if (r0 == r2 || r0 == r3) {
3845         movr(r0, rn(reg));
3846         jit_unget_reg(reg);
3847     }
3848 }
3849
3850 static void
3851 _iqmuli(jit_state_t *_jit, jit_int32_t r0, jit_int32_t r1,
3852         jit_int32_t r2, jit_word_t i0, jit_bool_t sign)
3853 {
3854     jit_int32_t         reg;
3855     reg = jit_get_reg(jit_class_gpr);
3856     movi(rn(reg), i0);
3857     iqmulr(r0, r1, r2, rn(reg), sign);
3858     jit_unget_reg(reg);
3859 }
3860
3861 static void
3862 _iqdivr(jit_state_t *_jit, jit_int32_t r0, jit_int32_t r1,
3863         jit_int32_t r2, jit_int32_t r3, jit_bool_t sign)
3864 {
3865     jit_int32_t         sv0, rg0;
3866     jit_int32_t         sv1, rg1;
3867
3868     if (r0 == r2 || r0 == r3) {
3869         sv0 = jit_get_reg(jit_class_gpr);
3870         rg0 = rn(sv0);
3871     }
3872     else
3873         rg0 = r0;
3874     if (r1 == r2 || r1 == r3) {
3875         sv1 = jit_get_reg(jit_class_gpr);
3876         rg1 = rn(sv1);
3877     }
3878     else
3879         rg1 = r1;
3880
3881     if (sign)
3882         divr(rg0, r2, r3);
3883     else
3884         divr_u(rg0, r2, r3);
3885     mulr(rg1, r3, rg0);
3886     subr(rg1, r2, rg1);
3887     if (rg0 != r0) {
3888         movr(r0, rg0);
3889         jit_unget_reg(sv0);
3890     }
3891     if (rg1 != r1) {
3892         movr(r1, rg1);
3893         jit_unget_reg(sv1);
3894     }
3895 }
3896
3897 static void
3898 _iqdivi(jit_state_t *_jit, jit_int32_t r0, jit_int32_t r1,
3899         jit_int32_t r2, jit_word_t i0, jit_bool_t sign)
3900 {
3901     jit_int32_t         reg;
3902     reg = jit_get_reg(jit_class_gpr);
3903     movi(rn(reg), i0);
3904     iqdivr(r0, r1, r2, rn(reg), sign);
3905     jit_unget_reg(reg);
3906 }
3907
3908 static void
3909 _andi(jit_state_t *_jit, jit_int32_t r0, jit_int32_t r1, jit_word_t i0)
3910 {
3911     jit_int32_t         reg;
3912     if (i0 >= -128 && i0 <= 127)
3913         ANDI(r0, i0, r1);
3914     else if (~i0 >= -128 && ~i0 <= 127)
3915         ANDCMI(r0, ~i0, r1);
3916     else {
3917         reg = jit_get_reg(jit_class_gpr);
3918         movi(rn(reg), i0);
3919         andr(r0, r1, rn(reg));
3920         jit_unget_reg(reg);
3921     }
3922 }
3923
3924 static void
3925 _ori(jit_state_t *_jit, jit_int32_t r0, jit_int32_t r1, jit_word_t i0)
3926 {
3927     jit_int32_t         reg;
3928     if (i0 >= -128 && i0 <= 127)
3929         ORI(r0, i0, r1);
3930     else {
3931         reg = jit_get_reg(jit_class_gpr);
3932         movi(rn(reg), i0);
3933         orr(r0, r1, rn(reg));
3934         jit_unget_reg(reg);
3935     }
3936 }
3937
3938 static void
3939 _xori(jit_state_t *_jit, jit_int32_t r0, jit_int32_t r1, jit_word_t i0)
3940 {
3941     jit_int32_t         reg;
3942     if (i0 >= -128 && i0 <= 127)
3943         XORI(r0, i0, r1);
3944     else {
3945         reg = jit_get_reg(jit_class_gpr);
3946         movi(rn(reg), i0);
3947         xorr(r0, r1, rn(reg));
3948         jit_unget_reg(reg);
3949     }
3950 }
3951
3952 #if __BYTE_ORDER == __LITTLE_ENDIAN
3953 static void
3954 _htonr_us(jit_state_t *_jit, jit_int32_t r0, jit_int32_t r1)
3955 {
3956     jit_int32_t         t0;
3957     t0 = jit_get_reg(jit_class_gpr);
3958     rshi(rn(t0), r1, 8);
3959     andi(r0, r1, 0xff);
3960     andi(rn(t0), rn(t0), 0xff);
3961     lshi(r0, r0, 8);
3962     orr(r0, r0, rn(t0));
3963     jit_unget_reg(t0);
3964 }
3965
3966 static void
3967 _htonr_ui(jit_state_t *_jit, jit_int32_t r0, jit_int32_t r1)
3968 {
3969     jit_int32_t         t0;
3970     jit_int32_t         t1;
3971     jit_int32_t         t2;
3972     t0 = jit_get_reg(jit_class_gpr);
3973     t1 = jit_get_reg(jit_class_gpr);
3974     t2 = jit_get_reg(jit_class_gpr);
3975     rshi(rn(t0), r1, 24);
3976     rshi(rn(t1), r1, 16);
3977     rshi(rn(t2), r1,  8);
3978     andi(rn(t0), rn(t0), 0xff);
3979     andi(rn(t1), rn(t1), 0xff);
3980     andi(rn(t2), rn(t2), 0xff);
3981     andi(r0, r1, 0xff);
3982     lshi(r0, r0, 24);
3983     lshi(rn(t1), rn(t1), 8);
3984     orr(r0, r0, rn(t0));
3985     lshi(rn(t2), rn(t2), 16);
3986     orr(r0, r0, rn(t1));
3987     orr(r0, r0, rn(t2));
3988     jit_unget_reg(t2);
3989     jit_unget_reg(t1);
3990     jit_unget_reg(t0);
3991 }
3992 #endif
3993
3994 static void
3995 _lshi(jit_state_t *_jit, jit_int32_t r0, jit_int32_t r1, jit_word_t i0)
3996 {
3997     jit_int32_t         reg;
3998     reg = jit_get_reg(jit_class_gpr);
3999     movi(rn(reg), i0);
4000     lshr(r0, r1, rn(reg));
4001     jit_unget_reg(reg);
4002 }
4003
4004 static void
4005 _rshi(jit_state_t *_jit, jit_int32_t r0, jit_int32_t r1, jit_word_t i0)
4006 {
4007     jit_int32_t         reg;
4008     reg = jit_get_reg(jit_class_gpr);
4009     movi(rn(reg), i0);
4010     rshr(r0, r1, rn(reg));
4011     jit_unget_reg(reg);
4012 }
4013
4014 static void
4015 _rshi_u(jit_state_t *_jit, jit_int32_t r0, jit_int32_t r1, jit_word_t i0)
4016 {
4017     jit_int32_t         reg;
4018     reg = jit_get_reg(jit_class_gpr);
4019     movi(rn(reg), i0);
4020     rshr_u(r0, r1, rn(reg));
4021     jit_unget_reg(reg);
4022 }
4023
4024 static void
4025 _ltr(jit_state_t *_jit, jit_int32_t r0, jit_int32_t r1, jit_int32_t r2)
4026 {
4027     CMP_LT(PR_6, PR_7, r1, r2);
4028     MOVI_p(r0, 1, PR_6);
4029     MOV_p(r0, GR_0, PR_7);
4030 }
4031
4032 static void
4033 _lti(jit_state_t *_jit, jit_int32_t r0, jit_int32_t r1, jit_word_t i0)
4034 {
4035     jit_int32_t         reg;
4036     if (i0 >= -127 && i0 <= 128)
4037         CMPI_LT(PR_7, PR_6, i0 - 1, r1);
4038     else {
4039         reg = jit_get_reg(jit_class_gpr);
4040         movi(rn(reg), i0);
4041         CMP_LT(PR_6, PR_7, r1, rn(reg));
4042         jit_unget_reg(reg);
4043     }
4044     MOVI_p(r0, 1, PR_6);
4045     MOV_p(r0, GR_0, PR_7);
4046 }
4047
4048 static void
4049 _ltr_u(jit_state_t *_jit, jit_int32_t r0, jit_int32_t r1, jit_int32_t r2)
4050 {
4051     CMP_LTU(PR_6, PR_7, r1, r2);
4052     MOVI_p(r0, 1, PR_6);
4053     MOV_p(r0, GR_0, PR_7);
4054 }
4055
4056 static void
4057 _lti_u(jit_state_t *_jit, jit_int32_t r0, jit_int32_t r1, jit_word_t i0)
4058 {
4059     jit_int32_t         reg;
4060     if (i0 >= -127 && i0 <= 128)
4061         CMPI_LTU(PR_7, PR_6, i0 - 1, r1);
4062     else {
4063         reg = jit_get_reg(jit_class_gpr);
4064         movi(rn(reg), i0);
4065         CMP_LTU(PR_6, PR_7, r1, rn(reg));
4066         jit_unget_reg(reg);
4067     }
4068     MOVI_p(r0, 1, PR_6);
4069     MOV_p(r0, GR_0, PR_7);
4070 }
4071
4072 static void
4073 _ler(jit_state_t *_jit, jit_int32_t r0, jit_int32_t r1, jit_int32_t r2)
4074 {
4075     CMP_LT(PR_6, PR_7, r2, r1);
4076     MOV_p(r0, GR_0, PR_6);
4077     MOVI_p(r0, 1, PR_7);
4078 }
4079
4080 static void
4081 _lei(jit_state_t *_jit, jit_int32_t r0, jit_int32_t r1, jit_word_t i0)
4082 {
4083     jit_int32_t         reg;
4084     reg = jit_get_reg(jit_class_gpr);
4085     movi(rn(reg), i0);
4086     ler(r0, r1, rn(reg));
4087     jit_unget_reg(reg);
4088 }
4089
4090 static void
4091 _ler_u(jit_state_t *_jit, jit_int32_t r0, jit_int32_t r1, jit_int32_t r2)
4092 {
4093     CMP_LTU(PR_6, PR_7, r2, r1);
4094     MOV_p(r0, GR_0, PR_6);
4095     MOVI_p(r0, 1, PR_7);
4096 }
4097
4098 static void
4099 _lei_u(jit_state_t *_jit, jit_int32_t r0, jit_int32_t r1, jit_word_t i0)
4100 {
4101     jit_int32_t         reg;
4102     reg = jit_get_reg(jit_class_gpr);
4103     movi(rn(reg), i0);
4104     ler_u(r0, r1, rn(reg));
4105     jit_unget_reg(reg);
4106 }
4107
4108 static void
4109 _eqr(jit_state_t *_jit, jit_int32_t r0, jit_int32_t r1, jit_int32_t r2)
4110 {
4111     CMP_EQ(PR_6, PR_7, r1, r2);
4112     MOVI_p(r0, 1, PR_6);
4113     MOV_p(r0, GR_0, PR_7);
4114 }
4115
4116 static void
4117 _eqi(jit_state_t *_jit, jit_int32_t r0, jit_int32_t r1, jit_word_t i0)
4118 {
4119     jit_int32_t         reg;
4120     if (i0 >= -128 && i0 <= 127)
4121         CMPI_EQ(PR_6, PR_7, i0, r1);
4122     else {
4123         reg = jit_get_reg(jit_class_gpr);
4124         movi(rn(reg), i0);
4125         CMP_EQ(PR_6, PR_7, r1, rn(reg));
4126         jit_unget_reg(reg);
4127     }
4128     MOVI_p(r0, 1, PR_6);
4129     MOV_p(r0, GR_0, PR_7);
4130 }
4131
4132 static void
4133 _ger(jit_state_t *_jit, jit_int32_t r0, jit_int32_t r1, jit_int32_t r2)
4134 {
4135     CMP_LT(PR_6, PR_7, r1, r2);
4136     MOV_p(r0, GR_0, PR_6);
4137     MOVI_p(r0, 1, PR_7);
4138 }
4139
4140 static void
4141 _gei(jit_state_t *_jit, jit_int32_t r0, jit_int32_t r1, jit_word_t i0)
4142 {
4143     jit_int32_t         reg;
4144     if (i0 >= -127 && i0 <= 128)
4145         CMPI_LT(PR_7, PR_6, i0 - 1, r1);
4146     else {
4147         reg = jit_get_reg(jit_class_gpr);
4148         movi(rn(reg), i0);
4149         CMP_LT(PR_6, PR_7, r1, rn(reg));
4150         jit_unget_reg(reg);
4151     }
4152     MOV_p(r0, GR_0, PR_6);
4153     MOVI_p(r0, 1, PR_7);
4154 }
4155
4156 static void
4157 _ger_u(jit_state_t *_jit, jit_int32_t r0, jit_int32_t r1, jit_int32_t r2)
4158 {
4159     CMP_LTU(PR_6, PR_7, r1, r2);
4160     MOV_p(r0, GR_0, PR_6);
4161     MOVI_p(r0, 1, PR_7);
4162 }
4163
4164 static void
4165 _gei_u(jit_state_t *_jit, jit_int32_t r0, jit_int32_t r1, jit_word_t i0)
4166 {
4167     jit_int32_t         reg;
4168     if (i0 >= -127 && i0 <= 128)
4169         CMPI_LTU(PR_7, PR_6, i0 - 1, r1);
4170     else {
4171         reg = jit_get_reg(jit_class_gpr);
4172         movi(rn(reg), i0);
4173         CMP_LTU(PR_6, PR_7, r1, rn(reg));
4174         jit_unget_reg(reg);
4175     }
4176     MOV_p(r0, GR_0, PR_6);
4177     MOVI_p(r0, 1, PR_7);
4178 }
4179
4180 static void
4181 _gtr(jit_state_t *_jit, jit_int32_t r0, jit_int32_t r1, jit_int32_t r2)
4182 {
4183     CMP_LT(PR_6, PR_7, r2, r1);
4184     MOVI_p(r0, 1, PR_6);
4185     MOV_p(r0, GR_0, PR_7);
4186 }
4187
4188 static void
4189 _gti(jit_state_t *_jit, jit_int32_t r0, jit_int32_t r1, jit_word_t i0)
4190 {
4191     jit_int32_t         reg;
4192     reg = jit_get_reg(jit_class_gpr);
4193     movi(rn(reg), i0);
4194     gtr(r0, r1, rn(reg));
4195     jit_unget_reg(reg);
4196 }
4197
4198 static void
4199 _gtr_u(jit_state_t *_jit, jit_int32_t r0, jit_int32_t r1, jit_int32_t r2)
4200 {
4201     CMP_LTU(PR_6, PR_7, r2, r1);
4202     MOVI_p(r0, 1, PR_6);
4203     MOV_p(r0, GR_0, PR_7);
4204 }
4205
4206 static void
4207 _gti_u(jit_state_t *_jit, jit_int32_t r0, jit_int32_t r1, jit_word_t i0)
4208 {
4209     jit_int32_t         reg;
4210     reg = jit_get_reg(jit_class_gpr);
4211     movi(rn(reg), i0);
4212     gtr_u(r0, r1, rn(reg));
4213     jit_unget_reg(reg);
4214 }
4215
4216 static void
4217 _ner(jit_state_t *_jit, jit_int32_t r0, jit_int32_t r1, jit_int32_t r2)
4218 {
4219     CMP_EQ(PR_6, PR_7, r1, r2);
4220     MOV_p(r0, GR_0, PR_6);
4221     MOVI_p(r0, 1, PR_7);
4222 }
4223
4224 static void
4225 _nei(jit_state_t *_jit, jit_int32_t r0, jit_int32_t r1, jit_word_t i0)
4226 {
4227     jit_int32_t         reg;
4228     if (i0 >= -128 && i0 <= 127)
4229         CMPI_EQ(PR_6, PR_7, i0, r1);
4230     else {
4231         reg = jit_get_reg(jit_class_gpr);
4232         movi(rn(reg), i0);
4233         CMP_EQ(PR_6, PR_7, r1, rn(reg));
4234         jit_unget_reg(reg);
4235     }
4236     MOV_p(r0, GR_0, PR_6);
4237     MOVI_p(r0, 1, PR_7);
4238 }
4239
4240 static void
4241 _ldr_c(jit_state_t *_jit, jit_int32_t r0, jit_int32_t r1)
4242 {
4243     ldr_uc(r0, r1);
4244     extr_c(r0, r0);
4245 }
4246
4247 static void
4248 _ldi_c(jit_state_t *_jit, jit_int32_t r0, jit_word_t i0)
4249 {
4250     jit_int32_t         reg;
4251     reg = jit_get_reg(jit_class_gpr);
4252     movi(rn(reg), i0);
4253     ldr_c(r0, rn(reg));
4254     jit_unget_reg(reg);
4255 }
4256
4257 static void
4258 _ldi_uc(jit_state_t *_jit, jit_int32_t r0, jit_word_t i0)
4259 {
4260     jit_int32_t         reg;
4261     reg = jit_get_reg(jit_class_gpr);
4262     movi(rn(reg), i0);
4263     ldr_uc(r0, rn(reg));
4264     jit_unget_reg(reg);
4265 }
4266
4267 static void
4268 _ldr_s(jit_state_t *_jit, jit_int32_t r0, jit_int32_t r1)
4269 {
4270     ldr_us(r0, r1);
4271     extr_s(r0, r0);
4272 }
4273
4274 static void
4275 _ldi_s(jit_state_t *_jit, jit_int32_t r0, jit_word_t i0)
4276 {
4277     jit_int32_t         reg;
4278     reg = jit_get_reg(jit_class_gpr);
4279     movi(rn(reg), i0);
4280     ldr_s(r0, rn(reg));
4281     jit_unget_reg(reg);
4282 }
4283
4284 static void
4285 _ldi_us(jit_state_t *_jit, jit_int32_t r0, jit_word_t i0)
4286 {
4287     jit_int32_t         reg;
4288     reg = jit_get_reg(jit_class_gpr);
4289     movi(rn(reg), i0);
4290     ldr_us(r0, rn(reg));
4291     jit_unget_reg(reg);
4292 }
4293
4294 static void
4295 _ldr_i(jit_state_t *_jit, jit_int32_t r0, jit_int32_t r1)
4296 {
4297     ldr_ui(r0, r1);
4298     extr_i(r0, r0);
4299 }
4300
4301 static void
4302 _ldi_i(jit_state_t *_jit, jit_int32_t r0, jit_word_t i0)
4303 {
4304     jit_int32_t         reg;
4305     reg = jit_get_reg(jit_class_gpr);
4306     movi(rn(reg), i0);
4307     ldr_i(r0, rn(reg));
4308     jit_unget_reg(reg);
4309 }
4310
4311 static void
4312 _ldi_ui(jit_state_t *_jit, jit_int32_t r0, jit_word_t i0)
4313 {
4314     jit_int32_t         reg;
4315     reg = jit_get_reg(jit_class_gpr);
4316     movi(rn(reg), i0);
4317     ldr_ui(r0, rn(reg));
4318     jit_unget_reg(reg);
4319 }
4320
4321 static void
4322 _ldi_l(jit_state_t *_jit, jit_int32_t r0, jit_word_t i0)
4323 {
4324     jit_int32_t         reg;
4325     reg = jit_get_reg(jit_class_gpr);
4326     movi(rn(reg), i0);
4327     ldr_l(r0, rn(reg));
4328     jit_unget_reg(reg);
4329 }
4330
4331 static void
4332 _ldxr_c(jit_state_t *_jit, jit_int32_t r0, jit_int32_t r1, jit_int32_t r2)
4333 {
4334     jit_int32_t         reg;
4335     reg = jit_get_reg(jit_class_gpr);
4336     addr(rn(reg), r1, r2);
4337     ldr_c(r0, rn(reg));
4338     jit_unget_reg(reg);
4339 }
4340
4341 static void
4342 _ldxi_c(jit_state_t *_jit, jit_int32_t r0, jit_int32_t r1, jit_word_t i0)
4343 {
4344     jit_int32_t         reg;
4345     if (i0) {
4346         reg = jit_get_reg(jit_class_gpr);
4347         movi(rn(reg), i0);
4348         ldxr_c(r0, r1, rn(reg));
4349         jit_unget_reg(reg);
4350     }
4351     else
4352         ldr_c(r0, r1);
4353 }
4354
4355 static void
4356 _ldxr_uc(jit_state_t *_jit, jit_int32_t r0, jit_int32_t r1, jit_int32_t r2)
4357 {
4358     jit_int32_t         reg;
4359     reg = jit_get_reg(jit_class_gpr);
4360     addr(rn(reg), r1, r2);
4361     ldr_uc(r0, rn(reg));
4362     jit_unget_reg(reg);
4363 }
4364
4365 static void
4366 _ldxi_uc(jit_state_t *_jit, jit_int32_t r0, jit_int32_t r1, jit_word_t i0)
4367 {
4368     jit_int32_t         reg;
4369     if (i0) {
4370         reg = jit_get_reg(jit_class_gpr);
4371         movi(rn(reg), i0);
4372         ldxr_uc(r0, r1, rn(reg));
4373         jit_unget_reg(reg);
4374     }
4375     else
4376         ldr_uc(r0, r1);
4377 }
4378
4379 static void
4380 _ldxr_s(jit_state_t *_jit, jit_int32_t r0, jit_int32_t r1, jit_int32_t r2)
4381 {
4382     jit_int32_t         reg;
4383     reg = jit_get_reg(jit_class_gpr);
4384     addr(rn(reg), r1, r2);
4385     ldr_s(r0, rn(reg));
4386     jit_unget_reg(reg);
4387 }
4388
4389 static void
4390 _ldxi_s(jit_state_t *_jit, jit_int32_t r0, jit_int32_t r1, jit_word_t i0)
4391 {
4392     jit_int32_t         reg;
4393     if (i0) {
4394         reg = jit_get_reg(jit_class_gpr);
4395         movi(rn(reg), i0);
4396         ldxr_s(r0, r1, rn(reg));
4397         jit_unget_reg(reg);
4398     }
4399     else
4400         ldr_s(r0, r1);
4401 }
4402
4403 static void
4404 _ldxr_us(jit_state_t *_jit, jit_int32_t r0, jit_int32_t r1, jit_int32_t r2)
4405 {
4406     jit_int32_t         reg;
4407     reg = jit_get_reg(jit_class_gpr);
4408     addr(rn(reg), r1, r2);
4409     ldr_us(r0, rn(reg));
4410     jit_unget_reg(reg);
4411 }
4412
4413 static void
4414 _ldxi_us(jit_state_t *_jit, jit_int32_t r0, jit_int32_t r1, jit_word_t i0)
4415 {
4416     jit_int32_t         reg;
4417     if (i0) {
4418         reg = jit_get_reg(jit_class_gpr);
4419         movi(rn(reg), i0);
4420         ldxr_us(r0, r1, rn(reg));
4421         jit_unget_reg(reg);
4422     }
4423     else
4424         ldr_us(r0, r1);
4425 }
4426
4427 static void
4428 _ldxr_i(jit_state_t *_jit, jit_int32_t r0, jit_int32_t r1, jit_int32_t r2)
4429 {
4430     jit_int32_t         reg;
4431     reg = jit_get_reg(jit_class_gpr);
4432     addr(rn(reg), r1, r2);
4433     ldr_i(r0, rn(reg));
4434     jit_unget_reg(reg);
4435 }
4436
4437 static void
4438 _ldxi_i(jit_state_t *_jit, jit_int32_t r0, jit_int32_t r1, jit_word_t i0)
4439 {
4440     jit_int32_t         reg;
4441     if (i0) {
4442         reg = jit_get_reg(jit_class_gpr);
4443         movi(rn(reg), i0);
4444         ldxr_i(r0, r1, rn(reg));
4445         jit_unget_reg(reg);
4446     }
4447     else
4448         ldr_i(r0, r1);
4449 }
4450
4451 static void
4452 _ldxr_ui(jit_state_t *_jit, jit_int32_t r0, jit_int32_t r1, jit_int32_t r2)
4453 {
4454     jit_int32_t         reg;
4455     reg = jit_get_reg(jit_class_gpr);
4456     addr(rn(reg), r1, r2);
4457     ldr_ui(r0, rn(reg));
4458     jit_unget_reg(reg);
4459 }
4460
4461 static void
4462 _ldxi_ui(jit_state_t *_jit, jit_int32_t r0, jit_int32_t r1, jit_word_t i0)
4463 {
4464     jit_int32_t         reg;
4465     if (i0) {
4466         reg = jit_get_reg(jit_class_gpr);
4467         movi(rn(reg), i0);
4468         ldxr_ui(r0, r1, rn(reg));
4469         jit_unget_reg(reg);
4470     }
4471     else
4472         ldr_ui(r0, r1);
4473 }
4474
4475 static void
4476 _ldxr_l(jit_state_t *_jit, jit_int32_t r0, jit_int32_t r1, jit_int32_t r2)
4477 {
4478     jit_int32_t         reg;
4479     reg = jit_get_reg(jit_class_gpr);
4480     addr(rn(reg), r1, r2);
4481     ldr_l(r0, rn(reg));
4482     jit_unget_reg(reg);
4483 }
4484
4485 static void
4486 _ldxi_l(jit_state_t *_jit, jit_int32_t r0, jit_int32_t r1, jit_word_t i0)
4487 {
4488     jit_int32_t         reg;
4489     if (i0) {
4490         reg = jit_get_reg(jit_class_gpr);
4491         movi(rn(reg), i0);
4492         ldxr_l(r0, r1, rn(reg));
4493         jit_unget_reg(reg);
4494     }
4495     else
4496         ldr_l(r0, r1);
4497 }
4498
4499 static void
4500 _sti_c(jit_state_t *_jit, jit_word_t i0, jit_int32_t r0)
4501 {
4502     jit_int32_t         reg;
4503     reg = jit_get_reg(jit_class_gpr);
4504     movi(rn(reg), i0);
4505     str_c(rn(reg), r0);
4506     jit_unget_reg(reg);
4507 }
4508
4509 static void
4510 _sti_s(jit_state_t *_jit, jit_word_t i0, jit_int32_t r0)
4511 {
4512     jit_int32_t         reg;
4513     reg = jit_get_reg(jit_class_gpr);
4514     movi(rn(reg), i0);
4515     str_s(rn(reg), r0);
4516     jit_unget_reg(reg);
4517 }
4518
4519 static void
4520 _sti_i(jit_state_t *_jit, jit_word_t i0, jit_int32_t r0)
4521 {
4522     jit_int32_t         reg;
4523     reg = jit_get_reg(jit_class_gpr);
4524     movi(rn(reg), i0);
4525     str_i(rn(reg), r0);
4526     jit_unget_reg(reg);
4527 }
4528
4529 static void
4530 _sti_l(jit_state_t *_jit, jit_word_t i0, jit_int32_t r0)
4531 {
4532     jit_int32_t         reg;
4533     reg = jit_get_reg(jit_class_gpr);
4534     movi(rn(reg), i0);
4535     str_l(rn(reg), r0);
4536     jit_unget_reg(reg);
4537 }
4538
4539 static void
4540 _stxr_c(jit_state_t *_jit, jit_int32_t r0, jit_int32_t r1, jit_int32_t r2)
4541 {
4542     jit_int32_t         reg;
4543     reg = jit_get_reg(jit_class_gpr);
4544     addr(rn(reg), r0, r1);
4545     str_c(rn(reg), r2);
4546     jit_unget_reg(reg);
4547 }
4548
4549 static void
4550 _stxi_c(jit_state_t *_jit, jit_word_t i0, jit_int32_t r0, jit_int32_t r1)
4551 {
4552     jit_int32_t         reg;
4553     if (i0) {
4554         reg = jit_get_reg(jit_class_gpr);
4555         addi(rn(reg), r0, i0);
4556         str_c(rn(reg), r1);
4557         jit_unget_reg(reg);
4558     }
4559     else
4560         str_c(r0, r1);
4561 }
4562
4563 static void
4564 _stxr_s(jit_state_t *_jit, jit_int32_t r0, jit_int32_t r1, jit_int32_t r2)
4565 {
4566     jit_int32_t         reg;
4567     reg = jit_get_reg(jit_class_gpr);
4568     addr(rn(reg), r0, r1);
4569     str_s(rn(reg), r2);
4570     jit_unget_reg(reg);
4571 }
4572
4573 static void
4574 _stxi_s(jit_state_t *_jit, jit_word_t i0, jit_int32_t r0, jit_int32_t r1)
4575 {
4576     jit_int32_t         reg;
4577     if (i0) {
4578         reg = jit_get_reg(jit_class_gpr);
4579         addi(rn(reg), r0, i0);
4580         str_s(rn(reg), r1);
4581         jit_unget_reg(reg);
4582     }
4583     else
4584         str_s(r0, r1);
4585 }
4586
4587 static void
4588 _stxr_i(jit_state_t *_jit, jit_int32_t r0, jit_int32_t r1, jit_int32_t r2)
4589 {
4590     jit_int32_t         reg;
4591     reg = jit_get_reg(jit_class_gpr);
4592     addr(rn(reg), r0, r1);
4593     str_i(rn(reg), r2);
4594     jit_unget_reg(reg);
4595 }
4596
4597 static void
4598 _stxi_i(jit_state_t *_jit, jit_word_t i0, jit_int32_t r0, jit_int32_t r1)
4599 {
4600     jit_int32_t         reg;
4601     if (i0) {
4602         reg = jit_get_reg(jit_class_gpr);
4603         addi(rn(reg), r0, i0);
4604         str_i(rn(reg), r1);
4605         jit_unget_reg(reg);
4606     }
4607     else
4608         str_i(r0, r1);
4609 }
4610
4611 static void
4612 _stxr_l(jit_state_t *_jit, jit_int32_t r0, jit_int32_t r1, jit_int32_t r2)
4613 {
4614     jit_int32_t         reg;
4615     reg = jit_get_reg(jit_class_gpr);
4616     addr(rn(reg), r0, r1);
4617     str_l(rn(reg), r2);
4618     jit_unget_reg(reg);
4619 }
4620
4621 static void
4622 _stxi_l(jit_state_t *_jit, jit_word_t i0, jit_int32_t r0, jit_int32_t r1)
4623 {
4624     jit_int32_t         reg;
4625     if (i0) {
4626         reg = jit_get_reg(jit_class_gpr);
4627         addi(rn(reg), r0, i0);
4628         str_l(rn(reg), r1);
4629         jit_unget_reg(reg);
4630     }
4631     else
4632         str_l(r0, r1);
4633 }
4634
4635 static jit_word_t
4636 _bltr(jit_state_t *_jit, jit_word_t i0, jit_int32_t r0, jit_int32_t r1)
4637 {
4638     jit_word_t          w;
4639     CMP_LT(PR_6, PR_7, r0, r1);
4640     sync();
4641     w = _jit->pc.w;
4642     BRI_COND((i0 - w) >> 4, PR_6);
4643     return (w);
4644 }
4645
4646 static jit_word_t
4647 _blti(jit_state_t *_jit, jit_word_t i0, jit_int32_t r0, jit_word_t i1)
4648 {
4649     jit_word_t          w;
4650     jit_int32_t         reg;
4651     if (i1 >= -127 && i1 <= 128)
4652         CMPI_LT(PR_7, PR_6, i1 - 1, r0);
4653     else {
4654         reg = jit_get_reg(jit_class_gpr);
4655         movi(rn(reg), i1);
4656         CMP_LT(PR_6, PR_7, r0, rn(reg));
4657         jit_unget_reg(reg);
4658     }
4659     sync();
4660     w = _jit->pc.w;
4661     BRI_COND((i0 - w) >> 4, PR_6);
4662     return (w);
4663 }
4664
4665 static jit_word_t
4666 _bltr_u(jit_state_t *_jit, jit_word_t i0, jit_int32_t r0, jit_int32_t r1)
4667 {
4668     jit_word_t          w;
4669     CMP_LTU(PR_6, PR_7, r0, r1);
4670     sync();
4671     w = _jit->pc.w;
4672     BRI_COND((i0 - w) >> 4, PR_6);
4673     return (w);
4674 }
4675
4676 static jit_word_t
4677 _blti_u(jit_state_t *_jit, jit_word_t i0, jit_int32_t r0, jit_word_t i1)
4678 {
4679     jit_word_t          w;
4680     jit_int32_t         reg;
4681     if (i1 >= -127 && i1 <= 128)
4682         CMPI_LTU(PR_7, PR_6, i1 - 1, r0);
4683     else {
4684         reg = jit_get_reg(jit_class_gpr);
4685         movi(rn(reg), i1);
4686         CMP_LTU(PR_6, PR_7, r0, rn(reg));
4687         jit_unget_reg(reg);
4688     }
4689     sync();
4690     w = _jit->pc.w;
4691     BRI_COND((i0 - w) >> 4, PR_6);
4692     return (w);
4693 }
4694
4695 static jit_word_t
4696 _bler(jit_state_t *_jit, jit_word_t i0, jit_int32_t r0, jit_int32_t r1)
4697 {
4698     jit_word_t          w;
4699     CMP_LT(PR_6, PR_7, r1, r0);
4700     sync();
4701     w = _jit->pc.w;
4702     BRI_COND((i0 - w) >> 4, PR_7);
4703     return (w);
4704 }
4705
4706 static jit_word_t
4707 _blei(jit_state_t *_jit, jit_word_t i0, jit_int32_t r0, jit_word_t i1)
4708 {
4709     jit_word_t          w;
4710     jit_int32_t         reg;
4711     reg = jit_get_reg(jit_class_gpr|jit_class_nospill);
4712     movi(rn(reg), i1);
4713     w = bler(i0, r0, rn(reg));
4714     jit_unget_reg(reg);
4715     return (w);
4716 }
4717
4718 static jit_word_t
4719 _bler_u(jit_state_t *_jit, jit_word_t i0, jit_int32_t r0, jit_int32_t r1)
4720 {
4721     jit_word_t          w;
4722     CMP_LTU(PR_6, PR_7, r1, r0);
4723     sync();
4724     w = _jit->pc.w;
4725     BRI_COND((i0 - w) >> 4, PR_7);
4726     return (w);
4727 }
4728
4729 static jit_word_t
4730 _blei_u(jit_state_t *_jit, jit_word_t i0, jit_int32_t r0, jit_word_t i1)
4731 {
4732     jit_word_t          w;
4733     jit_int32_t         reg;
4734     reg = jit_get_reg(jit_class_gpr|jit_class_nospill);
4735     movi(rn(reg), i1);
4736     w = bler_u(i0, r0, rn(reg));
4737     jit_unget_reg(reg);
4738     return (w);
4739 }
4740
4741 static jit_word_t
4742 _beqr(jit_state_t *_jit, jit_word_t i0, jit_int32_t r0, jit_int32_t r1)
4743 {
4744     jit_word_t          w;
4745     CMP_EQ(PR_6, PR_7, r0, r1);
4746     sync();
4747     w = _jit->pc.w;
4748     BRI_COND((i0 - w) >> 4, PR_6);
4749     return (w);
4750 }
4751
4752 static jit_word_t
4753 _beqi(jit_state_t *_jit, jit_word_t i0, jit_int32_t r0, jit_word_t i1)
4754 {
4755     jit_word_t          w;
4756     jit_int32_t         reg;
4757     if (i1 >= -128 && i1 <= 127)
4758         CMPI_EQ(PR_6, PR_7, i1, r0);
4759     else {
4760         reg = jit_get_reg(jit_class_gpr);
4761         movi(rn(reg), i1);
4762         CMP_EQ(PR_6, PR_7, r0, rn(reg));
4763         jit_unget_reg(reg);
4764     }
4765     sync();
4766     w = _jit->pc.w;
4767     BRI_COND((i0 - w) >> 4, PR_6);
4768     return (w);
4769 }
4770
4771 static jit_word_t
4772 _bger(jit_state_t *_jit, jit_word_t i0, jit_int32_t r0, jit_int32_t r1)
4773 {
4774     jit_word_t          w;
4775     CMP_LT(PR_6, PR_7, r0, r1);
4776     sync();
4777     w = _jit->pc.w;
4778     BRI_COND((i0 - w) >> 4, PR_7);
4779     return (w);
4780 }
4781
4782 static jit_word_t
4783 _bgei(jit_state_t *_jit, jit_word_t i0, jit_int32_t r0, jit_word_t i1)
4784 {
4785     jit_word_t          w;
4786     jit_int32_t         reg;
4787     if (i1 >= -127 && i1 <= 128)
4788         CMPI_LT(PR_7, PR_6, i1 - 1, r0);
4789     else {
4790         reg = jit_get_reg(jit_class_gpr);
4791         movi(rn(reg), i1);
4792         CMP_LT(PR_6, PR_7, r0, rn(reg));
4793         jit_unget_reg(reg);
4794     }
4795     sync();
4796     w = _jit->pc.w;
4797     BRI_COND((i0 - w) >> 4, PR_7);
4798     return (w);
4799 }
4800
4801 static jit_word_t
4802 _bger_u(jit_state_t *_jit, jit_word_t i0, jit_int32_t r0, jit_int32_t r1)
4803 {
4804     jit_word_t          w;
4805     CMP_LTU(PR_6, PR_7, r0, r1);
4806     sync();
4807     w = _jit->pc.w;
4808     BRI_COND((i0 - w) >> 4, PR_7);
4809     return (w);
4810 }
4811
4812 static jit_word_t
4813 _bgei_u(jit_state_t *_jit, jit_word_t i0, jit_int32_t r0, jit_word_t i1)
4814 {
4815     jit_word_t          w;
4816     jit_int32_t         reg;
4817     if (i1 >= -127 && i1 <= 128)
4818         CMPI_LTU(PR_7, PR_6, i1 - 1, r0);
4819     else {
4820         reg = jit_get_reg(jit_class_gpr);
4821         movi(rn(reg), i1);
4822         CMP_LTU(PR_6, PR_7, r0, rn(reg));
4823         jit_unget_reg(reg);
4824     }
4825     sync();
4826     w = _jit->pc.w;
4827     BRI_COND((i0 - w) >> 4, PR_7);
4828     return (w);
4829 }
4830
4831 static jit_word_t
4832 _bgtr(jit_state_t *_jit, jit_word_t i0, jit_int32_t r0, jit_int32_t r1)
4833 {
4834     jit_word_t          w;
4835     CMP_LT(PR_6, PR_7, r1, r0);
4836     sync();
4837     w = _jit->pc.w;
4838     BRI_COND((i0 - w) >> 4, PR_6);
4839     return (w);
4840 }
4841
4842 static jit_word_t
4843 _bgti(jit_state_t *_jit, jit_word_t i0, jit_int32_t r0, jit_word_t i1)
4844 {
4845     jit_word_t          w;
4846     jit_int32_t         reg;
4847     reg = jit_get_reg(jit_class_gpr|jit_class_nospill);
4848     movi(rn(reg), i1);
4849     w = bgtr(i0, r0, rn(reg));
4850     jit_unget_reg(reg);
4851     return (w);
4852 }
4853
4854 static jit_word_t
4855 _bgtr_u(jit_state_t *_jit, jit_word_t i0, jit_int32_t r0, jit_int32_t r1)
4856 {
4857     jit_word_t          w;
4858     CMP_LTU(PR_6, PR_7, r1, r0);
4859     sync();
4860     w = _jit->pc.w;
4861     BRI_COND((i0 - w) >> 4, PR_6);
4862     return (w);
4863 }
4864
4865 static jit_word_t
4866 _bgti_u(jit_state_t *_jit, jit_word_t i0, jit_int32_t r0, jit_word_t i1)
4867 {
4868     jit_word_t          w;
4869     jit_int32_t         reg;
4870     reg = jit_get_reg(jit_class_gpr|jit_class_nospill);
4871     movi(rn(reg), i1);
4872     w = bgtr_u(i0, r0, rn(reg));
4873     jit_unget_reg(reg);
4874     return (w);
4875 }
4876
4877 static jit_word_t
4878 _bner(jit_state_t *_jit, jit_word_t i0, jit_int32_t r0, jit_int32_t r1)
4879 {
4880     jit_word_t          w;
4881     CMP_EQ(PR_6, PR_7, r0, r1);
4882     sync();
4883     w = _jit->pc.w;
4884     BRI_COND((i0 - w) >> 4, PR_7);
4885     return (w);
4886 }
4887
4888 static jit_word_t
4889 _bnei(jit_state_t *_jit, jit_word_t i0, jit_int32_t r0, jit_word_t i1)
4890 {
4891     jit_word_t          w;
4892     jit_int32_t         reg;
4893     if (i1 >= -128 && i1 <= 127)
4894         CMPI_EQ(PR_6, PR_7, i1, r0);
4895     else {
4896         reg = jit_get_reg(jit_class_gpr);
4897         movi(rn(reg), i1);
4898         CMP_EQ(PR_6, PR_7, r0, rn(reg));
4899         jit_unget_reg(reg);
4900     }
4901     sync();
4902     w = _jit->pc.w;
4903     BRI_COND((i0 - w) >> 4, PR_7);
4904     return (w);
4905 }
4906
4907 static jit_word_t
4908 _bmsr(jit_state_t *_jit, jit_word_t i0, jit_int32_t r0, jit_int32_t r1)
4909 {
4910     jit_word_t          w;
4911     jit_int32_t         reg;
4912     reg = jit_get_reg(jit_class_gpr);
4913     andr(rn(reg), r0, r1);
4914     CMPI_EQ(PR_6, PR_7, 0, rn(reg));
4915     jit_unget_reg(reg);
4916     sync();
4917     w = _jit->pc.w;
4918     BRI_COND((i0 - w) >> 4, PR_7);
4919     return (w);
4920 }
4921
4922 static jit_word_t
4923 _bmsi(jit_state_t *_jit, jit_word_t i0, jit_int32_t r0, jit_word_t i1)
4924 {
4925     jit_word_t          w;
4926     jit_int32_t         reg;
4927     reg = jit_get_reg(jit_class_gpr);
4928     movi(rn(reg), i1);
4929     andr(rn(reg), r0, rn(reg));
4930     CMPI_EQ(PR_6, PR_7, 0, rn(reg));
4931     jit_unget_reg(reg);
4932     sync();
4933     w = _jit->pc.w;
4934     BRI_COND((i0 - w) >> 4, PR_7);
4935     return (w);
4936 }
4937
4938 static jit_word_t
4939 _bmcr(jit_state_t *_jit, jit_word_t i0, jit_int32_t r0, jit_int32_t r1)
4940 {
4941     jit_word_t          w;
4942     jit_int32_t         reg;
4943     reg = jit_get_reg(jit_class_gpr);
4944     andr(rn(reg), r0, r1);
4945     CMPI_EQ(PR_6, PR_7, 0, rn(reg));
4946     jit_unget_reg(reg);
4947     sync();
4948     w = _jit->pc.w;
4949     BRI_COND((i0 - w) >> 4, PR_6);
4950     return (w);
4951 }
4952
4953 static jit_word_t
4954 _bmci(jit_state_t *_jit, jit_word_t i0, jit_int32_t r0, jit_word_t i1)
4955 {
4956     jit_word_t          w;
4957     jit_int32_t         reg;
4958     reg = jit_get_reg(jit_class_gpr);
4959     movi(rn(reg), i1);
4960     andr(rn(reg), r0, rn(reg));
4961     CMPI_EQ(PR_6, PR_7, 0, rn(reg));
4962     jit_unget_reg(reg);
4963     sync();
4964     w = _jit->pc.w;
4965     BRI_COND((i0 - w) >> 4, PR_6);
4966     return (w);
4967 }
4968
4969 static jit_word_t
4970 _baddr(jit_state_t *_jit, jit_word_t i0, jit_int32_t r0, jit_int32_t r1,
4971        jit_bool_t carry)
4972 {
4973     jit_word_t          w;
4974     jit_int32_t         t0;
4975     jit_int32_t         t1;
4976     jit_int32_t         t2;
4977     /* t1 = r0 + r1;    overflow = r1 < 0 ? r0 < t1 : t1 < r0 */
4978     t0 = jit_get_reg(jit_class_gpr|jit_class_nospill);
4979     t1 = jit_get_reg(jit_class_gpr|jit_class_nospill);
4980     t2 = jit_get_reg(jit_class_gpr|jit_class_nospill);
4981     lti(rn(t0), r1, 0);                 /* t0 = r1 < 0 */
4982     addr(rn(t1), r0, r1);               /* t1 = r0 + r1 */
4983     ltr(rn(t2), rn(t1), r0);            /* t2 = t1 < r0 */
4984     ltr(rn(t1), r0, rn(t1));            /* t1 = r0 < t1 */
4985     CMPI_EQ(PR_6, PR_7, 0, rn(t0));
4986     CMPI_EQ_p(PR_8, PR_9, 0, rn(t2), PR_6);/* if (t0==0) p8=t2==0,p9=t2!=0; */
4987     CMPI_EQ_p(PR_8, PR_9, 0, rn(t1), PR_7);/* if (t0!=0) p8=t1==0,p9=t1!=0; */
4988     addr(r0, r0, r1);
4989     sync();
4990     w = _jit->pc.w;
4991     BRI_COND((i0 - w) >> 4, carry ? PR_9 : PR_8);
4992     jit_unget_reg(t2);
4993     jit_unget_reg(t1);
4994     jit_unget_reg(t0);
4995     return (w);
4996 }
4997
4998 static jit_word_t
4999 _baddi(jit_state_t *_jit, jit_word_t i0, jit_int32_t r0, jit_word_t i1,
5000        jit_bool_t carry)
5001 {
5002     jit_word_t          w;
5003     jit_int32_t         reg;
5004     reg = jit_get_reg(jit_class_gpr|jit_class_nospill);
5005     movi(rn(reg), i1);
5006     w = baddr(i0, r0, rn(reg), carry);
5007     jit_unget_reg(reg);
5008     return (w);
5009 }
5010
5011 static jit_word_t
5012 _baddr_u(jit_state_t *_jit, jit_word_t i0, jit_int32_t r0, jit_int32_t r1,
5013          jit_bool_t carry)
5014 {
5015     jit_word_t          w;
5016     jit_int32_t         t0;
5017     jit_int32_t         t1;
5018     t0 = jit_get_reg(jit_class_gpr|jit_class_nospill);
5019     t1 = jit_get_reg(jit_class_gpr|jit_class_nospill);
5020     addr(rn(t0), r0, r1);
5021     ltr_u(rn(t1), rn(t0), r0);
5022     CMPI_EQ(PR_6, PR_7, 0, rn(t1));
5023     MOV(r0, rn(t0));
5024     sync();
5025     w = _jit->pc.w;
5026     BRI_COND((i0 - w) >> 4, carry ? PR_7 : PR_6);
5027     jit_unget_reg(t1);
5028     jit_unget_reg(t0);
5029     return (w);
5030 }
5031
5032 static jit_word_t
5033 _baddi_u(jit_state_t *_jit, jit_word_t i0, jit_int32_t r0, jit_word_t i1,
5034          jit_bool_t carry)
5035 {
5036     jit_word_t          w;
5037     jit_int32_t         reg;
5038     reg = jit_get_reg(jit_class_gpr|jit_class_nospill);
5039     movi(rn(reg), i1);
5040     w = baddr_u(i0, r0, rn(reg), carry);
5041     jit_unget_reg(reg);
5042     return (w);
5043 }
5044
5045 static jit_word_t
5046 _bsubr(jit_state_t *_jit, jit_word_t i0, jit_int32_t r0, jit_int32_t r1,
5047        jit_bool_t carry)
5048 {
5049     jit_word_t          w;
5050     jit_int32_t         t0;
5051     jit_int32_t         t1;
5052     jit_int32_t         t2;
5053     /* t1 = r0 - r1;    overflow = 0 < r1 ? r0 < t1 : t1 < r0 */
5054     t0 = jit_get_reg(jit_class_gpr|jit_class_nospill);
5055     t1 = jit_get_reg(jit_class_gpr|jit_class_nospill);
5056     t2 = jit_get_reg(jit_class_gpr|jit_class_nospill);
5057     gti(rn(t0), r1, 0);                 /* t0 = r1 > 0 */
5058     subr(rn(t1), r0, r1);               /* t1 = r0 - r1 */
5059     ltr(rn(t2), rn(t1), r0);            /* t2 = t1 < r0 */
5060     ltr(rn(t1), r0, rn(t1));            /* t1 = r0 < t1 */
5061     CMPI_EQ(PR_6, PR_7, 0, rn(t0));
5062     CMPI_EQ_p(PR_8, PR_9, 0, rn(t2), PR_6);/* if (t0==0) p4=t2==0,p5=t2!=0; */
5063     CMPI_EQ_p(PR_8, PR_9, 0, rn(t1), PR_7);/* if (t0!=0) p4=t1==0,p5=t1!=0; */
5064     subr(r0, r0, r1);
5065     sync();
5066     w = _jit->pc.w;
5067     BRI_COND((i0 - w) >> 4, carry ? PR_9 : PR_8);
5068     jit_unget_reg(t2);
5069     jit_unget_reg(t1);
5070     jit_unget_reg(t0);
5071     return (w);
5072 }
5073
5074 static jit_word_t
5075 _bsubi(jit_state_t *_jit, jit_word_t i0, jit_int32_t r0, jit_word_t i1,
5076        jit_bool_t carry)
5077 {
5078     jit_word_t          w;
5079     jit_int32_t         reg;
5080     reg = jit_get_reg(jit_class_gpr|jit_class_nospill);
5081     movi(rn(reg), i1);
5082     w = bsubr(i0, r0, rn(reg), carry);
5083     jit_unget_reg(reg);
5084     return (w);
5085 }
5086
5087 static jit_word_t
5088 _bsubr_u(jit_state_t *_jit, jit_word_t i0, jit_int32_t r0, jit_int32_t r1,
5089          jit_bool_t carry)
5090 {
5091     jit_word_t          w;
5092     jit_int32_t         t0;
5093     jit_int32_t         t1;
5094     t0 = jit_get_reg(jit_class_gpr|jit_class_nospill);
5095     t1 = jit_get_reg(jit_class_gpr|jit_class_nospill);
5096     subr(rn(t0), r0, r1);
5097     ltr_u(rn(t1), r0, rn(t0));
5098     CMPI_EQ(PR_6, PR_7, 0, rn(t1));
5099     MOV(r0, rn(t0));
5100     sync();
5101     w = _jit->pc.w;
5102     BRI_COND((i0 - w) >> 4, carry ? PR_7 : PR_6);
5103     jit_unget_reg(t1);
5104     jit_unget_reg(t0);
5105     return (w);
5106 }
5107
5108 static jit_word_t
5109 _bsubi_u(jit_state_t *_jit, jit_word_t i0, jit_int32_t r0, jit_word_t i1,
5110          jit_bool_t carry)
5111 {
5112     jit_word_t          w;
5113     jit_int32_t         reg;
5114     reg = jit_get_reg(jit_class_gpr|jit_class_nospill);
5115     movi(rn(reg), i1);
5116     w = bsubr_u(i0, r0, rn(reg), carry);
5117     jit_unget_reg(reg);
5118     return (w);
5119 }
5120
5121 static void
5122 _jmpr(jit_state_t *_jit, jit_int32_t r0)
5123 {
5124     MOV_br_rn(BR_6, r0);
5125     BR(BR_6);
5126 }
5127
5128 static void
5129 _jmpi(jit_state_t *_jit, jit_word_t i0)
5130 {
5131     jit_word_t          d;
5132     sync();
5133     d = ((jit_word_t)i0 - _jit->pc.w) >> 4;
5134     if (d >= -16777216 && d <= 16777215)
5135         BRI(d);
5136     else
5137         BRL(d);
5138 }
5139
5140 static jit_word_t
5141 _jmpi_p(jit_state_t *_jit, jit_word_t i0)
5142 {
5143     jit_word_t          d, w;
5144     sync();
5145     w = _jit->pc.w;
5146     d = ((jit_word_t)i0 - w) >> 4;
5147     BRL(d);
5148     return (w);
5149 }
5150
5151 static void
5152 _callr(jit_state_t *_jit, jit_int32_t r0)
5153 {
5154     jit_int32_t         reg;
5155     reg = jit_get_reg(jit_class_gpr);
5156     LD8_inc(rn(reg), r0, 8);
5157     MOV_br_rn(BR_6, rn(reg));
5158     jit_unget_reg(reg);
5159     LD8(GR_1, r0);
5160     BR_CALL(BR_0, BR_6);
5161 }
5162
5163 static void
5164 _calli(jit_state_t *_jit, jit_word_t i0)
5165 {
5166     jit_int32_t         reg;
5167     reg = jit_get_reg(jit_class_gpr);
5168     movi(rn(reg), i0);
5169     callr(rn(reg));
5170     jit_unget_reg(reg);
5171 }
5172
5173 static jit_word_t
5174 _calli_p(jit_state_t *_jit, jit_word_t i0)
5175 {
5176     jit_word_t          w;
5177     jit_int32_t         reg;
5178     reg = jit_get_reg(jit_class_gpr);
5179     w = movi_p(rn(reg), i0);
5180     callr(rn(reg));
5181     jit_unget_reg(reg);
5182     return (w);
5183 }
5184
5185 static void
5186 _prolog(jit_state_t *_jit, jit_node_t *node)
5187 {
5188     jit_int32_t         reg, ruse, rout;
5189
5190     if (_jitc->function->define_frame || _jitc->function->assume_frame) {
5191         jit_int32_t     frame = -_jitc->function->frame;
5192         assert(_jitc->function->self.aoff >= frame);
5193         if (_jitc->function->assume_frame)
5194             return;
5195         _jitc->function->self.aoff = frame;
5196     }
5197     if (_jitc->function->allocar)
5198         _jitc->function->self.aoff &= -16;
5199     _jitc->function->stack = ((_jitc->function->self.alen -
5200                                _jitc->function->self.aoff) + 15) & -16;
5201
5202     /* First lowest unused register is first output register */
5203     for (reg = _R115; reg >= _R40; reg--) {
5204         if (jit_regset_tstbit(&_jitc->function->regset, reg))
5205             break;
5206     }
5207     _jitc->breg = rn(reg) + 1;
5208     _jitc->rout = _jitc->breg + 5;
5209     ruse = _jitc->rout - GR_32;
5210
5211     /* How many out argument registers required? */
5212     if (!_jitc->function->define_frame) {
5213         for (reg = _OUT7; reg >= _OUT0; --reg) {
5214             if (jit_regset_tstbit(&_jitc->function->regset, reg))
5215                 break;
5216         }
5217         rout = (reg + 1) - _OUT0;
5218     }
5219     else
5220         rout = 8;
5221
5222     /* Do not know if will call div/mod functions (sqrt* needs one) */
5223     if (rout < 2)
5224         rout = 2;
5225
5226     /* Match gcc prolog */
5227     ALLOC(_jitc->breg + 1, ruse, rout);
5228     MOV(_jitc->breg + 2, GR_12);
5229     MOV_rn_br(_jitc->breg, BR_0);
5230     MOV(_jitc->breg + 3, GR_1);
5231
5232     /* lightning specific, use r4 as frame pointer */
5233     MOV(_jitc->breg + 4, GR_4);
5234     addi(GR_4, GR_12, -(stack_framesize + params_offset));
5235
5236     /* adjust stack pointer */
5237     addi(GR_12, GR_12, -(stack_framesize +
5238                          (params_offset << 1) + _jitc->function->stack));
5239
5240     if (jit_regset_tstbit(&_jitc->function->regset, JIT_F0))
5241         STF_SPILL(GR_4, rn(JIT_F0));
5242     if (jit_regset_tstbit(&_jitc->function->regset, JIT_F1)) {
5243         addi(GR_2, GR_4, 16);
5244         STF_SPILL(GR_2, rn(JIT_F1));
5245     }
5246     if (jit_regset_tstbit(&_jitc->function->regset, JIT_F2)) {
5247         addi(GR_2, GR_4, 32);
5248         STF_SPILL(GR_2, rn(JIT_F2));
5249     }
5250     if (jit_regset_tstbit(&_jitc->function->regset, JIT_F3)) {
5251         addi(GR_2, GR_4, 48);
5252         STF_SPILL(GR_2, rn(JIT_F3));
5253     }
5254     if (jit_regset_tstbit(&_jitc->function->regset, JIT_F4)) {
5255         addi(GR_2, GR_4, 64);
5256         STF_SPILL(GR_2, rn(JIT_F4));
5257     }
5258     if (jit_regset_tstbit(&_jitc->function->regset, JIT_F5)) {
5259         addi(GR_2, GR_4, 80);
5260         STF_SPILL(GR_2, rn(JIT_F5));
5261     }
5262
5263     if (_jitc->function->allocar) {
5264         reg = jit_get_reg(jit_class_gpr);
5265         movi(rn(reg), _jitc->function->self.aoff);
5266         stxi_i(_jitc->function->aoffoff, GR_4, rn(reg));
5267         jit_unget_reg(reg);
5268     }
5269
5270     if (_jitc->function->self.call & jit_call_varargs) {
5271         for (reg = _jitc->function->vagp; reg < 8; ++reg)
5272             stxi(112 + reg * 8, GR_4, GR_32 + reg);
5273     }
5274 }
5275
5276 static void
5277 _epilog(jit_state_t *_jit, jit_node_t *node)
5278 {
5279     if (_jitc->function->assume_frame)
5280         return;
5281     if (jit_regset_tstbit(&_jitc->function->regset, JIT_F0))
5282         LDF_FILL(rn(JIT_F0), GR_4);
5283     if (jit_regset_tstbit(&_jitc->function->regset, JIT_F1)) {
5284         addi(GR_2, GR_4, 16);
5285         LDF_FILL(rn(JIT_F1), GR_2);
5286     }
5287     if (jit_regset_tstbit(&_jitc->function->regset, JIT_F2)) {
5288         addi(GR_2, GR_4, 32);
5289         LDF_FILL(rn(JIT_F2), GR_2);
5290     }
5291     if (jit_regset_tstbit(&_jitc->function->regset, JIT_F3)) {
5292         addi(GR_2, GR_4, 48);
5293         LDF_FILL(rn(JIT_F3), GR_2);
5294     }
5295     if (jit_regset_tstbit(&_jitc->function->regset, JIT_F4)) {
5296         addi(GR_2, GR_4, 64);
5297         LDF_FILL(rn(JIT_F4), GR_2);
5298     }
5299     if (jit_regset_tstbit(&_jitc->function->regset, JIT_F5)) {
5300         addi(GR_2, GR_4, 80);
5301         LDF_FILL(rn(JIT_F5), GR_2);
5302     }
5303     /* Match gcc epilog */
5304     MOV(GR_1, _jitc->breg + 3);
5305     MOV_I_ar_rn(AR_PFS, _jitc->breg + 1);
5306     MOV_br_rn(BR_0, _jitc->breg);
5307     MOV(GR_12, _jitc->breg + 2);
5308     /* Restore lightning specific r4 as frame pointer */
5309     MOV(GR_4, _jitc->breg + 4);
5310     BR_RET(BR_0);
5311     flush();
5312 }
5313
5314 static void
5315 _vastart(jit_state_t *_jit, jit_int32_t r0)
5316 {
5317     assert(_jitc->function->self.call & jit_call_varargs);
5318     /* Initialize va_list to the first stack argument. */
5319     if (_jitc->function->vagp < 8)
5320         addi(r0, GR_4, 112 + _jitc->function->vagp * 8);
5321     else
5322         addi(r0, GR_4, _jitc->function->self.size);
5323 }
5324
5325 static void
5326 _vaarg(jit_state_t *_jit, jit_int32_t r0, jit_int32_t r1)
5327 {
5328     assert(_jitc->function->self.call & jit_call_varargs);
5329     /* Load argument. */
5330     ldr(r0, r1);
5331     /* Update va_list. */
5332     addi(r1, r1, 8);
5333 }
5334
5335 static void
5336 _patch_at(jit_state_t *_jit, jit_code_t code,
5337           jit_word_t instr, jit_word_t label)
5338 {
5339     jit_word_t           l, h, tm, s0, s1, s2;
5340     union {
5341         jit_word_t       w;
5342         jit_word_t      *p;
5343     } c;
5344     jit_word_t           i1, i41, i20, ic, i5, i9, i7;
5345     c.w = instr;
5346     get_bundle(c.p, l, h, tm, s0, s1, s2);
5347     switch (code) {
5348         case jit_code_movi:
5349             /* Handle jit functions as C function, so that jit function
5350              * pointers can be passed to C code, and jit code does not
5351              * need to try to differentiate them. */
5352         case jit_code_calli:
5353             i1  = (label >> 63) &           0x1L;
5354             i41 = (label >> 22) & 0x1ffffffffffL;
5355             ic  = (label >> 21) &           0x1L;
5356             i5  = (label >> 16) &          0x1fL;
5357             i9  = (label >>  7) &         0x1ffL;
5358             i7  =  label        &          0x7fL;
5359             s1 = i41;
5360             assert((tm & ~1) == TM_M_L_X_ &&
5361                    (s2 & 0xfL<<37) == (6L<<37) &&
5362                    s0 == nop_m);
5363             s2 &= (6L<<37)|(0x7fL<<6);
5364             s2 |= (i1<<36)|(i9<<27)|(i5<<22)|(ic<<21)|(i7<<13);
5365             break;
5366         case jit_code_jmpi:
5367             if (_jitc->jump) {
5368                 /* kludge to hide function descriptors; check that gp
5369                  * is zero, what is done for all jit functions */
5370                 if (((long *)label)[1] == 0) {
5371                     for (ic = 0; ic < _jitc->prolog.offset; ic++) {
5372                         if (_jitc->prolog.ptr[ic] == label) {
5373                             label += 16;
5374                             break;
5375                         }
5376                     }
5377                 }
5378             }
5379             ic = (label - instr) >> 4;
5380             i1  = (ic >> 61) &           0x1L;
5381             i41 = (ic >> 22) & 0x1ffffffffffL;
5382             i20 =  ic        &       0xfffffL;
5383             assert((tm & ~1) == TM_M_L_X_ &&
5384                    (s2 & 0xfL<<37) == (0xcL<<37) &&
5385                    s0 == nop_m);
5386             s1 = i41;
5387             s2 &= (0xcL<<37)|(0x7L<<33)|(1L<<12);
5388             s2 |= (i1<<36)|(i20<<13);
5389             break;
5390         default:
5391             /* Only B1 in slot 0 expected due to need to either
5392              * a stop to update predicates, or a sync before
5393              * unconditional short branch */
5394             ic = (label - instr) >> 4;
5395             assert((s0 >> 37) == 4 && (s0 & (7 << 6)) == 0);
5396             s0 &= (4L<<37)|(7L<<33)|(1L<<12)|0x1f;
5397             s0 |= (((ic>>20)&1L)<<36)|((ic&0xfffffL)<<13);
5398             break;
5399     }
5400     set_bundle(c.p, l, h, tm, s0, s1, s2);
5401 }
5402 #endif