deps/lightning/lib/jit_ia64-cpu.c

   1 /*
   2  * Copyright (C) 2013-2023  Free Software Foundation, Inc.
   3  *
   4  * This file is part of GNU lightning.
   5  *
   6  * GNU lightning is free software; you can redistribute it and/or modify it
   7  * under the terms of the GNU Lesser General Public License as published
   8  * by the Free Software Foundation; either version 3, or (at your option)
   9  * any later version.
  10  *
  11  * GNU lightning is distributed in the hope that it will be useful, but
  12  * WITHOUT ANY WARRANTY; without even the implied warranty of MERCHANTABILITY
  13  * or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU Lesser General Public
  14  * License for more details.
  15  *
  16  * Authors:
  17  *      Paulo Cesar Pereira de Andrade
  18  */
  19
  20 #if PROTO
  21 #define stack_framesize                 144
  22 #define params_offset                   16
  23 #define INST_NONE                       0       /* should never be generated */
  24 #define INST_STOP                       1       /* or'ed if stop is required */
  25 #define INST_A                          2       /* M- or I- unit */
  26 #define INST_As                         3
  27 #define INST_I                          4
  28 #define INST_Is                         5
  29 #define INST_M                          6
  30 #define INST_Ms                         7
  31 #define INST_F                          8
  32 #define INST_Fs                         9
  33 #define INST_B                          10
  34 #define INST_Bs                         11
  35 #define INST_L                          12
  36 #define INST_Ls                         13
  37 #define INST_X                          14
  38 #define INST_Xs                         15
  39
  40 /* Data and instructions are referenced by 64-bit addresses. Instructions
  41  * are stored in memory in little endian byte order, in which the least
  42  * significant byte appears in the lowest addressed byte of a memory
  43  * location. For data, modes for both big and little endian byte order are
  44  * supported and can be controlled by a bit in the User Mask Register.
  45  */
  46 #define il(ii)                          *_jit->pc.ul++ = ii
  47 #define set_bundle(p, l, h, tm, s0, s1, s2)                             \
  48     do {                                                                \
  49         l = tm | ((s0 & 0x1ffffffffffL) << 5L) | (s1 << 46L);           \
  50         h = ((s1 >> 18L) & 0x7fffffLL) | (s2 << 23L);                   \
  51         p[0] = byte_swap_if_big_endian(l);                              \
  52         p[1] = byte_swap_if_big_endian(h);                              \
  53     } while (0)
  54 #define get_bundle(p, l, h, tm, s0, s1, s2)                             \
  55     do {                                                                \
  56         l = byte_swap_if_big_endian(p[0]);                              \
  57         h = byte_swap_if_big_endian(p[1]);                              \
  58         tm = l & 0x1f;                                                  \
  59         s0 = (l >> 5L) & 0x1ffffffffffL;                                \
  60         s1 = ((l >> 46L) & 0x3ffffL) | ((h & 0x7fffffL) << 18L);        \
  61         s2 = (h >> 23L) & 0x1ffffffffffL;                               \
  62     } while (0)
  63
  64 /*  Need to insert a stop if a modified register would (or could)
  65  *  be read in the same cycle.
  66  */
  67 #define TSTREG1(r0)                                                     \
  68     do {                                                                \
  69         if (jit_regset_tstbit(&_jitc->regs, r0))                        \
  70             stop();                                                     \
  71     } while (0)
  72 #define TSTREG2(r0, r1)                                                 \
  73     do {                                                                \
  74         if (jit_regset_tstbit(&_jitc->regs, r0) ||                      \
  75             jit_regset_tstbit(&_jitc->regs, r1))                        \
  76             stop();                                                     \
  77     } while (0)
  78 #define TSTPRED(p0)                                                     \
  79     do {                                                                \
  80         if (p0 && (_jitc->pred & (1 << p0)))                            \
  81             stop();                                                     \
  82     } while (0)
  83 /* Record register was modified */
  84 #define SETREG(r0)              jit_regset_setbit(&_jitc->regs, r0)
  85
  86 /* Avoid using constants in macros and code */
  87 typedef enum {
  88     GR_0,               GR_1,           GR_2,           GR_3,
  89     GR_4,               GR_5,           GR_6,           GR_7,
  90     GR_8,               GR_9,           GR_10,          GR_11,
  91     GR_12,              GR_13,          GR_14,          GR_15,
  92     GR_16,              GR_17,          GR_18,          GR_19,
  93     GR_20,              GR_21,          GR_22,          GR_23,
  94     GR_24,              GR_25,          GR_26,          GR_27,
  95     GR_28,              GR_29,          GR_30,          GR_31,
  96     GR_32,              GR_33,          GR_34,          GR_35,
  97     GR_36,              GR_37,          GR_38,          GR_39,
  98     GR_40,              GR_41,          GR_42,          GR_43,
  99     /* GR_44...GR_127 */
 100 } gr_reg_t;
 101
 102 typedef enum {
 103     PR_0,               /* constant - always 1 */
 104     /* p0-p5             -  preserved */
 105     PR_1,               PR_2,           PR_3,           PR_4,
 106     PR_5,
 107     /* p6-p15            - scratch */
 108     PR_6,               PR_7,           PR_8,           PR_9,
 109     PR_10,              PR_11,          PR_12,          PR_13,
 110     PR_14,              PR_15,
 111     /* p16-...           - preserved - rotating */
 112 } pr_reg_t;
 113
 114 typedef enum {
 115     BR_0,               /* scratch - Return link */
 116     /* b1-b5             - preserved */
 117     BR_1,               BR_2,           BR_3,           BR_4,
 118     BR_5,
 119     /* b6-b7            - scratch */
 120     BR_6,               BR_7
 121 } br_reg_t;
 122
 123 typedef enum {
 124     AR_KR0,             AR_KR1,         AR_KR2,         AR_KR3,
 125     AR_KR4,             AR_KR5,         AR_KR6,         AR_KR7,
 126     AR_8,               AR_9,           AR_10,          AR_11,
 127     AR_12,              AR_13,          AR_14,          AR_15,
 128     AR_RSC,             AR_BSP,         AR_BSPSTORE,    AR_RNAT,
 129     AR_20,              AR_FCR,         AR_22,          AR_23,
 130     AR_EFLAG,           AR_CSD,         AR_SSD,         AR_CFLG,
 131     AR_FSR,             AR_FIR,         AR_FDR,         AR_31,
 132     AR_CCV,             AR_33,          AR_34,          AR_35,
 133     AR_UNAT,            AR_37,          AR_38,          AR_39,
 134     AR_FPSR,            AR_41,          AR_42,          AR_43,
 135     AR_ITC,             AR_RUC,         AR_46,          AR_47,
 136     AR_48,              AR_49,          AR_50,          AR_51,
 137     AR_52,              AR_53,          AR_54,          AR_55,
 138     AR_56,              AR_57,          AR_58,          AR_59,
 139     AR_60,              AR_61,          AR_62,          AR_63,
 140     AR_PFS,             AR_LC,          AR_EC,
 141     /* AR_67 ... AR_127 */
 142 } ar_reg_t;
 143
 144 typedef enum {
 145     TM_M_I_I_,  TM_M_I_Is,      TM_M_IsI_,      TM_M_IsIs,
 146     TM_M_L_X_,  TM_M_L_Xs,      TM_ILL_06,      TM_ILL_07,
 147     TM_M_M_I_,  TM_M_M_Is,      TM_MsM_I_,      TM_MsM_Is,
 148     TM_M_F_I_,  TM_M_F_Is,      TM_M_M_F_,      TM_M_M_Fs,
 149     TM_M_I_B_,  TM_M_I_Bs,      TM_M_B_B_,      TM_M_B_Bs,
 150     TM_ILL_14,  TM_ILL_15,      TM_B_B_B_,      TM_B_B_Bs,
 151     TM_M_M_B_,  TM_M_M_Bs,      TM_ILL_1A,      TM_ILL_1B,
 152     TM_M_F_B_,  TM_M_F_Bs,      TM_ILL_1E,      TM_ILL_1F,
 153 } template_t;
 154
 155 #define MWH_SPTK                        0
 156 #define MWH_NONE                        1
 157 #define MWH_DPTK                        2
 158
 159 #define IH_NONE                         0
 160 #define IH_IMP                          1
 161
 162 #define LD_NONE                         0
 163 #define LD_NT1                          1
 164 #define LD_NTA                          3
 165
 166 #define ST_NONE                         0
 167 #define ST_NTA                          3
 168
 169 #define LF_NONE                         0
 170 #define LF_NT1                          1
 171 #define LF_NT2                          2
 172 #define LF_NTA                          3
 173
 174 #define BR_PH_FEW                       0
 175 #define BR_PH_MANY                      1
 176
 177 #define BR_BWH_SPTK                     0
 178 #define BR_BWH_SPNT                     1
 179 #define BR_BWH_DPTK                     2
 180 #define BR_BWH_DPNT                     3
 181
 182 #define BRI_BWH_SPTK                    1
 183 #define BRI_BWH_SPNT                    3
 184 #define BRI_BWH_DPTK                    5
 185 #define BRI_BWH_DPNT                    7
 186
 187 #define BR_DH_NONE                      0
 188 #define BR_DH_CLR                       1
 189
 190 #define BR_IH_NONE                      0
 191 #define BR_IH_IMP                       1
 192
 193 #define BR_IPWH_SPTK                    0
 194 #define BR_IPWH_LOOP                    1
 195 #define BR_IPWH_DPTK                    2
 196 #define BR_IPWH_EXIT                    3
 197
 198 #define BR_INDWH_SPTK                   0
 199 #define BR_INDWH_DPTK                   2
 200
 201 #define MUX_BRCST                       0
 202 #define MUX_REV                         11
 203 #define MUX_MIX                         8
 204 #define MUX_SHUF                        9
 205 #define MUX_ALT                         10
 206
 207 #define ldr(r0,r1)                      ldr_l(r0,r1)
 208 #define ldi(r0,i0)                      ldi_l(r0,i0)
 209 #define str(r0,r1)                      str_l(r0,r1)
 210 #define sti(i0,r0)                      str_l(i0,r0)
 211 #define ldxr(r0,r1,r2)                  ldxr_l(r0,r1,r2)
 212 #define ldxi(r0,r1,i0)                  ldxi_l(r0,r1,i0)
 213 #define stxr(r0,r1,r2)                  stxr_l(r0,r1,r2)
 214 #define stxi(i0,r0,r1)                  stxi_l(i0,r0,r1)
 215
 216 #if !HAVE_FFSL
 217 #  define ffsl(l)                       __builtin_ffsl(l)
 218 #endif
 219
 220 /* libgcc */
 221 #if defined(__GNUC__)
 222 extern long __divdi3(long,long);
 223 extern unsigned long __udivdi3(unsigned long,unsigned long);
 224 extern long __moddi3(long,long);
 225 extern unsigned long __umoddi3(unsigned long,unsigned long);
 226 #else
 227 static long __divdi3(long,long);
 228 static unsigned long __udivdi3(unsigned long,unsigned long);
 229 static long __moddi3(long,long);
 230 static unsigned long __umoddi3(unsigned long,unsigned long);
 231 #endif
 232 #define out(n,tm,s0,s1,s2)              _out(_jit,n,tm,s0,s1,s2)
 233 static void _out(jit_state_t*,int,int,jit_word_t,jit_word_t,jit_word_t);
 234 #define stop()                          _stop(_jit)
 235 static void _stop(jit_state_t*);
 236 #define sync()                          _sync(_jit)
 237 static void _sync(jit_state_t*);
 238 #define flush()                         _flush(_jit)
 239 static void _flush(jit_state_t*);
 240 #define inst(w, t)                      _inst(_jit, w, t)
 241 static void _inst(jit_state_t*, jit_word_t, jit_uint8_t);
 242 #define A1(x4,x2,r3,r2,r1)              _A1(_jit,0,x4,x2,r3,r2,r1)
 243 static void _A1(jit_state_t*, jit_word_t,
 244                 jit_word_t,jit_word_t,jit_word_t,jit_word_t,jit_word_t);
 245 #define A2(x4,ct,r3,r2,r1)              A1(x4,ct,r3,r2,r1)
 246 #define A3(x4,x2,r3,im,r1)              _A3(_jit,0,x4,x2,r3,im,r1)
 247 static void _A3(jit_state_t*, jit_word_t,
 248                 jit_word_t,jit_word_t,jit_word_t,jit_word_t,jit_word_t);
 249 #define A4(x2a,r3,im,r1)                _A4(_jit,0,x2a,r3,im,r1)
 250 static void _A4(jit_state_t*,jit_word_t,
 251                 jit_word_t,jit_word_t,jit_word_t,jit_word_t);
 252 #define A5(r3,im,r1)                    _A5(_jit,0,r3,im,r1)
 253 static void _A5(jit_state_t*,jit_word_t,
 254                 jit_word_t,jit_word_t,jit_word_t);
 255 #define A6(o,x2,ta,p2,r3,r2,c,p1)       _A6(_jit,0,o,x2,ta,p2,r3,r2,c,p1)
 256 static void _A6(jit_state_t*,jit_word_t,
 257                 jit_word_t,jit_word_t,jit_word_t,jit_word_t,
 258                 jit_word_t,jit_word_t,jit_word_t,jit_word_t);
 259 #define A7(o,x2,ta,p2,r3,c,p1)          _A7(_jit,0,o,x2,ta,p2,r3,c,p1)
 260 static void _A7(jit_state_t*,jit_word_t,
 261                 jit_word_t,jit_word_t,jit_word_t,
 262                 jit_word_t,jit_word_t,jit_word_t,jit_word_t)
 263     maybe_unused;
 264 #define A8(o,x2,ta,p2,r3,im,c,p1)       _A8(_jit,0,o,x2,ta,p2,r3,im,c,p1)
 265 static void _A8(jit_state_t*,jit_word_t,
 266                 jit_word_t,jit_word_t,jit_word_t,jit_word_t,
 267                 jit_word_t,jit_word_t,jit_word_t,jit_word_t);
 268 #define A9(za,zb,x4,x2,r3,r2,r1)        _A9(_jit,0,za,zb,x4,x2,r3,r2,r1)
 269 static void _A9(jit_state_t*,jit_word_t,
 270                 jit_word_t,jit_word_t,jit_word_t,
 271                 jit_word_t,jit_word_t,jit_word_t,jit_word_t)
 272     maybe_unused;
 273 #define A10(x4,ct,r3,r2,r1)             A9(0,1,x4,ct,r3,r2,r1)
 274 #define I1(ct,x2,r3,r2,r1)              _I1(_jit,0,ct,x2,r3,r2,r1)
 275 static void _I1(jit_state_t*,jit_word_t,
 276                 jit_word_t,jit_word_t,jit_word_t,jit_word_t,jit_word_t)
 277     maybe_unused;
 278 #define I2(za,x2a,zb,x2c,x2b,r3,r2,r1)  _I2(_jit,0,za,x2a,zb,x2c,x2b,r3,r2,r1)
 279 static void _I2(jit_state_t*,jit_word_t,
 280                 jit_word_t,jit_word_t,jit_word_t,jit_word_t,
 281                 jit_word_t,jit_word_t,jit_word_t,jit_word_t)
 282     maybe_unused;
 283 #define I3(mbt,r2,r1)                   _I3(_jit,0,mbt,r2,r1)
 284 static void _I3(jit_state_t*,jit_word_t,
 285                 jit_word_t,jit_word_t,jit_word_t)
 286 #if __BYTE_ORDER == __BIG_ENDIAN
 287     maybe_unused
 288 #endif
 289     ;
 290 #define I4(mht,r2,r1)                   _I4(_jit,0,mht,r2,r1)
 291 static void _I4(jit_state_t*,jit_word_t,
 292                 jit_word_t,jit_word_t,jit_word_t)
 293     maybe_unused;
 294 #define I5(za,zb,x2b,r3,r2,r1)          _I5(_jit,0,za,zb,x2b,r3,r2,r1)
 295 static void _I5(jit_state_t*,jit_word_t,
 296                 jit_word_t,jit_word_t,jit_word_t,
 297                 jit_word_t,jit_word_t,jit_word_t);
 298 #define I6(za,zb,x2b,r3,ct,r1)          _I6(_jit,0,za,zb,x2b,r3,ct,r1)
 299 static void _I6(jit_state_t*,jit_word_t,
 300                 jit_word_t,jit_word_t,jit_word_t,
 301                 jit_word_t,jit_word_t,jit_word_t)
 302     maybe_unused;
 303 #define I7(za,zb,r3,r2,r1)              _I7(_jit,0,za,zb,r3,r2,r1)
 304 static void _I7(jit_state_t*,jit_word_t,
 305                 jit_word_t,jit_word_t,jit_word_t,jit_word_t,jit_word_t);
 306 #define I8(za,zb,ct,r2,r1)              _I8(_jit,0,za,zb,ct,r2,r1)
 307 static void _I8(jit_state_t*,jit_word_t,
 308                 jit_word_t,jit_word_t,jit_word_t,jit_word_t,jit_word_t)
 309     maybe_unused;
 310 #define I9(x2c,r3,r1)                   _I9(_jit,0,x2c,r3,r1)
 311 static void _I9(jit_state_t*,jit_word_t,
 312                 jit_word_t,jit_word_t,jit_word_t)
 313     maybe_unused;
 314 #define I10(ct,r3,r2,r1)                _I10(_jit,0,ct,r3,r2,r1)
 315 static void _I10(jit_state_t*,jit_word_t,
 316                  jit_word_t,jit_word_t,jit_word_t,jit_word_t)
 317     maybe_unused;
 318 #define I11(len,r3,pos,y,r1)            _I11(_jit,0,len,r3,pos,y,r1)
 319 static void _I11(jit_state_t*,jit_word_t,
 320                  jit_word_t,jit_word_t,jit_word_t,jit_word_t,jit_word_t)
 321     maybe_unused;
 322 #define I12(len,pos,r2,r1)              _I12(_jit,0,len,pos,r2,r1)
 323 static void _I12(jit_state_t*,jit_word_t,
 324                  jit_word_t,jit_word_t,jit_word_t,jit_word_t)
 325     maybe_unused;
 326 #define I13(len,pos,im,r1)              _I13(_jit,0,len,pos,im,r1)
 327 static void _I13(jit_state_t*,jit_word_t,
 328                  jit_word_t,jit_word_t,jit_word_t,jit_word_t)
 329     maybe_unused;
 330 #define I14(s,len,r3,pos,r1)            _I14(_jit,0,s,len,r3,pos,r1)
 331 static void _I14(jit_state_t*,jit_word_t,
 332                  jit_word_t,jit_word_t,jit_word_t,jit_word_t,jit_word_t)
 333     maybe_unused;
 334 #define I15(pos,len,r3,r2,r1)           _I15(_jit,0,pos,len,r3,r2,r1)
 335 static void _I15(jit_state_t*,jit_word_t,
 336                  jit_word_t,jit_word_t,jit_word_t,jit_word_t,jit_word_t)
 337     maybe_unused;
 338 #define I16(tb,ta,p2,r3,pos,c,p1)       _I16(_jit,0,tb,ta,p2,r3,pos,c,p1)
 339 static void _I16(jit_state_t*,jit_word_t,
 340                  jit_word_t,jit_word_t,jit_word_t,
 341                  jit_word_t,jit_word_t,jit_word_t,jit_word_t)
 342     maybe_unused;
 343 #define I17(tb,ta,p2,r3,c,p1)           _I17(_jit,0,tb,ta,p2,r3,c,p1)
 344 static void _I17(jit_state_t*,jit_word_t,
 345                  jit_word_t,jit_word_t,jit_word_t,
 346                  jit_word_t,jit_word_t,jit_word_t)
 347     maybe_unused;
 348 #define I18(im,y)                       _I18(_jit,0,im,y)
 349 static void _I18(jit_state_t*,jit_word_t,
 350                  jit_word_t,jit_word_t)
 351     maybe_unused;
 352 #define I19(im)                         _I19(_jit,0,im)
 353 static void _I19(jit_state_t*,jit_word_t,
 354                  jit_word_t)
 355     maybe_unused;
 356 #define I20(r2,im)                      _I20(_jit,0,r2,im)
 357 static void _I20(jit_state_t*,jit_word_t,
 358                  jit_word_t,jit_word_t)
 359     maybe_unused;
 360 #define I21(im,ih,x,wh,r2,b1)           _I21(_jit,0,im,ih,x,wh,r2,b1)
 361 static void _I21(jit_state_t*,jit_word_t,
 362                  jit_word_t,jit_word_t,
 363                  jit_word_t,jit_word_t,jit_word_t,jit_word_t);
 364 #define I22(b2,r1)                      _I22(_jit,0,b2,r1)
 365 static void _I22(jit_state_t*,jit_word_t,
 366                  jit_word_t,jit_word_t);
 367 #define I23(r2,im)                      _I23(_jit,0,r2,im)
 368 static void _I23(jit_state_t*,jit_word_t,
 369                  jit_word_t,jit_word_t)
 370     maybe_unused;
 371 #define I24(im)                         _I24(_jit,0,im)
 372 static void _I24(jit_state_t*,jit_word_t,
 373                  jit_word_t)
 374     maybe_unused;
 375 #define I25(x6,r1)                      _I25(_jit,0,x6,r1)
 376 static void _I25(jit_state_t*,jit_word_t,
 377                  jit_word_t,jit_word_t)
 378     maybe_unused;
 379 #define I26(ar,r2)                      _I26(_jit,0,ar,r2)
 380 static void _I26(jit_state_t*,jit_word_t,
 381                  jit_word_t,jit_word_t);
 382 #define I27(ar,im)                      _I27(_jit,0,ar,im)
 383 static void _I27(jit_state_t*,jit_word_t,
 384                  jit_word_t,jit_word_t)
 385     maybe_unused;
 386 #define I28(ar,r1)                      _I28(_jit,0,ar,r1)
 387 static void _I28(jit_state_t*,jit_word_t,
 388                  jit_word_t,jit_word_t)
 389     maybe_unused;
 390 #define I29(x6,r3,r1)                   _I29(_jit,0,x6,r3,r1)
 391 static void _I29(jit_state_t*,jit_word_t,
 392                  jit_word_t,jit_word_t,jit_word_t);
 393 #define I30(tb,ta,p2,im,c,p1)           _I30(_jit,0,ta,tb,p2,im,c,p1)
 394 static void _I30(jit_state_t*,jit_word_t,
 395                  jit_word_t,jit_word_t,jit_word_t,
 396                  jit_word_t,jit_word_t,jit_word_t)
 397     maybe_unused;
 398 #define M1(x6,ht,x,r3,r1)               _M1(_jit,0,x6,ht,x,r3,r1)
 399 static void _M1(jit_state_t*,jit_word_t,
 400                 jit_word_t,jit_word_t,jit_word_t,jit_word_t,jit_word_t);
 401 #define M2(x6,ht,r3,r2,r1)              _M2(_jit,0,x6,ht,r3,r2,r1)
 402 static void _M2(jit_state_t*,jit_word_t,
 403                 jit_word_t,jit_word_t,jit_word_t,jit_word_t,jit_word_t)
 404     maybe_unused;
 405 #define M3(x6,ht,r3,im,r1)              _M3(_jit,0,x6,ht,r3,im,r1)
 406 static void _M3(jit_state_t*,jit_word_t,
 407                 jit_word_t,jit_word_t,jit_word_t,jit_word_t,jit_word_t);
 408 #define M5(x6,ht,r3,r2,im)              _M5(_jit,0,x6,ht,r3,r2,im)
 409 static void _M5(jit_state_t*,jit_word_t,
 410                 jit_word_t,jit_word_t,jit_word_t,jit_word_t,jit_word_t)
 411     maybe_unused;
 412 #define M6(x6,ht,x,r3,r2)               _M6(_jit,0,x6,ht,x,r3,r2)
 413 static void _M6(jit_state_t*,jit_word_t,
 414                 jit_word_t,jit_word_t,jit_word_t,jit_word_t,jit_word_t);
 415 #define M13(x6,ht,r3,f2)                _M13(_jit,0,x6,ht,r3,f2)
 416 static void _M13(jit_state_t*,jit_word_t,
 417                  jit_word_t,jit_word_t,jit_word_t,jit_word_t);
 418 #define M14(x6,ht,r3,r2)                _M14(_jit,0,x6,ht,r3,r2)
 419 static void _M14(jit_state_t*,jit_word_t,
 420                  jit_word_t,jit_word_t,jit_word_t,jit_word_t)
 421     maybe_unused;
 422 #define M15(x6,ht,r3,im)                _M15(_jit,0,x6,ht,r3,im)
 423 static void _M15(jit_state_t*,jit_word_t,
 424                  jit_word_t,jit_word_t,jit_word_t,jit_word_t)
 425     maybe_unused;
 426 #define M16(x6,ht,r3,r2,r1)             _M16(_jit,0,x6,ht,r3,r2,r1)
 427 static void _M16(jit_state_t*,jit_word_t,
 428                  jit_word_t,jit_word_t,jit_word_t,jit_word_t,jit_word_t)
 429     maybe_unused;
 430 #define M17(x6,ht,r3,im,r1)             _M17(_jit,0,x6,ht,r3,im,r1)
 431 static void _M17(jit_state_t*,jit_word_t,
 432                  jit_word_t,jit_word_t,jit_word_t,jit_word_t,jit_word_t)
 433     maybe_unused;
 434 #define M20(r2,im)                      M20x(0x1,r2,im)
 435 #define M20x(x3,r2,im)                  _M20x(_jit,0,x3,r2,im)
 436 static void _M20x(jit_state_t*,jit_word_t,
 437                   jit_word_t,jit_word_t,jit_word_t)
 438     maybe_unused;
 439 #define M22(x3,im,r1)                   M22x(x3,im,r1)
 440 #define M22x(x3,im,r1)                  _M22x(_jit,0,x3,im,r1)
 441 static void _M22x(jit_state_t*,jit_word_t,
 442                   jit_word_t,jit_word_t,jit_word_t)
 443     maybe_unused;
 444 #define M24(x2,x4)                      _M24(_jit,0,x2,x4)
 445 #define M25(x4)                         M24(0,x4)
 446 static void _M24(jit_state_t*,jit_word_t,
 447                  jit_word_t,jit_word_t)
 448     maybe_unused;
 449 #define M26(r1)                         M26x(2,r1)
 450 #define M26x(x4,r1)                     _M26x(_jit,0,x4,r1)
 451 static void _M26x(jit_state_t*,jit_word_t,
 452                   jit_word_t,jit_word_t)
 453     maybe_unused;
 454 #define M28(x,r3)                       _M28(_jit,0,x,r3)
 455 static void _M28(jit_state_t*,jit_word_t,
 456                  jit_word_t,jit_word_t)
 457     maybe_unused;
 458 #define M29(ar,r2)                      _M29(_jit,0,ar,r2)
 459 static void _M29(jit_state_t*,jit_word_t,
 460                  jit_word_t,jit_word_t)
 461     maybe_unused;
 462 #define M30(ar,im)                      _M30(_jit,0,ar,im)
 463 static void _M30(jit_state_t*,jit_word_t,jit_word_t,jit_word_t)
 464     maybe_unused;
 465 #define M31(ar,r1)                      _M31(_jit,0,ar,r1)
 466 static void _M31(jit_state_t*,jit_word_t,
 467                  jit_word_t,jit_word_t)
 468     maybe_unused;
 469 #define M32(cr,r2)                      _M32(_jit,0,cr,r2)
 470 static void _M32(jit_state_t*,jit_word_t,
 471                  jit_word_t,jit_word_t)
 472     maybe_unused;
 473 #define M33(cr,r1)                      _M33(_jit,0,cr,r1)
 474 static void _M33(jit_state_t*,jit_word_t,
 475                  jit_word_t,jit_word_t)
 476     maybe_unused;
 477 #define M34(sor,sol,sof,r1)             _M34(_jit,0,sor,sol,sof,r1)
 478 static void _M34(jit_state_t*,jit_word_t,
 479                  jit_word_t,jit_word_t,jit_word_t,jit_word_t);
 480 #define M35(x6,r2)                      _M35(_jit,0,x6,r2)
 481 static void _M35(jit_state_t*,jit_word_t,
 482                  jit_word_t,jit_word_t)
 483     maybe_unused;
 484 #define M36(x6,r1)                      _M36(_jit,0,x6,r1)
 485 static void _M36(jit_state_t*,jit_word_t,
 486                  jit_word_t,jit_word_t)
 487     maybe_unused;
 488 #define M37(im)                         _M37(_jit,0,im)
 489 static void _M37(jit_state_t*,jit_word_t,
 490                  jit_word_t)
 491     maybe_unused;
 492 #define M38(x6,r3,r2,r1)                _M38(_jit,0,x6,r3,r2,r1)
 493 static void _M38(jit_state_t*,jit_word_t,
 494                  jit_word_t,jit_word_t,jit_word_t,jit_word_t)
 495     maybe_unused;
 496 #define M39(x6,r3,im,r1)                _M39(_jit,0,x6,r3,im,r1)
 497 static void _M39(jit_state_t*,jit_word_t,
 498                  jit_word_t,jit_word_t,jit_word_t,jit_word_t)
 499     maybe_unused;
 500 #define M40(x6,r3,im)                   _M40(_jit,0,x6,r3,im)
 501 static void _M40(jit_state_t*,jit_word_t,
 502                  jit_word_t,jit_word_t,jit_word_t)
 503     maybe_unused;
 504 #define M41(x6,r2)                      _M41(_jit,0,x6,r2)
 505 static void _M41(jit_state_t*,jit_word_t,
 506                  jit_word_t,jit_word_t)
 507     maybe_unused;
 508 #define M42(x6,r3,r2)                   _M42(_jit,0,x6,r3,r2)
 509 static void _M42(jit_state_t*,jit_word_t,
 510                  jit_word_t,jit_word_t,jit_word_t)
 511     maybe_unused;
 512 #define M43(x6,r3,r1)                   _M43(_jit,0,x6,r3,r1)
 513 static void _M43(jit_state_t*,jit_word_t,
 514                  jit_word_t,jit_word_t,jit_word_t)
 515     maybe_unused;
 516 #define M44(x4,im)                      _M44(_jit,0,x4,im)
 517 static void _M44(jit_state_t*,jit_word_t,
 518                  jit_word_t,jit_word_t)
 519     maybe_unused;
 520 #define M45(x6,r3,r2)                   _M45(_jit,0,x6,r3,r2)
 521 static void _M45(jit_state_t*,jit_word_t,
 522                  jit_word_t,jit_word_t,jit_word_t)
 523     maybe_unused;
 524 #define M46(x6,r3,r1)                   _M46(_jit,0,x6,r3,r1)
 525 #define M47(x6,r3)                      M46(x6,r3,0)
 526 static void _M46(jit_state_t*,jit_word_t,
 527                  jit_word_t,jit_word_t,jit_word_t)
 528     maybe_unused;
 529 #define M48(y,im)                       _M48(_jit,0,y,im)
 530 static void _M48(jit_state_t*,jit_word_t,
 531                  jit_word_t,jit_word_t)
 532     maybe_unused;
 533 #define B1(d,wh,im,p,tp)                _B1(_jit,0,d,wh,im,p,tp)
 534 #define B2(d,wh,im,p,tp)                B1(d,wh,im,p,tp)
 535 static void _B1(jit_state_t*,jit_word_t,
 536                 jit_word_t,jit_word_t,jit_word_t,jit_word_t,jit_word_t);
 537 #define B3(d,wh,im,p,b)                 _B3(_jit,0,d,wh,im,p,b)
 538 static void _B3(jit_state_t*,jit_word_t,
 539                 jit_word_t,jit_word_t,jit_word_t,jit_word_t,jit_word_t)
 540     maybe_unused;
 541 #define B4(d,wh,x6,b,p,tp)              _B4(_jit,0,d,wh,x6,b,p,tp)
 542 static void _B4(jit_state_t*,jit_word_t,
 543                 jit_word_t,jit_word_t,jit_word_t,
 544                 jit_word_t,jit_word_t,jit_word_t);
 545 #define B5(d,wh,b2,p,b1)                _B5(_jit,0,d,wh,b2,p,b1)
 546 static void _B5(jit_state_t*,jit_word_t,
 547                 jit_word_t,jit_word_t,jit_word_t,jit_word_t,jit_word_t);
 548 #define B6(ih,im,tag,wh)                _B6(_jit,0,ih,im,tag,wh)
 549 static void _B6(jit_state_t*,jit_word_t,
 550                 jit_word_t,jit_word_t,jit_word_t,jit_word_t)
 551     maybe_unused;
 552 #define B7(ih,x6,b2,tag,wh)             _B7(_jit,0,ih,x6,b2,tag,wh)
 553 static void _B7(jit_state_t*,jit_word_t,
 554                 jit_word_t,jit_word_t,jit_word_t,jit_word_t,jit_word_t)
 555     maybe_unused;
 556 #define B8(x6)                          _B8(_jit,0,x6)
 557 static void _B8(jit_state_t*,jit_word_t,
 558                 jit_word_t)
 559     maybe_unused;
 560 #define B9(op,x6,im)                    _B9(_jit,0,op,x6,im)
 561 static void _B9(jit_state_t*,jit_word_t,
 562                 jit_word_t,jit_word_t,jit_word_t)
 563     maybe_unused;
 564 #define X1(im)                          _X1(_jit,0,im)
 565 static void _X1(jit_state_t*,jit_word_t,
 566                 jit_word_t)
 567     maybe_unused;
 568 #define X2(r1,im)                       _X2(_jit,0,r1,im)
 569 static void _X2(jit_state_t*,jit_word_t,
 570                 jit_word_t,jit_word_t);
 571 #define X3x(o,d,wh,p,tp,im)             _X3x(_jit,0,o,d,wh,p,tp,im)
 572 #define X3(d,wh,p,tp,im)                X3x(0xc,d,wh,p,tp,im)
 573 #define X4(d,wh,p,tp,im)                X3x(0xd,d,wh,p,tp,im)
 574 static void _X3x(jit_state_t*,jit_word_t,
 575                  jit_word_t,jit_word_t,jit_word_t,
 576                  jit_word_t,jit_word_t,jit_word_t);
 577 #define X5(y,im)                        _X5(_jit,0,y,im)
 578 static void _X5(jit_state_t*,jit_word_t,
 579                 jit_word_t,jit_word_t)
 580     maybe_unused;
 581
 582 /* add */
 583 #define ADD(r1,r2,r3)                   A1(0,0,r3,r2,r1)
 584 #define ADD1(r1,r2,r3)                  A1(0,1,r3,r2,r1)
 585 #define ADDS(r1,im,r3)                  A4(2,r3,im,r1)
 586 #define ADDS_p(r1,im,r3,_p)             _A4(_jit,_p,2,r3,im,r1)
 587 #define ADDL(r1,im,r3)                  A5(r3,im,r1)
 588 #define ADDL_p(r1,im,r3,_p)             _A5(_jit,_p,r3,im,r1)
 589 /* addp4 */
 590 #define ADDP4(r1,r2,r3)                 A1(2,0,r3,r2,r1)
 591 #define ADDIP4(r1,im,r3)                A4(3,r3,im,r1)
 592 /* alloc */
 593 #define ALLOCR(r1,i,l,o,r)              M34((r)>>3,(i)+(l),(i)+(l)+(o),r1)
 594 #define ALLOC(r1,i,o)                   ALLOCR(r1,i,0,o,0)
 595 /* and */
 596 #define AND(r1,r2,r3)                   A1(3,0,r3,r2,r1)
 597 #define ANDI(r1,im,r3)                  A3(0xb,0,r3,im,r1)
 598 /* andcm */
 599 #define ANDCM(r1,r2,r3)                 A1(3,1,r3,r2,r1)
 600 #define ANDCMI(r1,im,r3)                A3(0xb,1,r3,im,r1)
 601 /* br */
 602 #define BRI(im)                         B1(BR_DH_NONE,BR_BWH_SPTK,im,BR_PH_FEW,0)
 603 #define BRI_COND(im,_p)                 _B1(_jit,_p,BR_DH_NONE,BR_BWH_SPTK,im,BR_PH_FEW,0)
 604 #define BRI_WEXIT(im)                   B1(BR_DH_NONE,BR_BWH_SPTK,im,BR_PH_MANY,2)
 605 #define BRI_WTOP(im)                    B1(BR_DH_NONE,BR_BWH_SPTK,im,BR_PH_MANY,3)
 606 #define BRI_CALL(b,im)                  B3(BR_DH_NONE,BR_BWH_SPTK,im,BR_PH_MANY,b)
 607 #define BRI_CLOOP(im)                   B2(BR_DH_NONE,BR_BWH_SPTK,im,BR_PH_MANY,5)
 608 #define BRI_CEXIT(im)                   B2(BR_DH_NONE,BR_BWH_SPTK,im,BR_PH_MANY,6)
 609 #define BRI_CTOP(im)                    B2(BR_DH_NONE,BR_BWH_SPTK,im,BR_PH_MANY,7)
 610 #define BR_COND(b,_p)                   _B4(_jit,_p,BR_DH_NONE,BR_BWH_SPTK,0x20,b,BR_PH_FEW,0)
 611 #define BR(b)                           B4(BR_DH_NONE,BR_BWH_SPTK,0x20,b,BR_PH_FEW,0)
 612 #define BR_IA(b)                        B4(BR_DH_NONE,BR_BWH_SPTK,0x20,b,BR_PH_MANY,1)
 613 #define BR_RET(b)                       B4(BR_DH_NONE,BR_BWH_SPTK,0x21,b,BR_PH_MANY,4)
 614 #define BR_CALL(b1,b2)                  B5(BR_DH_NONE,BRI_BWH_SPTK,b2,BR_PH_MANY,b1)
 615 /* break */
 616 #define BREAK_I(im)                     I19(im)
 617 #define BREAK_M(im)                     M37(im)
 618 #define BREAK_B(im)                     B9(0,0,im)
 619 #define BREAK_X(im)                     X1(im)
 620 /* brl */
 621 #define BRL(im)                         X3(BR_DH_NONE,BR_BWH_SPTK,BR_PH_MANY,0,im)
 622 #define BRL_COND(im,_p)                 _X3(_jit,_p,BR_DH_NONE,BR_BWH_SPTK,BR_PH_MANY,0,im)
 623 #define BRL_CALL(b1,im)                 X4(BR_DH_NONE,BR_BWH_SPTK,BR_PH_MANY,b1,im)
 624 /* brp */
 625 #define BRP(im,tag)                     B6(BR_IH_NONE,im,tag,BR_IPWH_SPTK)
 626 #define BRPI(b2,tag)                    B7(BR_IH_NONE,0x10,b2,tag,BR_INDWH_SPTK)
 627 #define BRPI_RET(b2,tag)                B7(BR_IH_NONE,0x11,b2,tag,BR_INDWH_SPTK)
 628 /* bsw */
 629 #define BSW_0()                         B8(0x0c)
 630 #define BSW_1()                         B8(0x0d)
 631 /* chk */
 632 #define CHK_S_I(r2,im)                  I20(r2,im)
 633 #define CHK_S_M(r2,im)                  M20(r2,im)
 634 #define CHK_A_NC(r1,im)                 M22(0x4,im,r1)
 635 #define CHK_A_CLR(r1,im)                M22(0x5,im,r1)
 636 /* clrrrb */
 637 #define CLRRRB()                        B8(0x04)
 638 #define CLRRRB_PR()                     B8(0x05)
 639 /* clz */
 640 #define CLZ(r1,r3)                      I9(3,r3,r1)
 641 /* cmp */
 642 #define CMP_LT(p1,p2,r2,r3)             A6(0xc,0,0,p2,r3,r2,0,p1)
 643 #define CMP_LT_p(p1,p2,r2,r3,_p)        A6(_jit,_p,0xc,0,0,p2,r3,r2,0,p1)
 644 #define CMP_LTU(p1,p2,r2,r3)            A6(0xd,0,0,p2,r3,r2,0,p1)
 645 #define CMP_EQ(p1,p2,r2,r3)             A6(0xe,0,0,p2,r3,r2,0,p1)
 646 #define CMP_LT_UNC(p1,p2,r2,r3)         A6(0xc,0,0,p2,r3,r2,1,p1)
 647 #define CMP_LTU_UNC(p1,p2,r2,r3)        A6(0xd,0,0,p2,r3,r2,1,p1)
 648 #define CMP_EQ_UNC(p1,p2,r2,r3)         A6(0xe,0,0,p2,r3,r2,1,p1)
 649 #define CMP_EQ_AND(p1,p2,r2,r3)         A6(0xc,0,1,p2,r3,r2,0,p1)
 650 #define CMP_EQ_OR(p1,p2,r2,r3)          A6(0xd,0,1,p2,r3,r2,0,p1)
 651 #define CMP_EQ_OR_ANDCM(p1,p2,r2,r3)    A6(0xe,0,1,p2,r3,r2,0,p1)
 652 #define CMP_NE_AND(p1,p2,r2,r3)         A6(0xc,0,1,p2,r3,r2,1,p1)
 653 #define CMP_NE_OR(p1,p2,r2,r3)          A6(0xd,0,1,p2,r3,r2,1,p1)
 654 #define CMP_NE_OR_ANDCM(p1,p2,r2,r3)    A6(0xe,0,1,p2,r3,r2,1,p1)
 655 #define CMPI_LT(p1,p2,im,r3)            A8(0xc,2,0,p2,r3,im,0,p1)
 656 #define CMPI_LTU(p1,p2,im,r3)           A8(0xd,2,0,p2,r3,im,0,p1)
 657 #define CMPI_EQ(p1,p2,im,r3)            A8(0xe,2,0,p2,r3,im,0,p1)
 658 #define CMPI_EQ_p(p1,p2,im,r3,_p)       _A8(_jit,_p,0xe,2,0,p2,r3,im,0,p1)
 659 #define CMPI_LT_UNC(p1,p2,im,r3)        A8(0xc,2,0,p2,r3,im,1,p1)
 660 #define CMPI_LTU_UNC(p1,p2,im,r3)       A8(0xd,2,0,p2,r3,im,1,p1)
 661 #define CMPI_EQ_UNC(p1,p2,im,r3)        A8(0xe,2,0,p2,r3,im,1,p1)
 662 #define CMPI_EQ_AND(p1,p2,im,r3)        A8(0xc,2,1,p2,r3,im,0,p1)
 663 #define CMPI_EQ_OR(p1,p2,im,r3)         A8(0xd,2,1,p2,r3,im,0,p1)
 664 #define CMPI_EQ_ANDCM(p1,p2,im,r3)      A8(0xe,2,1,p2,r3,im,0,p1)
 665 #define CMPI_NE_AND(p1,p2,im,r3)        A8(0xc,2,1,p2,r3,im,1,p1)
 666 #define CMPI_NE_OR(p1,p2,im,r3)         A8(0xd,2,1,p2,r3,im,1,p1)
 667 #define CMPI_NE_ANDCM(p1,p2,im,r3)      A8(0xe,2,1,p2,r3,im,1,p1)
 668 #define ZCMP_GT_AND(p1,p2,r3)           A7(0xc,0,0,p2,r3,0,p1)
 669 #define ZCMP_GT_OR(p1,p2,r3)            A7(0xd,0,0,p2,r3,0,p1)
 670 #define ZCMP_GT_ANDCM(p1,p2,r3)         A7(0xe,0,0,p2,r3,0,p1)
 671 #define ZCMP_LE_AND(p1,p2,r3)           A7(0xc,0,0,p2,r3,1,p1)
 672 #define ZCMP_LE_OR(p1,p2,r3)            A7(0xd,0,0,p2,r3,1,p1)
 673 #define ZCMP_LE_ANDCM(p1,p2,r3)         A7(0xe,0,0,p2,r3,1,p1)
 674 #define ZCMP_GE_AND(p1,p2,r3)           A7(0xc,0,1,p2,r3,0,p1)
 675 #define ZCMP_GE_OR(p1,p2,r3)            A7(0xd,0,1,p2,r3,0,p1)
 676 #define ZCMP_GE_ANDCM(p1,p2,r3)         A7(0xe,0,1,p2,r3,0,p1)
 677 #define ZCMP_LT_AND(p1,p2,r3)           A7(0xc,0,1,p2,r3,1,p1)
 678 #define ZCMP_LT_OR(p1,p2,r3)            A7(0xd,0,1,p2,r3,1,p1)
 679 #define ZCMP_LT_ANDCM(p1,p2,r3)         A7(0xe,0,1,p2,r3,1,p1)
 680 /* cmp4 */
 681 #define CMP4_LT(p1,p2,r2,r3)            A6(0xc,1,0,p2,r3,r2,0,p1)
 682 #define CMP4_LTU(p1,p2,r2,r3)           A6(0xd,1,0,p2,r3,r2,0,p1)
 683 #define CMP4_EQ(p1,p2,r2,r3)            A6(0xe,1,0,p2,r3,r2,0,p1)
 684 #define CMP4_LT_UNC(p1,p2,r2,r3)        A6(0xc,1,0,p2,r3,r2,1,p1)
 685 #define CMP4_LTU_UNC(p1,p2,r2,r3)       A6(0xd,1,0,p2,r3,r2,1,p1)
 686 #define CMP4_EQ_UNC(p1,p2,r2,r3)        A6(0xe,1,0,p2,r3,r2,1,p1)
 687 #define CMP4_EQ_AND(p1,p2,r2,r3)        A6(0xc,1,1,p2,r3,r2,0,p1)
 688 #define CMP4_EQ_OR(p1,p2,r2,r3)         A6(0xd,1,1,p2,r3,r2,0,p1)
 689 #define CMP4_EQ_XOR(p1,p2,r2,r3)        A6(0xe,1,1,p2,r3,r2,0,p1)
 690 #define CMP4_NE_AND(p1,p2,r2,r3)        A6(0xc,1,1,p2,r3,r2,1,p1)
 691 #define CMP4_NE_OR(p1,p2,r2,r3)         A6(0xd,1,1,p2,r3,r2,1,p1)
 692 #define CMP4_NE_XOR(p1,p2,r2,r3)        A6(0xe,1,1,p2,r3,r2,1,p1)
 693 #define CMP4I_LT(p1,p2,im,r3)           A8(0xc,3,0,p2,r3,im,0,p1)
 694 #define CMP4I_LTU(p1,p2,im,r3)          A8(0xd,3,0,p2,r3,im,0,p1)
 695 #define CMP4I_EQ(p1,p2,im,r3)           A8(0xe,3,0,p2,r3,im,0,p1)
 696 #define CMP4I_LT_UNC(p1,p2,im,r3)       A8(0xc,3,0,p2,r3,im,1,p1)
 697 #define CMP4I_LTU_UNC(p1,p2,im,r3)      A8(0xd,3,0,p2,r3,im,1,p1)
 698 #define CMP4I_EQ_UNC(p1,p2,im,r3)       A8(0xe,3,0,p2,r3,im,1,p1)
 699 #define CMP4I_EQ_AND(p1,p2,im,r3)       A8(0xc,3,1,p2,r3,im,0,p1)
 700 #define CMP4I_EQ_OR(p1,p2,im,r3)        A8(0xd,3,1,p2,r3,im,0,p1)
 701 #define CMP4I_EQ_ANDCM(p1,p2,im,r3)     A8(0xe,3,1,p2,r3,im,0,p1)
 702 #define CMP4I_NE_AND(p1,p2,im,r3)       A8(0xc,3,1,p2,r3,im,1,p1)
 703 #define CMP4I_NE_OR(p1,p2,im,r3)        A8(0xd,3,1,p2,r3,im,1,p1)
 704 #define CMP4I_NE_ANDCM(p1,p2,im,r3)     A8(0xe,3,1,p2,r3,im,1,p1)
 705 #define ZCMP4_GT_AND(p1,p2,r3)          A7(0xc,1,0,p2,r3,0,p1)
 706 #define ZCMP4_GT_OR(p1,p2,r3)           A7(0xd,1,0,p2,r3,0,p1)
 707 #define ZCMP4_GT_ANDCM(p1,p2,r3)        A7(0xe,1,0,p2,r3,0,p1)
 708 #define ZCMP4_LE_AND(p1,p2,r3)          A7(0xc,1,0,p2,r3,1,p1)
 709 #define ZCMP4_LE_OR(p1,p2,r3)           A7(0xd,1,0,p2,r3,1,p1)
 710 #define ZCMP4_LE_ANDCM(p1,p2,r3)        A7(0xe,1,0,p2,r3,1,p1)
 711 #define ZCMP4_GE_AND(p1,p2,r3)          A7(0xc,1,1,p2,r3,0,p1)
 712 #define ZCMP4_GE_OR(p1,p2,r3)           A7(0xd,1,1,p2,r3,0,p1)
 713 #define ZCMP4_GE_ANDCM(p1,p2,r3)        A7(0xe,1,1,p2,r3,0,p1)
 714 #define ZCMP4_LT_AND(p1,p2,r3)          A7(0xc,1,1,p2,r3,1,p1)
 715 #define ZCMP4_LT_OR(p1,p2,r3)           A7(0xd,1,1,p2,r3,1,p1)
 716 #define ZCMP4_LT_ANDCM(p1,p2,r3)        A7(0xe,1,1,p2,r3,1,p1)
 717 /* cmpxchg */
 718 #define CMPXCHG1_ACQ(r1,r3,r2)          M16(0x00,LD_NONE,r3,r2,r1)
 719 #define CMPXCHG2_ACQ(r1,r3,r2)          M16(0x01,LD_NONE,r3,r2,r1)
 720 #define CMPXCHG4_ACQ(r1,r3,r2)          M16(0x02,LD_NONE,r3,r2,r1)
 721 #define CMPXCHG8_ACQ(r1,r3,r2)          M16(0x03,LD_NONE,r3,r2,r1)
 722 #define CMPXCHG1_REL(r1,r3,r2)          M16(0x04,LD_NONE,r3,r2,r1)
 723 #define CMPXCHG2_REL(r1,r3,r2)          M16(0x05,LD_NONE,r3,r2,r1)
 724 #define CMPXCHG4_REL(r1,r3,r2)          M16(0x06,LD_NONE,r3,r2,r1)
 725 #define CMPXCHG8_REL(r1,r3,r2)          M16(0x07,LD_NONE,r3,r2,r1)
 726 #define CMP8XCHG16_ACQ(r1,r3,r2)        M16(0x20,LD_NONE,r3,r2,r1)
 727 #define CMP8XCHG16_REL(r1,r3,r2)        M16(0x24,LD_NONE,r3,r2,r1)
 728 /* cover */
 729 #define COVER()                         B8(0x02)
 730 /* cxz */
 731 #define CZX1_L(r1,r3)                   I29(0x18,r3,r1)
 732 #define CZX2_L(r1,r3)                   I29(0x19,r3,r1)
 733 #define CZX1_R(r1,r3)                   I29(0x1c,r3,r1)
 734 #define CZX2_R(r1,r3)                   I29(0x1d,r3,r1)
 735 /* dep */
 736 #define DEP_Z(r1,r2,pos,len)            I12(len,pos,r2,r1)
 737 #define DEPI_Z(r1,im,pos,len)           I13(len,pos,im,r1)
 738 #define DEPs(r1,r2,r3,pos,len)          I14(1,len,r3,pos,r1)
 739 #define DEPu(r1,r2,r3,pos,len)          I14(0,len,r3,pos,r1)
 740 #define DEP(r1,r2,r3,pos,len)           I15(pos,len,r3,r2,r1)
 741 /* epc */
 742 #define EPC()                           B8(0x10)
 743 /* extr */
 744 #define EXTR(r1,r3,pos,len)             I11(len,r3,pos,1,r1)
 745 #define EXTR_U(r1,r3,pos,len)           I11(len,r3,pos,0,r1)
 746 /* fc */
 747 #define FC(r3)                          M28(0,r3)
 748 #define FC_I(r3)                        M28(1,r3)
 749 /* fetchadd */
 750 #define FETCHADD4_ACQ(r1,r3,im)         M17(0x12,LD_NONE,r3,im,r1)
 751 #define FETCHADD8_ACQ(r1,r3,im)         M17(0x13,LD_NONE,r3,im,r1)
 752 #define FETCHADD4_REL(r1,r3,im)         M17(0x16,LD_NONE,r3,im,r1)
 753 #define FETCHADD8_REL(r1,r3,im)         M17(0x17,LD_NONE,r3,im,r1)
 754 /* flushrs */
 755 #define FLUSHRS()                       M25(0xc)
 756 /* fwb */
 757 #define FWB()                           M24(2,0)
 758 /* hint */
 759 #define HINT_I(im)                      I18(im,1)
 760 #define HINT_M(im)                      M48(1,im)
 761 #define HINT_B(im)                      B9(2,1,im)
 762 #define HINT_X(im)                      X5(1,im)
 763 /* invala */
 764 #define INVALA()                        M24(1,0)
 765 #define INVALA_E(r1)                    M26(r1)
 766 /* itc */
 767 #define ITC_I(r2)                       M41(0x2f,r2)
 768 #define ITC_D(r2)                       M41(0x2e,r2)
 769 /* itr */
 770 #define ITR_I(r3,r2)                    M42(0x0f,r3,r2)
 771 #define ITR_D(r3,r2)                    M42(0x0e,r3,r2)
 772 /* ld */
 773 #define LD1(r1,r3)                      M1(0x00,LD_NONE,0,r3,r1)
 774 #define LD2(r1,r3)                      M1(0x01,LD_NONE,0,r3,r1)
 775 #define LD4(r1,r3)                      M1(0x02,LD_NONE,0,r3,r1)
 776 #define LD8(r1,r3)                      M1(0x03,LD_NONE,0,r3,r1)
 777 #define LD1_S(r1,r3)                    M1(0x04,LD_NONE,0,r3,r1)
 778 #define LD2_S(r1,r3)                    M1(0x05,LD_NONE,0,r3,r1)
 779 #define LD4_S(r1,r3)                    M1(0x06,LD_NONE,0,r3,r1)
 780 #define LD8_S(r1,r3)                    M1(0x07,LD_NONE,0,r3,r1)
 781 #define LD1_A(r1,r3)                    M1(0x08,LD_NONE,0,r3,r1)
 782 #define LD2_A(r1,r3)                    M1(0x09,LD_NONE,0,r3,r1)
 783 #define LD4_A(r1,r3)                    M1(0x0a,LD_NONE,0,r3,r1)
 784 #define LD8_A(r1,r3)                    M1(0x0b,LD_NONE,0,r3,r1)
 785 #define LD1_SA(r1,r3)                   M1(0x0c,LD_NONE,0,r3,r1)
 786 #define LD2_SA(r1,r3)                   M1(0x0d,LD_NONE,0,r3,r1)
 787 #define LD4_SA(r1,r3)                   M1(0x0e,LD_NONE,0,r3,r1)
 788 #define LD8_SA(r1,r3)                   M1(0x0f,LD_NONE,0,r3,r1)
 789 #define LD1_BIAS(r1,r3)                 M1(0x10,LD_NONE,0,r3,r1)
 790 #define LD2_BIAS(r1,r3)                 M1(0x11,LD_NONE,0,r3,r1)
 791 #define LD4_BIAS(r1,r3)                 M1(0x12,LD_NONE,0,r3,r1)
 792 #define LD8_BIAS(r1,r3)                 M1(0x13,LD_NONE,0,r3,r1)
 793 #define LD1_ACQ(r1,r3)                  M1(0x14,LD_NONE,0,r3,r1)
 794 #define LD2_ACQ(r1,r3)                  M1(0x15,LD_NONE,0,r3,r1)
 795 #define LD4_ACQ(r1,r3)                  M1(0x16,LD_NONE,0,r3,r1)
 796 #define LD8_ACQ(r1,r3)                  M1(0x17,LD_NONE,0,r3,r1)
 797 #define LD8_FILL(r1,r3)                 M1(0x1b,LD_NONE,0,r3,r1)
 798 #define LD1_C_CLR(r1,r3)                M1(0x20,LD_NONE,0,r3,r1)
 799 #define LD2_C_CLR(r1,r3)                M1(0x21,LD_NONE,0,r3,r1)
 800 #define LD4_C_CLR(r1,r3)                M1(0x22,LD_NONE,0,r3,r1)
 801 #define LD8_C_CLR(r1,r3)                M1(0x23,LD_NONE,0,r3,r1)
 802 #define LD1_C_NC(r1,r3)                 M1(0x24,LD_NONE,0,r3,r1)
 803 #define LD2_C_NC(r1,r3)                 M1(0x25,LD_NONE,0,r3,r1)
 804 #define LD4_C_NC(r1,r3)                 M1(0x26,LD_NONE,0,r3,r1)
 805 #define LD8_C_NC(r1,r3)                 M1(0x27,LD_NONE,0,r3,r1)
 806 #define LD1_C_CLR_ACQ(r1,r3)            M1(0x28,LD_NONE,0,r3,r1)
 807 #define LD2_C_CLR_ACQ(r1,r3)            M1(0x29,LD_NONE,0,r3,r1)
 808 #define LD4_C_CLR_ACQ(r1,r3)            M1(0x2a,LD_NONE,0,r3,r1)
 809 #define LD8_C_CLR_ACQ(r1,r3)            M1(0x2b,LD_NONE,0,r3,r1)
 810 #define LD16(r1,r3)                     M1(0x28,LD_NONE,1,r3,r1)
 811 #define LD16_ACQ(r1,r3)                 M1(0x2c,LD_NONE,1,r3,r1)
 812 #define LD1_inc(r1,r3,im)               M3(0x00,LD_NONE,r3,im,r1)
 813 #define LD2_inc(r1,r3,im)               M3(0x01,LD_NONE,r3,im,r1)
 814 #define LD4_inc(r1,r3,im)               M3(0x02,LD_NONE,r3,im,r1)
 815 #define LD8_inc(r1,r3,im)               M3(0x03,LD_NONE,r3,im,r1)
 816 #define LD1_S_inc(r1,r3,im)             M3(0x04,LD_NONE,r3,im,r1)
 817 #define LD2_S_inc(r1,r3,im)             M3(0x05,LD_NONE,r3,im,r1)
 818 #define LD4_S_inc(r1,r3,im)             M3(0x06,LD_NONE,r3,im,r1)
 819 #define LD8_S_inc(r1,r3,im)             M3(0x07,LD_NONE,r3,im,r1)
 820 #define LD1_A_inc(r1,r3,im)             M3(0x08,LD_NONE,r3,im,r1)
 821 #define LD2_A_inc(r1,r3,im)             M3(0x09,LD_NONE,r3,im,r1)
 822 #define LD4_A_inc(r1,r3,im)             M3(0x0a,LD_NONE,r3,im,r1)
 823 #define LD8_A_inc(r1,r3,im)             M3(0x0b,LD_NONE,r3,im,r1)
 824 #define LD1_SA_inc(r1,r3,im)            M3(0x0c,LD_NONE,r3,im,r1)
 825 #define LD2_SA_inc(r1,r3,im)            M3(0x0d,LD_NONE,r3,im,r1)
 826 #define LD4_SA_inc(r1,r3,im)            M3(0x0e,LD_NONE,r3,im,r1)
 827 #define LD8_SA_inc(r1,r3,im)            M3(0x0f,LD_NONE,r3,im,r1)
 828 #define LD1_BIAS_inc(r1,r3,im)          M3(0x10,LD_NONE,r3,im,r1)
 829 #define LD2_BIAS_inc(r1,r3,im)          M3(0x11,LD_NONE,r3,im,r1)
 830 #define LD4_BIAS_inc(r1,r3,im)          M3(0x12,LD_NONE,r3,im,r1)
 831 #define LD8_BIAS_inc(r1,r3,im)          M3(0x13,LD_NONE,r3,im,r1)
 832 #define LD1_ACQ_inc(r1,r3,im)           M3(0x14,LD_NONE,r3,im,r1)
 833 #define LD2_ACQ_inc(r1,r3,im)           M3(0x15,LD_NONE,r3,im,r1)
 834 #define LD4_ACQ_inc(r1,r3,im)           M3(0x16,LD_NONE,r3,im,r1)
 835 #define LD8_AVQ_inc(r1,r3,im)           M3(0x17,LD_NONE,r3,im,r1)
 836 #define LD8_FILL_inc(r1,r3,im)          M3(0x1b,LD_NONE,r3,im,r1)
 837 #define LD1_C_CLR_inc(r1,r3,im)         M3(0x20,LD_NONE,r3,im,r1)
 838 #define LD2_C_CLR_inc(r1,r3,im)         M3(0x21,LD_NONE,r3,im,r1)
 839 #define LD4_C_CLR_inc(r1,r3,im)         M3(0x22,LD_NONE,r3,im,r1)
 840 #define LD8_C_CLR_inc(r1,r3,im)         M3(0x23,LD_NONE,r3,im,r1)
 841 #define LD1_C_NC_inc(r1,r3,im)          M3(0x24,LD_NONE,r3,im,r1)
 842 #define LD2_C_NC_inc(r1,r3,im)          M3(0x25,LD_NONE,r3,im,r1)
 843 #define LD4_C_NC_inc(r1,r3,im)          M3(0x26,LD_NONE,r3,im,r1)
 844 #define LD8_C_NC_inc(r1,r3,im)          M3(0x27,LD_NONE,r3,im,r1)
 845 #define LD1_C_CLR_ACQ_inc(r1,r3,im)     M3(0x28,LD_NONE,r3,im,r1)
 846 #define LD2_C_CLR_ACQ_inc(r1,r3,im)     M3(0x29,LD_NONE,r3,im,r1)
 847 #define LD4_C_CLR_ACQ_inc(r1,r3,im)     M3(0x2a,LD_NONE,r3,im,r1)
 848 #define LD8_C_CLR_ACQ_inc(r1,r3,im)     M3(0x2b,LD_NONE,r3,im,r1)
 849 #define LDX1(r1,r3,r2)                  M2(0x00,LD_NONE,r3,r2,r1)
 850 #define LDX2(r1,r3,r2)                  M2(0x01,LD_NONE,r3,r2,r1)
 851 #define LDX4(r1,r3,r2)                  M2(0x02,LD_NONE,r3,r2,r1)
 852 #define LDX8(r1,r3,r2)                  M2(0x03,LD_NONE,r3,r2,r1)
 853 #define LDX1_S(r1,r3,r2)                M2(0x04,LD_NONE,r3,r2,r1)
 854 #define LDX2_S(r1,r3,r2)                M2(0x05,LD_NONE,r3,r2,r1)
 855 #define LDX4_S(r1,r3,r2)                M2(0x06,LD_NONE,r3,r2,r1)
 856 #define LDX8_S(r1,r3,r2)                M2(0x07,LD_NONE,r3,r2,r1)
 857 #define LDX1_A(r1,r3,r2)                M2(0x08,LD_NONE,r3,r2,r1)
 858 #define LDX2_A(r1,r3,r2)                M2(0x09,LD_NONE,r3,r2,r1)
 859 #define LDX4_A(r1,r3,r2)                M2(0x0a,LD_NONE,r3,r2,r1)
 860 #define LDX8_A(r1,r3,r2)                M2(0x0b,LD_NONE,r3,r2,r1)
 861 #define LDX1_SA(r1,r3,r2)               M2(0x0c,LD_NONE,r3,r2,r1)
 862 #define LDX2_SA(r1,r3,r2)               M2(0x0d,LD_NONE,r3,r2,r1)
 863 #define LDX4_SA(r1,r3,r2)               M2(0x0e,LD_NONE,r3,r2,r1)
 864 #define LDX8_SA(r1,r3,r2)               M2(0x0f,LD_NONE,r3,r2,r1)
 865 #define LDX1_BIAS(r1,r3,r2)             M2(0x10,LD_NONE,r3,r2,r1)
 866 #define LDX2_BIAS(r1,r3,r2)             M2(0x11,LD_NONE,r3,r2,r1)
 867 #define LDX4_BIAS(r1,r3,r2)             M2(0x12,LD_NONE,r3,r2,r1)
 868 #define LDX8_BIAS(r1,r3,r2)             M2(0x13,LD_NONE,r3,r2,r1)
 869 #define LDX1_ACQ(r1,r3,r2)              M2(0x14,LD_NONE,r3,r2,r1)
 870 #define LDX2_ACQ(r1,r3,r2)              M2(0x15,LD_NONE,r3,r2,r1)
 871 #define LDX4_ACQ(r1,r3,r2)              M2(0x16,LD_NONE,r3,r2,r1)
 872 #define LDX8_ACQ(r1,r3,r2)              M2(0x17,LD_NONE,r3,r2,r1)
 873 #define LDX8_FILL(r1,r3,r2)             M2(0x1b,LD_NONE,r3,r2,r1)
 874 #define LDX1_C_CLR(r1,r3,r2)            M2(0x20,LD_NONE,r3,r2,r1)
 875 #define LDX2_C_CLR(r1,r3,r2)            M2(0x21,LD_NONE,r3,r2,r1)
 876 #define LDX4_C_CLR(r1,r3,r2)            M2(0x22,LD_NONE,r3,r2,r1)
 877 #define LDX8_C_CLR(r1,r3,r2)            M2(0x23,LD_NONE,r3,r2,r1)
 878 #define LDX1_C_NC(r1,r3,r2)             M2(0x24,LD_NONE,r3,r2,r1)
 879 #define LDX2_C_NC(r1,r3,r2)             M2(0x25,LD_NONE,r3,r2,r1)
 880 #define LDX4_C_NC(r1,r3,r2)             M2(0x26,LD_NONE,r3,r2,r1)
 881 #define LDX8_C_NC(r1,r3,r2)             M2(0x27,LD_NONE,r3,r2,r1)
 882 #define LDX1_C_CLR_ACQ(r1,r3,r2)        M2(0x28,LD_NONE,r3,r2,r1)
 883 #define LDX2_C_CLR_ACQ(r1,r3,r2)        M2(0x29,LD_NONE,r3,r2,r1)
 884 #define LDX4_C_CLR_ACQ(r1,r3,r2)        M2(0x2a,LD_NONE,r3,r2,r1)
 885 #define LDX8_C_CLR_ACQ(r1,r3,r2)        M2(0x2b,LD_NONE,r3,r2,r1)
 886 /* lfetch */
 887 #define LFETCH_EXCL(r3)                 M13(0x2d,LF_NONE,r3,GR_0)
 888 #define LFETCH_FAULT(r3)                M13(0x2e,LF_NONE,r3,GR_0)
 889 #define LFETCH_FAULT_EXCL(r3)           M13(0x2f,LF_NONE,r3,GR_0)
 890 #define LXFETCH(r3,r2)                  M14(0x2c,LF_NONE,r3,r2)
 891 #define LXFETCH_EXCL(r3,r2)             M14(0x2d,LF_NONE,r3,r2)
 892 #define LXFETCH_FAULT(r3,r2)            M14(0x2e,LF_NONE,r3,r2)
 893 #define LXFETCH_FAULT_EXCL(r3,r2)       M14(0x2f,LF_NONE,r3,r2)
 894 #define LFETCHI(r3,im)                  M15(0x2c,LF_NONE,r3,im)
 895 #define LFETCHI_EXCL(r3,im)             M15(0x2d,LF_NONE,r3,im)
 896 #define LFETCHI_FAULT(r3,im)            M15(0x2e,LF_NONE,r3,im)
 897 #define LFETCHI_FAULT_EXCL(r3,im)       M15(0x2f,LF_NONE,r3,im)
 898 /* loadrs */
 899 #define LOADRS()                        M25(0xa)
 900 /* mf */
 901 #define MF()                            M24(2,2)
 902 #define MF_A()                          M24(2,3)
 903 /* mix */
 904 #define MIX1_R(r1,r2,r3)                I2(0,2,0,2,0,r3,r2,r1)
 905 #define MIX2_R(r1,r2,r3)                I2(0,2,1,2,0,r3,r2,r1)
 906 #define MIX4_R(r1,r2,r3)                I2(1,2,0,2,0,r3,r2,r1)
 907 #define MIX1_L(r1,r2,r3)                I2(0,2,0,2,2,r3,r2,r1)
 908 #define MIX2_L(r1,r2,r3)                I2(0,2,1,2,2,r3,r2,r1)
 909 #define MIX4_L(r1,r2,r3)                I2(1,2,0,2,2,r3,r2,r1)
 910 /* mov - Move Application Register */
 911 #define MOV_I_rn_ar(r1,ar)              I28(ar,r1)
 912 #define MOV_I_ar_rn(ar,r2)              I26(ar,r2)
 913 #define MOV_I_ar_im(ar,im)              I27(ar,im)
 914 #define MOV_M_rn_a(r1,ar)               M31(r1,ar)
 915 #define MOV_M_ar_rn(ar,r2)              M29(ar,r2)
 916 #define MOV_M_ar_im(ar,im)              M30(ar,im)
 917 /* mov - Move Branch Register */
 918 #define MOV_rn_br(r1,b2)                I22(b2,r1)
 919 #define MOV_br_rn_tg(b1,r2,tag)         I21(tag,IH_NONE,0,MWH_NONE,r2,b1)
 920 #define MOV_br_rn(b1,r2)                MOV_br_rn_tg(b1,r2,0)
 921 #define MOV_RET_br_rn_tg(b1,r2,tag)     I21(tag,IH_NONE,1,MWH_NONE,r2,b1)
 922 /* mov - Move Control Register */
 923 #define MOV_rn_cr(cr,r1)                M33(cr,r1)
 924 #define MOV_cr_rr(cr,r2)                M32(cr,r2)
 925 /* mov - Move General Register */
 926 #define MOV(r0,r1)                      ADDS(r0,0,r1)
 927 #define MOV_p(r0,r1,_p)                 ADDS_p(r0,0,r1,_p)
 928 /* mov - Move Immediate */
 929 #define MOVI(r1,im)                     ADDL(r1,im,GR_0)
 930 #define MOVI_p(r1,im,_p)                ADDL_p(r1,im,GR_0,_p)
 931 /* mov - Move Indirect Register */
 932 #define MOV_rn_RR(r1,r3)                M43(0x10,r3,r1)
 933 #define MOV_rn_DBR(r1,r3)               M43(0x11,r3,r1)
 934 #define MOV_rn_IBR(r1,r3)               M43(0x012,r3,r1)
 935 #define MOV_rn_PKR(r1,r3)               M43(0x13,r3,r1)
 936 #define MOV_rn_PMC(r1,r3)               M43(0x14,r3,r1)
 937 #define MOV_rn_PMD(r1,r3)               M43(0x15,r3,r1)
 938 #define MOV_rn_CPUID(r1,r3)             M43(0x17,r3,r1)
 939 #define MOV_RR_rn(r3,r2)                M42(0x00,r3,r2)
 940 #define MOV_DBR_rn(r3,r2)               M42(0x01,r3,r2)
 941 #define MOV_IBR_rn(r3,r2)               M42(0x02,r3,r2)
 942 #define MOV_PKR_rn(r3,r2)               M42(0x03,r3,r2)
 943 #define MOV_PMC_rn(r3,r2)               M42(0x04,r3,r2)
 944 #define MOV_PMD_rn(r3,r2)               M42(0x05,r3,r2)
 945 /* mov - Move Instruction Pointer */
 946 #define MOV_rn_ip(r1)                   I25(0x30,r1)
 947 /* mov - Move Predicates */
 948 #define MOV_rn_pr(r1)                   I25(0x33,r1)
 949 #define MOV_pr_rn(r2,im)                I23(r2,im)
 950 #define MOVI_pr(im)                     I24(im)
 951 /* mov - Move Processor Status Register */
 952 #define MOV_rn_psr(r1)                  M36(0x25,r1)
 953 #define MOV_psr_l_rn(r2)                M35(0x2d,r2)
 954 /* mov - Move User Mask */
 955 #define MOV_rn_psr_um(r1)               M36(0x21,r1)
 956 #define MOV_psr_um_rn(r2)               M35(0x29,r2)
 957 /* movl */
 958 #define MOVL(r1,im)                     X2(r1,im)
 959 /* mpy4 */
 960 #define MPY4(r1,r2,r3)                  I2(1,0,0,3,1,r3,r2,r1)
 961 /* mpyshl4 */
 962 #define MPYSHL4(r1,r2,r3)               I2(1,0,0,3,3,r3,r2,r1)
 963 /* mux */
 964 #define MUX1(r1,r2,mbt)                 I3(mbt,r2,r1)
 965 #define MUX2(r1,r2,mht)                 I4(mht,r2,r1)
 966 /* nop */
 967 #define NOP_I(im)                       I18(im,0)
 968 #define NOP_M(im)                       M48(0,im)
 969 #define NOP_B(im)                       B9(2,0,im)
 970 #define NOP_X(im)                       X5(0,im)
 971 /* or */
 972 #define OR(r1,r2,r3)                    A1(3,2,r3,r2,r1)
 973 #define ORI(r1,im,r3)                   A3(0xb,2,r3,im,r1)
 974 /* pack */
 975 #define PACK2_USS(r1,r2,r3)             I2(0,2,1,0,0,r3,r2,r1)
 976 #define PACK2_SSS(r1,r2,r3)             I2(0,2,1,0,2,r3,r2,r1)
 977 #define PACK4_SSS(r1,r2,r3)             I2(1,2,0,0,2,r3,r2,r1)
 978 /* padd */
 979 #define PADD1(r1,r2,r3)                 A9(0,0,0,0,r3,r2,r1)
 980 #define PADD1_SSS(r1,r2,r3)             A9(0,0,0,1,r3,r2,r1)
 981 #define PADD1_UUU(r1,r2,r3)             A9(0,0,0,2,r3,r2,r1)
 982 #define PADD1_UUS(r1,r2,r3)             A9(0,0,0,3,r3,r2,r1)
 983 #define PADD2(r1,r2,r3)                 A9(0,1,0,0,r3,r2,r1)
 984 #define PADD2_SSS(r1,r2,r3)             A9(0,1,0,1,r3,r2,r1)
 985 #define PADD2_UUU(r1,r2,r3)             A9(0,1,0,2,r3,r2,r1)
 986 #define PADD2_UUS(r1,r2,r3)             A9(0,1,0,3,r3,r2,r1)
 987 #define PADD4(r1,r2,r3)                 A9(1,0,0,0,r3,r2,r1)
 988 /* pavg */
 989 #define PAVG1(r1,r2,r3)                 A9(0,0,2,2,r3,r2,r1)
 990 #define PAVG2(r1,r2,r3)                 A9(0,1,2,2,r3,r2,r1)
 991 #define PAVG1_RAZ(r1,r2,r3)             A9(0,0,2,3,r3,r2,r1)
 992 #define PAVG2_RAZ(r1,r2,r3)             A9(0,1,2,3,r3,r2,r1)
 993 /* pavgsub */
 994 #define PAVGSUB1(r1,r2,r3)              A9(0,0,3,2,r3,r2,r1)
 995 #define PAVGSUB2(r1,r2,r3)              A9(0,1,3,2,r3,r2,r1)
 996 /* pcmp */
 997 #define PCMP1_EQ(r1,r2,r3)              A9(0,0,9,0,r3,r2,r1)
 998 #define PCMP2_EQ(r1,r2,r3)              A9(0,1,9,0,r3,r2,r1)
 999 #define PCMP4_EQ(r1,r2,r3)              A9(1,0,9,0,r3,r2,r1)
1000 #define PCMP1_GT(r1,r2,r3)              A9(0,0,9,1,r3,r2,r1)
1001 #define PCMP2_GT(r1,r2,r3)              A9(0,1,9,1,r3,r2,r1)
1002 #define PCMP4_GT(r1,r2,r3)              A9(1,0,9,1,r3,r2,r1)
1003 /* pmax */
1004 #define PMAX1_U(r1,r2,r3)               I2(0,2,0,1,1,r3,r2,r1)
1005 #define PMAX2(r1,r2,r3)                 I2(0,2,1,1,3,r3,r2,r1)
1006 /* pmin */
1007 #define PMIN1_U(r1,r2,r3)               I2(0,2,0,0,1,r3,r2,r1)
1008 #define PMIN2(r1,r2,r3)                 I2(0,2,1,0,3,r3,r2,r1)
1009 /* pmpy */
1010 #define PMPY2_R(r1,r2,r3)               I2(0,2,1,3,1,r3,r2,r1)
1011 #define PMPY2_L(r1,r2,r3)               I2(0,2,1,3,3,r3,r2,r1)
1012 /* pmpyshr */
1013 #define PMPYSHR2(r1,r2,r3,im)           I1(im,3,r3,r2,r1)
1014 #define PMPYSHR2_U(r1,r2,r3,im)         I1(im,1,r3,r2,r1)
1015 /* popcnt */
1016 #define POPCNT(r1,r3)                   I9(2,r3,r1)
1017 /* probe */
1018 #define PROBE_R(r1,r3,r2)               M38(0x38,r3,r2,r1)
1019 #define PROBE_W(r1,r3,r2)               M38(0x39,r3,r2,r1)
1020 #define PROBEI_R(r1,r3,im)              M39(0x18,r3,im,r1)
1021 #define PROBEI_W(r1,r3,im)              M39(0x19,r3,im,r1)
1022 #define PROBE_RW_FAULT(r3,im)           M40(0x31,r3,im)
1023 #define PROBE_R_FAULT(r3,im)            M40(0x32,r3,im)
1024 #define PROBE_W_FAULT(r3,im)            M40(0x33,r3,im)
1025 /* psad */
1026 #define PSAD1(r1,r2,r3)                 I2(0,2,0,2,3,r3,r2,r1)
1027 /* pshl */
1028 #define PSHL2(r1,r2,r3)                 I7(0,1,r3,r2,r1)
1029 #define PSHL4(r1,r2,r3)                 I7(1,0,r3,r2,r1)
1030 #define PSHL2I(r1,r2,im)                I8(0,1,im,r2,r1)
1031 #define PSHL4I(r1,r2,im)                I8(1,0,im,r2,r1)
1032 /* pshladd */
1033 #define PSHLADD2(r1,r2,im,r3)           A10(4,im,r3,r2,r1)
1034 /* pshr */
1035 #define PSHR2(r1,r3,r2)                 I5(0,1,2,r3,r2,r1)
1036 #define PSHR2I(r1,r3,im)                I6(0,1,3,r3,im,r1)
1037 #define PSHR2_U(r1,r3,r2)               I5(0,1,0,r3,r2,r1)
1038 #define PSHR2I_U(r1,r3,im)              I6(0,1,1,r3,im,r1)
1039 #define PSHR4(r1,r3,r2)                 I5(1,0,2,r3,r2,r1)
1040 #define PSHR4I(r1,r3,im)                I6(1,0,3,r3,im,r1)
1041 #define PSHR4_U(r1,r3,r2)               I5(1,0,0,r3,r2,r1)
1042 #define PSHR4I_U(r1,r3,im)              I6(1,0,1,r3,im,r1)
1043 /* pshradd */
1044 #define PSHRADD2(r1,r2,im,r3)           A10(6,im,r3,r2,r1)
1045 /* psub */
1046 #define PSUB1(r1,r2,r3)                 A9(0,0,1,0,r3,r2,r1)
1047 #define PSUB1_SSS(r1,r2,r3)             A9(0,0,1,1,r3,r2,r1)
1048 #define PSUB1_UUU(r1,r2,r3)             A9(0,0,1,2,r3,r2,r1)
1049 #define PSUB1_UUS(r1,r2,r3)             A9(0,0,1,3,r3,r2,r1)
1050 #define PSUB2(r1,r2,r3)                 A9(0,1,1,0,r3,r2,r1)
1051 #define PSUB2_SSS(r1,r2,r3)             A9(0,1,1,1,r3,r2,r1)
1052 #define PSUB2_UUU(r1,r2,r3)             A9(0,1,1,2,r3,r2,r1)
1053 #define PSUB2_UUS(r1,r2,r3)             A9(0,1,1,3,r3,r2,r1)
1054 #define PSUB4(r1,r2,r3)                 A9(1,0,1,0,r3,r2,r1)
1055 /* ptc.e */
1056 #define PTC_E(r3)                       M47(0x34,r3)
1057 /* ptc.g, ptc.ga */
1058 #define PTC_G(r3,r2)                    M45(0xa,r3,r2)
1059 #define PTC_GA(r3,r2)                   M45(0xb,r3,r2)
1060 /* ptc.l */
1061 #define PTC_L(r3,r2)                    M45(0x9,r3,r2)
1062 /* ptr */
1063 #define PTR_D(r3,r2)                    M45(0xc,r3,r2)
1064 #define PTR_I(r3,r2)                    M45(0xd,r3,r2)
1065 /* rfi */
1066 #define RFI()                           B8(0x08)
1067 /* rsm */
1068 #define RSM(im)                         M44(7,im)
1069 /* rum */
1070 #define RUM(im)                         M44(5,im)
1071 /* shl */
1072 #define SHL(r1,r2,r3)                   I7(1,1,r3,r2,r1)
1073 /* shladd */
1074 #define SHLADD(r1,r2,im,r3)             A2(4,im,r3,r2,r1)
1075 /* shladdp4 */
1076 #define SHLADDP4(r1,r2,im,r3)           A2(6,im,r3,r2,r1)
1077 /* shr */
1078 #define SHR(r1,r3,r2)                   I5(1,1,2,r3,r2,r1)
1079 #define SHR_U(r1,r3,r2)                 I5(1,1,0,r3,r2,r1)
1080 /* shrp */
1081 #define SHRP(r1,r2,r3,im)               I10(im,r3,r2,r1)
1082 /* srlz */
1083 #define SRLZ_I()                        M24(3,1)
1084 #define SRLZ_D()                        M24(3,0)
1085 /* ssm */
1086 #define SSM(im)                         M44(6,im)
1087 /* st */
1088 #define ST1(r3,r2)                      M6(0x30,ST_NONE,0,r3,r2)
1089 #define ST2(r3,r2)                      M6(0x31,ST_NONE,0,r3,r2)
1090 #define ST4(r3,r2)                      M6(0x32,ST_NONE,0,r3,r2)
1091 #define ST8(r3,r2)                      M6(0x33,ST_NONE,0,r3,r2)
1092 #define ST1_REL(r3,r2)                  M6(0x34,ST_NONE,0,r3,r2)
1093 #define ST2_REL(r3,r2)                  M6(0x35,ST_NONE,0,r3,r2)
1094 #define ST4_REL(r3,r2)                  M6(0x36,ST_NONE,0,r3,r2)
1095 #define ST8_REL(r3,r2)                  M6(0x37,ST_NONE,0,r3,r2)
1096 #define ST8_SPILL(r3,r2)                M6(0x3b,ST_NONE,0,r3,r2)
1097 #define ST16(r3,r2)                     M6(0x30,ST_NONE,1,r3,r2)
1098 #define ST16_REL(r3,r2)                 M6(0x34,ST_NONE,1,r3,r2)
1099 #define ST1_inc(r3,r2,im)               M5(0x30,ST_NONE,r3,r2,im)
1100 #define ST2_inc(r3,r2,im)               M5(0x31,ST_NONE,r3,r2,im)
1101 #define ST4_inc(r3,r2,im)               M5(0x32,ST_NONE,r3,r2,im)
1102 #define ST8_inc(r3,r2,im)               M5(0x33,ST_NONE,r3,r2,im)
1103 #define ST1_REL_inc(r3,r2,im)           M5(0x34,ST_NONE,r3,r2,im)
1104 #define ST2_REL_inc(r3,r2,im)           M5(0x35,ST_NONE,r3,r2,im)
1105 #define ST4_REL_inc(r3,r2,im)           M5(0x36,ST_NONE,r3,r2,im)
1106 #define ST8_REL_inc(r3,r2,im)           M5(0x37,ST_NONE,r3,r2,im)
1107 #define ST8_SPILL_inc(r3,r2,im)         M5(0x3b,ST_NONE,r3,r2,im)
1108 /* sub */
1109 #define SUB(r1,r2,r3)                   A1(1,1,r3,r2,r1)
1110 #define SUB1(r1,r2,r3)                  A1(1,0,r3,r2,r1)
1111 #define SUBI(r1,im,r3)                  A3(9,1,r3,im,r1)
1112 /* sum */
1113 #define SUM(im)                         M44(4,im)
1114 /* sxt */
1115 #define SXT1(r1,r3)                     I29(0x14,r3,r1)
1116 #define SXT2(r1,r3)                     I29(0x15,r3,r1)
1117 #define SXT4(r1,r3)                     I29(0x16,r3,r1)
1118 /* sync */
1119 #define SYNC_I()                        M24(3,3)
1120 /* tak */
1121 #define TAK(r1,r3)                      M46(0x1f,r3,r1)
1122 /* tbit */
1123 #define TBIT_Z(p1,p2,r3,pos)            I16(0,0,p2,r3,pos,0,p1)
1124 #define TBIT_Z_UNC(p1,p2,r3,pos)        I16(0,0,p2,r3,pos,1,p1)
1125 #define TBIT_Z_AND(p1,p2,r3,pos)        I16(1,0,p2,r3,pos,0,p1)
1126 #define TBIT_NZ_AND(p1,p2,r3,pos)       I16(1,0,p2,r3,pos,1,p1)
1127 #define TBIT_Z_OR(p1,p2,r3,pos)         I16(0,1,p2,r3,pos,0,p1)
1128 #define TBIT_NZ_OR(p1,p2,r3,pos)        I16(0,1,p2,r3,pos,1,p1)
1129 #define TBIT_Z_ANDCM(p1,p2,r3,pos)      I16(1,1,p2,r3,pos,0,p1)
1130 #define TBIT_NZ_ANDCM(p1,p2,r3,pos)     I16(1,1,p2,r3,pos,1,p1)
1131 /* tf */
1132 #define TF_Z(p1,p2,im)                  I30(0,0,p2,im,0,p1)
1133 #define TF_Z_UNC(p1,p2,im)              I30(0,0,p2,im,1,p1)
1134 #define TF_Z_AND(p1,p2,im)              I30(1,0,p2,im,0,p1)
1135 #define TF_NZ_AND(p1,p2,im)             I30(1,0,p2,im,1,p1)
1136 #define TF_Z_OR(p1,p2,im)               I30(0,1,p2,im,0,p1)
1137 #define TF_NZ_OR(p1,p2,im)              I30(0,1,p2,im,1,p1)
1138 #define TF_Z_ANDCM(p1,p2,im)            I30(1,1,p2,im,0,p1)
1139 #define TF_NZ_ANDCM(p1,p2,im)           I30(1,1,p2,im,1,p1)
1140 /* thash */
1141 #define THASH(r1,r3)                    M46(0x1a,r3,r1)
1142 /* tnat */
1143 #define TNAT_Z(p1,p2,r3)                I17(0,0,p2,r3,0,p1)
1144 #define TNAT_Z_UNC(p1,p2,r3)            I17(0,0,p2,r3,1,p1)
1145 #define TNAT_Z_AND(p1,p2,r3)            I17(1,0,p2,r3,0,p1)
1146 #define TNAT_NZ_AND(p1,p2,r3)           I17(1,0,p2,r3,1,p1)
1147 #define TNAT_Z_OR(p1,p2,r3)             I17(0,1,p2,r3,0,p1)
1148 #define TNAT_NZ_OR(p1,p2,r3)            I17(0,1,p2,r3,1,p1)
1149 #define TNAT_Z_ANDCM(p1,p2,r3)          I17(1,1,p2,r3,0,p1)
1150 #define TNAT_NZ_ANDCM(p1,p2,r3)         I17(1,1,p2,r3,1,p1)
1151 /* tpa */
1152 #define TPA(r1,r3)                      M46(0x1e,r3,r1)
1153 /* ttag */
1154 #define TTAG(r1,r3)                     M46(0x1b,r3,r1)
1155 /* unpack */
1156 #define UNPACK1_H(r1,r2,r3)             I2(0,2,0,1,0,r3,r2,r1)
1157 #define UNPACK2_H(r1,r2,r3)             I2(0,2,1,1,0,r3,r2,r1)
1158 #define UNPACK4_H(r1,r2,r3)             I2(1,2,0,1,0,r3,r2,r1)
1159 #define UNPACK1_L(r1,r2,r3)             I2(0,2,0,1,2,r3,r2,r1)
1160 #define UNPACK2_L(r1,r2,r3)             I2(0,2,1,1,2,r3,r2,r1)
1161 #define UNPACK4_L(r1,r2,r3)             I2(1,2,0,1,2,r3,r2,r1)
1162 /* vmsw */
1163 #define VMSW_0()                        B8(0x18)
1164 #define VMSW_1()                        B8(0x19)
1165 /* xchg */
1166 #define XCHG1_ACQ(r1,r3,r2)             M16(0x08,LD_NONE,r3,r2,r1)
1167 #define XCHG2_ACQ(r1,r3,r2)             M16(0x09,LD_NONE,r3,r2,r1)
1168 #define XCHG4_ACQ(r1,r3,r2)             M16(0x0a,LD_NONE,r3,r2,r1)
1169 #define XCHG8_ACQ(r1,r3,r2)             M16(0x0b,LD_NONE,r3,r2,r1)
1170 /* xor */
1171 #define XOR(r1,r2,r3)                   A1(3,3,r3,r2,r1)
1172 #define XORI(r1,im,r3)                  A3(0xb,3,r3,im,r1)
1173 /* zxt */
1174 #define ZXT1(r1,r3)                     I29(0x10,r3,r1)
1175 #define ZXT2(r1,r3)                     I29(0x11,r3,r1)
1176 #define ZXT4(r1,r3)                     I29(0x12,r3,r1)
1177
1178 #  define nop(i0)                       _nop(_jit,i0)
1179 static void _nop(jit_state_t*, jit_int32_t);
1180 #define addr(r0,r1,r2)                  ADD(r0,r1,r2)
1181 #define addi(r0,r1,i0)                  _addi(_jit,r0,r1,i0)
1182 static void _addi(jit_state_t*,jit_int32_t,jit_int32_t,jit_word_t);
1183 #define addcr(r0, r1, r2)               _addcr(_jit, r0, r1, r2)
1184 static void _addcr(jit_state_t*, jit_int32_t, jit_int32_t, jit_int32_t);
1185 #define addci(r0, r1, i0)               _addci(_jit, r0, r1, i0)
1186 static void _addci(jit_state_t*, jit_int32_t, jit_int32_t, jit_word_t);
1187 #  define addxr(r0, r1, r2)             _addxr(_jit, r0, r1, r2)
1188 static void _addxr(jit_state_t*, jit_int32_t, jit_int32_t, jit_int32_t);
1189 #  define addxi(r0, r1, i0)             _addxi(_jit, r0, r1, i0)
1190 static void _addxi(jit_state_t*, jit_int32_t, jit_int32_t, jit_word_t);
1191 #define subr(r0,r1,r2)                  SUB(r0,r1,r2)
1192 #define subi(r0,r1,i0)                  _subi(_jit,r0,r1,i0)
1193 static void _subi(jit_state_t*,jit_int32_t,jit_int32_t,jit_word_t);
1194 #  define subcr(r0, r1, r2)             _subcr(_jit, r0, r1, r2)
1195 static void _subcr(jit_state_t*,jit_int32_t,jit_int32_t,jit_int32_t);
1196 #  define subci(r0, r1, i0)             _subci(_jit, r0, r1, i0)
1197 static void _subci(jit_state_t*,jit_int32_t,jit_int32_t,jit_word_t);
1198 #  define subxr(r0, r1, r2)             _subxr(_jit, r0, r1, r2)
1199 static void _subxr(jit_state_t*,jit_int32_t,jit_int32_t,jit_int32_t);
1200 #  define subxi(r0, r1, i0)             _subxi(_jit, r0, r1, i0)
1201 static void _subxi(jit_state_t*,jit_int32_t,jit_int32_t,jit_word_t);
1202 #  define rsbi(r0, r1, i0)              _rsbi(_jit, r0, r1, i0)
1203 static void _rsbi(jit_state_t*,jit_int32_t,jit_int32_t,jit_word_t);
1204 #define mulr(r0,r1,r2)                  _mulr(_jit,r0,r1,r2)
1205 static void _mulr(jit_state_t*,jit_int32_t,jit_int32_t,jit_int32_t);
1206 #define muli(r0,r1,i0)                  _muli(_jit,r0,r1,i0)
1207 static void _muli(jit_state_t*,jit_int32_t,jit_int32_t,jit_word_t);
1208 #define divr(r0,r1,r2)                  _divr(_jit,r0,r1,r2)
1209 static void _divr(jit_state_t*,jit_int32_t,jit_int32_t,jit_int32_t);
1210 #define divi(r0,r1,i0)                  _divi(_jit,r0,r1,i0)
1211 static void _divi(jit_state_t*,jit_int32_t,jit_int32_t,jit_word_t);
1212 #define divr_u(r0,r1,r2)                _divr_u(_jit,r0,r1,r2)
1213 static void _divr_u(jit_state_t*,jit_int32_t,jit_int32_t,jit_int32_t);
1214 #define divi_u(r0,r1,i0)                _divi_u(_jit,r0,r1,i0)
1215 static void _divi_u(jit_state_t*,jit_int32_t,jit_int32_t,jit_word_t);
1216 #define remr(r0,r1,r2)                  _remr(_jit,r0,r1,r2)
1217 static void _remr(jit_state_t*,jit_int32_t,jit_int32_t,jit_int32_t);
1218 #define remi(r0,r1,i0)                  _remi(_jit,r0,r1,i0)
1219 static void _remi(jit_state_t*,jit_int32_t,jit_int32_t,jit_word_t);
1220 #define remr_u(r0,r1,r2)                _remr_u(_jit,r0,r1,r2)
1221 static void _remr_u(jit_state_t*,jit_int32_t,jit_int32_t,jit_int32_t);
1222 #define remi_u(r0,r1,i0)                _remi_u(_jit,r0,r1,i0)
1223 static void _remi_u(jit_state_t*,jit_int32_t,jit_int32_t,jit_word_t);
1224 #define mulh(r0,r1,r2,sign)             _mulh(_jit,r0,r1,r2,sign)
1225 static void _mulh(jit_state_t*,jit_bool_t,jit_int32_t,jit_int32_t,jit_int32_t);
1226 #define qmulr(r0,r1,r2,r3)              iqmulr(r0,r1,r2,r3,1)
1227 #define qmulr_u(r0,r1,r2,r3)            iqmulr(r0,r1,r2,r3,0)
1228 #define iqmulr(r0,r1,r2,r3,sign)        _iqmulr(_jit,r0,r1,r2,r3,sign)
1229 static void _iqmulr(jit_state_t*,jit_int32_t,jit_int32_t,
1230                     jit_int32_t,jit_int32_t,jit_bool_t);
1231 #define qmuli(r0,r1,r2,i0)              iqmuli(r0,r1,r2,i0,1)
1232 #define qmuli_u(r0,r1,r2,i0)            iqmuli(r0,r1,r2,i0,0)
1233 #define iqmuli(r0,r1,r2,i0,sign)        _iqmuli(_jit,r0,r1,r2,i0,sign)
1234 static void _iqmuli(jit_state_t*,jit_int32_t,jit_int32_t,
1235                     jit_int32_t,jit_word_t,jit_bool_t);
1236 #define qdivr(r0,r1,r2,r3)              iqdivr(r0,r1,r2,r3,1)
1237 #define qdivr_u(r0,r1,r2,r3)            iqdivr(r0,r1,r2,r3,0)
1238 #define iqdivr(r0,r1,r2,r3,sign)        _iqdivr(_jit,r0,r1,r2,r3,sign)
1239 static void _iqdivr(jit_state_t*,jit_int32_t,jit_int32_t,
1240                     jit_int32_t,jit_int32_t,jit_bool_t);
1241 #define qdivi(r0,r1,r2,i0)              iqdivi(r0,r1,r2,i0,1)
1242 #define qdivi_u(r0,r1,r2,i0)            iqdivi(r0,r1,r2,i0,0)
1243 #define iqdivi(r0,r1,r2,i0,sign)        _iqdivi(_jit,r0,r1,r2,i0,sign)
1244 static void _iqdivi(jit_state_t*,jit_int32_t,jit_int32_t,
1245                     jit_int32_t,jit_word_t,jit_bool_t);
1246 #define andr(r0,r1,r2)                  AND(r0,r1,r2)
1247 #define andi(r0,r1,i0)                  _andi(_jit,r0,r1,i0)
1248 static void _andi(jit_state_t*,jit_int32_t,jit_int32_t,jit_word_t);
1249 #define orr(r0,r1,r2)                   OR(r0,r1,r2)
1250 #define ori(r0,r1,i0)                   _ori(_jit,r0,r1,i0)
1251 static void _ori(jit_state_t*,jit_int32_t,jit_int32_t,jit_word_t);
1252 #define xorr(r0,r1,r2)                  XOR(r0,r1,r2)
1253 #define xori(r0,r1,i0)                  _xori(_jit,r0,r1,i0)
1254 static void _xori(jit_state_t*,jit_int32_t,jit_int32_t,jit_word_t);
1255 #define lshr(r0,r1,r2)                  SHL(r0,r1,r2)
1256 #define lshi(r0,r1,i0)                  _lshi(_jit,r0,r1,i0)
1257 static void _lshi(jit_state_t*,jit_int32_t,jit_int32_t,jit_word_t);
1258 #define rshr(r0,r1,r2)                  SHR(r0,r1,r2)
1259 #define rshi(r0,r1,i0)                  _rshi(_jit,r0,r1,i0)
1260 static void _rshi(jit_state_t*,jit_int32_t,jit_int32_t,jit_word_t);
1261 #define rshr_u(r0,r1,r2)                SHR_U(r0,r1,r2)
1262 #define rshi_u(r0,r1,i0)                _rshi_u(_jit,r0,r1,i0)
1263 static void _rshi_u(jit_state_t*,jit_int32_t,jit_int32_t,jit_word_t);
1264 #define ltr(r0,r1,r2)                   _ltr(_jit,r0,r1,r2)
1265 static void _ltr(jit_state_t*,jit_int32_t,jit_int32_t,jit_int32_t);
1266 #define lti(r0,r1,i0)                   _lti(_jit,r0,r1,i0)
1267 static void _lti(jit_state_t*,jit_int32_t,jit_int32_t,jit_word_t);
1268 #define ltr_u(r0,r1,r2)                 _ltr_u(_jit,r0,r1,r2)
1269 static void _ltr_u(jit_state_t*,jit_int32_t,jit_int32_t,jit_int32_t);
1270 #define lti_u(r0,r1,i0)                 _lti_u(_jit,r0,r1,i0)
1271 static void _lti_u(jit_state_t*,jit_int32_t,jit_int32_t,jit_word_t);
1272 #define ler(r0,r1,r2)                   _ler(_jit,r0,r1,r2)
1273 static void _ler(jit_state_t*,jit_int32_t,jit_int32_t,jit_int32_t);
1274 #define lei(r0,r1,i0)                   _lei(_jit,r0,r1,i0)
1275 static void _lei(jit_state_t*,jit_int32_t,jit_int32_t,jit_word_t);
1276 #define ler_u(r0,r1,r2)                 _ler_u(_jit,r0,r1,r2)
1277 static void _ler_u(jit_state_t*,jit_int32_t,jit_int32_t,jit_int32_t);
1278 #define lei_u(r0,r1,i0)                 _lei_u(_jit,r0,r1,i0)
1279 static void _lei_u(jit_state_t*,jit_int32_t,jit_int32_t,jit_word_t);
1280 #define eqr(r0,r1,r2)                   _eqr(_jit,r0,r1,r2)
1281 static void _eqr(jit_state_t*,jit_int32_t,jit_int32_t,jit_int32_t);
1282 #define eqi(r0,r1,i0)                   _eqi(_jit,r0,r1,i0)
1283 static void _eqi(jit_state_t*,jit_int32_t,jit_int32_t,jit_word_t);
1284 #define ger(r0,r1,r2)                   _ger(_jit,r0,r1,r2)
1285 static void _ger(jit_state_t*,jit_int32_t,jit_int32_t,jit_int32_t);
1286 #define gei(r0,r1,i0)                   _gei(_jit,r0,r1,i0)
1287 static void _gei(jit_state_t*,jit_int32_t,jit_int32_t,jit_word_t);
1288 #define ger_u(r0,r1,r2)                 _ger_u(_jit,r0,r1,r2)
1289 static void _ger_u(jit_state_t*,jit_int32_t,jit_int32_t,jit_int32_t);
1290 #define gei_u(r0,r1,i0)                 _gei_u(_jit,r0,r1,i0)
1291 static void _gei_u(jit_state_t*,jit_int32_t,jit_int32_t,jit_word_t);
1292 #define gtr(r0,r1,r2)                   _gtr(_jit,r0,r1,r2)
1293 static void _gtr(jit_state_t*,jit_int32_t,jit_int32_t,jit_int32_t);
1294 #define gti(r0,r1,i0)                   _gti(_jit,r0,r1,i0)
1295 static void _gti(jit_state_t*,jit_int32_t,jit_int32_t,jit_word_t);
1296 #define gtr_u(r0,r1,r2)                 _gtr_u(_jit,r0,r1,r2)
1297 static void _gtr_u(jit_state_t*,jit_int32_t,jit_int32_t,jit_int32_t);
1298 #define gti_u(r0,r1,i0)                 _gti_u(_jit,r0,r1,i0)
1299 static void _gti_u(jit_state_t*,jit_int32_t,jit_int32_t,jit_word_t);
1300 #define ner(r0,r1,r2)                   _ner(_jit,r0,r1,r2)
1301 static void _ner(jit_state_t*,jit_int32_t,jit_int32_t,jit_int32_t);
1302 #define nei(r0,r1,i0)                   _nei(_jit,r0,r1,i0)
1303 static void _nei(jit_state_t*,jit_int32_t,jit_int32_t,jit_word_t);
1304 #define bitswap(r0, r1)                 _bitswap(_jit, r0, r1)
1305 static void _bitswap(jit_state_t*, jit_int32_t, jit_int32_t);
1306 #define clor(r0, r1)                    _clor(_jit, r0, r1)
1307 static void _clor(jit_state_t*, jit_int32_t, jit_int32_t);
1308 #define clzr(r0, r1)                    _clzr(_jit, r0, r1)
1309 static void _clzr(jit_state_t*, jit_int32_t, jit_int32_t);
1310 #define ctor(r0, r1)                    _ctor(_jit, r0, r1)
1311 static void _ctor(jit_state_t*, jit_int32_t, jit_int32_t);
1312 #define ctzr(r0, r1)                    _ctzr(_jit, r0, r1)
1313 static void _ctzr(jit_state_t*, jit_int32_t, jit_int32_t);
1314 #define negr(r0,r1)                     subr(r0,0,r1)
1315 #define comr(r0,r1)                     ANDCMI(r0,-1,r1)
1316 #define movr(r0,r1)                     _movr(_jit,r0,r1)
1317 static void _movr(jit_state_t*,jit_int32_t,jit_int32_t);
1318 #define movi(r0,i0)                     _movi(_jit,r0,i0)
1319 static void _movi(jit_state_t*,jit_int32_t,jit_word_t);
1320 #define movi_p(r0,i0)                   _movi_p(_jit,r0,i0)
1321 static jit_word_t _movi_p(jit_state_t*,jit_int32_t,jit_word_t);
1322 #  define movnr(r0,r1,r2)               _movnr(_jit,r0,r1,r2)
1323 static void _movnr(jit_state_t*,jit_int32_t,jit_int32_t,jit_int32_t);
1324 #  define movzr(r0,r1,r2)               _movzr(_jit,r0,r1,r2)
1325 static void _movzr(jit_state_t*,jit_int32_t,jit_int32_t,jit_int32_t);
1326 #  define casx(r0, r1, r2, r3, i0)      _casx(_jit, r0, r1, r2, r3, i0)
1327 static void _casx(jit_state_t *_jit,jit_int32_t,jit_int32_t,
1328                   jit_int32_t,jit_int32_t,jit_word_t);
1329 #define casr(r0, r1, r2, r3)            casx(r0, r1, r2, r3, 0)
1330 #define casi(r0, i0, r1, r2)            casx(r0, _NOREG, r1, r2, i0)
1331 #  define bswapr_us(r0,r1)              _bswapr_us(_jit,r0,r1)
1332 static void _bswapr_us(jit_state_t*,jit_int32_t,jit_int32_t);
1333 #  define bswapr_ui(r0,r1)              _bswapr_ui(_jit,r0,r1)
1334 static void _bswapr_ui(jit_state_t*,jit_int32_t,jit_int32_t);
1335 #  define bswapr_ul(r0,r1)              MUX1(r0,r1,MUX_REV)
1336 #define extr_c(r0,r1)                   SXT1(r0,r1)
1337 #define extr_uc(r0,r1)                  ZXT1(r0,r1)
1338 #define extr_s(r0,r1)                   SXT2(r0,r1)
1339 #define extr_us(r0,r1)                  ZXT2(r0,r1)
1340 #define extr_i(r0,r1)                   SXT4(r0,r1)
1341 #define extr_ui(r0,r1)                  ZXT4(r0,r1)
1342 #define bltr(i0,r0,r1)                  _bltr(_jit,i0,r0,r1)
1343 static jit_word_t _bltr(jit_state_t*,jit_word_t,jit_int32_t,jit_int32_t);
1344 #define blti(i0,r0,i1)                  _blti(_jit,i0,r0,i1)
1345 static jit_word_t _blti(jit_state_t*,jit_word_t,jit_int32_t,jit_word_t);
1346 #define bltr_u(i0,r0,r1)                _bltr_u(_jit,i0,r0,r1)
1347 static jit_word_t _bltr_u(jit_state_t*,jit_word_t,jit_int32_t,jit_int32_t);
1348 #define blti_u(i0,r0,i1)                _blti_u(_jit,i0,r0,i1)
1349 static jit_word_t _blti_u(jit_state_t*,jit_word_t,jit_int32_t,jit_word_t);
1350 #define bler(i0,r0,r1)                  _bler(_jit,i0,r0,r1)
1351 static jit_word_t _bler(jit_state_t*,jit_word_t,jit_int32_t,jit_int32_t);
1352 #define blei(i0,r0,i1)                  _blei(_jit,i0,r0,i1)
1353 static jit_word_t _blei(jit_state_t*,jit_word_t,jit_int32_t,jit_word_t);
1354 #define bler_u(i0,r0,r1)                _bler_u(_jit,i0,r0,r1)
1355 static jit_word_t _bler_u(jit_state_t*,jit_word_t,jit_int32_t,jit_int32_t);
1356 #define blei_u(i0,r0,i1)                _blei_u(_jit,i0,r0,i1)
1357 static jit_word_t _blei_u(jit_state_t*,jit_word_t,jit_int32_t,jit_word_t);
1358 #define beqr(i0,r0,r1)                  _beqr(_jit,i0,r0,r1)
1359 static jit_word_t _beqr(jit_state_t*,jit_word_t,jit_int32_t,jit_int32_t);
1360 #define beqi(i0,r0,i1)                  _beqi(_jit,i0,r0,i1)
1361 static jit_word_t _beqi(jit_state_t*,jit_word_t,jit_int32_t,jit_word_t);
1362 #define bger(i0,r0,r1)                  _bger(_jit,i0,r0,r1)
1363 static jit_word_t _bger(jit_state_t*,jit_word_t,jit_int32_t,jit_int32_t);
1364 #define bgei(i0,r0,i1)                  _bgei(_jit,i0,r0,i1)
1365 static jit_word_t _bgei(jit_state_t*,jit_word_t,jit_int32_t,jit_word_t);
1366 #define bger_u(i0,r0,r1)                _bger_u(_jit,i0,r0,r1)
1367 static jit_word_t _bger_u(jit_state_t*,jit_word_t,jit_int32_t,jit_int32_t);
1368 #define bgei_u(i0,r0,i1)                _bgei_u(_jit,i0,r0,i1)
1369 static jit_word_t _bgei_u(jit_state_t*,jit_word_t,jit_int32_t,jit_word_t);
1370 #define bgtr(i0,r0,r1)                  _bgtr(_jit,i0,r0,r1)
1371 static jit_word_t _bgtr(jit_state_t*,jit_word_t,jit_int32_t,jit_int32_t);
1372 #define bgti(i0,r0,i1)                  _bgti(_jit,i0,r0,i1)
1373 static jit_word_t _bgti(jit_state_t*,jit_word_t,jit_int32_t,jit_word_t);
1374 #define bgtr_u(i0,r0,r1)                _bgtr_u(_jit,i0,r0,r1)
1375 static jit_word_t _bgtr_u(jit_state_t*,jit_word_t,jit_int32_t,jit_int32_t);
1376 #define bgti_u(i0,r0,i1)                _bgti_u(_jit,i0,r0,i1)
1377 static jit_word_t _bgti_u(jit_state_t*,jit_word_t,jit_int32_t,jit_word_t);
1378 #define bner(i0,r0,r1)                  _bner(_jit,i0,r0,r1)
1379 static jit_word_t _bner(jit_state_t*,jit_word_t,jit_int32_t,jit_int32_t);
1380 #define bnei(i0,r0,i1)                  _bnei(_jit,i0,r0,i1)
1381 static jit_word_t _bnei(jit_state_t*,jit_word_t,jit_int32_t,jit_word_t);
1382 #define bmsr(i0,r0,r1)                  _bmsr(_jit,i0,r0,r1)
1383 static jit_word_t _bmsr(jit_state_t*,jit_word_t,jit_int32_t,jit_int32_t);
1384 #define bmsi(i0,r0,i1)                  _bmsi(_jit,i0,r0,i1)
1385 static jit_word_t _bmsi(jit_state_t*,jit_word_t,jit_int32_t,jit_word_t);
1386 #define bmcr(i0,r0,r1)                  _bmcr(_jit,i0,r0,r1)
1387 static jit_word_t _bmcr(jit_state_t*,jit_word_t,jit_int32_t,jit_int32_t);
1388 #define bmci(i0,r0,i1)                  _bmci(_jit,i0,r0,i1)
1389 static jit_word_t _bmci(jit_state_t*,jit_word_t,jit_int32_t,jit_word_t);
1390 #define baddr(i0,r0,r1,cc)              _baddr(_jit,i0,r0,r1,cc)
1391 static jit_word_t _baddr(jit_state_t*,jit_word_t,
1392                          jit_int32_t,jit_int32_t,jit_bool_t);
1393 #define baddi(i0,r0,i1,cc)              _baddi(_jit,i0,r0,i1,cc)
1394 static jit_word_t _baddi(jit_state_t*,jit_word_t,
1395                          jit_int32_t,jit_word_t,jit_bool_t);
1396 #define baddr_u(i0,r0,r1,cc)            _baddr_u(_jit,i0,r0,r1,cc)
1397 static jit_word_t _baddr_u(jit_state_t*,jit_word_t,
1398                            jit_int32_t,jit_int32_t,jit_bool_t);
1399 #define baddi_u(i0,r0,i1,cc)            _baddi_u(_jit,i0,r0,i1,cc)
1400 static jit_word_t _baddi_u(jit_state_t*,jit_word_t,
1401                            jit_int32_t,jit_word_t,jit_bool_t);
1402 #define bsubr(i0,r0,r1,cc)              _bsubr(_jit,i0,r0,r1,cc)
1403 static jit_word_t _bsubr(jit_state_t*,jit_word_t,
1404                          jit_int32_t,jit_int32_t,jit_bool_t);
1405 #define bsubi(i0,r0,i1,cc)              _bsubi(_jit,i0,r0,i1,cc)
1406 static jit_word_t _bsubi(jit_state_t*,jit_word_t,
1407                          jit_int32_t,jit_word_t,jit_bool_t);
1408 #define bsubr_u(i0,r0,r1,cc)            _bsubr_u(_jit,i0,r0,r1,cc)
1409 static jit_word_t _bsubr_u(jit_state_t*,jit_word_t,
1410                            jit_int32_t,jit_int32_t,jit_bool_t);
1411 #define bsubi_u(i0,r0,i1,cc)            _bsubi_u(_jit,i0,r0,i1,cc)
1412 static jit_word_t _bsubi_u(jit_state_t*,jit_word_t,
1413                            jit_int32_t,jit_word_t,jit_bool_t);
1414 #define boaddr(i0,r0,r1)                baddr(i0,r0,r1,1)
1415 #define boaddi(i0,r0,i1)                baddi(i0,r0,i1,1)
1416 #define boaddr_u(i0,r0,r1)              baddr_u(i0,r0,r1,1)
1417 #define boaddi_u(i0,r0,i1)              baddi_u(i0,r0,i1,1)
1418 #define bxaddr(i0,r0,r1)                baddr(i0,r0,r1,0)
1419 #define bxaddi(i0,r0,i1)                baddi(i0,r0,i1,0)
1420 #define bxaddr_u(i0,r0,r1)              baddr_u(i0,r0,r1,0)
1421 #define bxaddi_u(i0,r0,i1)              baddi_u(i0,r0,i1,0)
1422 #define bosubr(i0,r0,r1)                bsubr(i0,r0,r1,1)
1423 #define bosubi(i0,r0,i1)                bsubi(i0,r0,i1,1)
1424 #define bosubr_u(i0,r0,r1)              bsubr_u(i0,r0,r1,1)
1425 #define bosubi_u(i0,r0,i1)              bsubi_u(i0,r0,i1,1)
1426 #define bxsubr(i0,r0,r1)                bsubr(i0,r0,r1,0)
1427 #define bxsubi(i0,r0,i1)                bsubi(i0,r0,i1,0)
1428 #define bxsubr_u(i0,r0,r1)              bsubr_u(i0,r0,r1,0)
1429 #define bxsubi_u(i0,r0,i1)              bsubi_u(i0,r0,i1,0)
1430 #define ldr_c(r0,r1)                    _ldr_c(_jit,r0,r1)
1431 static void _ldr_c(jit_state_t*,jit_int32_t,jit_int32_t);
1432 #define ldi_c(r0,i0)                    _ldi_c(_jit,r0,i0)
1433 static void _ldi_c(jit_state_t*,jit_int32_t,jit_word_t);
1434 #define ldr_uc(r0,r1)                   LD1(r0,r1)
1435 #define ldi_uc(r0,i0)                   _ldi_uc(_jit,r0,i0)
1436 static void _ldi_uc(jit_state_t*,jit_int32_t,jit_word_t);
1437 #define ldr_s(r0,r1)                    _ldr_s(_jit,r0,r1)
1438 static void _ldr_s(jit_state_t*,jit_int32_t,jit_int32_t);
1439 #define ldi_s(r0,i0)                    _ldi_s(_jit,r0,i0)
1440 static void _ldi_s(jit_state_t*,jit_int32_t,jit_word_t);
1441 #define ldr_us(r0,r1)                   LD2(r0,r1)
1442 #define ldi_us(r0,i0)                   _ldi_us(_jit,r0,i0)
1443 static void _ldi_us(jit_state_t*,jit_int32_t,jit_word_t);
1444 #define ldr_i(r0,r1)                    _ldr_i(_jit,r0,r1)
1445 static void _ldr_i(jit_state_t*,jit_int32_t,jit_int32_t);
1446 #define ldi_i(r0,i0)                    _ldi_i(_jit,r0,i0)
1447 static void _ldi_i(jit_state_t*,jit_int32_t,jit_word_t);
1448 #define ldr_ui(r0,r1)                   LD4(r0,r1)
1449 #define ldi_ui(r0,i0)                   _ldi_ui(_jit,r0,i0)
1450 static void _ldi_ui(jit_state_t*,jit_int32_t,jit_word_t);
1451 #define ldr_l(r0,r1)                    LD8(r0,r1)
1452 #define ldi_l(r0,i0)                    _ldi_l(_jit,r0,i0)
1453 static void _ldi_l(jit_state_t*,jit_int32_t,jit_word_t);
1454 #define ldxr_c(r0,r1,r2)                _ldxr_c(_jit,r0,r1,r2)
1455 static void _ldxr_c(jit_state_t*,jit_int32_t,jit_int32_t,jit_int32_t);
1456 #define ldxi_c(r0,r1,i0)                _ldxi_c(_jit,r0,r1,i0)
1457 static void _ldxi_c(jit_state_t*,jit_int32_t,jit_int32_t,jit_word_t);
1458 #define ldxr_uc(r0,r1,r2)               _ldxr_uc(_jit,r0,r1,r2)
1459 static void _ldxr_uc(jit_state_t*,jit_int32_t,jit_int32_t,jit_int32_t);
1460 #define ldxi_uc(r0,r1,i0)               _ldxi_uc(_jit,r0,r1,i0)
1461 static void _ldxi_uc(jit_state_t*,jit_int32_t,jit_int32_t,jit_word_t);
1462 #define ldxr_s(r0,r1,r2)                _ldxr_s(_jit,r0,r1,r2)
1463 static void _ldxr_s(jit_state_t*,jit_int32_t,jit_int32_t,jit_int32_t);
1464 #define ldxi_s(r0,r1,i0)                _ldxi_s(_jit,r0,r1,i0)
1465 static void _ldxi_s(jit_state_t*,jit_int32_t,jit_int32_t,jit_word_t);
1466 #define ldxr_us(r0,r1,r2)               _ldxr_us(_jit,r0,r1,r2)
1467 static void _ldxr_us(jit_state_t*,jit_int32_t,jit_int32_t,jit_int32_t);
1468 #define ldxi_us(r0,r1,i0)               _ldxi_us(_jit,r0,r1,i0)
1469 static void _ldxi_us(jit_state_t*,jit_int32_t,jit_int32_t,jit_word_t);
1470 #define ldxr_i(r0,r1,r2)                _ldxr_i(_jit,r0,r1,r2)
1471 static void _ldxr_i(jit_state_t*,jit_int32_t,jit_int32_t,jit_int32_t);
1472 #define ldxi_i(r0,r1,i0)                _ldxi_i(_jit,r0,r1,i0)
1473 static void _ldxi_i(jit_state_t*,jit_int32_t,jit_int32_t,jit_word_t);
1474 #define ldxr_ui(r0,r1,r2)               _ldxr_ui(_jit,r0,r1,r2)
1475 static void _ldxr_ui(jit_state_t*,jit_int32_t,jit_int32_t,jit_int32_t);
1476 #define ldxi_ui(r0,r1,i0)               _ldxi_ui(_jit,r0,r1,i0)
1477 static void _ldxi_ui(jit_state_t*,jit_int32_t,jit_int32_t,jit_word_t);
1478 #define ldxr_l(r0,r1,r2)                _ldxr_l(_jit,r0,r1,r2)
1479 static void _ldxr_l(jit_state_t*,jit_int32_t,jit_int32_t,jit_int32_t);
1480 #define ldxi_l(r0,r1,i0)                _ldxi_l(_jit,r0,r1,i0)
1481 static void _ldxi_l(jit_state_t*,jit_int32_t,jit_int32_t,jit_word_t);
1482 #define str_c(r0,r1)                    ST1(r0,r1)
1483 #define sti_c(i0,r0)                    _sti_c(_jit,i0,r0)
1484 static void _sti_c(jit_state_t*,jit_word_t,jit_int32_t);
1485 #define str_s(r0,r1)                    ST2(r0,r1)
1486 #define sti_s(i0,r0)                    _sti_s(_jit,i0,r0)
1487 static void _sti_s(jit_state_t*,jit_word_t,jit_int32_t);
1488 #define str_i(r0,r1)                    ST4(r0,r1)
1489 #define sti_i(i0,r0)                    _sti_i(_jit,i0,r0)
1490 static void _sti_i(jit_state_t*,jit_word_t,jit_int32_t);
1491 #define str_l(r0,r1)                    ST8(r0,r1)
1492 #define sti_l(i0,r0)                    _sti_l(_jit,i0,r0)
1493 static void _sti_l(jit_state_t*,jit_word_t,jit_int32_t);
1494 #define stxr_c(r0,r1,r2)                _stxr_c(_jit,r0,r1,r2)
1495 static void _stxr_c(jit_state_t*,jit_int32_t,jit_int32_t,jit_int32_t);
1496 #define stxi_c(i0,r0,r1)                _stxi_c(_jit,i0,r0,r1)
1497 static void _stxi_c(jit_state_t*,jit_word_t,jit_int32_t,jit_int32_t);
1498 #define stxr_s(r0,r1,r2)                _stxr_s(_jit,r0,r1,r2)
1499 static void _stxr_s(jit_state_t*,jit_int32_t,jit_int32_t,jit_int32_t);
1500 #define stxi_s(i0,r0,r1)                _stxi_s(_jit,i0,r0,r1)
1501 static void _stxi_s(jit_state_t*,jit_word_t,jit_int32_t,jit_int32_t);
1502 #define stxr_i(r0,r1,r2)                _stxr_i(_jit,r0,r1,r2)
1503 static void _stxr_i(jit_state_t*,jit_int32_t,jit_int32_t,jit_int32_t);
1504 #define stxi_i(i0,r0,r1)                _stxi_i(_jit,i0,r0,r1)
1505 static void _stxi_i(jit_state_t*,jit_word_t,jit_int32_t,jit_int32_t);
1506 #define stxr_l(r0,r1,r2)                _stxr_l(_jit,r0,r1,r2)
1507 static void _stxr_l(jit_state_t*,jit_int32_t,jit_int32_t,jit_int32_t);
1508 #define stxi_l(i0,r0,r1)                _stxi_l(_jit,i0,r0,r1)
1509 static void _stxi_l(jit_state_t*,jit_word_t,jit_int32_t,jit_int32_t);
1510 #define jmpr(r0)                        _jmpr(_jit,r0)
1511 static void _jmpr(jit_state_t*,jit_int32_t);
1512 #define jmpi(i0)                        _jmpi(_jit,i0)
1513 static jit_word_t _jmpi(jit_state_t*,jit_word_t);
1514 #define jmpi_p(i0)                      _jmpi_p(_jit,i0)
1515 static jit_word_t _jmpi_p(jit_state_t*,jit_word_t);
1516 #define callr(r0)                       _callr(_jit,r0)
1517 static void _callr(jit_state_t*,jit_int32_t);
1518 #define calli(i0)                       _calli(_jit,i0)
1519 static void _calli(jit_state_t*,jit_word_t);
1520 #define calli_p(i0)                     _calli_p(_jit,i0)
1521 static jit_word_t _calli_p(jit_state_t*,jit_word_t);
1522 #define prolog(node)                    _prolog(_jit,node)
1523 static void _prolog(jit_state_t*,jit_node_t*);
1524 #define epilog(node)                    _epilog(_jit,node)
1525 static void _epilog(jit_state_t*,jit_node_t*);
1526 #  define vastart(r0)                   _vastart(_jit, r0)
1527 static void _vastart(jit_state_t*, jit_int32_t);
1528 #  define vaarg(r0, r1)                 _vaarg(_jit, r0, r1)
1529 static void _vaarg(jit_state_t*, jit_int32_t, jit_int32_t);
1530 #define patch_at(code,instr,label)      _patch_at(_jit,code,instr,label)
1531 static void _patch_at(jit_state_t*,jit_code_t,jit_word_t,jit_word_t);
1532 #endif
1533
1534 #if CODE
1535 #if __BYTE_ORDER == __BIG_ENDIAN
1536 static jit_word_t
1537 byte_swap_if_big_endian(jit_word_t w)
1538 {
1539     union {
1540         char    c[8];
1541         long    w;
1542     } l, h;
1543     l.w = w;
1544     h.c[0] = l.c[7];
1545     h.c[1] = l.c[6];
1546     h.c[2] = l.c[5];
1547     h.c[3] = l.c[4];
1548     h.c[4] = l.c[3];
1549     h.c[5] = l.c[2];
1550     h.c[6] = l.c[1];
1551     h.c[7] = l.c[0];
1552     return (h.w);
1553 }
1554 #else
1555 #define byte_swap_if_big_endian(w)              (w)
1556 #endif
1557
1558 static void
1559 _out(jit_state_t *_jit, int n, int tm,
1560      jit_word_t s0, jit_word_t s1, jit_word_t s2)
1561 {
1562     int                  i;
1563     jit_word_t           l, h, *w;
1564     set_bundle(_jit->pc.ul, l, h, tm, s0, s1, s2);
1565     _jit->pc.ul += 2;
1566     w = (jit_word_t *)_jitc->inst;
1567     for (i = n; i < _jitc->ioff; i++)
1568         w[i - n] = w[i];
1569     _jitc->ioff -= n;
1570 }
1571
1572 #define nop_m           0x0008000000L
1573 #define nop_i           0x0008000000L
1574 #define nop_b           0x4000000000L
1575 static void
1576 _stop(jit_state_t *_jit)
1577 {
1578     /* Clear set of live registers */
1579     jit_regset_set_ui(&_jitc->regs, 0);
1580     _jitc->pred = 0;
1581     /* Flag a stop is required */
1582     if (_jitc->ioff)
1583         _jitc->inst[_jitc->ioff - 1].t |= INST_STOP;
1584     else
1585         inst(nop_m, INST_Ms);
1586 }
1587
1588 static void
1589 _sync(jit_state_t *_jit)
1590 {
1591     /* Taken branches are supposed to not need a stop, so, it
1592      * should not be required to stop if no registers live in
1593      * sequential code */
1594     if (jit_regset_cmp_ui(&_jitc->regs, 0) != 0 || _jitc->pred)
1595         stop();
1596     do
1597         flush();
1598     while (_jitc->ioff);
1599 }
1600
1601 #define A_0             INST_A
1602 #define As0             INST_As
1603 #define I_0             INST_I
1604 #define Is0             INST_Is
1605 #define M_0             INST_M
1606 #define Ms0             INST_Ms
1607 #define F_0             INST_F
1608 #define Fs0             INST_Fs
1609 #define B_0             INST_B
1610 #define Bs0             INST_Bs
1611 #define L_0             INST_L
1612 #define Ls0             INST_Ls
1613 #define X_0             INST_X
1614 #define Xs0             INST_Xs
1615 #define A_1             (INST_A<<4)
1616 #define As1             (INST_As<<4)
1617 #define I_1             (INST_I<<4)
1618 #define Is1             (INST_Is<<4)
1619 #define M_1             (INST_M<<4)
1620 #define Ms1             (INST_Ms<<4)
1621 #define F_1             (INST_F<<4)
1622 #define Fs1             (INST_Fs<<4)
1623 #define B_1             (INST_B<<4)
1624 #define Bs1             (INST_Bs<<4)
1625 #define L_1             (INST_L<<4)
1626 #define Ls1             (INST_Ls<<4)
1627 #define X_1             (INST_X<<4)
1628 #define Xs1             (INST_Xs<<4)
1629 #define A_2             (INST_A<<8)
1630 #define As2             (INST_As<<8)
1631 #define I_2             (INST_I<<8)
1632 #define Is2             (INST_Is<<8)
1633 #define M_2             (INST_M<<8)
1634 #define Ms2             (INST_Ms<<8)
1635 #define F_2             (INST_F<<8)
1636 #define Fs2             (INST_Fs<<8)
1637 #define B_2             (INST_B<<8)
1638 #define Bs2             (INST_Bs<<8)
1639 #define L_2             (INST_L<<8)
1640 #define Ls2             (INST_Ls<<8)
1641 #define X_2             (INST_X<<8)
1642 #define Xs2             (INST_Xs<<8)
1643
1644 #define I_              I_0
1645 #define I_I_            I_0|I_1
1646 #define I_Is            I_0|Is1
1647 #define I_B_            I_0|B_1
1648 #define I_Bs            I_0|Bs1
1649 #define Is              Is0
1650 #define IsI_            Is0|I_1
1651 #define IsIs            Is0|Is1
1652 #define M_              M_0
1653 #define M_I_            M_0|I_1
1654 #define M_Is            M_0|Is1
1655 #define M_M_            M_0|M_1
1656 #define M_Ms            M_0|Ms1
1657 #define M_F_            M_0|F_1
1658 #define M_Fs            M_0|Fs1
1659 #define M_B_            M_0|B_1
1660 #define M_Bs            M_0|Bs1
1661 #define M_I_I_          M_0|I_1|I_2
1662 #define M_I_Is          M_0|I_1|Is2
1663 #define M_I_B_          M_0|I_1|B_2
1664 #define M_I_Bs          M_0|I_1|Bs2
1665 #define M_IsI_          M_0|Is1|I_2
1666 #define M_IsIs          M_0|Is1|Is2
1667 #define M_M_I_          M_0|M_1|I_2
1668 #define M_M_Is          M_0|M_1|Is2
1669 #define M_M_F_          M_0|M_1|F_2
1670 #define M_M_Fs          M_0|M_1|Fs2
1671 #define M_M_B_          M_0|M_1|B_2
1672 #define M_M_Bs          M_0|M_1|Bs2
1673 #define M_F_I_          M_0|F_1|I_2
1674 #define M_F_Is          M_0|F_1|Is2
1675 #define M_F_B_          M_0|F_1|B_2
1676 #define M_F_Bs          M_0|F_1|Bs2
1677 #define M_B_B_          M_0|B_1|B_2
1678 #define M_B_Bs          M_0|B_1|Bs2
1679 #define M_L_X_          M_0|L_1|X_2
1680 #define M_L_Xs          M_0|L_1|Xs2
1681 #define Ms              Ms0
1682 #define MsI_            Ms0|I_1
1683 #define MsIs            Ms0|Is1
1684 #define MsM_            Ms0|M_1
1685 #define MsMs            Ms0|Ms1
1686 #define MsM_I_          Ms0|M_1|I_2
1687 #define MsM_Is          Ms0|M_1|Is2
1688 #define F_              F_0
1689 #define F_I_            F_0|I_1
1690 #define F_Is            F_0|Is1
1691 #define F_B_            F_0|B_1
1692 #define F_Bs            F_0|Bs1
1693 #define Fs              Fs0
1694 #define B_              B_0
1695 #define B_B_            B_0|B_1
1696 #define B_Bs            B_0|Bs1
1697 #define B_B_B_          B_0|B_1|B_2
1698 #define B_B_Bs          B_0|B_1|Bs2
1699 #define Bs              Bs0
1700 #define L_X_            L_0|X_1
1701 #define L_Xs            L_0|Xs1
1702
1703 static jit_word_t
1704 templat(jit_word_t cc)
1705 {
1706     switch (cc) {
1707         case I_:
1708         case I_I_:      case I_Is:
1709         case I_B_:      case I_Bs:
1710         case Is:
1711         case IsI_:      case IsIs:
1712         case M_:
1713         case M_I_:      case M_Is:
1714         case M_M_:      case M_Ms:
1715         case M_F_:      case M_Fs:
1716         case M_B_:      case M_Bs:
1717         case M_I_I_:    case M_I_Is:
1718         case M_I_B_:    case M_I_Bs:
1719         case M_IsI_:    case M_IsIs:
1720         case M_M_I_:    case M_M_Is:
1721         case M_M_F_:    case M_M_Fs:
1722         case M_M_B_:    case M_M_Bs:
1723         case M_F_I_:    case M_F_Is:
1724         case M_F_B_:    case M_F_Bs:
1725         case M_B_B_:    case M_B_Bs:
1726         case M_L_X_:    case M_L_Xs:
1727         case Ms:
1728         case MsI_:      case MsIs:
1729         case MsM_:      case MsMs:
1730         case MsM_I_:    case MsM_Is:
1731         case F_:
1732         case F_I_:      case F_Is:
1733         case F_B_:      case F_Bs:
1734         case Fs:
1735         case B_:
1736         case B_B_:      case B_Bs:
1737         case B_B_B_:    case B_B_Bs:
1738         case Bs:
1739         case L_X_:      case L_Xs:
1740             return (cc);
1741         default:
1742             return (0);
1743     }
1744 }
1745
1746 /* match* functions recurse attempting to find a template for A-
1747  * instructions, that may be executed in M- or I- unit.
1748  * It also uses an heuristic of trying first M- for slot 0 and 2,
1749  * and I- for slot 1, but tries all possible matches.
1750  */
1751 static jit_word_t
1752 match_2(jit_word_t cc)
1753 {
1754     jit_word_t          t;
1755     if ((t = templat(cc | M_2)))
1756         return (t);
1757     if ((t = templat(cc | I_2)))
1758         return (t);
1759     return (0);
1760 }
1761
1762 static jit_word_t
1763 matchs2(jit_word_t cc)
1764 {
1765     jit_word_t          t;
1766     if ((t = templat(cc | Ms2)))
1767         return (t);
1768     if ((t = templat(cc | Is2)))
1769         return (t);
1770     return (0);
1771 }
1772
1773 static jit_word_t
1774 match2(jit_word_t cc)
1775 {
1776     if ((cc & 0xf00) == A_2)
1777         return (match_2(cc & ~0xf00));
1778     if ((cc & 0xf00) == As2)
1779         return (matchs2(cc & ~0xf00));
1780     return (0);
1781 }
1782
1783 static jit_word_t
1784 match_1(jit_word_t cc)
1785 {
1786     jit_word_t          t;
1787     if ((t = templat(cc | I_1)))
1788         return (t);
1789     if ((t = templat(cc | M_1)))
1790         return (t);
1791     if ((t = match2(cc | I_1)))
1792         return (t);
1793     if ((t = match2(cc | M_1)))
1794         return (t);
1795     return (0);
1796 }
1797
1798 static jit_word_t
1799 matchs1(jit_word_t cc)
1800 {
1801     jit_word_t          t;
1802     if ((t = templat(cc | Is1)))
1803         return (t);
1804     if ((t = templat(cc | Ms1)))
1805         return (t);
1806     if ((t = match2(cc | Is1)))
1807         return (t);
1808     if ((t = match2(cc | Ms1)))
1809         return (t);
1810     return (0);
1811 }
1812
1813 static jit_word_t
1814 match1(jit_word_t cc)
1815 {
1816     if ((cc & 0x0f0) == A_1)
1817         return (match_1(cc & ~0x0f0));
1818     if ((cc & 0x0f0) == As1)
1819         return (matchs1(cc & ~0x0f0));
1820     return (0);
1821 }
1822
1823 static jit_word_t
1824 match_0(jit_word_t cc)
1825 {
1826     jit_word_t          t;
1827     if ((t = templat(cc | M_0)))
1828         return (t);
1829     if ((t = templat(cc | I_0)))
1830         return (t);
1831     if ((t = match1(cc | M_0)))
1832         return (t);
1833     if ((t = match1(cc | I_0)))
1834         return (t);
1835     return (0);
1836 }
1837
1838 static jit_word_t
1839 matchs0(jit_word_t cc)
1840 {
1841     jit_word_t          t;
1842     if ((t = templat(cc | Ms0)))
1843         return (t);
1844     if ((t = templat(cc | Is0)))
1845         return (t);
1846     if ((t = match1(cc | Ms0)))
1847         return (t);
1848     if ((t = match1(cc | Is0)))
1849         return (t);
1850     return (0);
1851 }
1852
1853 static jit_word_t
1854 match0(jit_word_t cc)
1855 {
1856     if ((cc & 0x00f) == A_0)
1857         return (match_0(cc & ~0x00f));
1858     if ((cc & 0x00f) == As0)
1859         return (matchs0(cc & ~0x00f));
1860     return (0);
1861 }
1862
1863 static void
1864 _flush(jit_state_t *_jit)
1865 {
1866     int                 n, soff;
1867     jit_word_t          t, cc, tm, s0, s1, s2;
1868
1869     if (!_jitc->ioff)
1870         return;
1871     for (cc = 0, n = soff = 0; n < _jitc->ioff; n++, soff += 4)
1872         cc |= (jit_uword_t)(_jitc->inst[n].t) << soff;
1873
1874     soff = 0xf00;
1875     while (soff) {
1876         /* Try to find a template, or reduce down
1877          * to one instruction if no template match */
1878         if ((t = templat(cc))) {
1879             cc = t;
1880             break;
1881         }
1882         /* A- instructions may be execute in M- or I- unit */
1883         if ((t = match0(cc))) {
1884             cc = t;
1885             break;
1886         }
1887         cc &= ~soff;
1888         soff >>= 4;
1889     }
1890     assert(soff);
1891
1892     /* Prefer tail nop if need to add some nop, so that patching is easier */
1893 #define ii(n)           _jitc->inst[n].i
1894     switch (cc) {
1895         case I_:
1896             n = 1;              tm = TM_M_I_I_;
1897             s0 = nop_m;         s1 = ii(0);             s2 = nop_i;
1898             break;
1899         case I_I_:
1900             n = 2;              tm = TM_M_I_I_;
1901             s0 = nop_m;         s1 = ii(0);             s2 = ii(1);
1902             break;
1903         case I_Is:
1904             n = 2;              tm = TM_M_I_Is;
1905             s0 = nop_m;         s1 = ii(0);             s2 = ii(1);
1906             break;
1907         case I_B_:
1908             n = 2;              tm = TM_M_I_B_;
1909             s0 = nop_m;         s1 = ii(0);             s2 = ii(1);
1910             break;
1911         case I_Bs:
1912             n = 2;              tm = TM_M_I_Bs;
1913             s0 = nop_m;         s1 = ii(0);             s2 = ii(1);
1914             break;
1915         case Is:
1916             n = 1;              tm = TM_M_IsI_;
1917             s0 = nop_m;         s1 = ii(0);             s2 = nop_i;
1918             break;
1919         case IsI_:
1920             n = 2;              tm = TM_M_IsI_;
1921             s0 = nop_m;         s1 = ii(0);             s2 = ii(1);
1922             break;
1923         case IsIs:
1924             n = 2;              tm = TM_M_IsIs;
1925             s0 = nop_m;         s1 = ii(0);             s2 = ii(1);
1926             break;
1927         case M_:
1928             n = 1;              tm = TM_M_I_I_;
1929             s0 = ii(0);         s1 = nop_i;             s2 = nop_i;
1930             break;
1931         case M_I_:
1932             n = 2;              tm = TM_M_I_I_;
1933             s0 = ii(0);         s1 = ii(1);             s2 = nop_i;
1934             break;
1935         case M_Is:
1936             n = 2;              tm = TM_M_IsI_;
1937             s0 = ii(0);         s1 = ii(1);             s2 = nop_i;
1938             break;
1939         case M_M_:
1940             n = 2;              tm = TM_M_M_I_;
1941             s0 = ii(0);         s1 = ii(1);             s2 = nop_i;
1942             break;
1943         case M_Ms:
1944             n = 2;              tm = TM_M_M_Is;
1945             s0 = ii(0);         s1 = ii(1);             s2 = nop_i;
1946             break;
1947         case M_F_:
1948             n = 2;              tm = TM_M_F_I_;
1949             s0 = ii(0);         s1 = ii(1);             s2 = nop_i;
1950             break;
1951         case M_Fs:
1952             n = 2;              tm = TM_M_F_Is;
1953             s0 = ii(0);         s1 = ii(1);             s2 = nop_i;
1954             break;
1955         case M_B_:
1956             n = 2;              tm = TM_M_B_B_;
1957             s0 = ii(0);         s1 = ii(1);             s2 = nop_b;
1958             break;
1959         case M_Bs:
1960             n = 2;              tm = TM_M_B_Bs;
1961             s0 = ii(0);         s1 = ii(1);             s2 = nop_b;
1962             break;
1963         case M_I_I_:
1964             n = 3;              tm = TM_M_I_I_;
1965             s0 = ii(0);         s1 = ii(1);             s2 = ii(2);
1966             break;
1967         case M_I_Is:
1968             n = 3;              tm = TM_M_I_Is;
1969             s0 = ii(0);         s1 = ii(1);             s2 = ii(2);
1970             break;
1971         case M_I_B_:
1972             n = 3;              tm = TM_M_I_B_;
1973             s0 = ii(0);         s1 = ii(1);             s2 = ii(2);
1974             break;
1975         case M_I_Bs:
1976             n = 3;              tm = TM_M_I_Bs;
1977             s0 = ii(0);         s1 = ii(1);             s2 = ii(2);
1978             break;
1979         case M_IsI_:
1980             n = 3;              tm = TM_M_IsI_;
1981             s0 = ii(0);         s1 = ii(1);             s2 = ii(2);
1982             break;
1983         case M_IsIs:
1984             n = 3;              tm = TM_M_IsIs;
1985             s0 = ii(0);         s1 = ii(1);             s2 = ii(2);
1986             break;
1987         case M_M_I_:
1988             n = 3;              tm = TM_M_M_I_;
1989             s0 = ii(0);         s1 = ii(1);             s2 = ii(2);
1990             break;
1991         case M_M_Is:
1992             n = 3;              tm = TM_M_M_Is;
1993             s0 = ii(0);         s1 = ii(1);             s2 = ii(2);
1994             break;
1995         case M_M_F_:
1996             n = 3;              tm = TM_M_M_F_;
1997             s0 = ii(0);         s1 = ii(1);             s2 = ii(2);
1998             break;
1999         case M_M_Fs:
2000             n = 3;              tm = TM_M_M_Fs;
2001             s0 = ii(0);         s1 = ii(1);             s2 = ii(2);
2002             break;
2003         case M_M_B_:
2004             n = 3;              tm = TM_M_M_B_;
2005             s0 = ii(0);         s1 = ii(1);             s2 = ii(2);
2006             break;
2007         case M_M_Bs:
2008             n = 3;              tm = TM_M_M_Bs;
2009             s0 = ii(0);         s1 = ii(1);             s2 = ii(2);
2010             break;
2011         case M_F_I_:
2012             n = 3;              tm = TM_M_F_I_;
2013             s0 = ii(0);         s1 = ii(1);             s2 = ii(2);
2014             break;
2015         case M_F_Is:
2016             n = 3;              tm = TM_M_F_Is;
2017             s0 = ii(0);         s1 = ii(1);             s2 = ii(2);
2018             break;
2019         case M_F_B_:
2020             n = 3;              tm = TM_M_F_B_;
2021             s0 = ii(0);         s1 = ii(1);             s2 = ii(2);
2022             break;
2023         case M_F_Bs:
2024             n = 3;              tm = TM_M_F_Bs;
2025             s0 = ii(0);         s1 = ii(1);             s2 = ii(2);
2026             break;
2027         case M_B_B_:
2028             n = 3;              tm = TM_M_B_B_;
2029             s0 = ii(0);         s1 = ii(1);             s2 = ii(2);
2030             break;
2031         case M_B_Bs:
2032             n = 3;              tm = TM_M_B_Bs;
2033             s0 = ii(0);         s1 = ii(1);             s2 = ii(2);
2034             break;
2035         case M_L_X_:
2036             n = 3;              tm = TM_M_L_X_;
2037             s0 = ii(0);         s1 = ii(1);             s2 = ii(2);
2038             break;
2039         case M_L_Xs:
2040             n = 3;              tm = TM_M_L_Xs;
2041             s0 = ii(0);         s1 = ii(1);             s2 = ii(2);
2042             break;
2043         case Ms:
2044             n = 1;              tm = TM_MsM_I_;
2045             s0 = ii(0);         s1 = nop_m;             s2 = nop_i;
2046             break;
2047         case MsI_:
2048             n = 2;              tm = TM_MsM_I_;
2049             s0 = ii(0);         s1 = nop_m;             s2 = ii(1);
2050             break;
2051         case MsIs:
2052             n = 2;              tm = TM_MsM_Is;
2053             s0 = ii(0);         s1 = nop_m;             s2 = ii(1);
2054             break;
2055         case MsM_:
2056             n = 2;              tm = TM_MsM_I_;
2057             s0 = ii(0);         s1 = ii(1);             s2 = nop_i;
2058             break;
2059         case MsMs:
2060             n = 2;              tm = TM_MsM_Is;
2061             s0 = ii(0);         s1 = ii(1);             s2 = nop_i;
2062             break;
2063         case MsM_I_:
2064             n = 3;              tm = TM_MsM_I_;
2065             s0 = ii(0);         s1 = ii(1);             s2 = ii(2);
2066             break;
2067         case MsM_Is:
2068             n = 3;              tm = TM_MsM_Is;
2069             s0 = ii(0);         s1 = ii(1);             s2 = ii(2);
2070             break;
2071         case F_:
2072             n = 1;              tm = TM_M_F_I_;
2073             s0 = nop_m;         s1 = ii(0);             s2 = nop_i;
2074             break;
2075         case F_I_:
2076             n = 2;              tm = TM_M_F_I_;
2077             s0 = nop_m;         s1 = ii(0);             s2 = ii(1);
2078             break;
2079         case F_Is:
2080             n = 2;              tm = TM_M_F_Is;
2081             s0 = nop_m;         s1 = ii(0);             s2 = ii(1);
2082             break;
2083         case F_B_:
2084             n = 2;              tm = TM_M_F_B_;
2085             s0 = nop_m;         s1 = ii(0);             s2 = ii(1);
2086             break;
2087         case F_Bs:
2088             n = 2;              tm = TM_M_F_Bs;
2089             s0 = nop_m;         s1 = ii(0);             s2 = ii(1);
2090             break;
2091         case Fs:
2092             n = 1;              tm = TM_M_F_Is;
2093             s0 = nop_m;         s1 = ii(0);             s2 = nop_i;
2094             break;
2095         case B_:
2096             n = 1;              tm = TM_B_B_B_;
2097             s0 = ii(0);         s1 = nop_b;             s2 = nop_b;
2098             break;
2099         case B_B_:
2100             n = 2;              tm = TM_B_B_B_;
2101             s0 = ii(0);         s1 = ii(1);             s2 = nop_b;
2102             break;
2103         case B_Bs:
2104             n = 2;              tm = TM_B_B_Bs;
2105             s0 = ii(0);         s1 = ii(1);             s2 = nop_b;
2106             break;
2107         case B_B_B_:
2108             n = 3;              tm = TM_B_B_B_;
2109             s0 = ii(0);         s1 = ii(1);             s2 = ii(2);
2110             break;
2111         case B_B_Bs:
2112             n = 3;              tm = TM_B_B_Bs;
2113             s0 = ii(0);         s1 = ii(1);             s2 = ii(2);
2114             break;
2115         case Bs:
2116             n = 1;              tm = TM_B_B_Bs;
2117             s0 = ii(0);         s1 = nop_b;             s2 = nop_b;
2118             break;
2119         case L_X_:
2120             n = 2;              tm = TM_M_L_X_;
2121             s0 = nop_m;         s1 = ii(0);             s2 = ii(1);
2122             break;
2123         case L_Xs:
2124             n = 2;              tm = TM_M_L_Xs;
2125             s0 = nop_m;         s1 = ii(0);             s2 = ii(1);
2126             break;
2127         default:
2128             abort();
2129     }
2130     out(n, tm, s0, s1, s2);
2131 }
2132
2133 static void
2134 _inst(jit_state_t *_jit, jit_word_t i, jit_uint8_t t)
2135 {
2136     if (_jitc->ioff > 2)
2137         flush();
2138     assert(!(i & 0x11111e0000000000L));
2139     _jitc->inst[_jitc->ioff].i = i;
2140     _jitc->inst[_jitc->ioff].t = t;
2141     ++_jitc->ioff;
2142 }
2143
2144 static void
2145 _A1(jit_state_t *_jit, jit_word_t _p,
2146     jit_word_t x4, jit_word_t x2, jit_word_t r3, jit_word_t r2, jit_word_t r1)
2147 {
2148     assert(!(_p & ~0x3fL));
2149     assert(!(x4 &  ~0xfL));
2150     assert(!(x2 &  ~0x3L));
2151     assert(!(r3 & ~0x7fL));
2152     assert(!(r2 & ~0x7fL));
2153     assert(!(r1 & ~0x7fL));
2154     TSTREG2(r2, r3);
2155     TSTPRED(_p);
2156     TSTREG1(r1);
2157     inst((8L<<37)|(x4<<29)|(x2<<27)|(r3<<20)|(r2<<13)|(r1<<6)|_p, INST_A);
2158     SETREG(r1);
2159 }
2160
2161 static void
2162 _A3(jit_state_t *_jit, jit_word_t _p,
2163     jit_word_t x4, jit_word_t x2, jit_word_t r3, jit_word_t im, jit_word_t r1)
2164 {
2165     assert(!(_p & ~0x3fL));
2166     assert(!(x4 &  ~0xfL));
2167     assert(!(x2 &  ~0x3L));
2168     assert(!(r3 & ~0x7fL));
2169     assert(im >= -128 && im <= 127);
2170     assert(!(r1 & ~0x7f));
2171     TSTREG1(r3);
2172     TSTPRED(_p);
2173     TSTREG1(r1);
2174     inst((8L<<37)|(((im>>7)&1L)<<36)|(x4<<29)|(x2<<27)|
2175          (r3<<20)|((im&0x7fL)<<13)|(r1<<6)|_p, INST_A);
2176     SETREG(r1);
2177 }
2178
2179 static void
2180 _A4(jit_state_t *_jit, jit_word_t _p,
2181     jit_word_t x2, jit_word_t r3, jit_word_t im, jit_word_t r1)
2182 {
2183     assert(!(_p & ~0x3fL));
2184     assert(!(x2 &  ~0x3L));
2185     assert(!(r3 & ~0x7fL));
2186     assert(im >= -8192 && im <= 8191);
2187     assert(!(r1  & ~0x7f));
2188     TSTREG1(r3);
2189     TSTPRED(_p);
2190     TSTREG1(r1);
2191     inst((8L<<37)|(((im>>13)&1L)<<36)|(x2<<34)|(((im>>7)&0x3fL)<<27)|
2192          (r3<<20)|((im&0x7fL)<<13)|(r1<<6)|_p, INST_A);
2193     SETREG(r1);
2194 }
2195
2196 static void
2197 _A5(jit_state_t *_jit, jit_word_t _p,
2198     jit_word_t r3, jit_word_t im, jit_word_t r1)
2199 {
2200     jit_word_t          s, i5, i9, i7;
2201     assert(!(_p & ~0x3fL));
2202     assert(!(r3  & ~0x3L));
2203     assert(im >= -2097152 && im <= 2097151);
2204     assert(!(r1  & ~0x7fL));
2205     /* imm22 = sign_ext(s << 21 | imm5c << 16 | imm9d << 7 | imm7b, 22) */
2206     s  = (im & 0x200000) >> 21;
2207     i5 = (im & 0x1f0000) >> 16;
2208     i9 = (im &   0xff80) >>  7;
2209     i7 =  im &     0x7f;
2210     TSTREG1(r3);
2211     TSTPRED(_p);
2212     TSTREG1(r1);
2213     inst((9L<<37)|(s<<36)|(i9<<27)|(i5<<22)|
2214          (r3<<20)|(i7<<13)|(r1<<6)|_p, INST_A);
2215     SETREG(r1);
2216 }
2217
2218 static void
2219 _A6(jit_state_t *_jit, jit_word_t _p,
2220     jit_word_t o, jit_word_t x2, jit_word_t ta, jit_word_t p2,
2221     jit_word_t r3, jit_word_t r2, jit_word_t c, jit_word_t p1)
2222 {
2223     assert(!(_p & ~0x3fL));
2224     assert(!(o  &  ~0xfL));
2225     assert(!(x2 &  ~0x3L));
2226     assert(!(ta &  ~0x1L));
2227     assert(!(p2 & ~0x7fL));
2228     assert(!(r3 & ~0x7fL));
2229     assert(!(r2 & ~0x7fL));
2230     assert(!(c  &  ~0x1L));
2231     assert(!(p1 & ~0x3fL));
2232     TSTREG2(r2, r3);
2233     TSTPRED(_p);
2234     inst((o<<37)|(x2<<34)|(ta<<33)|(p2<<27)|(r3<<20)|
2235          (r2<<13)|(c<<12)|(p1<<6)|_p, INST_A);
2236     if (p1)
2237         _jitc->pred |= 1 << p1;
2238     if (p2)
2239         _jitc->pred |= 1 << p2;
2240 }
2241
2242 static void
2243 _A7(jit_state_t *_jit, jit_word_t _p,
2244     jit_word_t o, jit_word_t x2, jit_word_t ta,
2245     jit_word_t p2, jit_word_t r3, jit_word_t c, jit_word_t p1)
2246 {
2247     assert(!(_p & ~0x3fL));
2248     assert(!(o  &  ~0xfL));
2249     assert(!(x2 &  ~0x3L));
2250     assert(!(ta &  ~0x1L));
2251     assert(!(p2 & ~0x7fL));
2252     assert(!(r3 & ~0x7fL));
2253     assert(!(c  &  ~0x1L));
2254     assert(!(p1 & ~0x3fL));
2255     TSTREG1(r3);
2256     TSTPRED(_p);
2257     inst((o<<37)|(1L<<36)|(x2<<34)|(ta<<33)|
2258          (p2<<27)|(r3<<20)|(c<<12)|(p1<<6)|_p, INST_A);
2259     if (p1)
2260         _jitc->pred |= 1 << p1;
2261     if (p2)
2262         _jitc->pred |= 1 << p2;
2263 }
2264
2265 static void
2266 _A8(jit_state_t *_jit, jit_word_t _p,
2267     jit_word_t o, jit_word_t x2, jit_word_t ta, jit_word_t p2,
2268     jit_word_t r3, jit_word_t im, jit_word_t c, jit_word_t p1)
2269 {
2270     assert(!(_p & ~0x3fL));
2271     assert(!(o  &  ~0xfL));
2272     assert(!(x2 &  ~0x3L));
2273     assert(!(ta &  ~0x1L));
2274     assert(!(p2 & ~0x7fL));
2275     assert(!(r3 & ~0x7fL));
2276     assert(im >= -128 && im <= 127);
2277     assert(!(c  &  ~0x1L));
2278     assert(!(p1 & ~0x3fL));
2279     TSTREG1(r3);
2280     TSTPRED(_p);
2281     inst((o<<37)|(((im>>7)&1L)<<36)|(x2<<34)|(ta<<33)|(p2<<27)|(r3<<20)|
2282          ((im&0x7fL)<<13)|(c<<12)|(p1<<6)|_p, INST_A);
2283     if (p1)
2284         _jitc->pred |= 1 << p1;
2285     if (p2)
2286         _jitc->pred |= 1 << p2;
2287 }
2288
2289 static void
2290 _A9(jit_state_t *_jit, jit_word_t _p,
2291     jit_word_t za, jit_word_t zb, jit_word_t x4,
2292     jit_word_t x2, jit_word_t r3, jit_word_t r2, jit_word_t r1)
2293 {
2294     assert(!(_p & ~0x3fL));
2295     assert(!(za &  ~0x1L));
2296     assert(!(zb &  ~0x1L));
2297     assert(!(x4 &  ~0xfL));
2298     assert(!(x2 &  ~0x3L));
2299     assert(!(r3 & ~0x7fL));
2300     assert(!(r2 & ~0x7fL));
2301     assert(!(r1 & ~0x7fL));
2302     TSTREG2(r2, r3);
2303     TSTPRED(_p);
2304     TSTREG1(r1);
2305     inst((8L<<37)|(za<<36)|(1L<<34)|(zb<<33)|(x4<<29)|(x2<<27)|
2306          (r3<<20)|(r2<<13)|(r1<<6)|_p, INST_A);
2307     SETREG(r1);
2308 }
2309
2310 static void
2311 _I1(jit_state_t *_jit, jit_word_t _p,
2312     jit_word_t ct, jit_word_t x2, jit_word_t r3, jit_word_t r2, jit_word_t r1)
2313 {
2314     assert(!(_p & ~0x3fL));
2315     assert(!(ct &  ~0x3L));
2316     assert(!(x2 &  ~0x3L));
2317     assert(!(r3 & ~0x7fL));
2318     assert(!(r2 & ~0x7fL));
2319     assert(!(r1 & ~0x7fL));
2320     TSTREG2(r2, r3);
2321     TSTPRED(_p);
2322     TSTREG1(r1);
2323     inst((7L<<37)|(1L<<33)|(ct<<30)|(x2<<28)|
2324          (r3<<20)|(r2<<13)|(r1<<6)|_p, INST_I);
2325     SETREG(r1);
2326 }
2327
2328 static void
2329 _I2(jit_state_t *_jit, jit_word_t _p,
2330     jit_word_t za, jit_word_t xa, jit_word_t zb, jit_word_t xc,
2331     jit_word_t xb ,jit_word_t r3, jit_word_t r2, jit_word_t r1)
2332 {
2333     assert(!(_p & ~0x3fL));
2334     assert(!(za &  ~0x1L));
2335     assert(!(xa &  ~0x3L));
2336     assert(!(zb &  ~0x1L));
2337     assert(!(xc &  ~0x3L));
2338     assert(!(xb &  ~0x3L));
2339     assert(!(r3 & ~0x7fL));
2340     assert(!(r2 & ~0x7fL));
2341     assert(!(r1 & ~0x7fL));
2342     TSTREG2(r2, r3);
2343     TSTPRED(_p);
2344     TSTREG1(r1);
2345     inst((7L<<37)|(za<<36)|(xa<<34)|(zb<<33)|(xc<<30)|
2346          (xb<<28)|(r3<<20)|(r2<<13)|(r1<<6), INST_I);
2347     SETREG(r1);
2348 }
2349
2350 static void
2351 _I3(jit_state_t *_jit, jit_word_t _p,
2352     jit_word_t mb, jit_word_t r2, jit_word_t r1)
2353 {
2354     assert(!(_p & ~0x3fL));
2355     assert(!(mb &  ~0xfL));
2356     assert(!(r2 & ~0x7fL));
2357     assert(!(r1 & ~0x7fL));
2358     TSTREG1(r2);
2359     TSTPRED(_p);
2360     TSTREG1(r1);
2361     inst((7L<<37)|(3L<<34)|(2L<<30)|(2L<<28)|
2362          (mb<<20)|(r2<<13)|(r1<<6)|_p, INST_I);
2363     SETREG(r1);
2364 }
2365
2366 static void
2367 _I4(jit_state_t *_jit, jit_word_t _p,
2368     jit_word_t mh, jit_word_t r2, jit_word_t r1)
2369 {
2370     assert(!(_p & ~0x3fL));
2371     assert(!(mh & ~0xffL));
2372     assert(!(r2 & ~0x7fL));
2373     assert(!(r1 & ~0x7fL));
2374     TSTREG1(r2);
2375     TSTPRED(_p);
2376     TSTREG1(r1);
2377     inst((7L<<37)|(3L<<34)|(1L<<33)|(2L<<30)|
2378          (2L<<28)|(mh<<20)|(r2<<13)|(r1<<6)|_p, INST_I);
2379     SETREG(r1);
2380 }
2381
2382 static void
2383 _I5(jit_state_t *_jit, jit_word_t _p,
2384     jit_word_t za, jit_word_t zb, jit_word_t x2,
2385     jit_word_t r3, jit_word_t r2, jit_word_t r1)
2386 {
2387     assert(!(_p & ~0x3fL));
2388     assert(!(za &  ~0x1L));
2389     assert(!(zb &  ~0x1L));
2390     assert(!(x2 &  ~0x3L));
2391     assert(!(r3 & ~0x7fL));
2392     assert(!(r2 & ~0x7fL));
2393     assert(!(r1 & ~0x7fL));
2394     TSTREG2(r2, r3);
2395     TSTPRED(_p);
2396     TSTREG1(r1);
2397     inst((7L<<37)|(za<<36)|(zb<<33)|(x2<<28)|
2398          (r3<<20)|(r2<<13)|(r1<<6)|_p, INST_I);
2399     SETREG(r1);
2400 }
2401
2402 static void
2403 _I6(jit_state_t *_jit, jit_word_t _p,
2404     jit_word_t za, jit_word_t zb, jit_word_t x2,
2405     jit_word_t r3, jit_word_t ct, jit_word_t r1)
2406 {
2407     assert(!(_p & ~0x3fL));
2408     assert(!(za &  ~0x1L));
2409     assert(!(zb &  ~0x1L));
2410     assert(!(x2 &  ~0x3L));
2411     assert(!(r3 & ~0x7fL));
2412     assert(!(ct & ~0x1fL));
2413     assert(!(r1 & ~0x7fL));
2414     TSTREG1(r3);
2415     TSTPRED(_p);
2416     TSTREG1(r1);
2417     inst((7L<<37)|(za<<36)|(1L<<34)|(zb<<33)|
2418          (x2<<28)|(r3<<20)|(ct<<14)|(r1<<6)|_p, INST_I);
2419     SETREG(r1);
2420 }
2421
2422 static void
2423 _I7(jit_state_t *_jit, jit_word_t _p,
2424     jit_word_t za, jit_word_t zb, jit_word_t r3, jit_word_t r2, jit_word_t r1)
2425 {
2426     assert(!(_p & ~0x3fL));
2427     assert(!(za &  ~0x1L));
2428     assert(!(zb &  ~0x1L));
2429     assert(!(r3 & ~0x7fL));
2430     assert(!(r2 & ~0x7fL));
2431     assert(!(r1 & ~0x7fL));
2432     TSTREG2(r2, r3);
2433     TSTPRED(_p);
2434     TSTREG1(r1);
2435     inst((7L<<37)|(za<<36)|(zb<<33)|(1L<<30)|
2436          (r3<<20)|(r2<<13)|(r1<<6)|_p, INST_I);
2437     SETREG(r1);
2438 }
2439
2440 static void
2441 _I8(jit_state_t *_jit, jit_word_t _p,
2442     jit_word_t za, jit_word_t zb, jit_word_t im, jit_word_t r2, jit_word_t r1)
2443 {
2444     assert(!(_p & ~0x3fL));
2445     assert(!(za &  ~0x1L));
2446     assert(!(zb &  ~0x1L));
2447     assert(!(im & ~0x1fL));
2448     assert(!(r2 & ~0x7fL));
2449     assert(!(r1 & ~0x7fL));
2450     TSTREG1(r2);
2451     TSTPRED(_p);
2452     TSTREG1(r1);
2453     inst((7L<<37)|(za<<36)|(3L<<34)|(zb<<33)|(1L<<30)|(1L<<28)|
2454          (im<<20)|(r2<<13)|(r1<<6), INST_I);
2455     SETREG(r1);
2456 }
2457
2458 static void
2459 _I9(jit_state_t *_jit, jit_word_t _p,
2460     jit_word_t x2, jit_word_t r3, jit_word_t r1)
2461 {
2462     assert(!(_p & ~0x3fL));
2463     assert(!(x2 &  ~0x3L));
2464     assert(!(r3 & ~0x7fL));
2465     assert(!(r1 & ~0x7fL));
2466     TSTREG1(r3);
2467     TSTPRED(_p);
2468     TSTREG1(r1);
2469     inst((7L<<37)|(1L<<34)|(1L<<33)|
2470          (x2<<30)|(1L<<28)|(r3<<20)|(r1<<6)|_p, INST_I);
2471     SETREG(r1);
2472 }
2473
2474 static void
2475 _I10(jit_state_t *_jit, jit_word_t _p,
2476      jit_word_t ct, jit_word_t r3, jit_word_t r2, jit_word_t r1)
2477 {
2478     assert(!(_p & ~0x3fL));
2479     assert(!(ct & ~0x3fL));
2480     assert(!(r3 & ~0x7fL));
2481     assert(!(r2 & ~0x7fL));
2482     assert(!(r1 & ~0x7fL));
2483     TSTREG2(r2, r3);
2484     TSTPRED(_p);
2485     TSTREG1(r1);
2486     inst((5L<<37)|(3L<<34)|(ct<<27)|(r3<<20)|(r2<<13)|(r1<<6)|_p, INST_I);
2487     SETREG(r1);
2488 }
2489
2490 static void
2491 _I11(jit_state_t *_jit, jit_word_t _p,
2492      jit_word_t len, jit_word_t r3,
2493      jit_word_t pos, jit_word_t y, jit_word_t r1)
2494 {
2495     assert(!(_p  & ~0x3fL));
2496     assert(!(len & ~0x3fL));
2497     assert(!(r3  & ~0x7fL));
2498     assert(!(pos & ~0x1fL));
2499     assert(!(y   &  ~0x1L));
2500     assert(!(r1  & ~0x7fL));
2501     TSTREG1(r3);
2502     TSTPRED(_p);
2503     TSTREG1(r1);
2504     inst((5L<<37)|(1L<<34)|(len<<27)|(r3<<20)|
2505          (pos<<14)|(y<<13)|(r1<<6)|_p, INST_I);
2506     SETREG(r1);
2507 }
2508
2509 static void
2510 _I12(jit_state_t *_jit, jit_word_t _p,
2511      jit_word_t len, jit_word_t pos, jit_word_t r2, jit_word_t r1)
2512 {
2513     assert(!(_p  & ~0x3fL));
2514     assert(!(len & ~0x3fL));
2515     assert(!(pos & ~0x3fL));
2516     assert(!(r2  & ~0x7fL));
2517     assert(!(r1  & ~0x7fL));
2518     TSTREG1(r2);
2519     TSTPRED(_p);
2520     TSTREG1(r1);
2521     inst((5L<<37)|(1L<<34)|(1L<<33)|(len<<27)|
2522          (pos<<20)|(r2<<13)|(r1<<6)|_p, INST_I);
2523     SETREG(r1);
2524 }
2525
2526 static void
2527 _I13(jit_state_t *_jit, jit_word_t _p,
2528      jit_word_t len, jit_word_t pos, jit_word_t im, jit_word_t r1)
2529 {
2530     assert(!(_p  & ~0x3fL));
2531     assert(!(len & ~0x3fL));
2532     assert(!(pos & ~0x3fL));
2533     assert(!(im  & ~0x7fL));
2534     assert(!(r1  & ~0x7fL));
2535     TSTPRED(_p);
2536     TSTREG1(r1);
2537     inst((5L<<37)|(((im>>7)&1L)<<36)|(1L<<34)|(1L<<33)|(len<<27)|
2538          (1L<<26)|(pos<<20)|((im&0x7fL)<<13)|(r1<<6)|_p, INST_I);
2539     SETREG(r1);
2540 }
2541
2542 static void
2543 _I14(jit_state_t *_jit, jit_word_t _p,
2544      jit_word_t s, jit_word_t len, jit_word_t r3, jit_word_t pos, jit_word_t r1)
2545 {
2546     assert(!(_p  & ~0x3fL));
2547     assert(!(s   &  ~0x1L));
2548     assert(!(len & ~0x3fL));
2549     assert(!(r3  & ~0x7fL));
2550     assert(!(pos & ~0x1fL));
2551     assert(!(r1  & ~0x7fL));
2552     TSTREG1(r3);
2553     TSTPRED(_p);
2554     TSTREG1(r1);
2555     inst((5L<<37)|(s<<36)|(3L<<34)|(1L<<33)|
2556          (len<<27)|(r3<<20)|(pos<<14)|(r1<<6)|_p, INST_I);
2557     SETREG(r1);
2558 }
2559
2560 static void
2561 _I15(jit_state_t *_jit, jit_word_t _p,
2562      jit_word_t pos, jit_word_t len,
2563      jit_word_t r3, jit_word_t r2, jit_word_t r1)
2564 {
2565     assert(!(_p  & ~0x3fL));
2566     assert(!(pos & ~0x3fL));
2567     assert(!(len &  ~0xfL));
2568     assert(!(r3  & ~0x7fL));
2569     assert(!(r2  & ~0x7fL));
2570     assert(!(r1  & ~0x7fL));
2571     TSTREG2(r2, r3);
2572     TSTPRED(_p);
2573     TSTREG1(r1);
2574     inst((4L<<37)|(pos<<31)|(len<<27)|(r3<<20)|(r2<<13)|(r1<<6)|_p, INST_I);
2575     SETREG(r1);
2576 }
2577
2578 static void
2579 _I16(jit_state_t *_jit, jit_word_t _p,
2580      jit_word_t tb, jit_word_t ta, jit_word_t p2,
2581      jit_word_t r3, jit_word_t ps, jit_word_t c, jit_word_t p1)
2582 {
2583     assert(!(_p & ~0x3fL));
2584     assert(!(tb &  ~0x1L));
2585     assert(!(ta &  ~0x1L));
2586     assert(!(p2 & ~0x7fL));
2587     assert(!(r3 & ~0x7fL));
2588     assert(!(ps & ~0x3fL));
2589     assert(!(c  &  ~0x1L));
2590     assert(!(p1 & ~0x3fL));
2591     TSTREG1(r3);
2592     TSTPRED(_p);
2593     inst((5L<<37)|(tb<<36)|(ta<<33)|(p2<<27)|
2594          (r3<<20)|(ps<<14)|(c<<12)|(p1<<6), INST_I);
2595     if (p1)
2596         _jitc->pred |= 1 << p1;
2597     if (p2)
2598         _jitc->pred |= 1 << p2;
2599 }
2600
2601 static void
2602 _I17(jit_state_t *_jit, jit_word_t _p,
2603      jit_word_t tb, jit_word_t ta, jit_word_t p2,
2604      jit_word_t r3, jit_word_t c, jit_word_t p1)
2605 {
2606     assert(!(_p & ~0x3fL));
2607     assert(!(tb &  ~0x1L));
2608     assert(!(ta &  ~0x1L));
2609     assert(!(p2 & ~0x7fL));
2610     assert(!(r3 & ~0x7fL));
2611     assert(!(c  &  ~0x1L));
2612     assert(!(p1 & ~0x3fL));
2613     TSTREG1(r3);
2614     TSTPRED(_p);
2615     inst((5L<<37)|(tb<<36)|(ta<<33)|(p2<<27)|
2616          (r3<<20)|(1L<<13)|(c<<12)|(p1<<6)|_p, INST_I);
2617     if (p1)
2618         _jitc->pred |= 1 << p1;
2619     if (p2)
2620         _jitc->pred |= 1 << p2;
2621 }
2622
2623 static void
2624 _I18(jit_state_t *_jit, jit_word_t _p,
2625      jit_word_t im, jit_word_t y)
2626 {
2627     assert(!(_p &     ~0x3fL));
2628     assert(!(im & ~0x1fffffL));
2629     assert(!(y  &      ~0x1L));
2630     TSTPRED(_p);
2631     inst((((im>>20)&1L)<<26)|(1L<<27)|(y<<26)|((im&0xffffL)<<6)|_p, INST_I);
2632 }
2633
2634 static void
2635 _I19(jit_state_t *_jit, jit_word_t _p,
2636      jit_word_t im)
2637 {
2638     assert(!(_p &     ~0x3fL));
2639     assert(!(im & ~0x1fffffL));
2640     TSTPRED(_p);
2641     inst(((im>>20)&1L)|((im&0xffffL)<<6)|_p, INST_I);
2642 }
2643
2644 static void
2645 _I20(jit_state_t *_jit, jit_word_t _p,
2646      jit_word_t r2, jit_word_t im)
2647 {
2648     assert(!(_p &     ~0x3fL));
2649     assert(!(r2 &     ~0x7fL));
2650     assert(!(im & ~0x1fffffL));
2651     TSTREG1(r2);
2652     TSTPRED(_p);
2653     inst(((im>>20)&1L)|(1L<<33)|(((im>>7)&0x1fffL)<<20)|
2654          (r2<<13)|((im&0x7fL)<<6)|_p, INST_I);
2655 }
2656
2657 static void
2658 _I21(jit_state_t *_jit, jit_word_t _p,
2659      jit_word_t im, jit_word_t ih, jit_word_t x,
2660      jit_word_t wh, jit_word_t r2, jit_word_t b1)
2661 {
2662     assert(!(_p &     ~0x3fL));
2663     assert(!(im &    ~0x1ffL));
2664     assert(!(ih &      ~0x1L));
2665     assert(!(x  &      ~0x1L));
2666     assert(!(wh &      ~0x3L));
2667     assert(!(r2 &     ~0x7fL));
2668     assert(!(b1 &      ~0x7L));
2669     TSTREG1(r2);
2670     TSTPRED(_p);
2671     inst((7L<<33)|(im<<24)|(ih<<23)|(x<<22)|(wh<<20)|
2672          (r2<<13)|(b1<<6), INST_I);
2673 }
2674
2675 static void
2676 _I22(jit_state_t *_jit, jit_word_t _p,
2677      jit_word_t b2, jit_word_t r1)
2678 {
2679     assert(!(_p & ~0x3fL));
2680     assert(!(b2 &  ~0x7L));
2681     assert(!(r1 & ~0x7fL));
2682     TSTPRED(_p);
2683     inst((0x31L<<27)|(b2<<13)|(r1<<6)|_p, INST_I);
2684 }
2685
2686 static void
2687 _I23(jit_state_t *_jit, jit_word_t _p,
2688      jit_word_t r2, jit_word_t im)
2689 {
2690     assert(!(_p &   ~0x3fL));
2691     assert(!(r2 &   ~0x7fL));
2692     assert(!(im & ~0xffffL));
2693     TSTREG1(r2);
2694     TSTPRED(_p);
2695     inst((((im>>15)&1L)<<36)|(3L<<33)|(((im>>7)&0xffL)<<24)|
2696          (r2<<13)|(im&0x7fL)|_p, INST_I);
2697 }
2698
2699 static void
2700 _I24(jit_state_t *_jit, jit_word_t _p,
2701      jit_word_t im)
2702 {
2703     jit_uint8_t         cc = INST_I;
2704     assert(!(_p &      ~0x3fL));
2705     assert(!(im & ~0xfffffffL));
2706     TSTPRED(_p);
2707     inst((((im>>27)&1L)<<36)|(2L<<33)|((im&0x7ffffffL)<<6)|_p, cc);
2708 }
2709
2710 static void
2711 _I25(jit_state_t *_jit, jit_word_t _p,
2712      jit_word_t x6, jit_word_t r1)
2713 {
2714     assert(!(_p & ~0x3fL));
2715     assert(!(x6 & ~0x3fL));
2716     assert(!(r1 & ~0x7fL));
2717     TSTPRED(_p);
2718     TSTREG1(r1);
2719     inst((x6<<27)|(r1<<6)|_p, INST_I);
2720     SETREG(r1);
2721 }
2722
2723 static void
2724 _I26(jit_state_t *_jit, jit_word_t _p,
2725      jit_word_t ar,jit_word_t r2)
2726 {
2727     assert(!(_p & ~0x3fL));
2728     assert(!(ar & ~0x7fL));
2729     assert(!(r2 & ~0x7fL));
2730     TSTREG1(r2);
2731     TSTPRED(_p);
2732     inst((0x2aL<<27)|(ar<<20)|(r2<<13)|_p, INST_I);
2733 }
2734
2735 static void
2736 _I27(jit_state_t *_jit, jit_word_t _p,
2737      jit_word_t ar,jit_word_t im)
2738 {
2739     assert(!(_p & ~0x3fL));
2740     assert(!(ar & ~0x7fL));
2741     assert(!(im & ~0xffL));
2742     TSTPRED(_p);
2743     inst((((im>>7)&1L)<<36)|(0xaL<<27)|(ar<<20)|((im&0x7fL)<<13)|_p, INST_I);
2744 }
2745
2746 static void
2747 _I28(jit_state_t *_jit, jit_word_t _p,
2748      jit_word_t ar, jit_word_t r1)
2749 {
2750     assert(!(_p & ~0x3fL));
2751     assert(!(ar & ~0x7fL));
2752     assert(!(r1 & ~0x7fL));
2753     TSTPRED(_p);
2754     TSTREG1(r1);
2755     inst((0x32L<<27)|(ar<<20)|(r1<<6)|_p, INST_I);
2756     SETREG(r1);
2757 }
2758
2759 static void
2760 _I29(jit_state_t *_jit, jit_word_t _p,
2761      jit_word_t x6, jit_word_t r3,jit_word_t r1)
2762 {
2763     assert(!(_p & ~0x3fL));
2764     assert(!(x6 & ~0x3fL));
2765     assert(!(r3 & ~0x7fL));
2766     assert(!(r1 & ~0x7fL));
2767     TSTREG1(r3);
2768     TSTPRED(_p);
2769     TSTREG1(r1);
2770     inst((x6<<27)|(r3<<20)|(r1<<6)|_p, INST_I);
2771     SETREG(r1);
2772 }
2773
2774 static void
2775 _I30(jit_state_t *_jit, jit_word_t _p,
2776      jit_word_t ta, jit_word_t tb, jit_word_t p2,
2777      jit_word_t im, jit_word_t c, jit_word_t p1)
2778 {
2779     assert(!(_p & ~0x3fL));
2780     assert(!(ta &  ~0x1L));
2781     assert(!(tb &  ~0x1L));
2782     assert(!(p2 & ~0x3fL));
2783     assert(!(im & ~0x1fL));
2784     assert(!(c  &  ~0x1L));
2785     assert(!(p1 & ~0x1fL));
2786     TSTPRED(_p);
2787     inst((5L<<37)|(tb<<36)|(ta<<33)|(1L<<19)|(im<<14)|
2788          (1L<<13)|(c<<12)|(p1<<6)|_p, INST_I);
2789     if (p1)
2790         _jitc->pred |= 1 << p1;
2791     if (p2)
2792         _jitc->pred |= 1 << p2;
2793 }
2794
2795 static void
2796 _M1(jit_state_t *_jit, jit_word_t _p,
2797     jit_word_t x6, jit_word_t ht, jit_word_t x, jit_word_t r3, jit_word_t r1)
2798 {
2799     assert(!(_p & ~0x3fL));
2800     assert(!(x6 & ~0x3fL));
2801     assert(!(ht &  ~0x3L));
2802     assert(!(x  &  ~0x1L));
2803     assert(!(r3 & ~0x7fL));
2804     assert(!(r1 & ~0x7fL));
2805     TSTREG1(r3);
2806     TSTPRED(_p);
2807     TSTREG1(r1);
2808     inst((4L<<37)|(x6<<30)|(ht<<28)|(x<<27)|(r3<<20)|(r1<<6)|_p, INST_M);
2809     SETREG(r1);
2810 }
2811
2812 static void
2813 _M2(jit_state_t *_jit, jit_word_t _p,
2814     jit_word_t x6, jit_word_t ht, jit_word_t r3, jit_word_t r2, jit_word_t r1)
2815 {
2816     assert(!(_p & ~0x3fL));
2817     assert(!(x6 & ~0x3fL));
2818     assert(!(ht &  ~0x3L));
2819     assert(!(r3 & ~0x7fL));
2820     assert(!(r2 & ~0x7fL));
2821     assert(!(r1 & ~0x7fL));
2822     TSTREG2(r2, r3);
2823     TSTPRED(_p);
2824     TSTREG1(r1);
2825     inst((4L<<37)|(1L<<36)|(x6<<30)|(ht<<28)|
2826          (r3<<20)|(r2<<13)|(r1<<6)|_p, INST_M);
2827     SETREG(r1);
2828     SETREG(r3);
2829 }
2830
2831 static void
2832 _M3(jit_state_t *_jit, jit_word_t _p,
2833     jit_word_t x6, jit_word_t ht, jit_word_t r3, jit_word_t im, jit_word_t r1)
2834 {
2835     jit_uint8_t         cc = INST_M;
2836     assert(!(_p &  ~0x3fL));
2837     assert(!(x6 &  ~0x3fL));
2838     assert(!(ht &   ~0x3L));
2839     assert(!(r3 &  ~0x7fL));
2840     assert(im > -256 && im <= 255);
2841     assert(!(r1 &  ~0x7fL));
2842     TSTREG1(r3);
2843     TSTPRED(_p);
2844     TSTREG1(r1);
2845     inst((5L<<37)|(((im>>8)&1L)<<36)|(x6<<30)|(ht<<28)|
2846          (((im>>7)&1L)<<27)|(r3<<20)|((im&0x7fL)<<13)|(r1<<6)|_p, cc);
2847     SETREG(r1);
2848     SETREG(r3);
2849 }
2850
2851 static void
2852 _M5(jit_state_t *_jit, jit_word_t _p,
2853     jit_word_t x6, jit_word_t ht, jit_word_t r3, jit_word_t r2, jit_word_t im)
2854 {
2855     assert(!(_p &  ~0x3fL));
2856     assert(!(x6 &  ~0x3fL));
2857     assert(!(ht &   ~0x3L));
2858     assert(!(r3 &  ~0x7fL));
2859     assert(!(r2 &  ~0x7fL));
2860     assert(im > -256 && im <= 255);
2861     TSTREG2(r2, r3);
2862     TSTPRED(_p);
2863     inst((5L<<37)|(((im>>8)&1L)<<36)|(x6<<30)|(ht<<28)|
2864          (((im>>7)&1L)<<27)|(r3<<20)|(r2<<13)|((im&0x7fL)<<6)|_p, INST_M);
2865     SETREG(r3);
2866 }
2867
2868 static void
2869 _M6(jit_state_t *_jit, jit_word_t _p,
2870     jit_word_t x6, jit_word_t ht, jit_word_t x, jit_word_t r3, jit_word_t r2)
2871 {
2872     assert(!(_p & ~0x3fL));
2873     assert(!(x6 & ~0x3fL));
2874     assert(!(ht &  ~0x3L));
2875     assert(!(x  &  ~0x1L));
2876     assert(!(r3 & ~0x7fL));
2877     assert(!(r2 & ~0x7fL));
2878     TSTREG2(r2, r3);
2879     TSTPRED(_p);
2880     inst((4L<<37)|(x6<<30)|(ht<<28)|(x<<27)|(r3<<20)|(r2<<13)|_p, INST_M);
2881 }
2882
2883 static void
2884 _M13(jit_state_t *_jit, jit_word_t _p,
2885      jit_word_t x6, jit_word_t ht, jit_word_t r3, jit_word_t r2)
2886 {
2887     assert(!(_p & ~0x3fL));
2888     assert(!(x6 & ~0x3fL));
2889     assert(!(ht &  ~0x3L));
2890     assert(!(r3 & ~0x7fL));
2891     assert(!(r2 & ~0x7fL));
2892     TSTREG1(r3);
2893     if (r2)
2894         TSTFREG1(r2);
2895     TSTPRED(_p);
2896     inst((6L<<37)|(x6<<30)|(ht<<28)|(r3<<20)|(r2<<13)|_p, INST_M);
2897 }
2898
2899 static void
2900 _M14(jit_state_t *_jit, jit_word_t _p,
2901      jit_word_t x6, jit_word_t ht, jit_word_t r3, jit_word_t r2)
2902 {
2903     assert(!(_p & ~0x3fL));
2904     assert(!(x6 & ~0x3fL));
2905     assert(!(ht &  ~0x3L));
2906     assert(!(r3 & ~0x7fL));
2907     assert(!(r2 & ~0x7fL));
2908     TSTREG2(r2, r3);
2909     TSTPRED(_p);
2910     inst((6L<<37)|(1L<<36)|(x6<<30)|(ht<<28)|(r3<<20)|(r2<<13)|_p, INST_M);
2911 }
2912
2913 static void
2914 _M15(jit_state_t *_jit, jit_word_t _p,
2915      jit_word_t x6, jit_word_t ht, jit_word_t r3, jit_word_t im)
2916 {
2917     assert(!(_p &  ~0x3fL));
2918     assert(!(x6 &  ~0x3fL));
2919     assert(!(ht &   ~0x3L));
2920     assert(!(r3 &  ~0x7fL));
2921     assert(!(im & ~0x1ffL));
2922     TSTREG1(r3);
2923     TSTPRED(_p);
2924     inst((7L<<37)|(((im>>8)&1L)<<36)|(x6<<30)|(ht<<28)|
2925          (((im>>7)&1L)<<27)|(r3<<20)|((im&0x7fL)<<13)|_p, INST_M);
2926 }
2927
2928 static void
2929 _M16(jit_state_t *_jit, jit_word_t _p,
2930      jit_word_t x6, jit_word_t ht, jit_word_t r3, jit_word_t r2, jit_word_t r1)
2931 {
2932     assert(!(_p & ~0x3fL));
2933     assert(!(x6 & ~0x3fL));
2934     assert(!(ht &  ~0x3L));
2935     assert(!(r3 & ~0x7fL));
2936     assert(!(r2 & ~0x7fL));
2937     assert(!(r1 & ~0x7fL));
2938     TSTREG2(r2, r3);
2939     TSTPRED(_p);
2940     TSTREG1(r1);
2941     inst((4L<<37)|(x6<<30)|(ht<<28)|(1L<<27)|
2942          (r3<<20)|(r2<<13)|(r1<<6)|_p, INST_M);
2943     SETREG(r1);
2944 }
2945
2946 static void
2947 _M17(jit_state_t *_jit, jit_word_t _p,
2948      jit_word_t x6, jit_word_t ht, jit_word_t r3, jit_word_t im, jit_word_t r1)
2949 {
2950     assert(!(_p & ~0x3fL));
2951     assert(!(x6 & ~0x3fL));
2952     assert(!(ht &  ~0x3L));
2953     assert(!(r3 & ~0x7fL));
2954     assert(!(im &  ~0x7L));
2955     assert(!(r1 & ~0x7fL));
2956     TSTREG1(r3);
2957     TSTPRED(_p);
2958     TSTREG1(r1);
2959     inst((4L<<37)|(x6<<30)|(ht<<28)|(1L<<27)|
2960          (r3<<20)|(im<<13)|(r1<<6)|_p, INST_M);
2961     SETREG(r1);
2962 }
2963
2964 static void
2965 _M20x(jit_state_t *_jit, jit_word_t _p,
2966       jit_word_t x3, jit_word_t r2, jit_word_t im)
2967 {
2968     assert(!(_p &     ~0x3fL));
2969     assert(!(x3 &      ~0x7L));
2970     assert(!(r2 &     ~0x7fL));
2971     assert(!(im & ~0x1fffffL));
2972     if (x3 == 1)
2973         TSTREG1(r2);
2974     else
2975         TSTFREG1(r2);
2976     TSTPRED(_p);
2977     inst((1L<<37)|(((im>>20)&1L)<<36)|(x3<<33)|
2978          (((im>>7)&0x1fffL)<<20)|(r2<<13)|((im&0x7fL)<<6)|_p, INST_M);
2979 }
2980
2981 static void
2982 _M22x(jit_state_t *_jit, jit_word_t _p,
2983       jit_word_t x3, jit_word_t im, jit_word_t r1)
2984 {
2985     assert(!(_p &     ~0x3fL));
2986     assert(!(x3 &      ~0x7L));
2987     assert(!(im & ~0x1fffffL));
2988     assert(!(r1 &     ~0x7fL));
2989     TSTPRED(_p);
2990     if (x3 < 6)
2991         TSTREG1(r1);
2992     else
2993         TSTFREG1(r1);
2994     inst((((im>>20)&1L)<<36)|(x3<<33)|((im&0xffffL)<<13)|(r1<<6)|_p, INST_M);
2995     if (x3 < 6)
2996         SETREG(r1);
2997     else
2998         SETFREG(r1);
2999 }
3000
3001 static void
3002 _M24(jit_state_t *_jit, jit_word_t _p,
3003      jit_word_t x2, jit_word_t x4)
3004 {
3005     assert(!(_p & ~0x3fL));
3006     assert(!(x2 &  ~0x3L));
3007     assert(!(x4 &  ~0xfL));
3008     TSTPRED(_p);
3009     inst((x2<<31)|(x4<<27)|_p, INST_M);
3010 }
3011
3012 static void
3013 _M26x(jit_state_t *_jit, jit_word_t _p,
3014       jit_word_t x4, jit_word_t r1)
3015 {
3016     assert(!(_p & ~0x3fL));
3017     assert(!(x4 &  ~0xfL));
3018     assert(!(r1 & ~0x7fL));
3019     if (x4 == 2)
3020         TSTREG1(r1);
3021     else
3022         TSTFREG1(r1);
3023     TSTPRED(_p);
3024     inst((1L<<31)|(x4<<27)|(r1<<6)|_p, INST_M);
3025 }
3026
3027 static void
3028 _M28(jit_state_t *_jit, jit_word_t _p,
3029      jit_word_t x, jit_word_t r3)
3030 {
3031     assert(!(_p & ~0x3fL));
3032     assert(!(x  &  ~0x1L));
3033     assert(!(r3 & ~0x7fL));
3034     TSTREG1(r3);
3035     TSTPRED(_p);
3036     inst((1L<<37)|(x<<36)|(0x30L<<27)|(r3<<20)|_p, INST_M);
3037 }
3038
3039 static void
3040 _M29(jit_state_t *_jit, jit_word_t _p,
3041      jit_word_t ar, jit_word_t r2)
3042 {
3043     assert(!(_p & ~0x3fL));
3044     assert(!(ar & ~0x7fL));
3045     assert(!(r2 & ~0x7fL));
3046     TSTREG1(r2);
3047     TSTPRED(_p);
3048     inst((1L<<37)|(0x2aL<<27)|(ar<<20)|(r2<<13)|_p, INST_M);
3049 }
3050
3051 static void
3052 _M30(jit_state_t *_jit, jit_word_t _p,
3053      jit_word_t ar, jit_word_t im)
3054 {
3055     assert(!(_p & ~0x3fL));
3056     assert(!(ar  & ~0x7L));
3057     assert(!(im & ~0xffL));
3058     TSTPRED(_p);
3059     inst((((im>>7)&1L)<<36)|(2L<<31)|(0x8L<<27)|
3060          (ar<<20)|((im&0x7fL)<<13)|_p, INST_M);
3061 }
3062
3063 static void
3064 _M31(jit_state_t *_jit, jit_word_t _p,
3065      jit_word_t ar, jit_word_t r1)
3066 {
3067     assert(!(_p & ~0x3fL));
3068     assert(!(ar  & ~0x7L));
3069     assert(!(r1 & ~0x7fL));
3070     TSTPRED(_p);
3071     TSTREG1(r1);
3072     inst((1L<<37)|(0x22L<<27)|(ar<<20)|(r1<<6)|_p, INST_M);
3073     SETREG(r1);
3074 }
3075
3076 static void
3077 _M32(jit_state_t *_jit, jit_word_t _p,
3078      jit_word_t cr, jit_word_t r2)
3079 {
3080     assert(!(_p & ~0x3fL));
3081     assert(!(cr  & ~0x7L));
3082     assert(!(r2 & ~0x7fL));
3083     TSTREG1(r2);
3084     TSTPRED(_p);
3085     inst((1L<<37)|(0x2cL<<27)|(cr<<20)|(r2<<13)|_p, INST_M);
3086 }
3087
3088 static void
3089 _M33(jit_state_t *_jit, jit_word_t _p,
3090      jit_word_t cr, jit_word_t r1)
3091 {
3092     assert(!(_p & ~0x3fL));
3093     assert(!(cr  & ~0x7L));
3094     assert(!(r1 & ~0x7fL));
3095     TSTPRED(_p);
3096     TSTREG1(r1);
3097     inst((1L<<37)|(0x24L<<27)|(cr<<20)|(r1<<6)|_p, INST_M);
3098     SETREG(r1);
3099 }
3100
3101 static void
3102 _M34(jit_state_t *_jit, jit_word_t _p,
3103      jit_word_t sor, jit_word_t sol, jit_word_t sof, jit_word_t r1)
3104 {
3105     assert(!(_p  & ~0x3fL));
3106     assert(!(sor &  ~0xfL));
3107     assert(!(sol & ~0x7fL));
3108     assert(!(sof & ~0x7fL));
3109     assert(!(r1  & ~0x7fL));
3110     TSTPRED(_p);
3111     inst((1L<<37)|(6L<<33)|(sor<<27)|(sol<<20)|(sof<<13)|(r1<<6)|_p, INST_M);
3112 }
3113
3114 static void
3115 _M35(jit_state_t *_jit, jit_word_t _p,
3116      jit_word_t x6, jit_word_t r2)
3117 {
3118     assert(!(_p & ~0x3fL));
3119     assert(!(x6 & ~0x3fL));
3120     assert(!(r2 & ~0x7fL));
3121     TSTREG1(r2);
3122     TSTPRED(_p);
3123     inst((1L<<37)|(x6<<27)|(r2<<13)|_p, INST_M);
3124 }
3125
3126 static void
3127 _M36(jit_state_t *_jit, jit_word_t _p,
3128      jit_word_t x6, jit_word_t r1)
3129 {
3130     assert(!(_p & ~0x3fL));
3131     assert(!(x6 & ~0x3fL));
3132     assert(!(r1 & ~0x7fL));
3133     TSTPRED(_p);
3134     TSTREG1(r1);
3135     inst((1L<<37)|(x6<<27)|(r1<<6)|_p, INST_M);
3136     SETREG(r1);
3137 }
3138
3139 static void
3140 _M37(jit_state_t *_jit, jit_word_t _p,
3141      jit_word_t im)
3142 {
3143     assert(!(_p &    ~0x3fL));
3144     assert(!(im & ~0x1ffffL));
3145     TSTPRED(_p);
3146     inst((((im>>20)&1L)<<36)|((im&0xffffL)<<6)|_p, INST_M);
3147 }
3148
3149 static void
3150 _M38(jit_state_t *_jit, jit_word_t _p,
3151      jit_word_t x6, jit_word_t r3, jit_word_t r2, jit_word_t r1)
3152 {
3153     assert(!(_p &  ~0x3fL));
3154     assert(!(x6 &  ~0x3fL));
3155     assert(!(r3 &  ~0x7fL));
3156     assert(!(r2 &  ~0x7fL));
3157     assert(!(r1 &  ~0x7fL));
3158     TSTREG2(r2, r3);
3159     TSTPRED(_p);
3160     TSTREG1(r1);
3161     inst((1L<<37)|(x6<<27)|(r3<<20)|(r2<<13)|(r1<<6)|_p, INST_M);
3162     SETREG(r1);
3163 }
3164
3165 static void
3166 _M39(jit_state_t *_jit, jit_word_t _p,
3167      jit_word_t x6, jit_word_t r3, jit_word_t im, jit_word_t r1)
3168 {
3169     assert(!(_p &  ~0x3fL));
3170     assert(!(x6 &  ~0x3fL));
3171     assert(!(r3 &  ~0x7fL));
3172     assert(!(im &   ~0x7L));
3173     assert(!(r1 &  ~0x7fL));
3174     TSTREG1(r3);
3175     TSTPRED(_p);
3176     TSTREG1(r1);
3177     inst((1L<<37)|(x6<<27)|(r3<<20)|(im<<13)|(r1<<6)|_p, INST_M);
3178     SETREG(r1);
3179 }
3180
3181 static void
3182 _M40(jit_state_t *_jit, jit_word_t _p,
3183      jit_word_t x6, jit_word_t r3, jit_word_t im)
3184 {
3185     assert(!(_p &  ~0x3fL));
3186     assert(!(x6 &  ~0x3fL));
3187     assert(!(r3 &  ~0x7fL));
3188     assert(!(im &   ~0x7L));
3189     TSTREG1(r3);
3190     TSTPRED(_p);
3191     inst((1L<<37)|(x6<<27)|(r3<<20)|(im<<13)|_p, INST_M);
3192 }
3193
3194 static void
3195 _M41(jit_state_t *_jit, jit_word_t _p,
3196      jit_word_t x6, jit_word_t r2)
3197 {
3198     assert(!(_p &  ~0x3fL));
3199     assert(!(x6 &  ~0x3fL));
3200     assert(!(r2 &  ~0x7fL));
3201     TSTREG1(r2);
3202     TSTPRED(_p);
3203     inst((1L<<37)|(x6<<27)|(r2<<13)|_p, INST_M);
3204 }
3205
3206 static void
3207 _M42(jit_state_t *_jit, jit_word_t _p,
3208      jit_word_t x6, jit_word_t r3, jit_word_t r2)
3209 {
3210     assert(!(_p &  ~0x3fL));
3211     assert(!(x6 &  ~0x3fL));
3212     assert(!(r3 &  ~0x7fL));
3213     assert(!(r2 &  ~0x7fL));
3214     TSTREG1(r2);
3215     TSTPRED(_p);
3216     inst((1L<<37)|(x6<<27)|(r3<<20)|(r2<<13)|_p, INST_M);
3217 }
3218
3219 static void
3220 _M43(jit_state_t *_jit, jit_word_t _p,
3221      jit_word_t x6, jit_word_t r3, jit_word_t r1)
3222 {
3223     assert(!(_p &  ~0x3fL));
3224     assert(!(x6 &  ~0x3fL));
3225     assert(!(r3 &  ~0x7fL));
3226     assert(!(r1 &  ~0x7fL));
3227     TSTPRED(_p);
3228     TSTREG1(r1);
3229     inst((1L<<37)|(x6<<27)|(r3<<20)|(r1<<6)|_p, INST_M);
3230     SETREG(r1);
3231 }
3232
3233 static void
3234 _M44(jit_state_t *_jit, jit_word_t _p,
3235      jit_word_t x4, jit_word_t im)
3236 {
3237     assert(!(_p &    ~0x3fL));
3238     assert(!(x4 &     ~0xfL));
3239     assert(!(im & ~0xfffffL));
3240     TSTPRED(_p);
3241     inst((((im>>23)&1L)<<36)|(((im>>21)&3L)<<31)|
3242          (x4<<27)|((im&0x1ffffL)<<6)|_p, INST_M);
3243 }
3244
3245 static void
3246 _M45(jit_state_t *_jit, jit_word_t _p,
3247      jit_word_t x6, jit_word_t r3, jit_word_t r2)
3248 {
3249     assert(!(_p &  ~0x3fL));
3250     assert(!(x6 &  ~0x3fL));
3251     assert(!(r3 &  ~0x7fL));
3252     assert(!(r2 &  ~0x7fL));
3253     TSTREG2(r2, r3);
3254     TSTPRED(_p);
3255     inst((1L<<37)|(x6<<27)|(r3<<20)|(r2<<13)|_p, INST_M);
3256 }
3257
3258 static void
3259 _M46(jit_state_t *_jit, jit_word_t _p,
3260      jit_word_t x6, jit_word_t r3, jit_word_t r1)
3261 {
3262     assert(!(_p & ~0x3fL));
3263     assert(!(x6 & ~0x3fL));
3264     assert(!(r3 & ~0x7fL));
3265     assert(!(r1 & ~0x7fL));
3266     TSTREG1(r3);
3267     TSTPRED(_p);
3268     if (r1)     TSTREG1(r1);
3269     inst((1L<<37)|(x6<<27)|(r3<<20)|(r1<<6)|_p, INST_M);
3270     if (r1)     SETREG(r1);
3271 }
3272
3273 static void
3274 _M48(jit_state_t *_jit, jit_word_t _p,
3275      jit_word_t y, jit_word_t im)
3276 {
3277     assert(!(_p &    ~0x3fL));
3278     assert(!(y  &     ~0x1L));
3279     assert(!(im & ~0x1ffffL));
3280     TSTPRED(_p);
3281     inst((((im>>20)&1L)<<36)|(1L<<27)|(y<<26)|((im&0xffffL)<<6)|_p, INST_M);
3282 }
3283
3284 static void
3285 _B1(jit_state_t *_jit, jit_word_t _p,
3286     jit_word_t d, jit_word_t wh, jit_word_t im, jit_word_t p, jit_word_t tp)
3287 {
3288     assert(!(_p &    ~0x3fL));
3289     assert(!(d  &     ~0x1L));
3290     assert(!(wh &     ~0x3L));
3291     assert(im >= -1048576 && im <= 1048575);
3292     assert(!(p  &     ~0x1L));
3293     assert(!(tp &     ~0x7L));
3294     TSTPRED(_p);
3295     inst((4L<<37)|(((im>>20)&1L)<<36)|(d<<35)|(wh<<33)|
3296          ((im&0xfffffL)<<13)|(p<<12)|(tp<<6)|_p, INST_B);
3297 }
3298
3299 static void
3300 _B3(jit_state_t *_jit, jit_word_t _p,
3301     jit_word_t d, jit_word_t wh, jit_word_t im, jit_word_t p, jit_word_t b)
3302 {
3303     assert(!(_p &    ~0x3fL));
3304     assert(!(d  &     ~0x1L));
3305     assert(!(wh &     ~0x3L));
3306     assert(im >= -1048576 && im <= 1048575);
3307     assert(!(p  &     ~0x1L));
3308     assert(!(b  &     ~0x3L));
3309     TSTPRED(_p);
3310     inst((5L<<37)|(((im>>20)&1L)<<36)|(d<<35)|(wh<<33)|
3311          ((im&0xfffffL)<<13)|(p<<12)|(b<<6)|_p, INST_B);
3312 }
3313
3314 static void
3315 _B4(jit_state_t *_jit, jit_word_t _p,
3316     jit_word_t d, jit_word_t wh, jit_word_t x6,
3317     jit_word_t b, jit_word_t p, jit_word_t tp)
3318 {
3319     assert(!(_p & ~0x3fL));
3320     assert(!(d  &  ~0x1L));
3321     assert(!(wh &  ~0x3L));
3322     assert(!(x6 & ~0x3fL));
3323     assert(!(b  &  ~0x7L));
3324     assert(!(p  &  ~0x1L));
3325     assert(!(tp &  ~0x7L));
3326     TSTPRED(_p);
3327     inst((d<<35)|(wh<<33)|(x6<<27)|(b<<13)|(p<<12)|(tp<<6)|_p, INST_B);
3328 }
3329
3330 static void
3331 _B5(jit_state_t *_jit, jit_word_t _p,
3332     jit_word_t d, jit_word_t wh, jit_word_t b2, jit_word_t p, jit_word_t b1)
3333 {
3334     assert(!(_p & ~0x3fL));
3335     assert(!(d  &  ~0x1L));
3336     assert(!(wh &  ~0x3L));
3337     assert(!(b2 &  ~0x7L));
3338     assert(!(p  &  ~0x1L));
3339     assert(!(b1 &  ~0x7L));
3340     TSTPRED(_p);
3341     inst((1L<<37)|(d<<35)|(wh<<32)|(b2<<13)|(p<<12)|(b1<<6)|_p, INST_B);
3342 }
3343
3344 static void
3345 _B6(jit_state_t *_jit, jit_word_t _p,
3346     jit_word_t ih, jit_word_t im, jit_word_t tag, jit_word_t wh)
3347 {
3348     assert(!(_p &     ~0x3fL));
3349     assert(!(ih  &     ~0x1L));
3350     assert(!(im  & ~0x1ffffL));
3351     assert(!(tag &   ~0x1ffL));
3352     assert(!(wh  &     ~0x3L));
3353     TSTPRED(_p);
3354     inst((7L<<37)|(((im>>20)&1L)<<36)|(ih<<35)|(((tag>>7)&3L)<<33)|
3355          ((im&0xfffffL)<<13)|((tag&0x7fL)<<6)|(wh<<3)|_p, INST_B);
3356 }
3357
3358 static void
3359 _B7(jit_state_t *_jit, jit_word_t _p,
3360     jit_word_t ih, jit_word_t x6, jit_word_t b2, jit_word_t tag, jit_word_t wh)
3361 {
3362     assert(!(_p &   ~0x3fL));
3363     assert(!(ih  &   ~0x1L));
3364     assert(!(x6  &  ~0x3fL));
3365     assert(!(b2  &   ~0x7L));
3366     assert(!(tag & ~0x1ffL));
3367     assert(!(wh  &   ~0x3L));
3368     TSTPRED(_p);
3369     inst((2L<<37)|(ih<<35)|(((tag>>7)&3L)<<33)|(x6<<27)|
3370          (b2<<13)|((tag&0x7fL)<<6)|(wh<<3)|_p, INST_B);
3371 }
3372
3373 static void
3374 _B8(jit_state_t *_jit, jit_word_t _p,
3375     jit_word_t x6)
3376 {
3377     assert(!(_p & ~0x3fL));
3378     assert(!(x6 & ~0x3fL));
3379     TSTPRED(_p);
3380     inst((x6<<27)|_p, INST_B);
3381 }
3382
3383 static void
3384 _B9(jit_state_t *_jit, jit_word_t _p,
3385     jit_word_t op, jit_word_t x6, jit_word_t im)
3386 {
3387     assert(!(_p &    ~0x3fL));
3388     assert(!(op &     ~0xfL));
3389     assert(!(x6 &    ~0x3fL));
3390     assert(!(im & ~0x1ffffL));
3391     TSTPRED(_p);
3392     inst((op<<37)|(((im>>20)&1L)<<36)|(x6<<27)|((im&0xffffL)<<6)|_p, INST_B);
3393 }
3394
3395 static void
3396 _X1(jit_state_t *_jit, jit_word_t _p,
3397     jit_word_t im)
3398 {
3399     jit_word_t          i41, i1, i20;
3400     assert(!(_p &               ~0x3fL));
3401     assert(im > -0x2000000000000000 && im <= 0x1fffffffffffffff);
3402     i41 = (im >> 22) & 0x1ffffffffffL;
3403     i1  = (im >> 21) &           0x1L;
3404     i20 =  im        &       0xfffffL;
3405     TSTPRED(_p);
3406     inst(i41, INST_L);
3407     inst((i1<<36)|(i20<<6)|_p, INST_X);
3408 }
3409
3410 static void
3411 _X2(jit_state_t *_jit, jit_word_t _p,
3412     jit_word_t r1, jit_word_t im)
3413 {
3414     jit_word_t          i1, i41, ic, i5, i9, i7;
3415     assert(!(_p & ~0x3fL));
3416     assert(!(r1 & ~0x7fL));
3417     i1  = (im >> 63) &           0x1L;
3418     i41 = (im >> 22) & 0x1ffffffffffL;
3419     ic  = (im >> 21) &           0x1L;
3420     i5  = (im >> 16) &          0x1fL;
3421     i9  = (im >>  7) &         0x1ffL;
3422     i7  =  im        &          0x7fL;
3423     TSTPRED(_p);
3424     TSTREG1(r1);
3425     inst(i41, INST_L);
3426     inst((6L<<37)|(i1<<36)|(i9<<27)|(i5<<22)|
3427          (ic<<21)|(i7<<13)|(r1<<6)|_p, INST_X);
3428     SETREG(r1);
3429 }
3430
3431 static void
3432 _X3x(jit_state_t *_jit, jit_word_t _p,
3433      jit_word_t op, jit_word_t d, jit_word_t wh,
3434      jit_word_t p, jit_word_t tp, jit_word_t im)
3435 {
3436     /* target64 = IP + ((i1 << 59 | imm39 << 20 | imm20b) << 4) */
3437     jit_word_t          i1, i41, i20;
3438     assert(!(_p &               ~0x3fL));
3439     assert(!(op &                ~0xfL));
3440     assert(!(d  &                ~0x1L));
3441     assert(!(wh &                ~0x3L));
3442     assert(!(p  &                ~0x1L));
3443     assert(!(tp &                ~0x7L));
3444     i1  = (im >> 61) &           0x1L;
3445     i41 = (im >> 22) & 0x1ffffffffffL;
3446     i20 =  im        &       0xfffffL;
3447     TSTPRED(_p);
3448     inst(i41, INST_L);
3449     inst((op<<37)|(i1<<36)|(d<<35)|(wh<<33)|
3450          (i20<<13)|(p<<12)|(tp<<6)|_p, INST_X);
3451 }
3452
3453 static void
3454 _X5(jit_state_t *_jit, jit_word_t _p,
3455     jit_word_t y, jit_word_t im)
3456 {
3457     jit_word_t          i41, i1, i20;
3458     assert(!(_p &               ~0x3fL));
3459     assert(im > -0x2000000000000000 && im <= 0x1fffffffffffffff);
3460     i41 = (im >> 22) & 0x1ffffffffffL;
3461     i1  = (im >> 21) &           0x1L;
3462     i20 =  im        &       0xfffffL;
3463     TSTPRED(_p);
3464     inst(i41, INST_L);
3465     inst((i1<<36)|(1L<<27)|(y<<26)|(i20<<6)|_p, INST_X);
3466 }
3467
3468 static void
3469 _nop(jit_state_t *_jit, jit_int32_t i0)
3470 {
3471     for (; i0 > 0; i0 -= 8) {
3472         NOP_M(0);
3473         sync();
3474     }
3475     assert(i0 == 0);
3476 }
3477
3478 static void
3479 _bitswap(jit_state_t *_jit, jit_int32_t r0, jit_int32_t r1)
3480 {
3481     jit_int32_t         t0, t1, t2, t3, t4;
3482     movr(r0, r1);
3483     t0 = jit_get_reg(jit_class_gpr);
3484     t1 = jit_get_reg(jit_class_gpr);
3485     t2 = jit_get_reg(jit_class_gpr);
3486     movi(rn(t0), __WORDSIZE == 32 ? 0x55555555L : 0x5555555555555555L);
3487     rshi_u(rn(t1), r0, 1);              /* t1 = v >> 1 */
3488     andr(rn(t1), rn(t1), rn(t0));       /* t1 &= t0 */
3489     andr(rn(t2), r0, rn(t0));           /* t2 = v & t0*/
3490     lshi(rn(t2), rn(t2), 1);            /* t2 <<= 1 */
3491     orr(r0, rn(t1), rn(t2));            /* v = t1 | t2 */
3492     movi(rn(t0), __WORDSIZE == 32 ? 0x33333333L : 0x3333333333333333L);
3493     rshi_u(rn(t1), r0, 2);              /* t1 = v >> 2 */
3494     andr(rn(t1), rn(t1), rn(t0));       /* t1 &= t0 */
3495     andr(rn(t2), r0, rn(t0));           /* t2 = v & t0*/
3496     lshi(rn(t2), rn(t2), 2);            /* t2 <<= 2 */
3497     orr(r0, rn(t1), rn(t2));            /* v = t1 | t2 */
3498     movi(rn(t0), __WORDSIZE == 32 ? 0x0f0f0f0fL : 0x0f0f0f0f0f0f0f0fL);
3499     rshi_u(rn(t1), r0, 4);              /* t1 = v >> 4 */
3500     andr(rn(t1), rn(t1), rn(t0));       /* t1 &= t0 */
3501     andr(rn(t2), r0, rn(t0));           /* t2 = v & t0*/
3502     lshi(rn(t2), rn(t2), 4);            /* t2 <<= 4 */
3503     orr(r0, rn(t1), rn(t2));            /* v = t1 | t2 */
3504     movi(rn(t0), __WORDSIZE == 32 ?  0x00ff00ffL : 0x00ff00ff00ff00ffL);
3505     rshi_u(rn(t1), r0, 8);              /* t1 = v >> 8 */
3506     andr(rn(t1), rn(t1), rn(t0));       /* t1 &= t0 */
3507     andr(rn(t2), r0, rn(t0));           /* t2 = v & t0*/
3508     lshi(rn(t2), rn(t2), 8);            /* t2 <<= 8 */
3509     orr(r0, rn(t1), rn(t2));            /* v = t1 | t2 */
3510     movi(rn(t0), 0x0000ffff0000ffffL);
3511     rshi_u(rn(t1), r0, 16);             /* t1 = v >> 16 */
3512     andr(rn(t1), rn(t1), rn(t0));       /* t1 &= t0 */
3513     andr(rn(t2), r0, rn(t0));           /* t2 = v & t0*/
3514     lshi(rn(t2), rn(t2), 16);           /* t2 <<= 16 */
3515     orr(r0, rn(t1), rn(t2));            /* v = t1 | t2 */
3516     rshi_u(rn(t1), r0, 32);             /* t1 = v >> 32 */
3517     lshi(rn(t2), r0, 32);               /* t2 = v << 32 */
3518     orr(r0, rn(t1), rn(t2));            /* v = t1 | t2 */
3519     jit_unget_reg(t2);
3520     jit_unget_reg(t1);
3521     jit_unget_reg(t0);
3522 }
3523
3524 static void
3525 _clzr(jit_state_t *_jit, jit_int32_t r0, jit_int32_t r1)
3526 {
3527     if (jit_cpu.clz)
3528         CLZ(r0, r1);
3529     else
3530         fallback_clz(r0, r1);
3531 }
3532
3533 static void
3534 _clor(jit_state_t *_jit, jit_int32_t r0, jit_int32_t r1)
3535 {
3536     if (jit_cpu.clz) {
3537         comr(r0, r1);
3538         clzr(r0, r0);
3539     }
3540     else
3541         fallback_clo(r0, r1);
3542 }
3543
3544 static void
3545 _ctor(jit_state_t *_jit, jit_int32_t r0, jit_int32_t r1)
3546 {
3547     if (jit_cpu.clz) {
3548         bitswap(r0, r1);
3549         clor(r0, r0);
3550     }
3551     else
3552         fallback_cto(r0, r1);
3553 }
3554
3555 static void
3556 _ctzr(jit_state_t *_jit, jit_int32_t r0, jit_int32_t r1)
3557 {
3558     if (jit_cpu.clz) {
3559         bitswap(r0, r1);
3560         clzr(r0, r0);
3561     }
3562     else
3563         fallback_ctz(r0, r1);
3564 }
3565
3566 static void
3567 _movr(jit_state_t *_jit, jit_int32_t r0, jit_int32_t r1)
3568 {
3569     /* patch pushargr */
3570     if (r0 >= 120)
3571         r0 = _jitc->rout + (r0 - 120);
3572
3573     MOV(r0, r1);
3574 }
3575
3576 static void
3577 _movi(jit_state_t *_jit, jit_int32_t r0, jit_word_t i0)
3578 {
3579     /* patch pushargi */
3580     if (r0 >= 120)
3581         r0 = _jitc->rout + (r0 - 120);
3582
3583     if (i0 >= -2097152 && i0 <= 2097151)
3584         MOVI(r0, i0);
3585     else
3586         MOVL(r0, i0);
3587 }
3588
3589 static jit_word_t
3590 _movi_p(jit_state_t *_jit, jit_int32_t r0, jit_word_t i0)
3591 {
3592     jit_word_t          w;
3593     sync();
3594     w = _jit->pc.w;
3595     MOVL(r0, i0);
3596     return (w);
3597 }
3598
3599 static void
3600 _movnr(jit_state_t *_jit, jit_int32_t r0, jit_int32_t r1, jit_int32_t r2)
3601 {
3602     CMP_EQ(PR_6, PR_7, r2, GR_0);
3603     MOV_p(r0, r1, PR_7);
3604 }
3605
3606 static void
3607 _movzr(jit_state_t *_jit, jit_int32_t r0, jit_int32_t r1, jit_int32_t r2)
3608 {
3609     CMP_EQ(PR_6, PR_7, r2, GR_0);
3610     MOV_p(r0, r1, PR_6);
3611 }
3612
3613 static void
3614 _casx(jit_state_t *_jit, jit_int32_t r0, jit_int32_t r1,
3615       jit_int32_t r2, jit_int32_t r3, jit_word_t i0)
3616 {
3617     jit_int32_t         r1_reg, iscasi;
3618     if ((iscasi = (r1 == _NOREG))) {
3619         r1_reg = jit_get_reg(jit_class_gpr);
3620         r1 = rn(r1_reg);
3621         movi(r1, i0);
3622     }
3623     sync();
3624     MOV_M_ar_rn(AR_CCV, r2);
3625     CMPXCHG8_ACQ(r0, r1, r3);
3626     eqr(r0, r0, r2);
3627     if (iscasi)
3628         jit_unget_reg(r1_reg);
3629 }
3630
3631
3632 static void
3633 _bswapr_us(jit_state_t *_jit, jit_int32_t r0, jit_int32_t r1)
3634 {
3635         bswapr_ul(r0, r1);
3636         rshi_u(r0, r0, 48);
3637 }
3638
3639 static void
3640 _bswapr_ui(jit_state_t *_jit, jit_int32_t r0, jit_int32_t r1)
3641 {
3642         bswapr_ul(r0, r1);
3643         rshi_u(r0, r0, 32);
3644 }
3645
3646 static void
3647 _addi(jit_state_t *_jit, jit_int32_t r0, jit_int32_t r1, jit_word_t i0)
3648 {
3649     jit_int32_t         reg;
3650     if (i0 >= -8192 && i0 <= 8191)
3651         ADDS(r0, i0, r1);
3652     else if (!(r1 & ~3) && i0 >= -2097152 && i0 <= 2097151)
3653         ADDL(r1, i0, r0);
3654     else {
3655         reg = jit_get_reg(jit_class_gpr);
3656         movi(rn(reg), i0);
3657         addr(r0, r1, rn(reg));
3658         jit_unget_reg(reg);
3659     }
3660 }
3661
3662 static void
3663 _addcr(jit_state_t *_jit, jit_int32_t r0, jit_int32_t r1, jit_int32_t r2)
3664 {
3665     jit_int32_t         reg;
3666     if (jit_carry == _NOREG)
3667         jit_carry = jit_get_reg(jit_class_gpr);
3668     if (r0 == r1) {
3669         reg = jit_get_reg(jit_class_gpr);
3670         addr(rn(reg), r1, r2);
3671         ltr_u(rn(jit_carry), rn(reg), r1);
3672         movr(r0, rn(reg));
3673         jit_unget_reg(reg);
3674     }
3675     else {
3676         addr(r0, r1, r2);
3677         ltr_u(rn(jit_carry), r0, r1);
3678     }
3679 }
3680
3681 static void
3682 _addci(jit_state_t *_jit, jit_int32_t r0, jit_int32_t r1, jit_word_t i0)
3683 {
3684     jit_int32_t         reg;
3685     if (jit_carry == _NOREG)
3686         jit_carry = jit_get_reg(jit_class_gpr);
3687     if (r0 == r1) {
3688         reg = jit_get_reg(jit_class_gpr);
3689         addi(rn(reg), r1, i0);
3690         ltr_u(rn(jit_carry), rn(reg), r1);
3691         movr(r0, rn(reg));
3692         jit_unget_reg(reg);
3693     }
3694     else {
3695         addi(r0, r1, i0);
3696         ltr_u(rn(jit_carry), r0, r1);
3697     }
3698 }
3699
3700 static void
3701 _addxr(jit_state_t *_jit, jit_int32_t r0, jit_int32_t r1, jit_int32_t r2)
3702 {
3703     jit_int32_t         reg;
3704     assert(jit_carry != _NOREG);
3705     reg = jit_get_reg(jit_class_gpr);
3706     movr(rn(reg), rn(jit_carry));
3707     addcr(r0, r1, r2);
3708     addcr(r0, r0, rn(reg));
3709     jit_unget_reg(reg);
3710 }
3711
3712 static void
3713 _addxi(jit_state_t *_jit, jit_int32_t r0, jit_int32_t r1, jit_word_t i0)
3714 {
3715     jit_int32_t         reg;
3716     assert(jit_carry != _NOREG);
3717     reg = jit_get_reg(jit_class_gpr);
3718     movr(rn(reg), rn(jit_carry));
3719     addci(r0, r1, i0);
3720     addcr(r0, r0, rn(reg));
3721     jit_unget_reg(reg);
3722 }
3723
3724 static void
3725 _subi(jit_state_t *_jit, jit_int32_t r0, jit_int32_t r1, jit_word_t i0)
3726 {
3727     jit_int32_t         reg;
3728     reg = jit_get_reg(jit_class_gpr);
3729     movi(rn(reg), i0);
3730     subr(r0, r1, rn(reg));
3731     jit_unget_reg(reg);
3732 }
3733
3734 static void
3735 _subcr(jit_state_t *_jit, jit_int32_t r0, jit_int32_t r1, jit_int32_t r2)
3736 {
3737     jit_int32_t         reg;
3738     if (jit_carry == _NOREG)
3739         jit_carry = jit_get_reg(jit_class_gpr);
3740     if (r0 == r1) {
3741         reg = jit_get_reg(jit_class_gpr);
3742         subr(rn(reg), r1, r2);
3743         ltr_u(rn(jit_carry), r1, rn(reg));
3744         movr(r0, rn(reg));
3745         jit_unget_reg(reg);
3746     }
3747     else {
3748         subr(r0, r1, r2);
3749         ltr_u(rn(jit_carry), r1, r0);
3750     }
3751 }
3752
3753 static void
3754 _subci(jit_state_t *_jit, jit_int32_t r0, jit_int32_t r1, jit_word_t i0)
3755 {
3756     jit_int32_t         reg;
3757     if (jit_carry == _NOREG)
3758         jit_carry = jit_get_reg(jit_class_gpr);
3759     if (r0 == r1) {
3760         reg = jit_get_reg(jit_class_gpr);
3761         addi(rn(reg), r1, -i0);
3762         ltr_u(rn(jit_carry), r1, rn(reg));
3763         movr(r0, rn(reg));
3764         jit_unget_reg(reg);
3765     }
3766     else {
3767         addi(r0, r1, -i0);
3768         ltr_u(rn(jit_carry), r1, r0);
3769     }
3770 }
3771
3772 static void
3773 _subxr(jit_state_t *_jit, jit_int32_t r0, jit_int32_t r1, jit_int32_t r2)
3774 {
3775     jit_int32_t         reg;
3776     assert(jit_carry != _NOREG);
3777     reg = jit_get_reg(jit_class_gpr);
3778     movr(rn(reg), rn(jit_carry));
3779     subcr(r0, r1, r2);
3780     subcr(r0, r0, rn(reg));
3781     jit_unget_reg(reg);
3782 }
3783
3784 static void
3785 _subxi(jit_state_t *_jit, jit_int32_t r0, jit_int32_t r1, jit_word_t i0)
3786 {
3787     jit_int32_t         reg;
3788     assert(jit_carry != _NOREG);
3789     reg = jit_get_reg(jit_class_gpr);
3790     movr(rn(reg), rn(jit_carry));
3791     subci(r0, r1, i0);
3792     subcr(r0, r0, rn(reg));
3793     jit_unget_reg(reg);
3794 }
3795
3796 static void
3797 _rsbi(jit_state_t *_jit, jit_int32_t r0, jit_int32_t r1, jit_word_t i0)
3798 {
3799     jit_int32_t         reg;
3800     if (i0 >= -128 && i0 <= 127)
3801         SUBI(r0, i0, r1);
3802     else if (!(r1 & ~3) && i0 >= -2097151 && i0 <= 2097152)
3803         ADDL(r1, -i0, r0);
3804     else {
3805         reg = jit_get_reg(jit_class_gpr);
3806         movi(rn(reg), i0);
3807         subr(r0, rn(reg), r1);
3808         jit_unget_reg(reg);
3809     }
3810 }
3811
3812 static void
3813 _mulr(jit_state_t *_jit, jit_int32_t r0, jit_int32_t r1, jit_int32_t r2)
3814 {
3815     jit_int32_t         f0, f1;
3816     f0 = jit_get_reg(jit_class_fpr);
3817     f1 = jit_get_reg(jit_class_fpr);
3818     SETF_SIG(rn(f0), r1);
3819     SETF_SIG(rn(f1), r2);
3820     XMPY_L(rn(f0), rn(f0), rn(f1));
3821     GETF_SIG(r0, rn(f0));
3822     jit_unget_reg(f0);
3823     jit_unget_reg(f1);
3824 }
3825
3826 static void
3827 _muli(jit_state_t *_jit, jit_int32_t r0, jit_int32_t r1, jit_word_t i0)
3828 {
3829     jit_int32_t         reg;
3830     reg = jit_get_reg(jit_class_gpr);
3831     movi(rn(reg), i0);
3832     mulr(r0, r1, rn(reg));
3833     jit_unget_reg(reg);
3834 }
3835
3836 #if !defined(__GNUC__)
3837 static long
3838 __divdi3(long u, long v)
3839 {
3840     return (u / v);
3841 }
3842
3843 static unsigned long
3844 __udivdi3(unsigned long u, unsigned long v)
3845 {
3846     return (u / v);
3847 }
3848
3849 static long
3850 __moddi3(long u, long v)
3851 {
3852     return (u % v);
3853 }
3854
3855 static unsigned long
3856 __umoddi3(unsigned long u, unsigned long v)
3857 {
3858     return (u % v);
3859 }
3860 #endif
3861
3862 static void
3863 _divr(jit_state_t *_jit, jit_int32_t r0, jit_int32_t r1, jit_int32_t r2)
3864 {
3865     MOV(_jitc->rout, r1);
3866     MOV(_jitc->rout + 1, r2);
3867     calli((jit_word_t)__divdi3);
3868     MOV(r0, GR_8);
3869 }
3870
3871 static void
3872 _divi(jit_state_t *_jit, jit_int32_t r0, jit_int32_t r1, jit_word_t i0)
3873 {
3874     jit_int32_t         reg;
3875     switch (i0) {
3876         case 1:
3877             movr(r0, r1);
3878             return;
3879         case -1:
3880             negr(r0, r1);
3881             return;
3882         default:
3883             if (i0 > 0 && !(i0 & (i0 - 1))) {
3884                 movr(r0, r1);
3885                 rshi(r0, r0, ffsl(i0) - 1);
3886                 return;
3887             }
3888             break;
3889     }
3890     reg = jit_get_reg(jit_class_gpr);
3891     movi(rn(reg), i0);
3892     divr(r0, r1, rn(reg));
3893     jit_unget_reg(reg);
3894 }
3895
3896 static void
3897 _divr_u(jit_state_t *_jit, jit_int32_t r0, jit_int32_t r1, jit_int32_t r2)
3898 {
3899     MOV(_jitc->rout, r1);
3900     MOV(_jitc->rout + 1, r2);
3901     calli((jit_word_t)__udivdi3);
3902     MOV(r0, GR_8);
3903 }
3904
3905 static void
3906 _divi_u(jit_state_t *_jit, jit_int32_t r0, jit_int32_t r1, jit_word_t i0)
3907 {
3908     jit_int32_t         reg;
3909     if (i0 == 1) {
3910         movr(r0, r1);
3911         return;
3912     }
3913     else if (i0 > 0 && !(i0 & (i0 - 1))) {
3914         movr(r0, r1);
3915         rshi_u(r0, r0, ffsl(i0) - 1);
3916         return;
3917     }
3918     reg = jit_get_reg(jit_class_gpr);
3919     movi(rn(reg), i0);
3920     divr_u(r0, r1, rn(reg));
3921     jit_unget_reg(reg);
3922 }
3923
3924 static void
3925 _remr(jit_state_t *_jit, jit_int32_t r0, jit_int32_t r1, jit_int32_t r2)
3926 {
3927     MOV(_jitc->rout, r1);
3928     MOV(_jitc->rout + 1, r2);
3929     calli((jit_word_t)__moddi3);
3930     MOV(r0, GR_8);
3931 }
3932
3933 static void
3934 _remi(jit_state_t *_jit, jit_int32_t r0, jit_int32_t r1, jit_word_t i0)
3935 {
3936     jit_int32_t         reg;
3937     if (i0 == 1 || i0 == -1) {
3938         MOV(r0, GR_0);
3939         return;
3940     }
3941     reg = jit_get_reg(jit_class_gpr);
3942     movi(rn(reg), i0);
3943     remr(r0, r1, rn(reg));
3944     jit_unget_reg(reg);
3945 }
3946
3947 static void
3948 _remr_u(jit_state_t *_jit, jit_int32_t r0, jit_int32_t r1, jit_int32_t r2)
3949 {
3950     MOV(_jitc->rout, r1);
3951     MOV(_jitc->rout + 1, r2);
3952     calli((jit_word_t)__umoddi3);
3953     MOV(r0, GR_8);
3954 }
3955
3956 static void
3957 _remi_u(jit_state_t *_jit, jit_int32_t r0, jit_int32_t r1, jit_word_t i0)
3958 {
3959     jit_int32_t         reg;
3960     if (i0 == 1) {
3961         MOV(r0, GR_0);
3962         return;
3963     }
3964     else if (i0 > 0 && !(i0 & (i0 - 1))) {
3965         andi(r0, r1, i0 - 1);
3966         return;
3967     }
3968     reg = jit_get_reg(jit_class_gpr);
3969     movi(rn(reg), i0);
3970     remr_u(r0, r1, rn(reg));
3971     jit_unget_reg(reg);
3972 }
3973
3974 static void
3975 _mulh(jit_state_t *_jit,
3976       jit_int32_t r0, jit_int32_t r1, jit_int32_t r2, jit_bool_t sign)
3977 {
3978     jit_int32_t         f0, f1;
3979     f0 = jit_get_reg(jit_class_fpr);
3980     f1 = jit_get_reg(jit_class_fpr);
3981     SETF_SIG(rn(f0), r1);
3982     SETF_SIG(rn(f1), r2);
3983     if (sign)
3984         XMPY_H(rn(f0), rn(f0), rn(f1));
3985     else
3986         XMPY_HU(rn(f0), rn(f0), rn(f1));
3987     GETF_SIG(r0, rn(f0));
3988     jit_unget_reg(f0);
3989     jit_unget_reg(f1);
3990 }
3991
3992 static void
3993 _iqmulr(jit_state_t *_jit, jit_int32_t r0, jit_int32_t r1,
3994         jit_int32_t r2, jit_int32_t r3, jit_bool_t sign)
3995 {
3996     jit_int32_t         reg;
3997     if (r0 == r2 || r0 == r3) {
3998         reg = jit_get_reg(jit_class_gpr);
3999         mulr(rn(reg), r2, r3);
4000     }
4001     else
4002         mulr(r0, r2, r3);
4003     mulh(r1, r2, r3, sign);
4004     if (r0 == r2 || r0 == r3) {
4005         movr(r0, rn(reg));
4006         jit_unget_reg(reg);
4007     }
4008 }
4009
4010 static void
4011 _iqmuli(jit_state_t *_jit, jit_int32_t r0, jit_int32_t r1,
4012         jit_int32_t r2, jit_word_t i0, jit_bool_t sign)
4013 {
4014     jit_int32_t         reg;
4015     reg = jit_get_reg(jit_class_gpr);
4016     movi(rn(reg), i0);
4017     iqmulr(r0, r1, r2, rn(reg), sign);
4018     jit_unget_reg(reg);
4019 }
4020
4021 static void
4022 _iqdivr(jit_state_t *_jit, jit_int32_t r0, jit_int32_t r1,
4023         jit_int32_t r2, jit_int32_t r3, jit_bool_t sign)
4024 {
4025     jit_int32_t         sv0, rg0;
4026     jit_int32_t         sv1, rg1;
4027
4028     if (r0 == r2 || r0 == r3) {
4029         sv0 = jit_get_reg(jit_class_gpr);
4030         rg0 = rn(sv0);
4031     }
4032     else
4033         rg0 = r0;
4034     if (r1 == r2 || r1 == r3) {
4035         sv1 = jit_get_reg(jit_class_gpr);
4036         rg1 = rn(sv1);
4037     }
4038     else
4039         rg1 = r1;
4040
4041     if (sign)
4042         divr(rg0, r2, r3);
4043     else
4044         divr_u(rg0, r2, r3);
4045     mulr(rg1, r3, rg0);
4046     subr(rg1, r2, rg1);
4047     if (rg0 != r0) {
4048         movr(r0, rg0);
4049         jit_unget_reg(sv0);
4050     }
4051     if (rg1 != r1) {
4052         movr(r1, rg1);
4053         jit_unget_reg(sv1);
4054     }
4055 }
4056
4057 static void
4058 _iqdivi(jit_state_t *_jit, jit_int32_t r0, jit_int32_t r1,
4059         jit_int32_t r2, jit_word_t i0, jit_bool_t sign)
4060 {
4061     jit_int32_t         reg;
4062     reg = jit_get_reg(jit_class_gpr);
4063     movi(rn(reg), i0);
4064     iqdivr(r0, r1, r2, rn(reg), sign);
4065     jit_unget_reg(reg);
4066 }
4067
4068 static void
4069 _andi(jit_state_t *_jit, jit_int32_t r0, jit_int32_t r1, jit_word_t i0)
4070 {
4071     jit_int32_t         reg;
4072     if (i0 >= -128 && i0 <= 127)
4073         ANDI(r0, i0, r1);
4074     else if (~i0 >= -128 && ~i0 <= 127)
4075         ANDCMI(r0, ~i0, r1);
4076     else {
4077         reg = jit_get_reg(jit_class_gpr);
4078         movi(rn(reg), i0);
4079         andr(r0, r1, rn(reg));
4080         jit_unget_reg(reg);
4081     }
4082 }
4083
4084 static void
4085 _ori(jit_state_t *_jit, jit_int32_t r0, jit_int32_t r1, jit_word_t i0)
4086 {
4087     jit_int32_t         reg;
4088     if (i0 >= -128 && i0 <= 127)
4089         ORI(r0, i0, r1);
4090     else {
4091         reg = jit_get_reg(jit_class_gpr);
4092         movi(rn(reg), i0);
4093         orr(r0, r1, rn(reg));
4094         jit_unget_reg(reg);
4095     }
4096 }
4097
4098 static void
4099 _xori(jit_state_t *_jit, jit_int32_t r0, jit_int32_t r1, jit_word_t i0)
4100 {
4101     jit_int32_t         reg;
4102     if (i0 >= -128 && i0 <= 127)
4103         XORI(r0, i0, r1);
4104     else {
4105         reg = jit_get_reg(jit_class_gpr);
4106         movi(rn(reg), i0);
4107         xorr(r0, r1, rn(reg));
4108         jit_unget_reg(reg);
4109     }
4110 }
4111
4112 static void
4113 _lshi(jit_state_t *_jit, jit_int32_t r0, jit_int32_t r1, jit_word_t i0)
4114 {
4115     jit_int32_t         reg;
4116     reg = jit_get_reg(jit_class_gpr);
4117     movi(rn(reg), i0);
4118     lshr(r0, r1, rn(reg));
4119     jit_unget_reg(reg);
4120 }
4121
4122 static void
4123 _rshi(jit_state_t *_jit, jit_int32_t r0, jit_int32_t r1, jit_word_t i0)
4124 {
4125     jit_int32_t         reg;
4126     reg = jit_get_reg(jit_class_gpr);
4127     movi(rn(reg), i0);
4128     rshr(r0, r1, rn(reg));
4129     jit_unget_reg(reg);
4130 }
4131
4132 static void
4133 _rshi_u(jit_state_t *_jit, jit_int32_t r0, jit_int32_t r1, jit_word_t i0)
4134 {
4135     jit_int32_t         reg;
4136     reg = jit_get_reg(jit_class_gpr);
4137     movi(rn(reg), i0);
4138     rshr_u(r0, r1, rn(reg));
4139     jit_unget_reg(reg);
4140 }
4141
4142 static void
4143 _ltr(jit_state_t *_jit, jit_int32_t r0, jit_int32_t r1, jit_int32_t r2)
4144 {
4145     CMP_LT(PR_6, PR_7, r1, r2);
4146     MOVI_p(r0, 1, PR_6);
4147     MOV_p(r0, GR_0, PR_7);
4148 }
4149
4150 static void
4151 _lti(jit_state_t *_jit, jit_int32_t r0, jit_int32_t r1, jit_word_t i0)
4152 {
4153     jit_int32_t         reg;
4154     if (i0 >= -127 && i0 <= 128)
4155         CMPI_LT(PR_7, PR_6, i0 - 1, r1);
4156     else {
4157         reg = jit_get_reg(jit_class_gpr);
4158         movi(rn(reg), i0);
4159         CMP_LT(PR_6, PR_7, r1, rn(reg));
4160         jit_unget_reg(reg);
4161     }
4162     MOVI_p(r0, 1, PR_6);
4163     MOV_p(r0, GR_0, PR_7);
4164 }
4165
4166 static void
4167 _ltr_u(jit_state_t *_jit, jit_int32_t r0, jit_int32_t r1, jit_int32_t r2)
4168 {
4169     CMP_LTU(PR_6, PR_7, r1, r2);
4170     MOVI_p(r0, 1, PR_6);
4171     MOV_p(r0, GR_0, PR_7);
4172 }
4173
4174 static void
4175 _lti_u(jit_state_t *_jit, jit_int32_t r0, jit_int32_t r1, jit_word_t i0)
4176 {
4177     jit_int32_t         reg;
4178     if (i0 >= -127 && i0 <= 128)
4179         CMPI_LTU(PR_7, PR_6, i0 - 1, r1);
4180     else {
4181         reg = jit_get_reg(jit_class_gpr);
4182         movi(rn(reg), i0);
4183         CMP_LTU(PR_6, PR_7, r1, rn(reg));
4184         jit_unget_reg(reg);
4185     }
4186     MOVI_p(r0, 1, PR_6);
4187     MOV_p(r0, GR_0, PR_7);
4188 }
4189
4190 static void
4191 _ler(jit_state_t *_jit, jit_int32_t r0, jit_int32_t r1, jit_int32_t r2)
4192 {
4193     CMP_LT(PR_6, PR_7, r2, r1);
4194     MOV_p(r0, GR_0, PR_6);
4195     MOVI_p(r0, 1, PR_7);
4196 }
4197
4198 static void
4199 _lei(jit_state_t *_jit, jit_int32_t r0, jit_int32_t r1, jit_word_t i0)
4200 {
4201     jit_int32_t         reg;
4202     reg = jit_get_reg(jit_class_gpr);
4203     movi(rn(reg), i0);
4204     ler(r0, r1, rn(reg));
4205     jit_unget_reg(reg);
4206 }
4207
4208 static void
4209 _ler_u(jit_state_t *_jit, jit_int32_t r0, jit_int32_t r1, jit_int32_t r2)
4210 {
4211     CMP_LTU(PR_6, PR_7, r2, r1);
4212     MOV_p(r0, GR_0, PR_6);
4213     MOVI_p(r0, 1, PR_7);
4214 }
4215
4216 static void
4217 _lei_u(jit_state_t *_jit, jit_int32_t r0, jit_int32_t r1, jit_word_t i0)
4218 {
4219     jit_int32_t         reg;
4220     reg = jit_get_reg(jit_class_gpr);
4221     movi(rn(reg), i0);
4222     ler_u(r0, r1, rn(reg));
4223     jit_unget_reg(reg);
4224 }
4225
4226 static void
4227 _eqr(jit_state_t *_jit, jit_int32_t r0, jit_int32_t r1, jit_int32_t r2)
4228 {
4229     CMP_EQ(PR_6, PR_7, r1, r2);
4230     MOVI_p(r0, 1, PR_6);
4231     MOV_p(r0, GR_0, PR_7);
4232 }
4233
4234 static void
4235 _eqi(jit_state_t *_jit, jit_int32_t r0, jit_int32_t r1, jit_word_t i0)
4236 {
4237     jit_int32_t         reg;
4238     if (i0 >= -128 && i0 <= 127)
4239         CMPI_EQ(PR_6, PR_7, i0, r1);
4240     else {
4241         reg = jit_get_reg(jit_class_gpr);
4242         movi(rn(reg), i0);
4243         CMP_EQ(PR_6, PR_7, r1, rn(reg));
4244         jit_unget_reg(reg);
4245     }
4246     MOVI_p(r0, 1, PR_6);
4247     MOV_p(r0, GR_0, PR_7);
4248 }
4249
4250 static void
4251 _ger(jit_state_t *_jit, jit_int32_t r0, jit_int32_t r1, jit_int32_t r2)
4252 {
4253     CMP_LT(PR_6, PR_7, r1, r2);
4254     MOV_p(r0, GR_0, PR_6);
4255     MOVI_p(r0, 1, PR_7);
4256 }
4257
4258 static void
4259 _gei(jit_state_t *_jit, jit_int32_t r0, jit_int32_t r1, jit_word_t i0)
4260 {
4261     jit_int32_t         reg;
4262     if (i0 >= -127 && i0 <= 128)
4263         CMPI_LT(PR_7, PR_6, i0 - 1, r1);
4264     else {
4265         reg = jit_get_reg(jit_class_gpr);
4266         movi(rn(reg), i0);
4267         CMP_LT(PR_6, PR_7, r1, rn(reg));
4268         jit_unget_reg(reg);
4269     }
4270     MOV_p(r0, GR_0, PR_6);
4271     MOVI_p(r0, 1, PR_7);
4272 }
4273
4274 static void
4275 _ger_u(jit_state_t *_jit, jit_int32_t r0, jit_int32_t r1, jit_int32_t r2)
4276 {
4277     CMP_LTU(PR_6, PR_7, r1, r2);
4278     MOV_p(r0, GR_0, PR_6);
4279     MOVI_p(r0, 1, PR_7);
4280 }
4281
4282 static void
4283 _gei_u(jit_state_t *_jit, jit_int32_t r0, jit_int32_t r1, jit_word_t i0)
4284 {
4285     jit_int32_t         reg;
4286     if (i0 >= -127 && i0 <= 128)
4287         CMPI_LTU(PR_7, PR_6, i0 - 1, r1);
4288     else {
4289         reg = jit_get_reg(jit_class_gpr);
4290         movi(rn(reg), i0);
4291         CMP_LTU(PR_6, PR_7, r1, rn(reg));
4292         jit_unget_reg(reg);
4293     }
4294     MOV_p(r0, GR_0, PR_6);
4295     MOVI_p(r0, 1, PR_7);
4296 }
4297
4298 static void
4299 _gtr(jit_state_t *_jit, jit_int32_t r0, jit_int32_t r1, jit_int32_t r2)
4300 {
4301     CMP_LT(PR_6, PR_7, r2, r1);
4302     MOVI_p(r0, 1, PR_6);
4303     MOV_p(r0, GR_0, PR_7);
4304 }
4305
4306 static void
4307 _gti(jit_state_t *_jit, jit_int32_t r0, jit_int32_t r1, jit_word_t i0)
4308 {
4309     jit_int32_t         reg;
4310     reg = jit_get_reg(jit_class_gpr);
4311     movi(rn(reg), i0);
4312     gtr(r0, r1, rn(reg));
4313     jit_unget_reg(reg);
4314 }
4315
4316 static void
4317 _gtr_u(jit_state_t *_jit, jit_int32_t r0, jit_int32_t r1, jit_int32_t r2)
4318 {
4319     CMP_LTU(PR_6, PR_7, r2, r1);
4320     MOVI_p(r0, 1, PR_6);
4321     MOV_p(r0, GR_0, PR_7);
4322 }
4323
4324 static void
4325 _gti_u(jit_state_t *_jit, jit_int32_t r0, jit_int32_t r1, jit_word_t i0)
4326 {
4327     jit_int32_t         reg;
4328     reg = jit_get_reg(jit_class_gpr);
4329     movi(rn(reg), i0);
4330     gtr_u(r0, r1, rn(reg));
4331     jit_unget_reg(reg);
4332 }
4333
4334 static void
4335 _ner(jit_state_t *_jit, jit_int32_t r0, jit_int32_t r1, jit_int32_t r2)
4336 {
4337     CMP_EQ(PR_6, PR_7, r1, r2);
4338     MOV_p(r0, GR_0, PR_6);
4339     MOVI_p(r0, 1, PR_7);
4340 }
4341
4342 static void
4343 _nei(jit_state_t *_jit, jit_int32_t r0, jit_int32_t r1, jit_word_t i0)
4344 {
4345     jit_int32_t         reg;
4346     if (i0 >= -128 && i0 <= 127)
4347         CMPI_EQ(PR_6, PR_7, i0, r1);
4348     else {
4349         reg = jit_get_reg(jit_class_gpr);
4350         movi(rn(reg), i0);
4351         CMP_EQ(PR_6, PR_7, r1, rn(reg));
4352         jit_unget_reg(reg);
4353     }
4354     MOV_p(r0, GR_0, PR_6);
4355     MOVI_p(r0, 1, PR_7);
4356 }
4357
4358 static void
4359 _ldr_c(jit_state_t *_jit, jit_int32_t r0, jit_int32_t r1)
4360 {
4361     ldr_uc(r0, r1);
4362     extr_c(r0, r0);
4363 }
4364
4365 static void
4366 _ldi_c(jit_state_t *_jit, jit_int32_t r0, jit_word_t i0)
4367 {
4368     jit_int32_t         reg;
4369     reg = jit_get_reg(jit_class_gpr);
4370     movi(rn(reg), i0);
4371     ldr_c(r0, rn(reg));
4372     jit_unget_reg(reg);
4373 }
4374
4375 static void
4376 _ldi_uc(jit_state_t *_jit, jit_int32_t r0, jit_word_t i0)
4377 {
4378     jit_int32_t         reg;
4379     reg = jit_get_reg(jit_class_gpr);
4380     movi(rn(reg), i0);
4381     ldr_uc(r0, rn(reg));
4382     jit_unget_reg(reg);
4383 }
4384
4385 static void
4386 _ldr_s(jit_state_t *_jit, jit_int32_t r0, jit_int32_t r1)
4387 {
4388     ldr_us(r0, r1);
4389     extr_s(r0, r0);
4390 }
4391
4392 static void
4393 _ldi_s(jit_state_t *_jit, jit_int32_t r0, jit_word_t i0)
4394 {
4395     jit_int32_t         reg;
4396     reg = jit_get_reg(jit_class_gpr);
4397     movi(rn(reg), i0);
4398     ldr_s(r0, rn(reg));
4399     jit_unget_reg(reg);
4400 }
4401
4402 static void
4403 _ldi_us(jit_state_t *_jit, jit_int32_t r0, jit_word_t i0)
4404 {
4405     jit_int32_t         reg;
4406     reg = jit_get_reg(jit_class_gpr);
4407     movi(rn(reg), i0);
4408     ldr_us(r0, rn(reg));
4409     jit_unget_reg(reg);
4410 }
4411
4412 static void
4413 _ldr_i(jit_state_t *_jit, jit_int32_t r0, jit_int32_t r1)
4414 {
4415     ldr_ui(r0, r1);
4416     extr_i(r0, r0);
4417 }
4418
4419 static void
4420 _ldi_i(jit_state_t *_jit, jit_int32_t r0, jit_word_t i0)
4421 {
4422     jit_int32_t         reg;
4423     reg = jit_get_reg(jit_class_gpr);
4424     movi(rn(reg), i0);
4425     ldr_i(r0, rn(reg));
4426     jit_unget_reg(reg);
4427 }
4428
4429 static void
4430 _ldi_ui(jit_state_t *_jit, jit_int32_t r0, jit_word_t i0)
4431 {
4432     jit_int32_t         reg;
4433     reg = jit_get_reg(jit_class_gpr);
4434     movi(rn(reg), i0);
4435     ldr_ui(r0, rn(reg));
4436     jit_unget_reg(reg);
4437 }
4438
4439 static void
4440 _ldi_l(jit_state_t *_jit, jit_int32_t r0, jit_word_t i0)
4441 {
4442     jit_int32_t         reg;
4443     reg = jit_get_reg(jit_class_gpr);
4444     movi(rn(reg), i0);
4445     ldr_l(r0, rn(reg));
4446     jit_unget_reg(reg);
4447 }
4448
4449 static void
4450 _ldxr_c(jit_state_t *_jit, jit_int32_t r0, jit_int32_t r1, jit_int32_t r2)
4451 {
4452     jit_int32_t         reg;
4453     reg = jit_get_reg(jit_class_gpr);
4454     addr(rn(reg), r1, r2);
4455     ldr_c(r0, rn(reg));
4456     jit_unget_reg(reg);
4457 }
4458
4459 static void
4460 _ldxi_c(jit_state_t *_jit, jit_int32_t r0, jit_int32_t r1, jit_word_t i0)
4461 {
4462     jit_int32_t         reg;
4463     if (i0) {
4464         reg = jit_get_reg(jit_class_gpr);
4465         movi(rn(reg), i0);
4466         ldxr_c(r0, r1, rn(reg));
4467         jit_unget_reg(reg);
4468     }
4469     else
4470         ldr_c(r0, r1);
4471 }
4472
4473 static void
4474 _ldxr_uc(jit_state_t *_jit, jit_int32_t r0, jit_int32_t r1, jit_int32_t r2)
4475 {
4476     jit_int32_t         reg;
4477     reg = jit_get_reg(jit_class_gpr);
4478     addr(rn(reg), r1, r2);
4479     ldr_uc(r0, rn(reg));
4480     jit_unget_reg(reg);
4481 }
4482
4483 static void
4484 _ldxi_uc(jit_state_t *_jit, jit_int32_t r0, jit_int32_t r1, jit_word_t i0)
4485 {
4486     jit_int32_t         reg;
4487     if (i0) {
4488         reg = jit_get_reg(jit_class_gpr);
4489         movi(rn(reg), i0);
4490         ldxr_uc(r0, r1, rn(reg));
4491         jit_unget_reg(reg);
4492     }
4493     else
4494         ldr_uc(r0, r1);
4495 }
4496
4497 static void
4498 _ldxr_s(jit_state_t *_jit, jit_int32_t r0, jit_int32_t r1, jit_int32_t r2)
4499 {
4500     jit_int32_t         reg;
4501     reg = jit_get_reg(jit_class_gpr);
4502     addr(rn(reg), r1, r2);
4503     ldr_s(r0, rn(reg));
4504     jit_unget_reg(reg);
4505 }
4506
4507 static void
4508 _ldxi_s(jit_state_t *_jit, jit_int32_t r0, jit_int32_t r1, jit_word_t i0)
4509 {
4510     jit_int32_t         reg;
4511     if (i0) {
4512         reg = jit_get_reg(jit_class_gpr);
4513         movi(rn(reg), i0);
4514         ldxr_s(r0, r1, rn(reg));
4515         jit_unget_reg(reg);
4516     }
4517     else
4518         ldr_s(r0, r1);
4519 }
4520
4521 static void
4522 _ldxr_us(jit_state_t *_jit, jit_int32_t r0, jit_int32_t r1, jit_int32_t r2)
4523 {
4524     jit_int32_t         reg;
4525     reg = jit_get_reg(jit_class_gpr);
4526     addr(rn(reg), r1, r2);
4527     ldr_us(r0, rn(reg));
4528     jit_unget_reg(reg);
4529 }
4530
4531 static void
4532 _ldxi_us(jit_state_t *_jit, jit_int32_t r0, jit_int32_t r1, jit_word_t i0)
4533 {
4534     jit_int32_t         reg;
4535     if (i0) {
4536         reg = jit_get_reg(jit_class_gpr);
4537         movi(rn(reg), i0);
4538         ldxr_us(r0, r1, rn(reg));
4539         jit_unget_reg(reg);
4540     }
4541     else
4542         ldr_us(r0, r1);
4543 }
4544
4545 static void
4546 _ldxr_i(jit_state_t *_jit, jit_int32_t r0, jit_int32_t r1, jit_int32_t r2)
4547 {
4548     jit_int32_t         reg;
4549     reg = jit_get_reg(jit_class_gpr);
4550     addr(rn(reg), r1, r2);
4551     ldr_i(r0, rn(reg));
4552     jit_unget_reg(reg);
4553 }
4554
4555 static void
4556 _ldxi_i(jit_state_t *_jit, jit_int32_t r0, jit_int32_t r1, jit_word_t i0)
4557 {
4558     jit_int32_t         reg;
4559     if (i0) {
4560         reg = jit_get_reg(jit_class_gpr);
4561         movi(rn(reg), i0);
4562         ldxr_i(r0, r1, rn(reg));
4563         jit_unget_reg(reg);
4564     }
4565     else
4566         ldr_i(r0, r1);
4567 }
4568
4569 static void
4570 _ldxr_ui(jit_state_t *_jit, jit_int32_t r0, jit_int32_t r1, jit_int32_t r2)
4571 {
4572     jit_int32_t         reg;
4573     reg = jit_get_reg(jit_class_gpr);
4574     addr(rn(reg), r1, r2);
4575     ldr_ui(r0, rn(reg));
4576     jit_unget_reg(reg);
4577 }
4578
4579 static void
4580 _ldxi_ui(jit_state_t *_jit, jit_int32_t r0, jit_int32_t r1, jit_word_t i0)
4581 {
4582     jit_int32_t         reg;
4583     if (i0) {
4584         reg = jit_get_reg(jit_class_gpr);
4585         movi(rn(reg), i0);
4586         ldxr_ui(r0, r1, rn(reg));
4587         jit_unget_reg(reg);
4588     }
4589     else
4590         ldr_ui(r0, r1);
4591 }
4592
4593 static void
4594 _ldxr_l(jit_state_t *_jit, jit_int32_t r0, jit_int32_t r1, jit_int32_t r2)
4595 {
4596     jit_int32_t         reg;
4597     reg = jit_get_reg(jit_class_gpr);
4598     addr(rn(reg), r1, r2);
4599     ldr_l(r0, rn(reg));
4600     jit_unget_reg(reg);
4601 }
4602
4603 static void
4604 _ldxi_l(jit_state_t *_jit, jit_int32_t r0, jit_int32_t r1, jit_word_t i0)
4605 {
4606     jit_int32_t         reg;
4607     if (i0) {
4608         reg = jit_get_reg(jit_class_gpr);
4609         movi(rn(reg), i0);
4610         ldxr_l(r0, r1, rn(reg));
4611         jit_unget_reg(reg);
4612     }
4613     else
4614         ldr_l(r0, r1);
4615 }
4616
4617 static void
4618 _sti_c(jit_state_t *_jit, jit_word_t i0, jit_int32_t r0)
4619 {
4620     jit_int32_t         reg;
4621     reg = jit_get_reg(jit_class_gpr);
4622     movi(rn(reg), i0);
4623     str_c(rn(reg), r0);
4624     jit_unget_reg(reg);
4625 }
4626
4627 static void
4628 _sti_s(jit_state_t *_jit, jit_word_t i0, jit_int32_t r0)
4629 {
4630     jit_int32_t         reg;
4631     reg = jit_get_reg(jit_class_gpr);
4632     movi(rn(reg), i0);
4633     str_s(rn(reg), r0);
4634     jit_unget_reg(reg);
4635 }
4636
4637 static void
4638 _sti_i(jit_state_t *_jit, jit_word_t i0, jit_int32_t r0)
4639 {
4640     jit_int32_t         reg;
4641     reg = jit_get_reg(jit_class_gpr);
4642     movi(rn(reg), i0);
4643     str_i(rn(reg), r0);
4644     jit_unget_reg(reg);
4645 }
4646
4647 static void
4648 _sti_l(jit_state_t *_jit, jit_word_t i0, jit_int32_t r0)
4649 {
4650     jit_int32_t         reg;
4651     reg = jit_get_reg(jit_class_gpr);
4652     movi(rn(reg), i0);
4653     str_l(rn(reg), r0);
4654     jit_unget_reg(reg);
4655 }
4656
4657 static void
4658 _stxr_c(jit_state_t *_jit, jit_int32_t r0, jit_int32_t r1, jit_int32_t r2)
4659 {
4660     jit_int32_t         reg;
4661     reg = jit_get_reg(jit_class_gpr);
4662     addr(rn(reg), r0, r1);
4663     str_c(rn(reg), r2);
4664     jit_unget_reg(reg);
4665 }
4666
4667 static void
4668 _stxi_c(jit_state_t *_jit, jit_word_t i0, jit_int32_t r0, jit_int32_t r1)
4669 {
4670     jit_int32_t         reg;
4671     if (i0) {
4672         reg = jit_get_reg(jit_class_gpr);
4673         addi(rn(reg), r0, i0);
4674         str_c(rn(reg), r1);
4675         jit_unget_reg(reg);
4676     }
4677     else
4678         str_c(r0, r1);
4679 }
4680
4681 static void
4682 _stxr_s(jit_state_t *_jit, jit_int32_t r0, jit_int32_t r1, jit_int32_t r2)
4683 {
4684     jit_int32_t         reg;
4685     reg = jit_get_reg(jit_class_gpr);
4686     addr(rn(reg), r0, r1);
4687     str_s(rn(reg), r2);
4688     jit_unget_reg(reg);
4689 }
4690
4691 static void
4692 _stxi_s(jit_state_t *_jit, jit_word_t i0, jit_int32_t r0, jit_int32_t r1)
4693 {
4694     jit_int32_t         reg;
4695     if (i0) {
4696         reg = jit_get_reg(jit_class_gpr);
4697         addi(rn(reg), r0, i0);
4698         str_s(rn(reg), r1);
4699         jit_unget_reg(reg);
4700     }
4701     else
4702         str_s(r0, r1);
4703 }
4704
4705 static void
4706 _stxr_i(jit_state_t *_jit, jit_int32_t r0, jit_int32_t r1, jit_int32_t r2)
4707 {
4708     jit_int32_t         reg;
4709     reg = jit_get_reg(jit_class_gpr);
4710     addr(rn(reg), r0, r1);
4711     str_i(rn(reg), r2);
4712     jit_unget_reg(reg);
4713 }
4714
4715 static void
4716 _stxi_i(jit_state_t *_jit, jit_word_t i0, jit_int32_t r0, jit_int32_t r1)
4717 {
4718     jit_int32_t         reg;
4719     if (i0) {
4720         reg = jit_get_reg(jit_class_gpr);
4721         addi(rn(reg), r0, i0);
4722         str_i(rn(reg), r1);
4723         jit_unget_reg(reg);
4724     }
4725     else
4726         str_i(r0, r1);
4727 }
4728
4729 static void
4730 _stxr_l(jit_state_t *_jit, jit_int32_t r0, jit_int32_t r1, jit_int32_t r2)
4731 {
4732     jit_int32_t         reg;
4733     reg = jit_get_reg(jit_class_gpr);
4734     addr(rn(reg), r0, r1);
4735     str_l(rn(reg), r2);
4736     jit_unget_reg(reg);
4737 }
4738
4739 static void
4740 _stxi_l(jit_state_t *_jit, jit_word_t i0, jit_int32_t r0, jit_int32_t r1)
4741 {
4742     jit_int32_t         reg;
4743     if (i0) {
4744         reg = jit_get_reg(jit_class_gpr);
4745         addi(rn(reg), r0, i0);
4746         str_l(rn(reg), r1);
4747         jit_unget_reg(reg);
4748     }
4749     else
4750         str_l(r0, r1);
4751 }
4752
4753 static jit_word_t
4754 _bltr(jit_state_t *_jit, jit_word_t i0, jit_int32_t r0, jit_int32_t r1)
4755 {
4756     jit_word_t          w;
4757     CMP_LT(PR_6, PR_7, r0, r1);
4758     sync();
4759     w = _jit->pc.w;
4760     BRI_COND((i0 - w) >> 4, PR_6);
4761     return (w);
4762 }
4763
4764 static jit_word_t
4765 _blti(jit_state_t *_jit, jit_word_t i0, jit_int32_t r0, jit_word_t i1)
4766 {
4767     jit_word_t          w;
4768     jit_int32_t         reg;
4769     if (i1 >= -127 && i1 <= 128)
4770         CMPI_LT(PR_7, PR_6, i1 - 1, r0);
4771     else {
4772         reg = jit_get_reg(jit_class_gpr);
4773         movi(rn(reg), i1);
4774         CMP_LT(PR_6, PR_7, r0, rn(reg));
4775         jit_unget_reg(reg);
4776     }
4777     sync();
4778     w = _jit->pc.w;
4779     BRI_COND((i0 - w) >> 4, PR_6);
4780     return (w);
4781 }
4782
4783 static jit_word_t
4784 _bltr_u(jit_state_t *_jit, jit_word_t i0, jit_int32_t r0, jit_int32_t r1)
4785 {
4786     jit_word_t          w;
4787     CMP_LTU(PR_6, PR_7, r0, r1);
4788     sync();
4789     w = _jit->pc.w;
4790     BRI_COND((i0 - w) >> 4, PR_6);
4791     return (w);
4792 }
4793
4794 static jit_word_t
4795 _blti_u(jit_state_t *_jit, jit_word_t i0, jit_int32_t r0, jit_word_t i1)
4796 {
4797     jit_word_t          w;
4798     jit_int32_t         reg;
4799     if (i1 >= -127 && i1 <= 128)
4800         CMPI_LTU(PR_7, PR_6, i1 - 1, r0);
4801     else {
4802         reg = jit_get_reg(jit_class_gpr);
4803         movi(rn(reg), i1);
4804         CMP_LTU(PR_6, PR_7, r0, rn(reg));
4805         jit_unget_reg(reg);
4806     }
4807     sync();
4808     w = _jit->pc.w;
4809     BRI_COND((i0 - w) >> 4, PR_6);
4810     return (w);
4811 }
4812
4813 static jit_word_t
4814 _bler(jit_state_t *_jit, jit_word_t i0, jit_int32_t r0, jit_int32_t r1)
4815 {
4816     jit_word_t          w;
4817     CMP_LT(PR_6, PR_7, r1, r0);
4818     sync();
4819     w = _jit->pc.w;
4820     BRI_COND((i0 - w) >> 4, PR_7);
4821     return (w);
4822 }
4823
4824 static jit_word_t
4825 _blei(jit_state_t *_jit, jit_word_t i0, jit_int32_t r0, jit_word_t i1)
4826 {
4827     jit_word_t          w;
4828     jit_int32_t         reg;
4829     reg = jit_get_reg(jit_class_gpr|jit_class_nospill);
4830     movi(rn(reg), i1);
4831     w = bler(i0, r0, rn(reg));
4832     jit_unget_reg(reg);
4833     return (w);
4834 }
4835
4836 static jit_word_t
4837 _bler_u(jit_state_t *_jit, jit_word_t i0, jit_int32_t r0, jit_int32_t r1)
4838 {
4839     jit_word_t          w;
4840     CMP_LTU(PR_6, PR_7, r1, r0);
4841     sync();
4842     w = _jit->pc.w;
4843     BRI_COND((i0 - w) >> 4, PR_7);
4844     return (w);
4845 }
4846
4847 static jit_word_t
4848 _blei_u(jit_state_t *_jit, jit_word_t i0, jit_int32_t r0, jit_word_t i1)
4849 {
4850     jit_word_t          w;
4851     jit_int32_t         reg;
4852     reg = jit_get_reg(jit_class_gpr|jit_class_nospill);
4853     movi(rn(reg), i1);
4854     w = bler_u(i0, r0, rn(reg));
4855     jit_unget_reg(reg);
4856     return (w);
4857 }
4858
4859 static jit_word_t
4860 _beqr(jit_state_t *_jit, jit_word_t i0, jit_int32_t r0, jit_int32_t r1)
4861 {
4862     jit_word_t          w;
4863     CMP_EQ(PR_6, PR_7, r0, r1);
4864     sync();
4865     w = _jit->pc.w;
4866     BRI_COND((i0 - w) >> 4, PR_6);
4867     return (w);
4868 }
4869
4870 static jit_word_t
4871 _beqi(jit_state_t *_jit, jit_word_t i0, jit_int32_t r0, jit_word_t i1)
4872 {
4873     jit_word_t          w;
4874     jit_int32_t         reg;
4875     if (i1 >= -128 && i1 <= 127)
4876         CMPI_EQ(PR_6, PR_7, i1, r0);
4877     else {
4878         reg = jit_get_reg(jit_class_gpr);
4879         movi(rn(reg), i1);
4880         CMP_EQ(PR_6, PR_7, r0, rn(reg));
4881         jit_unget_reg(reg);
4882     }
4883     sync();
4884     w = _jit->pc.w;
4885     BRI_COND((i0 - w) >> 4, PR_6);
4886     return (w);
4887 }
4888
4889 static jit_word_t
4890 _bger(jit_state_t *_jit, jit_word_t i0, jit_int32_t r0, jit_int32_t r1)
4891 {
4892     jit_word_t          w;
4893     CMP_LT(PR_6, PR_7, r0, r1);
4894     sync();
4895     w = _jit->pc.w;
4896     BRI_COND((i0 - w) >> 4, PR_7);
4897     return (w);
4898 }
4899
4900 static jit_word_t
4901 _bgei(jit_state_t *_jit, jit_word_t i0, jit_int32_t r0, jit_word_t i1)
4902 {
4903     jit_word_t          w;
4904     jit_int32_t         reg;
4905     if (i1 >= -127 && i1 <= 128)
4906         CMPI_LT(PR_7, PR_6, i1 - 1, r0);
4907     else {
4908         reg = jit_get_reg(jit_class_gpr);
4909         movi(rn(reg), i1);
4910         CMP_LT(PR_6, PR_7, r0, rn(reg));
4911         jit_unget_reg(reg);
4912     }
4913     sync();
4914     w = _jit->pc.w;
4915     BRI_COND((i0 - w) >> 4, PR_7);
4916     return (w);
4917 }
4918
4919 static jit_word_t
4920 _bger_u(jit_state_t *_jit, jit_word_t i0, jit_int32_t r0, jit_int32_t r1)
4921 {
4922     jit_word_t          w;
4923     CMP_LTU(PR_6, PR_7, r0, r1);
4924     sync();
4925     w = _jit->pc.w;
4926     BRI_COND((i0 - w) >> 4, PR_7);
4927     return (w);
4928 }
4929
4930 static jit_word_t
4931 _bgei_u(jit_state_t *_jit, jit_word_t i0, jit_int32_t r0, jit_word_t i1)
4932 {
4933     jit_word_t          w;
4934     jit_int32_t         reg;
4935     if (i1 >= -127 && i1 <= 128)
4936         CMPI_LTU(PR_7, PR_6, i1 - 1, r0);
4937     else {
4938         reg = jit_get_reg(jit_class_gpr);
4939         movi(rn(reg), i1);
4940         CMP_LTU(PR_6, PR_7, r0, rn(reg));
4941         jit_unget_reg(reg);
4942     }
4943     sync();
4944     w = _jit->pc.w;
4945     BRI_COND((i0 - w) >> 4, PR_7);
4946     return (w);
4947 }
4948
4949 static jit_word_t
4950 _bgtr(jit_state_t *_jit, jit_word_t i0, jit_int32_t r0, jit_int32_t r1)
4951 {
4952     jit_word_t          w;
4953     CMP_LT(PR_6, PR_7, r1, r0);
4954     sync();
4955     w = _jit->pc.w;
4956     BRI_COND((i0 - w) >> 4, PR_6);
4957     return (w);
4958 }
4959
4960 static jit_word_t
4961 _bgti(jit_state_t *_jit, jit_word_t i0, jit_int32_t r0, jit_word_t i1)
4962 {
4963     jit_word_t          w;
4964     jit_int32_t         reg;
4965     reg = jit_get_reg(jit_class_gpr|jit_class_nospill);
4966     movi(rn(reg), i1);
4967     w = bgtr(i0, r0, rn(reg));
4968     jit_unget_reg(reg);
4969     return (w);
4970 }
4971
4972 static jit_word_t
4973 _bgtr_u(jit_state_t *_jit, jit_word_t i0, jit_int32_t r0, jit_int32_t r1)
4974 {
4975     jit_word_t          w;
4976     CMP_LTU(PR_6, PR_7, r1, r0);
4977     sync();
4978     w = _jit->pc.w;
4979     BRI_COND((i0 - w) >> 4, PR_6);
4980     return (w);
4981 }
4982
4983 static jit_word_t
4984 _bgti_u(jit_state_t *_jit, jit_word_t i0, jit_int32_t r0, jit_word_t i1)
4985 {
4986     jit_word_t          w;
4987     jit_int32_t         reg;
4988     reg = jit_get_reg(jit_class_gpr|jit_class_nospill);
4989     movi(rn(reg), i1);
4990     w = bgtr_u(i0, r0, rn(reg));
4991     jit_unget_reg(reg);
4992     return (w);
4993 }
4994
4995 static jit_word_t
4996 _bner(jit_state_t *_jit, jit_word_t i0, jit_int32_t r0, jit_int32_t r1)
4997 {
4998     jit_word_t          w;
4999     CMP_EQ(PR_6, PR_7, r0, r1);
5000     sync();
5001     w = _jit->pc.w;
5002     BRI_COND((i0 - w) >> 4, PR_7);
5003     return (w);
5004 }
5005
5006 static jit_word_t
5007 _bnei(jit_state_t *_jit, jit_word_t i0, jit_int32_t r0, jit_word_t i1)
5008 {
5009     jit_word_t          w;
5010     jit_int32_t         reg;
5011     if (i1 >= -128 && i1 <= 127)
5012         CMPI_EQ(PR_6, PR_7, i1, r0);
5013     else {
5014         reg = jit_get_reg(jit_class_gpr);
5015         movi(rn(reg), i1);
5016         CMP_EQ(PR_6, PR_7, r0, rn(reg));
5017         jit_unget_reg(reg);
5018     }
5019     sync();
5020     w = _jit->pc.w;
5021     BRI_COND((i0 - w) >> 4, PR_7);
5022     return (w);
5023 }
5024
5025 static jit_word_t
5026 _bmsr(jit_state_t *_jit, jit_word_t i0, jit_int32_t r0, jit_int32_t r1)
5027 {
5028     jit_word_t          w;
5029     jit_int32_t         reg;
5030     reg = jit_get_reg(jit_class_gpr);
5031     andr(rn(reg), r0, r1);
5032     CMPI_EQ(PR_6, PR_7, 0, rn(reg));
5033     jit_unget_reg(reg);
5034     sync();
5035     w = _jit->pc.w;
5036     BRI_COND((i0 - w) >> 4, PR_7);
5037     return (w);
5038 }
5039
5040 static jit_word_t
5041 _bmsi(jit_state_t *_jit, jit_word_t i0, jit_int32_t r0, jit_word_t i1)
5042 {
5043     jit_word_t          w;
5044     jit_int32_t         reg;
5045     reg = jit_get_reg(jit_class_gpr);
5046     movi(rn(reg), i1);
5047     andr(rn(reg), r0, rn(reg));
5048     CMPI_EQ(PR_6, PR_7, 0, rn(reg));
5049     jit_unget_reg(reg);
5050     sync();
5051     w = _jit->pc.w;
5052     BRI_COND((i0 - w) >> 4, PR_7);
5053     return (w);
5054 }
5055
5056 static jit_word_t
5057 _bmcr(jit_state_t *_jit, jit_word_t i0, jit_int32_t r0, jit_int32_t r1)
5058 {
5059     jit_word_t          w;
5060     jit_int32_t         reg;
5061     reg = jit_get_reg(jit_class_gpr);
5062     andr(rn(reg), r0, r1);
5063     CMPI_EQ(PR_6, PR_7, 0, rn(reg));
5064     jit_unget_reg(reg);
5065     sync();
5066     w = _jit->pc.w;
5067     BRI_COND((i0 - w) >> 4, PR_6);
5068     return (w);
5069 }
5070
5071 static jit_word_t
5072 _bmci(jit_state_t *_jit, jit_word_t i0, jit_int32_t r0, jit_word_t i1)
5073 {
5074     jit_word_t          w;
5075     jit_int32_t         reg;
5076     reg = jit_get_reg(jit_class_gpr);
5077     movi(rn(reg), i1);
5078     andr(rn(reg), r0, rn(reg));
5079     CMPI_EQ(PR_6, PR_7, 0, rn(reg));
5080     jit_unget_reg(reg);
5081     sync();
5082     w = _jit->pc.w;
5083     BRI_COND((i0 - w) >> 4, PR_6);
5084     return (w);
5085 }
5086
5087 static jit_word_t
5088 _baddr(jit_state_t *_jit, jit_word_t i0, jit_int32_t r0, jit_int32_t r1,
5089        jit_bool_t carry)
5090 {
5091     jit_word_t          w;
5092     jit_int32_t         t0;
5093     jit_int32_t         t1;
5094     jit_int32_t         t2;
5095     /* t1 = r0 + r1;    overflow = r1 < 0 ? r0 < t1 : t1 < r0 */
5096     t0 = jit_get_reg(jit_class_gpr|jit_class_nospill);
5097     t1 = jit_get_reg(jit_class_gpr|jit_class_nospill);
5098     t2 = jit_get_reg(jit_class_gpr|jit_class_nospill);
5099     lti(rn(t0), r1, 0);                 /* t0 = r1 < 0 */
5100     addr(rn(t1), r0, r1);               /* t1 = r0 + r1 */
5101     ltr(rn(t2), rn(t1), r0);            /* t2 = t1 < r0 */
5102     ltr(rn(t1), r0, rn(t1));            /* t1 = r0 < t1 */
5103     CMPI_EQ(PR_6, PR_7, 0, rn(t0));
5104     CMPI_EQ_p(PR_8, PR_9, 0, rn(t2), PR_6);/* if (t0==0) p8=t2==0,p9=t2!=0; */
5105     CMPI_EQ_p(PR_8, PR_9, 0, rn(t1), PR_7);/* if (t0!=0) p8=t1==0,p9=t1!=0; */
5106     addr(r0, r0, r1);
5107     sync();
5108     w = _jit->pc.w;
5109     BRI_COND((i0 - w) >> 4, carry ? PR_9 : PR_8);
5110     jit_unget_reg(t2);
5111     jit_unget_reg(t1);
5112     jit_unget_reg(t0);
5113     return (w);
5114 }
5115
5116 static jit_word_t
5117 _baddi(jit_state_t *_jit, jit_word_t i0, jit_int32_t r0, jit_word_t i1,
5118        jit_bool_t carry)
5119 {
5120     jit_word_t          w;
5121     jit_int32_t         reg;
5122     reg = jit_get_reg(jit_class_gpr|jit_class_nospill);
5123     movi(rn(reg), i1);
5124     w = baddr(i0, r0, rn(reg), carry);
5125     jit_unget_reg(reg);
5126     return (w);
5127 }
5128
5129 static jit_word_t
5130 _baddr_u(jit_state_t *_jit, jit_word_t i0, jit_int32_t r0, jit_int32_t r1,
5131          jit_bool_t carry)
5132 {
5133     jit_word_t          w;
5134     jit_int32_t         t0;
5135     jit_int32_t         t1;
5136     t0 = jit_get_reg(jit_class_gpr|jit_class_nospill);
5137     t1 = jit_get_reg(jit_class_gpr|jit_class_nospill);
5138     addr(rn(t0), r0, r1);
5139     ltr_u(rn(t1), rn(t0), r0);
5140     CMPI_EQ(PR_6, PR_7, 0, rn(t1));
5141     MOV(r0, rn(t0));
5142     sync();
5143     w = _jit->pc.w;
5144     BRI_COND((i0 - w) >> 4, carry ? PR_7 : PR_6);
5145     jit_unget_reg(t1);
5146     jit_unget_reg(t0);
5147     return (w);
5148 }
5149
5150 static jit_word_t
5151 _baddi_u(jit_state_t *_jit, jit_word_t i0, jit_int32_t r0, jit_word_t i1,
5152          jit_bool_t carry)
5153 {
5154     jit_word_t          w;
5155     jit_int32_t         reg;
5156     reg = jit_get_reg(jit_class_gpr|jit_class_nospill);
5157     movi(rn(reg), i1);
5158     w = baddr_u(i0, r0, rn(reg), carry);
5159     jit_unget_reg(reg);
5160     return (w);
5161 }
5162
5163 static jit_word_t
5164 _bsubr(jit_state_t *_jit, jit_word_t i0, jit_int32_t r0, jit_int32_t r1,
5165        jit_bool_t carry)
5166 {
5167     jit_word_t          w;
5168     jit_int32_t         t0;
5169     jit_int32_t         t1;
5170     jit_int32_t         t2;
5171     /* t1 = r0 - r1;    overflow = 0 < r1 ? r0 < t1 : t1 < r0 */
5172     t0 = jit_get_reg(jit_class_gpr|jit_class_nospill);
5173     t1 = jit_get_reg(jit_class_gpr|jit_class_nospill);
5174     t2 = jit_get_reg(jit_class_gpr|jit_class_nospill);
5175     gti(rn(t0), r1, 0);                 /* t0 = r1 > 0 */
5176     subr(rn(t1), r0, r1);               /* t1 = r0 - r1 */
5177     ltr(rn(t2), rn(t1), r0);            /* t2 = t1 < r0 */
5178     ltr(rn(t1), r0, rn(t1));            /* t1 = r0 < t1 */
5179     CMPI_EQ(PR_6, PR_7, 0, rn(t0));
5180     CMPI_EQ_p(PR_8, PR_9, 0, rn(t2), PR_6);/* if (t0==0) p4=t2==0,p5=t2!=0; */
5181     CMPI_EQ_p(PR_8, PR_9, 0, rn(t1), PR_7);/* if (t0!=0) p4=t1==0,p5=t1!=0; */
5182     subr(r0, r0, r1);
5183     sync();
5184     w = _jit->pc.w;
5185     BRI_COND((i0 - w) >> 4, carry ? PR_9 : PR_8);
5186     jit_unget_reg(t2);
5187     jit_unget_reg(t1);
5188     jit_unget_reg(t0);
5189     return (w);
5190 }
5191
5192 static jit_word_t
5193 _bsubi(jit_state_t *_jit, jit_word_t i0, jit_int32_t r0, jit_word_t i1,
5194        jit_bool_t carry)
5195 {
5196     jit_word_t          w;
5197     jit_int32_t         reg;
5198     reg = jit_get_reg(jit_class_gpr|jit_class_nospill);
5199     movi(rn(reg), i1);
5200     w = bsubr(i0, r0, rn(reg), carry);
5201     jit_unget_reg(reg);
5202     return (w);
5203 }
5204
5205 static jit_word_t
5206 _bsubr_u(jit_state_t *_jit, jit_word_t i0, jit_int32_t r0, jit_int32_t r1,
5207          jit_bool_t carry)
5208 {
5209     jit_word_t          w;
5210     jit_int32_t         t0;
5211     jit_int32_t         t1;
5212     t0 = jit_get_reg(jit_class_gpr|jit_class_nospill);
5213     t1 = jit_get_reg(jit_class_gpr|jit_class_nospill);
5214     subr(rn(t0), r0, r1);
5215     ltr_u(rn(t1), r0, rn(t0));
5216     CMPI_EQ(PR_6, PR_7, 0, rn(t1));
5217     MOV(r0, rn(t0));
5218     sync();
5219     w = _jit->pc.w;
5220     BRI_COND((i0 - w) >> 4, carry ? PR_7 : PR_6);
5221     jit_unget_reg(t1);
5222     jit_unget_reg(t0);
5223     return (w);
5224 }
5225
5226 static jit_word_t
5227 _bsubi_u(jit_state_t *_jit, jit_word_t i0, jit_int32_t r0, jit_word_t i1,
5228          jit_bool_t carry)
5229 {
5230     jit_word_t          w;
5231     jit_int32_t         reg;
5232     reg = jit_get_reg(jit_class_gpr|jit_class_nospill);
5233     movi(rn(reg), i1);
5234     w = bsubr_u(i0, r0, rn(reg), carry);
5235     jit_unget_reg(reg);
5236     return (w);
5237 }
5238
5239 static void
5240 _jmpr(jit_state_t *_jit, jit_int32_t r0)
5241 {
5242     MOV_br_rn(BR_6, r0);
5243     BR(BR_6);
5244 }
5245
5246 static jit_word_t
5247 _jmpi(jit_state_t *_jit, jit_word_t i0)
5248 {
5249     jit_word_t          d, w;
5250     sync();
5251     w = _jit->pc.w;
5252     d = ((jit_word_t)i0 - w) >> 4;
5253     if (d >= -16777216 && d <= 16777215)
5254         BRI(d);
5255     else
5256         BRL(d);
5257     return (w);
5258 }
5259
5260 static jit_word_t
5261 _jmpi_p(jit_state_t *_jit, jit_word_t i0)
5262 {
5263     jit_word_t          d, w;
5264     sync();
5265     w = _jit->pc.w;
5266     d = ((jit_word_t)i0 - w) >> 4;
5267     BRL(d);
5268     return (w);
5269 }
5270
5271 static void
5272 _callr(jit_state_t *_jit, jit_int32_t r0)
5273 {
5274     jit_int32_t         reg;
5275     reg = jit_get_reg(jit_class_gpr);
5276     LD8_inc(rn(reg), r0, 8);
5277     MOV_br_rn(BR_6, rn(reg));
5278     jit_unget_reg(reg);
5279     LD8(GR_1, r0);
5280     BR_CALL(BR_0, BR_6);
5281 }
5282
5283 static void
5284 _calli(jit_state_t *_jit, jit_word_t i0)
5285 {
5286     jit_int32_t         reg;
5287     reg = jit_get_reg(jit_class_gpr);
5288     movi(rn(reg), i0);
5289     callr(rn(reg));
5290     jit_unget_reg(reg);
5291 }
5292
5293 static jit_word_t
5294 _calli_p(jit_state_t *_jit, jit_word_t i0)
5295 {
5296     jit_word_t          w;
5297     jit_int32_t         reg;
5298     reg = jit_get_reg(jit_class_gpr);
5299     w = movi_p(rn(reg), i0);
5300     callr(rn(reg));
5301     jit_unget_reg(reg);
5302     return (w);
5303 }
5304
5305 static void
5306 _prolog(jit_state_t *_jit, jit_node_t *node)
5307 {
5308     jit_int32_t         reg, ruse, rout;
5309
5310     if (_jitc->function->define_frame || _jitc->function->assume_frame) {
5311         jit_int32_t     frame = -_jitc->function->frame;
5312         assert(_jitc->function->self.aoff >= frame);
5313         if (_jitc->function->assume_frame)
5314             return;
5315         _jitc->function->self.aoff = frame;
5316     }
5317     if (_jitc->function->allocar)
5318         _jitc->function->self.aoff &= -16;
5319     _jitc->function->stack = ((_jitc->function->self.alen -
5320                                _jitc->function->self.aoff) + 15) & -16;
5321
5322     /* First lowest unused register is first output register */
5323     for (reg = _R115; reg >= _R40; reg--) {
5324         if (jit_regset_tstbit(&_jitc->function->regset, reg))
5325             break;
5326     }
5327     _jitc->breg = rn(reg) + 1;
5328     _jitc->rout = _jitc->breg + 5;
5329     ruse = _jitc->rout - GR_32;
5330
5331     /* How many out argument registers required? */
5332     if (!_jitc->function->define_frame) {
5333         for (reg = _OUT7; reg >= _OUT0; --reg) {
5334             if (jit_regset_tstbit(&_jitc->function->regset, reg))
5335                 break;
5336         }
5337         rout = (reg + 1) - _OUT0;
5338     }
5339     else
5340         rout = 8;
5341
5342     /* Do not know if will call div/mod functions (sqrt* needs one) */
5343     if (rout < 2)
5344         rout = 2;
5345
5346     /* Match gcc prolog */
5347     ALLOC(_jitc->breg + 1, ruse, rout);
5348     MOV(_jitc->breg + 2, GR_12);
5349     MOV_rn_br(_jitc->breg, BR_0);
5350     MOV(_jitc->breg + 3, GR_1);
5351
5352     /* lightning specific, use r4 as frame pointer */
5353     MOV(_jitc->breg + 4, GR_4);
5354     addi(GR_4, GR_12, -(stack_framesize + params_offset));
5355
5356     /* adjust stack pointer */
5357     addi(GR_12, GR_12, -(stack_framesize +
5358                          (params_offset << 1) + _jitc->function->stack));
5359
5360     if (jit_regset_tstbit(&_jitc->function->regset, JIT_F0))
5361         STF_SPILL(GR_4, rn(JIT_F0));
5362     if (jit_regset_tstbit(&_jitc->function->regset, JIT_F1)) {
5363         addi(GR_2, GR_4, 16);
5364         STF_SPILL(GR_2, rn(JIT_F1));
5365     }
5366     if (jit_regset_tstbit(&_jitc->function->regset, JIT_F2)) {
5367         addi(GR_2, GR_4, 32);
5368         STF_SPILL(GR_2, rn(JIT_F2));
5369     }
5370     if (jit_regset_tstbit(&_jitc->function->regset, JIT_F3)) {
5371         addi(GR_2, GR_4, 48);
5372         STF_SPILL(GR_2, rn(JIT_F3));
5373     }
5374     if (jit_regset_tstbit(&_jitc->function->regset, JIT_F4)) {
5375         addi(GR_2, GR_4, 64);
5376         STF_SPILL(GR_2, rn(JIT_F4));
5377     }
5378     if (jit_regset_tstbit(&_jitc->function->regset, JIT_F5)) {
5379         addi(GR_2, GR_4, 80);
5380         STF_SPILL(GR_2, rn(JIT_F5));
5381     }
5382
5383     if (_jitc->function->allocar) {
5384         reg = jit_get_reg(jit_class_gpr);
5385         movi(rn(reg), _jitc->function->self.aoff);
5386         stxi_i(_jitc->function->aoffoff, GR_4, rn(reg));
5387         jit_unget_reg(reg);
5388     }
5389
5390     if (_jitc->function->self.call & jit_call_varargs) {
5391         for (reg = _jitc->function->vagp; reg < 8; ++reg)
5392             stxi(112 + reg * 8, GR_4, GR_32 + reg);
5393     }
5394 }
5395
5396 static void
5397 _epilog(jit_state_t *_jit, jit_node_t *node)
5398 {
5399     if (_jitc->function->assume_frame)
5400         return;
5401     if (jit_regset_tstbit(&_jitc->function->regset, JIT_F0))
5402         LDF_FILL(rn(JIT_F0), GR_4);
5403     if (jit_regset_tstbit(&_jitc->function->regset, JIT_F1)) {
5404         addi(GR_2, GR_4, 16);
5405         LDF_FILL(rn(JIT_F1), GR_2);
5406     }
5407     if (jit_regset_tstbit(&_jitc->function->regset, JIT_F2)) {
5408         addi(GR_2, GR_4, 32);
5409         LDF_FILL(rn(JIT_F2), GR_2);
5410     }
5411     if (jit_regset_tstbit(&_jitc->function->regset, JIT_F3)) {
5412         addi(GR_2, GR_4, 48);
5413         LDF_FILL(rn(JIT_F3), GR_2);
5414     }
5415     if (jit_regset_tstbit(&_jitc->function->regset, JIT_F4)) {
5416         addi(GR_2, GR_4, 64);
5417         LDF_FILL(rn(JIT_F4), GR_2);
5418     }
5419     if (jit_regset_tstbit(&_jitc->function->regset, JIT_F5)) {
5420         addi(GR_2, GR_4, 80);
5421         LDF_FILL(rn(JIT_F5), GR_2);
5422     }
5423     /* Match gcc epilog */
5424     MOV(GR_1, _jitc->breg + 3);
5425     MOV_I_ar_rn(AR_PFS, _jitc->breg + 1);
5426     MOV_br_rn(BR_0, _jitc->breg);
5427     MOV(GR_12, _jitc->breg + 2);
5428     /* Restore lightning specific r4 as frame pointer */
5429     MOV(GR_4, _jitc->breg + 4);
5430     BR_RET(BR_0);
5431     flush();
5432 }
5433
5434 static void
5435 _vastart(jit_state_t *_jit, jit_int32_t r0)
5436 {
5437     assert(_jitc->function->self.call & jit_call_varargs);
5438     /* Initialize va_list to the first stack argument. */
5439     if (_jitc->function->vagp < 8)
5440         addi(r0, GR_4, 112 + _jitc->function->vagp * 8);
5441     else
5442         addi(r0, GR_4, _jitc->function->self.size);
5443 }
5444
5445 static void
5446 _vaarg(jit_state_t *_jit, jit_int32_t r0, jit_int32_t r1)
5447 {
5448     assert(_jitc->function->self.call & jit_call_varargs);
5449     /* Load argument. */
5450     ldr(r0, r1);
5451     /* Update va_list. */
5452     addi(r1, r1, 8);
5453 }
5454
5455 static void
5456 _patch_at(jit_state_t *_jit, jit_code_t code,
5457           jit_word_t instr, jit_word_t label)
5458 {
5459     jit_word_t           l, h, tm, s0, s1, s2;
5460     union {
5461         jit_word_t       w;
5462         jit_word_t      *p;
5463     } c;
5464     jit_word_t           i1, i41, i20, ic, i5, i9, i7;
5465     c.w = instr;
5466     get_bundle(c.p, l, h, tm, s0, s1, s2);
5467     switch (code) {
5468         case jit_code_movi:
5469             /* Handle jit functions as C function, so that jit function
5470              * pointers can be passed to C code, and jit code does not
5471              * need to try to differentiate them. */
5472         case jit_code_calli:
5473             i1  = (label >> 63) &           0x1L;
5474             i41 = (label >> 22) & 0x1ffffffffffL;
5475             ic  = (label >> 21) &           0x1L;
5476             i5  = (label >> 16) &          0x1fL;
5477             i9  = (label >>  7) &         0x1ffL;
5478             i7  =  label        &          0x7fL;
5479             s1 = i41;
5480             assert((tm & ~1) == TM_M_L_X_ &&
5481                    (s2 & 0xfL<<37) == (6L<<37) &&
5482                    s0 == nop_m);
5483             s2 &= (6L<<37)|(0x7fL<<6);
5484             s2 |= (i1<<36)|(i9<<27)|(i5<<22)|(ic<<21)|(i7<<13);
5485             break;
5486         case jit_code_jmpi:
5487             if (_jitc->jump) {
5488                 /* kludge to hide function descriptors; check that gp
5489                  * is zero, what is done for all jit functions */
5490                 if (((long *)label)[1] == 0) {
5491                     for (ic = 0; ic < _jitc->prolog.offset; ic++) {
5492                         if (_jitc->prolog.ptr[ic] == label) {
5493                             label += 16;
5494                             break;
5495                         }
5496                     }
5497                 }
5498             }
5499             ic = (label - instr) >> 4;
5500             i1  = (ic >> 61) &           0x1L;
5501             i41 = (ic >> 22) & 0x1ffffffffffL;
5502             i20 =  ic        &       0xfffffL;
5503             if (!((tm & ~1) == TM_M_L_X_ &&
5504                    (s2 & 0xfL<<37) == (0xcL<<37) &&
5505                   s0 == nop_m))
5506                 goto short_jump;
5507             s1 = i41;
5508             s2 &= (0xcL<<37)|(0x7L<<33)|(1L<<12);
5509             s2 |= (i1<<36)|(i20<<13);
5510             break;
5511         default:
5512         short_jump:
5513             /* Only B1 in slot 0 expected due to need to either
5514              * a stop to update predicates, or a sync before
5515              * unconditional short branch */
5516             ic = (label - instr) >> 4;
5517             assert((s0 >> 37) == 4 && (s0 & (7 << 6)) == 0);
5518             s0 &= (4L<<37)|(7L<<33)|(1L<<12)|0x1f;
5519             s0 |= (((ic>>20)&1L)<<36)|((ic&0xfffffL)<<13);
5520             break;
5521     }
5522     set_bundle(c.p, l, h, tm, s0, s1, s2);
5523 }
5524 #endif