deps/lightrec/emitter.c

   1 // SPDX-License-Identifier: LGPL-2.1-or-later
   2 /*
   3  * Copyright (C) 2014-2021 Paul Cercueil <paul@crapouillou.net>
   4  */
   5
   6 #include "blockcache.h"
   7 #include "debug.h"
   8 #include "disassembler.h"
   9 #include "emitter.h"
  10 #include "lightning-wrapper.h"
  11 #include "optimizer.h"
  12 #include "regcache.h"
  13
  14 #include <stdbool.h>
  15 #include <stddef.h>
  16
  17 #define LIGHTNING_UNALIGNED_32BIT 4
  18
  19 typedef void (*lightrec_rec_func_t)(struct lightrec_cstate *, const struct block *, u16);
  20
  21 /* Forward declarations */
  22 static void rec_SPECIAL(struct lightrec_cstate *state, const struct block *block, u16 offset);
  23 static void rec_REGIMM(struct lightrec_cstate *state, const struct block *block, u16 offset);
  24 static void rec_CP0(struct lightrec_cstate *state, const struct block *block, u16 offset);
  25 static void rec_CP2(struct lightrec_cstate *state, const struct block *block, u16 offset);
  26 static void rec_META(struct lightrec_cstate *state, const struct block *block, u16 offset);
  27 static void rec_cp2_do_mtc2(struct lightrec_cstate *state,
  28                             const struct block *block, u16 offset, u8 reg, u8 in_reg);
  29 static void rec_cp2_do_mfc2(struct lightrec_cstate *state,
  30                             const struct block *block, u16 offset,
  31                             u8 reg, u8 out_reg);
  32
  33 static void
  34 lightrec_jump_to_fn(jit_state_t *_jit, void (*fn)(void))
  35 {
  36         /* Prevent jit_jmpi() from using our cycles register as a temporary */
  37         jit_live(LIGHTREC_REG_CYCLE);
  38
  39         jit_patch_abs(jit_jmpi(), fn);
  40 }
  41
  42 static void
  43 lightrec_jump_to_eob(struct lightrec_cstate *state, jit_state_t *_jit)
  44 {
  45         lightrec_jump_to_fn(_jit, state->state->eob_wrapper_func);
  46 }
  47
  48 static void
  49 lightrec_jump_to_ds_check(struct lightrec_cstate *state, jit_state_t *_jit)
  50 {
  51         lightrec_jump_to_fn(_jit, state->state->ds_check_func);
  52 }
  53
  54 static void update_ra_register(struct regcache *reg_cache, jit_state_t *_jit,
  55                                u8 ra_reg, u32 pc, u32 link)
  56 {
  57         u8 link_reg;
  58
  59         link_reg = lightrec_alloc_reg_out(reg_cache, _jit, ra_reg, 0);
  60         lightrec_load_imm(reg_cache, _jit, link_reg, pc, link);
  61         lightrec_free_reg(reg_cache, link_reg);
  62 }
  63
  64 static void lightrec_emit_end_of_block(struct lightrec_cstate *state,
  65                                        const struct block *block, u16 offset,
  66                                        s8 reg_new_pc, u32 imm, u8 ra_reg,
  67                                        u32 link, bool update_cycles)
  68 {
  69         struct regcache *reg_cache = state->reg_cache;
  70         jit_state_t *_jit = block->_jit;
  71         const struct opcode *op = &block->opcode_list[offset],
  72                             *ds = get_delay_slot(block->opcode_list, offset);
  73         u32 cycles = state->cycles + lightrec_cycles_of_opcode(state->state, op->c);
  74         bool has_ds = has_delay_slot(op->c);
  75
  76         jit_note(__FILE__, __LINE__);
  77
  78         if (link && ra_reg != reg_new_pc)
  79                 update_ra_register(reg_cache, _jit, ra_reg, block->pc, link);
  80
  81         if (reg_new_pc < 0)
  82                 lightrec_load_next_pc_imm(reg_cache, _jit, block->pc, imm);
  83         else
  84                 lightrec_load_next_pc(reg_cache, _jit, reg_new_pc);
  85
  86         if (link && ra_reg == reg_new_pc) {
  87                 /* Handle the special case: JALR $r0, $r0
  88                  * In that case the target PC should be the old value of the
  89                  * register. */
  90                 update_ra_register(reg_cache, _jit, ra_reg, block->pc, link);
  91         }
  92
  93         if (has_ds && !op_flag_no_ds(op->flags) && !op_flag_local_branch(op->flags)) {
  94                 cycles += lightrec_cycles_of_opcode(state->state, ds->c);
  95
  96                 /* Recompile the delay slot */
  97                 if (ds->c.opcode)
  98                         lightrec_rec_opcode(state, block, offset + 1);
  99         }
 100
 101         /* Clean the remaining registers */
 102         lightrec_clean_regs(reg_cache, _jit);
 103
 104         if (cycles && update_cycles) {
 105                 jit_subi(LIGHTREC_REG_CYCLE, LIGHTREC_REG_CYCLE, cycles);
 106                 pr_debug("EOB: %u cycles\n", cycles);
 107         }
 108
 109         if (has_ds && op_flag_load_delay(ds->flags)
 110             && opcode_has_load_delay(ds->c) && !state->no_load_delay) {
 111                 /* If the delay slot is a load opcode, its target register
 112                  * will be written after the first opcode of the target is
 113                  * executed. Handle this by jumping to a special section of
 114                  * the dispatcher. It expects the loaded value to be in
 115                  * REG_TEMP, and the target register number to be in JIT_V1.*/
 116                 jit_movi(JIT_V1, ds->c.i.rt);
 117
 118                 lightrec_jump_to_ds_check(state, _jit);
 119         } else {
 120                 lightrec_jump_to_eob(state, _jit);
 121         }
 122
 123         lightrec_regcache_reset(reg_cache);
 124 }
 125
 126 void lightrec_emit_jump_to_interpreter(struct lightrec_cstate *state,
 127                                        const struct block *block, u16 offset)
 128 {
 129         struct regcache *reg_cache = state->reg_cache;
 130         jit_state_t *_jit = block->_jit;
 131
 132         lightrec_clean_regs(reg_cache, _jit);
 133
 134         /* Call the interpreter with the block's address in JIT_V1 and the
 135          * PC (which might have an offset) in JIT_V0. */
 136         lightrec_load_imm(reg_cache, _jit, JIT_V0, block->pc,
 137                           block->pc + (offset << 2));
 138         if (lightrec_store_next_pc()) {
 139               jit_stxi_i(offsetof(struct lightrec_state, next_pc),
 140                          LIGHTREC_REG_STATE, JIT_V0);
 141         }
 142
 143         jit_movi(JIT_V1, (uintptr_t)block);
 144
 145         jit_subi(LIGHTREC_REG_CYCLE, LIGHTREC_REG_CYCLE, state->cycles);
 146         lightrec_jump_to_fn(_jit, state->state->interpreter_func);
 147 }
 148
 149 static void lightrec_emit_eob(struct lightrec_cstate *state,
 150                               const struct block *block, u16 offset)
 151 {
 152         struct regcache *reg_cache = state->reg_cache;
 153         jit_state_t *_jit = block->_jit;
 154
 155         lightrec_clean_regs(reg_cache, _jit);
 156
 157         lightrec_load_imm(reg_cache, _jit, JIT_V0, block->pc,
 158                           block->pc + (offset << 2));
 159         if (lightrec_store_next_pc()) {
 160               jit_stxi_i(offsetof(struct lightrec_state, next_pc),
 161                          LIGHTREC_REG_STATE, JIT_V0);
 162         }
 163
 164         jit_subi(LIGHTREC_REG_CYCLE, LIGHTREC_REG_CYCLE, state->cycles);
 165
 166         lightrec_jump_to_eob(state, _jit);
 167 }
 168
 169 static void rec_special_JR(struct lightrec_cstate *state, const struct block *block, u16 offset)
 170 {
 171         union code c = block->opcode_list[offset].c;
 172
 173         _jit_name(block->_jit, __func__);
 174         lightrec_emit_end_of_block(state, block, offset, c.r.rs, 0, 31, 0, true);
 175 }
 176
 177 static void rec_special_JALR(struct lightrec_cstate *state, const struct block *block, u16 offset)
 178 {
 179         union code c = block->opcode_list[offset].c;
 180
 181         _jit_name(block->_jit, __func__);
 182         lightrec_emit_end_of_block(state, block, offset, c.r.rs, 0, c.r.rd,
 183                                    get_branch_pc(block, offset, 2), true);
 184 }
 185
 186 static void rec_J(struct lightrec_cstate *state, const struct block *block, u16 offset)
 187 {
 188         union code c = block->opcode_list[offset].c;
 189
 190         _jit_name(block->_jit, __func__);
 191         lightrec_emit_end_of_block(state, block, offset, -1,
 192                                    (block->pc & 0xf0000000) | (c.j.imm << 2),
 193                                    31, 0, true);
 194 }
 195
 196 static void rec_JAL(struct lightrec_cstate *state, const struct block *block, u16 offset)
 197 {
 198         union code c = block->opcode_list[offset].c;
 199
 200         _jit_name(block->_jit, __func__);
 201         lightrec_emit_end_of_block(state, block, offset, -1,
 202                                    (block->pc & 0xf0000000) | (c.j.imm << 2),
 203                                    31, get_branch_pc(block, offset, 2), true);
 204 }
 205
 206 static void lightrec_do_early_unload(struct lightrec_cstate *state,
 207                                      const struct block *block, u16 offset)
 208 {
 209         struct regcache *reg_cache = state->reg_cache;
 210         const struct opcode *op = &block->opcode_list[offset];
 211         jit_state_t *_jit = block->_jit;
 212         unsigned int i;
 213         u8 reg;
 214         struct {
 215                 u8 reg, op;
 216         } reg_ops[3] = {
 217                 { op->r.rd, LIGHTREC_FLAGS_GET_RD(op->flags), },
 218                 { op->i.rt, LIGHTREC_FLAGS_GET_RT(op->flags), },
 219                 { op->i.rs, LIGHTREC_FLAGS_GET_RS(op->flags), },
 220         };
 221
 222         for (i = 0; i < ARRAY_SIZE(reg_ops); i++) {
 223                 reg = reg_ops[i].reg;
 224
 225                 switch (reg_ops[i].op) {
 226                 case LIGHTREC_REG_UNLOAD:
 227                         lightrec_clean_reg_if_loaded(reg_cache, _jit, reg, true);
 228                         break;
 229
 230                 case LIGHTREC_REG_DISCARD:
 231                         lightrec_discard_reg_if_loaded(reg_cache, reg);
 232                         break;
 233
 234                 case LIGHTREC_REG_CLEAN:
 235                         lightrec_clean_reg_if_loaded(reg_cache, _jit, reg, false);
 236                         break;
 237                 default:
 238                         break;
 239                 };
 240         }
 241 }
 242
 243 static void rec_b(struct lightrec_cstate *state, const struct block *block, u16 offset,
 244                   jit_code_t code, jit_code_t code2, u32 link, bool unconditional, bool bz)
 245 {
 246         struct regcache *reg_cache = state->reg_cache;
 247         struct native_register *regs_backup;
 248         jit_state_t *_jit = block->_jit;
 249         struct lightrec_branch *branch;
 250         const struct opcode *op = &block->opcode_list[offset],
 251                             *ds = get_delay_slot(block->opcode_list, offset);
 252         jit_node_t *addr;
 253         bool is_forward = (s16)op->i.imm >= 0;
 254         int op_cycles = lightrec_cycles_of_opcode(state->state, op->c);
 255         u32 target_offset, cycles = state->cycles + op_cycles;
 256         bool no_indirection = false;
 257         u32 next_pc;
 258         u8 rs, rt;
 259
 260         jit_note(__FILE__, __LINE__);
 261
 262         if (!op_flag_no_ds(op->flags))
 263                 cycles += lightrec_cycles_of_opcode(state->state, ds->c);
 264
 265         state->cycles = -op_cycles;
 266
 267         if (!unconditional) {
 268                 rs = lightrec_alloc_reg_in(reg_cache, _jit, op->i.rs, REG_EXT);
 269                 rt = bz ? 0 : lightrec_alloc_reg_in(reg_cache,
 270                                                     _jit, op->i.rt, REG_EXT);
 271
 272                 /* Unload dead registers before evaluating the branch */
 273                 if (OPT_EARLY_UNLOAD)
 274                         lightrec_do_early_unload(state, block, offset);
 275
 276                 if (op_flag_local_branch(op->flags) &&
 277                     (op_flag_no_ds(op->flags) || !ds->opcode) &&
 278                     is_forward && !lightrec_has_dirty_regs(reg_cache))
 279                         no_indirection = true;
 280
 281                 if (no_indirection)
 282                         pr_debug("Using no indirection for branch at offset 0x%hx\n", offset << 2);
 283         }
 284
 285         if (cycles)
 286                 jit_subi(LIGHTREC_REG_CYCLE, LIGHTREC_REG_CYCLE, cycles);
 287
 288         if (!unconditional) {
 289                 /* Generate the branch opcode */
 290                 if (!no_indirection)
 291                         addr = jit_new_node_pww(code, NULL, rs, rt);
 292
 293                 lightrec_free_regs(reg_cache);
 294                 regs_backup = lightrec_regcache_enter_branch(reg_cache);
 295         }
 296
 297         if (op_flag_local_branch(op->flags)) {
 298                 /* Recompile the delay slot */
 299                 if (!op_flag_no_ds(op->flags) && ds->opcode) {
 300                         /* Never handle load delays with local branches. */
 301                         state->no_load_delay = true;
 302                         lightrec_rec_opcode(state, block, offset + 1);
 303                 }
 304
 305                 if (link)
 306                         update_ra_register(reg_cache, _jit, 31, block->pc, link);
 307
 308                 /* Clean remaining registers */
 309                 lightrec_clean_regs(reg_cache, _jit);
 310
 311                 target_offset = offset + 1 + (s16)op->i.imm
 312                         - !!op_flag_no_ds(op->flags);
 313                 pr_debug("Adding local branch to offset 0x%x\n",
 314                          target_offset << 2);
 315                 branch = &state->local_branches[
 316                         state->nb_local_branches++];
 317
 318                 branch->target = target_offset;
 319
 320                 if (no_indirection)
 321                         branch->branch = jit_new_node_pww(code2, NULL, rs, rt);
 322                 else if (is_forward)
 323                         branch->branch = jit_b();
 324                 else
 325                         branch->branch = jit_bgti(LIGHTREC_REG_CYCLE, 0);
 326         }
 327
 328         if (!op_flag_local_branch(op->flags) || !is_forward) {
 329                 next_pc = get_branch_pc(block, offset, 1 + (s16)op->i.imm);
 330                 state->no_load_delay = op_flag_local_branch(op->flags);
 331                 lightrec_emit_end_of_block(state, block, offset, -1, next_pc,
 332                                            31, link, false);
 333         }
 334
 335         if (!unconditional) {
 336                 if (!no_indirection)
 337                         jit_patch(addr);
 338
 339                 lightrec_regcache_leave_branch(reg_cache, regs_backup);
 340
 341                 if (bz && link)
 342                         update_ra_register(reg_cache, _jit, 31, block->pc, link);
 343
 344                 if (!op_flag_no_ds(op->flags) && ds->opcode) {
 345                         state->no_load_delay = true;
 346                         lightrec_rec_opcode(state, block, offset + 1);
 347                 }
 348         }
 349 }
 350
 351 static void rec_BNE(struct lightrec_cstate *state,
 352                     const struct block *block, u16 offset)
 353 {
 354         union code c = block->opcode_list[offset].c;
 355
 356         _jit_name(block->_jit, __func__);
 357
 358         if (c.i.rt == 0)
 359                 rec_b(state, block, offset, jit_code_beqi, jit_code_bnei, 0, false, true);
 360         else
 361                 rec_b(state, block, offset, jit_code_beqr, jit_code_bner, 0, false, false);
 362 }
 363
 364 static void rec_BEQ(struct lightrec_cstate *state,
 365                     const struct block *block, u16 offset)
 366 {
 367         union code c = block->opcode_list[offset].c;
 368
 369         _jit_name(block->_jit, __func__);
 370
 371         if (c.i.rt == 0)
 372                 rec_b(state, block, offset, jit_code_bnei, jit_code_beqi, 0, c.i.rs == 0, true);
 373         else
 374                 rec_b(state, block, offset, jit_code_bner, jit_code_beqr, 0, c.i.rs == c.i.rt, false);
 375 }
 376
 377 static void rec_BLEZ(struct lightrec_cstate *state,
 378                      const struct block *block, u16 offset)
 379 {
 380         union code c = block->opcode_list[offset].c;
 381
 382         _jit_name(block->_jit, __func__);
 383         rec_b(state, block, offset, jit_code_bgti, jit_code_blei, 0, c.i.rs == 0, true);
 384 }
 385
 386 static void rec_BGTZ(struct lightrec_cstate *state,
 387                      const struct block *block, u16 offset)
 388 {
 389         _jit_name(block->_jit, __func__);
 390         rec_b(state, block, offset, jit_code_blei, jit_code_bgti, 0, false, true);
 391 }
 392
 393 static void rec_regimm_BLTZ(struct lightrec_cstate *state,
 394                             const struct block *block, u16 offset)
 395 {
 396         _jit_name(block->_jit, __func__);
 397         rec_b(state, block, offset, jit_code_bgei, jit_code_blti, 0, false, true);
 398 }
 399
 400 static void rec_regimm_BLTZAL(struct lightrec_cstate *state,
 401                               const struct block *block, u16 offset)
 402 {
 403         _jit_name(block->_jit, __func__);
 404         rec_b(state, block, offset, jit_code_bgei, jit_code_blti,
 405               get_branch_pc(block, offset, 2), false, true);
 406 }
 407
 408 static void rec_regimm_BGEZ(struct lightrec_cstate *state,
 409                             const struct block *block, u16 offset)
 410 {
 411         union code c = block->opcode_list[offset].c;
 412
 413         _jit_name(block->_jit, __func__);
 414         rec_b(state, block, offset, jit_code_blti, jit_code_bgei, 0, !c.i.rs, true);
 415 }
 416
 417 static void rec_regimm_BGEZAL(struct lightrec_cstate *state,
 418                               const struct block *block, u16 offset)
 419 {
 420         const struct opcode *op = &block->opcode_list[offset];
 421         _jit_name(block->_jit, __func__);
 422         rec_b(state, block, offset, jit_code_blti, jit_code_bgei,
 423               get_branch_pc(block, offset, 2),
 424               !op->i.rs, true);
 425 }
 426
 427 static void rec_alloc_rs_rd(struct regcache *reg_cache,
 428                             jit_state_t *_jit,
 429                             const struct opcode *op,
 430                             u8 rs, u8 rd,
 431                             u8 in_flags, u8 out_flags,
 432                             u8 *rs_out, u8 *rd_out)
 433 {
 434         bool unload, discard;
 435         u32 unload_flags;
 436
 437         if (OPT_EARLY_UNLOAD) {
 438                 unload_flags = LIGHTREC_FLAGS_GET_RS(op->flags);
 439                 unload = unload_flags == LIGHTREC_REG_UNLOAD;
 440                 discard = unload_flags == LIGHTREC_REG_DISCARD;
 441         }
 442
 443         if (OPT_EARLY_UNLOAD && rs && rd != rs && (unload || discard)) {
 444                 rs = lightrec_alloc_reg_in(reg_cache, _jit, rs, in_flags);
 445                 lightrec_remap_reg(reg_cache, _jit, rs, rd, discard);
 446                 lightrec_set_reg_out_flags(reg_cache, rs, out_flags);
 447                 rd = rs;
 448         } else {
 449                 rs = lightrec_alloc_reg_in(reg_cache, _jit, rs, in_flags);
 450                 rd = lightrec_alloc_reg_out(reg_cache, _jit, rd, out_flags);
 451         }
 452
 453         *rs_out = rs;
 454         *rd_out = rd;
 455 }
 456
 457 static void rec_alu_imm(struct lightrec_cstate *state, const struct block *block,
 458                         u16 offset, jit_code_t code, bool slti)
 459 {
 460         struct regcache *reg_cache = state->reg_cache;
 461         union code c = block->opcode_list[offset].c;
 462         jit_state_t *_jit = block->_jit;
 463         u8 rs, rt, out_flags = REG_EXT;
 464
 465         if (slti)
 466                 out_flags |= REG_ZEXT;
 467
 468         jit_note(__FILE__, __LINE__);
 469
 470         rec_alloc_rs_rd(reg_cache, _jit, &block->opcode_list[offset],
 471                         c.i.rs, c.i.rt, REG_EXT, out_flags, &rs, &rt);
 472
 473         jit_new_node_www(code, rt, rs, (s32)(s16) c.i.imm);
 474
 475         lightrec_free_reg(reg_cache, rs);
 476         lightrec_free_reg(reg_cache, rt);
 477 }
 478
 479 static void rec_alu_special(struct lightrec_cstate *state, const struct block *block,
 480                             u16 offset, jit_code_t code, bool out_ext)
 481 {
 482         struct regcache *reg_cache = state->reg_cache;
 483         union code c = block->opcode_list[offset].c;
 484         jit_state_t *_jit = block->_jit;
 485         u8 rd, rt, rs;
 486
 487         jit_note(__FILE__, __LINE__);
 488
 489         rt = lightrec_alloc_reg_in(reg_cache, _jit, c.r.rt, REG_EXT);
 490         rec_alloc_rs_rd(reg_cache, _jit, &block->opcode_list[offset],
 491                         c.r.rs, c.r.rd, REG_EXT,
 492                         out_ext ? REG_EXT | REG_ZEXT : 0, &rs, &rd);
 493
 494         jit_new_node_www(code, rd, rs, rt);
 495
 496         lightrec_free_reg(reg_cache, rs);
 497         lightrec_free_reg(reg_cache, rt);
 498         lightrec_free_reg(reg_cache, rd);
 499 }
 500
 501 static void rec_alu_shiftv(struct lightrec_cstate *state, const struct block *block,
 502                            u16 offset, jit_code_t code)
 503 {
 504         struct regcache *reg_cache = state->reg_cache;
 505         union code c = block->opcode_list[offset].c;
 506         jit_state_t *_jit = block->_jit;
 507         u8 rd, rt, rs, temp, flags = 0;
 508
 509         jit_note(__FILE__, __LINE__);
 510
 511         if (code == jit_code_rshr)
 512                 flags = REG_EXT;
 513         else if (code == jit_code_rshr_u)
 514                 flags = REG_ZEXT;
 515
 516         rs = lightrec_alloc_reg_in(reg_cache, _jit, c.r.rs, 0);
 517         rec_alloc_rs_rd(reg_cache, _jit, &block->opcode_list[offset],
 518                         c.r.rt, c.r.rd, flags, flags, &rt, &rd);
 519
 520         if (rt != rd) {
 521                 jit_andi(rd, rs, 0x1f);
 522                 jit_new_node_www(code, rd, rt, rd);
 523         } else {
 524                 temp = lightrec_alloc_reg_temp(reg_cache, _jit);
 525                 jit_andi(temp, rs, 0x1f);
 526                 jit_new_node_www(code, rd, rt, temp);
 527                 lightrec_free_reg(reg_cache, temp);
 528         }
 529
 530         lightrec_free_reg(reg_cache, rs);
 531         lightrec_free_reg(reg_cache, rt);
 532         lightrec_free_reg(reg_cache, rd);
 533 }
 534
 535 static void rec_movi(struct lightrec_cstate *state,
 536                      const struct block *block, u16 offset)
 537 {
 538         struct regcache *reg_cache = state->reg_cache;
 539         union code c = block->opcode_list[offset].c;
 540         jit_state_t *_jit = block->_jit;
 541         u16 flags = REG_EXT;
 542         s32 value = (s32)(s16) c.i.imm;
 543         u8 rt;
 544
 545         if (block->opcode_list[offset].flags & LIGHTREC_MOVI)
 546                 value += (s32)((u32)state->movi_temp[c.i.rt] << 16);
 547
 548         if (value >= 0)
 549                 flags |= REG_ZEXT;
 550
 551         rt = lightrec_alloc_reg_out(reg_cache, _jit, c.i.rt, flags);
 552
 553         jit_movi(rt, value);
 554
 555         lightrec_free_reg(reg_cache, rt);
 556 }
 557
 558 static void rec_ADDIU(struct lightrec_cstate *state,
 559                       const struct block *block, u16 offset)
 560 {
 561         const struct opcode *op = &block->opcode_list[offset];
 562
 563         _jit_name(block->_jit, __func__);
 564
 565         if (op->i.rs && !(op->flags & LIGHTREC_MOVI))
 566                 rec_alu_imm(state, block, offset, jit_code_addi, false);
 567         else
 568                 rec_movi(state, block, offset);
 569 }
 570
 571 static void rec_ADDI(struct lightrec_cstate *state,
 572                      const struct block *block, u16 offset)
 573 {
 574         /* TODO: Handle the exception? */
 575         _jit_name(block->_jit, __func__);
 576         rec_ADDIU(state, block, offset);
 577 }
 578
 579 static void rec_SLTIU(struct lightrec_cstate *state,
 580                       const struct block *block, u16 offset)
 581 {
 582         _jit_name(block->_jit, __func__);
 583         rec_alu_imm(state, block, offset, jit_code_lti_u, true);
 584 }
 585
 586 static void rec_SLTI(struct lightrec_cstate *state,
 587                      const struct block *block, u16 offset)
 588 {
 589         _jit_name(block->_jit, __func__);
 590         rec_alu_imm(state, block, offset, jit_code_lti, true);
 591 }
 592
 593 static void rec_ANDI(struct lightrec_cstate *state,
 594                      const struct block *block, u16 offset)
 595 {
 596         struct regcache *reg_cache = state->reg_cache;
 597         union code c = block->opcode_list[offset].c;
 598         jit_state_t *_jit = block->_jit;
 599         u8 rs, rt;
 600
 601         _jit_name(block->_jit, __func__);
 602         jit_note(__FILE__, __LINE__);
 603
 604         rec_alloc_rs_rd(reg_cache, _jit, &block->opcode_list[offset],
 605                         c.i.rs, c.i.rt, 0, REG_EXT | REG_ZEXT, &rs, &rt);
 606
 607         /* PSX code uses ANDI 0xff / ANDI 0xffff a lot, which are basically
 608          * casts to uint8_t / uint16_t. */
 609         if (c.i.imm == 0xff)
 610                 jit_extr_uc(rt, rs);
 611         else if (c.i.imm == 0xffff)
 612                 jit_extr_us(rt, rs);
 613         else
 614                 jit_andi(rt, rs, (u32)(u16) c.i.imm);
 615
 616         lightrec_free_reg(reg_cache, rs);
 617         lightrec_free_reg(reg_cache, rt);
 618 }
 619
 620 static void rec_alu_or_xor(struct lightrec_cstate *state, const struct block *block,
 621                            u16 offset, jit_code_t code)
 622 {
 623         struct regcache *reg_cache = state->reg_cache;
 624         union code c = block->opcode_list[offset].c;
 625         jit_state_t *_jit = block->_jit;
 626         u8 rs, rt, flags;
 627
 628         jit_note(__FILE__, __LINE__);
 629
 630         rec_alloc_rs_rd(reg_cache, _jit, &block->opcode_list[offset],
 631                         c.i.rs, c.i.rt, 0, 0, &rs, &rt);
 632
 633         flags = lightrec_get_reg_in_flags(reg_cache, rs);
 634         lightrec_set_reg_out_flags(reg_cache, rt, flags);
 635
 636         jit_new_node_www(code, rt, rs, (u32)(u16) c.i.imm);
 637
 638         lightrec_free_reg(reg_cache, rs);
 639         lightrec_free_reg(reg_cache, rt);
 640 }
 641
 642
 643 static void rec_ORI(struct lightrec_cstate *state,
 644                     const struct block *block, u16 offset)
 645 {
 646         const struct opcode *op = &block->opcode_list[offset];
 647         struct regcache *reg_cache = state->reg_cache;
 648         jit_state_t *_jit = block->_jit;
 649         s32 val;
 650         u8 rt;
 651
 652         _jit_name(_jit, __func__);
 653
 654         if (op->flags & LIGHTREC_MOVI) {
 655                 rt = lightrec_alloc_reg_out(reg_cache, _jit, op->i.rt, REG_EXT);
 656
 657                 val = ((u32)state->movi_temp[op->i.rt] << 16) | op->i.imm;
 658                 jit_movi(rt, val);
 659
 660                 lightrec_free_reg(reg_cache, rt);
 661         } else {
 662                 rec_alu_or_xor(state, block, offset, jit_code_ori);
 663         }
 664 }
 665
 666 static void rec_XORI(struct lightrec_cstate *state,
 667                      const struct block *block, u16 offset)
 668 {
 669         _jit_name(block->_jit, __func__);
 670         rec_alu_or_xor(state, block, offset, jit_code_xori);
 671 }
 672
 673 static void rec_LUI(struct lightrec_cstate *state,
 674                     const struct block *block, u16 offset)
 675 {
 676         struct regcache *reg_cache = state->reg_cache;
 677         union code c = block->opcode_list[offset].c;
 678         jit_state_t *_jit = block->_jit;
 679         u8 rt, flags = REG_EXT;
 680
 681         if (block->opcode_list[offset].flags & LIGHTREC_MOVI) {
 682                 state->movi_temp[c.i.rt] = c.i.imm;
 683                 return;
 684         }
 685
 686         jit_name(__func__);
 687         jit_note(__FILE__, __LINE__);
 688
 689         if (!(c.i.imm & BIT(15)))
 690                 flags |= REG_ZEXT;
 691
 692         rt = lightrec_alloc_reg_out(reg_cache, _jit, c.i.rt, flags);
 693
 694         jit_movi(rt, (s32)(c.i.imm << 16));
 695
 696         lightrec_free_reg(reg_cache, rt);
 697 }
 698
 699 static void rec_special_ADDU(struct lightrec_cstate *state,
 700                              const struct block *block, u16 offset)
 701 {
 702         _jit_name(block->_jit, __func__);
 703         rec_alu_special(state, block, offset, jit_code_addr, false);
 704 }
 705
 706 static void rec_special_ADD(struct lightrec_cstate *state,
 707                             const struct block *block, u16 offset)
 708 {
 709         /* TODO: Handle the exception? */
 710         _jit_name(block->_jit, __func__);
 711         rec_alu_special(state, block, offset, jit_code_addr, false);
 712 }
 713
 714 static void rec_special_SUBU(struct lightrec_cstate *state,
 715                              const struct block *block, u16 offset)
 716 {
 717         _jit_name(block->_jit, __func__);
 718         rec_alu_special(state, block, offset, jit_code_subr, false);
 719 }
 720
 721 static void rec_special_SUB(struct lightrec_cstate *state,
 722                             const struct block *block, u16 offset)
 723 {
 724         /* TODO: Handle the exception? */
 725         _jit_name(block->_jit, __func__);
 726         rec_alu_special(state, block, offset, jit_code_subr, false);
 727 }
 728
 729 static void rec_special_AND(struct lightrec_cstate *state,
 730                             const struct block *block, u16 offset)
 731 {
 732         struct regcache *reg_cache = state->reg_cache;
 733         union code c = block->opcode_list[offset].c;
 734         jit_state_t *_jit = block->_jit;
 735         u8 rd, rt, rs, flags_rs, flags_rt, flags_rd;
 736
 737         _jit_name(block->_jit, __func__);
 738         jit_note(__FILE__, __LINE__);
 739
 740         rt = lightrec_alloc_reg_in(reg_cache, _jit, c.r.rt, 0);
 741         rec_alloc_rs_rd(reg_cache, _jit, &block->opcode_list[offset],
 742                         c.r.rs, c.r.rd, 0, 0, &rs, &rd);
 743
 744         flags_rs = lightrec_get_reg_in_flags(reg_cache, rs);
 745         flags_rt = lightrec_get_reg_in_flags(reg_cache, rt);
 746
 747         /* Z(rd) = Z(rs) | Z(rt) */
 748         flags_rd = REG_ZEXT & (flags_rs | flags_rt);
 749
 750         /* E(rd) = (E(rt) & Z(rt)) | (E(rs) & Z(rs)) | (E(rs) & E(rt)) */
 751         if (((flags_rs & REG_EXT) && (flags_rt & REG_ZEXT)) ||
 752             ((flags_rt & REG_EXT) && (flags_rs & REG_ZEXT)) ||
 753             (REG_EXT & flags_rs & flags_rt))
 754                 flags_rd |= REG_EXT;
 755
 756         lightrec_set_reg_out_flags(reg_cache, rd, flags_rd);
 757
 758         jit_andr(rd, rs, rt);
 759
 760         lightrec_free_reg(reg_cache, rs);
 761         lightrec_free_reg(reg_cache, rt);
 762         lightrec_free_reg(reg_cache, rd);
 763 }
 764
 765 static void rec_special_or_nor(struct lightrec_cstate *state,
 766                                const struct block *block, u16 offset, bool nor)
 767 {
 768         struct regcache *reg_cache = state->reg_cache;
 769         union code c = block->opcode_list[offset].c;
 770         jit_state_t *_jit = block->_jit;
 771         u8 rd, rt, rs, flags_rs, flags_rt, flags_rd = 0;
 772
 773         jit_note(__FILE__, __LINE__);
 774
 775         rt = lightrec_alloc_reg_in(reg_cache, _jit, c.r.rt, 0);
 776         rec_alloc_rs_rd(reg_cache, _jit, &block->opcode_list[offset],
 777                         c.r.rs, c.r.rd, 0, 0, &rs, &rd);
 778
 779         flags_rs = lightrec_get_reg_in_flags(reg_cache, rs);
 780         flags_rt = lightrec_get_reg_in_flags(reg_cache, rt);
 781
 782         /* or: Z(rd) = Z(rs) & Z(rt)
 783          * nor: Z(rd) = 0 */
 784         if (!nor)
 785                 flags_rd = REG_ZEXT & flags_rs & flags_rt;
 786
 787         /* E(rd) = E(rs) & E(rt) */
 788         if (REG_EXT & flags_rs & flags_rt)
 789                 flags_rd |= REG_EXT;
 790
 791         lightrec_set_reg_out_flags(reg_cache, rd, flags_rd);
 792
 793         jit_orr(rd, rs, rt);
 794
 795         if (nor)
 796                 jit_comr(rd, rd);
 797
 798         lightrec_free_reg(reg_cache, rs);
 799         lightrec_free_reg(reg_cache, rt);
 800         lightrec_free_reg(reg_cache, rd);
 801 }
 802
 803 static void rec_special_OR(struct lightrec_cstate *state,
 804                            const struct block *block, u16 offset)
 805 {
 806         _jit_name(block->_jit, __func__);
 807         rec_special_or_nor(state, block, offset, false);
 808 }
 809
 810 static void rec_special_NOR(struct lightrec_cstate *state,
 811                             const struct block *block, u16 offset)
 812 {
 813         _jit_name(block->_jit, __func__);
 814         rec_special_or_nor(state, block, offset, true);
 815 }
 816
 817 static void rec_special_XOR(struct lightrec_cstate *state,
 818                             const struct block *block, u16 offset)
 819 {
 820         struct regcache *reg_cache = state->reg_cache;
 821         union code c = block->opcode_list[offset].c;
 822         jit_state_t *_jit = block->_jit;
 823         u8 rd, rt, rs, flags_rs, flags_rt, flags_rd;
 824
 825         _jit_name(block->_jit, __func__);
 826
 827         jit_note(__FILE__, __LINE__);
 828
 829         rt = lightrec_alloc_reg_in(reg_cache, _jit, c.r.rt, 0);
 830         rec_alloc_rs_rd(reg_cache, _jit, &block->opcode_list[offset],
 831                         c.r.rs, c.r.rd, 0, 0, &rs, &rd);
 832
 833         flags_rs = lightrec_get_reg_in_flags(reg_cache, rs);
 834         flags_rt = lightrec_get_reg_in_flags(reg_cache, rt);
 835
 836         /* Z(rd) = Z(rs) & Z(rt) */
 837         flags_rd = REG_ZEXT & flags_rs & flags_rt;
 838
 839         /* E(rd) = E(rs) & E(rt) */
 840         flags_rd |= REG_EXT & flags_rs & flags_rt;
 841
 842         lightrec_set_reg_out_flags(reg_cache, rd, flags_rd);
 843
 844         jit_xorr(rd, rs, rt);
 845
 846         lightrec_free_reg(reg_cache, rs);
 847         lightrec_free_reg(reg_cache, rt);
 848         lightrec_free_reg(reg_cache, rd);
 849 }
 850
 851 static void rec_special_SLTU(struct lightrec_cstate *state,
 852                              const struct block *block, u16 offset)
 853 {
 854         _jit_name(block->_jit, __func__);
 855         rec_alu_special(state, block, offset, jit_code_ltr_u, true);
 856 }
 857
 858 static void rec_special_SLT(struct lightrec_cstate *state,
 859                             const struct block *block, u16 offset)
 860 {
 861         _jit_name(block->_jit, __func__);
 862         rec_alu_special(state, block, offset, jit_code_ltr, true);
 863 }
 864
 865 static void rec_special_SLLV(struct lightrec_cstate *state,
 866                              const struct block *block, u16 offset)
 867 {
 868         _jit_name(block->_jit, __func__);
 869         rec_alu_shiftv(state, block, offset, jit_code_lshr);
 870 }
 871
 872 static void rec_special_SRLV(struct lightrec_cstate *state,
 873                              const struct block *block, u16 offset)
 874 {
 875         _jit_name(block->_jit, __func__);
 876         rec_alu_shiftv(state, block, offset, jit_code_rshr_u);
 877 }
 878
 879 static void rec_special_SRAV(struct lightrec_cstate *state,
 880                              const struct block *block, u16 offset)
 881 {
 882         _jit_name(block->_jit, __func__);
 883         rec_alu_shiftv(state, block, offset, jit_code_rshr);
 884 }
 885
 886 static void rec_alu_shift(struct lightrec_cstate *state, const struct block *block,
 887                           u16 offset, jit_code_t code)
 888 {
 889         struct regcache *reg_cache = state->reg_cache;
 890         union code c = block->opcode_list[offset].c;
 891         jit_state_t *_jit = block->_jit;
 892         u8 rd, rt, flags = 0, out_flags = 0;
 893
 894         jit_note(__FILE__, __LINE__);
 895
 896         if (code == jit_code_rshi)
 897                 flags = REG_EXT;
 898         else if (code == jit_code_rshi_u)
 899                 flags = REG_ZEXT;
 900
 901         /* Input reg is zero-extended, if we SRL at least by one bit, we know
 902          * the output reg will be both zero-extended and sign-extended. */
 903         out_flags = flags;
 904         if (code == jit_code_rshi_u && c.r.imm)
 905                 out_flags |= REG_EXT;
 906
 907         rec_alloc_rs_rd(reg_cache, _jit, &block->opcode_list[offset],
 908                         c.r.rt, c.r.rd, flags, out_flags, &rt, &rd);
 909
 910         jit_new_node_www(code, rd, rt, c.r.imm);
 911
 912         lightrec_free_reg(reg_cache, rt);
 913         lightrec_free_reg(reg_cache, rd);
 914 }
 915
 916 static void rec_special_SLL(struct lightrec_cstate *state,
 917                             const struct block *block, u16 offset)
 918 {
 919         _jit_name(block->_jit, __func__);
 920         rec_alu_shift(state, block, offset, jit_code_lshi);
 921 }
 922
 923 static void rec_special_SRL(struct lightrec_cstate *state,
 924                             const struct block *block, u16 offset)
 925 {
 926         _jit_name(block->_jit, __func__);
 927         rec_alu_shift(state, block, offset, jit_code_rshi_u);
 928 }
 929
 930 static void rec_special_SRA(struct lightrec_cstate *state,
 931                             const struct block *block, u16 offset)
 932 {
 933         _jit_name(block->_jit, __func__);
 934         rec_alu_shift(state, block, offset, jit_code_rshi);
 935 }
 936
 937 static void rec_alu_mult(struct lightrec_cstate *state,
 938                          const struct block *block, u16 offset, bool is_signed)
 939 {
 940         struct regcache *reg_cache = state->reg_cache;
 941         union code c = block->opcode_list[offset].c;
 942         u32 flags = block->opcode_list[offset].flags;
 943         u8 reg_lo = get_mult_div_lo(c);
 944         u8 reg_hi = get_mult_div_hi(c);
 945         jit_state_t *_jit = block->_jit;
 946         u8 lo, hi, rs, rt, rflags = 0;
 947         bool no_lo = op_flag_no_lo(flags);
 948         bool no_hi = op_flag_no_hi(flags);
 949
 950         jit_note(__FILE__, __LINE__);
 951
 952         if (is_signed)
 953                 rflags = REG_EXT;
 954         else
 955                 rflags = REG_ZEXT;
 956
 957         rs = lightrec_alloc_reg_in(reg_cache, _jit, c.r.rs, rflags);
 958         rt = lightrec_alloc_reg_in(reg_cache, _jit, c.r.rt, rflags);
 959
 960         if (!no_lo)
 961                 lo = lightrec_alloc_reg_out(reg_cache, _jit, reg_lo, 0);
 962
 963         if (!no_hi)
 964                 hi = lightrec_alloc_reg_out(reg_cache, _jit, reg_hi, REG_EXT);
 965
 966         if (__WORDSIZE == 32) {
 967                 /* On 32-bit systems, do a 32*32->64 bit operation, or a 32*32->32 bit
 968                  * operation if the MULT was detected a 32-bit only. */
 969                 if (no_lo) {
 970                         if (is_signed)
 971                                 jit_hmulr(hi, rs, rt);
 972                         else
 973                                 jit_hmulr_u(hi, rs, rt);
 974                 } else if (no_hi) {
 975                         jit_mulr(lo, rs, rt);
 976                 } else if (is_signed) {
 977                         jit_qmulr(lo, hi, rs, rt);
 978                 } else {
 979                         jit_qmulr_u(lo, hi, rs, rt);
 980                 }
 981         } else {
 982                 /* On 64-bit systems, do a 64*64->64 bit operation. */
 983                 if (no_lo) {
 984                         jit_mulr(hi, rs, rt);
 985                         jit_rshi(hi, hi, 32);
 986                 } else {
 987                         jit_mulr(lo, rs, rt);
 988
 989                         /* The 64-bit output value is in $lo, store the upper 32 bits in $hi */
 990                         if (!no_hi)
 991                                 jit_rshi(hi, lo, 32);
 992                 }
 993         }
 994
 995         lightrec_free_reg(reg_cache, rs);
 996         lightrec_free_reg(reg_cache, rt);
 997         if (!no_lo)
 998                 lightrec_free_reg(reg_cache, lo);
 999         if (!no_hi)
1000                 lightrec_free_reg(reg_cache, hi);
1001 }
1002
1003 static void rec_alu_div(struct lightrec_cstate *state,
1004                         const struct block *block, u16 offset, bool is_signed)
1005 {
1006         struct regcache *reg_cache = state->reg_cache;
1007         union code c = block->opcode_list[offset].c;
1008         u32 flags = block->opcode_list[offset].flags;
1009         bool no_check = op_flag_no_div_check(flags);
1010         u8 reg_lo = get_mult_div_lo(c);
1011         u8 reg_hi = get_mult_div_hi(c);
1012         jit_state_t *_jit = block->_jit;
1013         jit_node_t *branch, *to_end;
1014         u8 lo = 0, hi = 0, rs, rt, rflags = 0;
1015
1016         jit_note(__FILE__, __LINE__);
1017
1018         if (is_signed)
1019                 rflags = REG_EXT;
1020         else
1021                 rflags = REG_ZEXT;
1022
1023         rs = lightrec_alloc_reg_in(reg_cache, _jit, c.r.rs, rflags);
1024         rt = lightrec_alloc_reg_in(reg_cache, _jit, c.r.rt, rflags);
1025
1026         if (!op_flag_no_lo(flags))
1027                 lo = lightrec_alloc_reg_out(reg_cache, _jit, reg_lo, 0);
1028
1029         if (!op_flag_no_hi(flags))
1030                 hi = lightrec_alloc_reg_out(reg_cache, _jit, reg_hi, 0);
1031
1032         /* Jump to special handler if dividing by zero  */
1033         if (!no_check)
1034                 branch = jit_beqi(rt, 0);
1035
1036         if (op_flag_no_lo(flags)) {
1037                 if (is_signed)
1038                         jit_remr(hi, rs, rt);
1039                 else
1040                         jit_remr_u(hi, rs, rt);
1041         } else if (op_flag_no_hi(flags)) {
1042                 if (is_signed)
1043                         jit_divr(lo, rs, rt);
1044                 else
1045                         jit_divr_u(lo, rs, rt);
1046         } else {
1047                 if (is_signed)
1048                         jit_qdivr(lo, hi, rs, rt);
1049                 else
1050                         jit_qdivr_u(lo, hi, rs, rt);
1051         }
1052
1053         if (!no_check) {
1054                 /* Jump above the div-by-zero handler */
1055                 to_end = jit_b();
1056
1057                 jit_patch(branch);
1058
1059                 if (!op_flag_no_lo(flags)) {
1060                         if (is_signed) {
1061                                 jit_ltr(lo, rs, rt);
1062                                 jit_lshi(lo, lo, 1);
1063                                 jit_subi(lo, lo, 1);
1064                         } else {
1065                                 jit_subi(lo, rt, 1);
1066                         }
1067                 }
1068
1069                 if (!op_flag_no_hi(flags))
1070                         jit_movr(hi, rs);
1071
1072                 jit_patch(to_end);
1073         }
1074
1075         lightrec_free_reg(reg_cache, rs);
1076         lightrec_free_reg(reg_cache, rt);
1077
1078         if (!op_flag_no_lo(flags))
1079                 lightrec_free_reg(reg_cache, lo);
1080
1081         if (!op_flag_no_hi(flags))
1082                 lightrec_free_reg(reg_cache, hi);
1083 }
1084
1085 static void rec_special_MULT(struct lightrec_cstate *state,
1086                              const struct block *block, u16 offset)
1087 {
1088         _jit_name(block->_jit, __func__);
1089         rec_alu_mult(state, block, offset, true);
1090 }
1091
1092 static void rec_special_MULTU(struct lightrec_cstate *state,
1093                               const struct block *block, u16 offset)
1094 {
1095         _jit_name(block->_jit, __func__);
1096         rec_alu_mult(state, block, offset, false);
1097 }
1098
1099 static void rec_special_DIV(struct lightrec_cstate *state,
1100                             const struct block *block, u16 offset)
1101 {
1102         _jit_name(block->_jit, __func__);
1103         rec_alu_div(state, block, offset, true);
1104 }
1105
1106 static void rec_special_DIVU(struct lightrec_cstate *state,
1107                              const struct block *block, u16 offset)
1108 {
1109         _jit_name(block->_jit, __func__);
1110         rec_alu_div(state, block, offset, false);
1111 }
1112
1113 static void rec_alu_mv_lo_hi(struct lightrec_cstate *state,
1114                              const struct block *block, u16 offset,
1115                              u8 dst, u8 src)
1116 {
1117         struct regcache *reg_cache = state->reg_cache;
1118         jit_state_t *_jit = block->_jit;
1119
1120         jit_note(__FILE__, __LINE__);
1121
1122         rec_alloc_rs_rd(reg_cache, _jit, &block->opcode_list[offset],
1123                         src, dst, 0, REG_EXT, &src, &dst);
1124
1125         jit_extr_i(dst, src);
1126
1127         lightrec_free_reg(reg_cache, src);
1128         lightrec_free_reg(reg_cache, dst);
1129 }
1130
1131 static void rec_special_MFHI(struct lightrec_cstate *state,
1132                              const struct block *block, u16 offset)
1133 {
1134         union code c = block->opcode_list[offset].c;
1135
1136         _jit_name(block->_jit, __func__);
1137         rec_alu_mv_lo_hi(state, block, offset, c.r.rd, REG_HI);
1138 }
1139
1140 static void rec_special_MTHI(struct lightrec_cstate *state,
1141                              const struct block *block, u16 offset)
1142 {
1143         union code c = block->opcode_list[offset].c;
1144
1145         _jit_name(block->_jit, __func__);
1146         rec_alu_mv_lo_hi(state, block, offset, REG_HI, c.r.rs);
1147 }
1148
1149 static void rec_special_MFLO(struct lightrec_cstate *state,
1150                              const struct block *block, u16 offset)
1151 {
1152         union code c = block->opcode_list[offset].c;
1153
1154         _jit_name(block->_jit, __func__);
1155         rec_alu_mv_lo_hi(state, block, offset, c.r.rd, REG_LO);
1156 }
1157
1158 static void rec_special_MTLO(struct lightrec_cstate *state,
1159                              const struct block *block, u16 offset)
1160 {
1161         union code c = block->opcode_list[offset].c;
1162
1163         _jit_name(block->_jit, __func__);
1164         rec_alu_mv_lo_hi(state, block, offset, REG_LO, c.r.rs);
1165 }
1166
1167 static void call_to_c_wrapper(struct lightrec_cstate *state,
1168                               const struct block *block, u32 arg,
1169                               enum c_wrappers wrapper)
1170 {
1171         struct regcache *reg_cache = state->reg_cache;
1172         jit_state_t *_jit = block->_jit;
1173         s8 tmp, tmp2;
1174
1175         /* Make sure JIT_R1 is not mapped; it will be used in the C wrapper. */
1176         tmp2 = lightrec_alloc_reg(reg_cache, _jit, JIT_R1);
1177
1178         tmp = lightrec_get_reg_with_value(reg_cache,
1179                                           (intptr_t) state->state->wrappers_eps[wrapper]);
1180         if (tmp < 0) {
1181                 tmp = lightrec_alloc_reg_temp(reg_cache, _jit);
1182                 jit_ldxi(tmp, LIGHTREC_REG_STATE,
1183                          offsetof(struct lightrec_state, wrappers_eps[wrapper]));
1184
1185                 lightrec_temp_set_value(reg_cache, tmp,
1186                                         (intptr_t) state->state->wrappers_eps[wrapper]);
1187         }
1188
1189         lightrec_free_reg(reg_cache, tmp2);
1190
1191 #ifdef __mips__
1192         /* On MIPS, register t9 is always used as the target register for JALR.
1193          * Therefore if it does not contain the target address we must
1194          * invalidate it. */
1195         if (tmp != _T9)
1196                 lightrec_unload_reg(reg_cache, _jit, _T9);
1197 #endif
1198
1199         jit_prepare();
1200         jit_pushargi(arg);
1201
1202         lightrec_regcache_mark_live(reg_cache, _jit);
1203         jit_callr(tmp);
1204
1205         lightrec_free_reg(reg_cache, tmp);
1206         lightrec_regcache_mark_live(reg_cache, _jit);
1207 }
1208
1209 static void rec_io(struct lightrec_cstate *state,
1210                    const struct block *block, u16 offset,
1211                    bool load_rt, bool read_rt)
1212 {
1213         struct regcache *reg_cache = state->reg_cache;
1214         jit_state_t *_jit = block->_jit;
1215         union code c = block->opcode_list[offset].c;
1216         u32 flags = block->opcode_list[offset].flags;
1217         bool is_tagged = LIGHTREC_FLAGS_GET_IO_MODE(flags);
1218         u32 lut_entry;
1219         u8 zero;
1220
1221         jit_note(__FILE__, __LINE__);
1222
1223         lightrec_clean_reg_if_loaded(reg_cache, _jit, c.i.rs, false);
1224
1225         if (read_rt && likely(c.i.rt))
1226                 lightrec_clean_reg_if_loaded(reg_cache, _jit, c.i.rt, true);
1227         else if (load_rt)
1228                 lightrec_clean_reg_if_loaded(reg_cache, _jit, c.i.rt, false);
1229
1230         if (op_flag_load_delay(flags) && !state->no_load_delay) {
1231                 /* Clear state->in_delay_slot_n. This notifies the lightrec_rw
1232                  * wrapper that it should write the REG_TEMP register instead of
1233                  * the actual output register of the opcode. */
1234                 zero = lightrec_alloc_reg_in(reg_cache, _jit, 0, 0);
1235                 jit_stxi_c(offsetof(struct lightrec_state, in_delay_slot_n),
1236                             LIGHTREC_REG_STATE, zero);
1237                 lightrec_free_reg(reg_cache, zero);
1238         }
1239
1240         if (is_tagged) {
1241                 call_to_c_wrapper(state, block, c.opcode, C_WRAPPER_RW);
1242         } else {
1243                 lut_entry = lightrec_get_lut_entry(block);
1244                 call_to_c_wrapper(state, block, (lut_entry << 16) | offset,
1245                                   C_WRAPPER_RW_GENERIC);
1246         }
1247 }
1248
1249 static u32 rec_ram_mask(const struct lightrec_state *state)
1250 {
1251         return (RAM_SIZE << (state->mirrors_mapped * 2)) - 1;
1252 }
1253
1254 static u32 rec_io_mask(const struct lightrec_state *state)
1255 {
1256         u32 length = state->maps[PSX_MAP_HW_REGISTERS].length;
1257
1258         return 0x1f800000 | GENMASK(31 - clz32(length - 1), 0);
1259 }
1260
1261 static void rec_add_offset(struct lightrec_cstate *cstate,
1262                            jit_state_t *_jit, u8 reg_out, u8 reg_in,
1263                            uintptr_t offset)
1264 {
1265         struct regcache *reg_cache = cstate->reg_cache;
1266         u8 reg_imm;
1267
1268         reg_imm = lightrec_alloc_reg_temp_with_value(reg_cache, _jit, offset);
1269         jit_addr(reg_out, reg_in, reg_imm);
1270
1271         lightrec_free_reg(reg_cache, reg_imm);
1272 }
1273
1274 static void rec_and_mask(struct lightrec_cstate *cstate,
1275                          jit_state_t *_jit, u8 reg_out, u8 reg_in, u32 mask)
1276 {
1277         struct regcache *reg_cache = cstate->reg_cache;
1278         u8 reg_imm;
1279
1280         reg_imm = lightrec_alloc_reg_temp_with_value(reg_cache, _jit, mask);
1281         jit_andr(reg_out, reg_in, reg_imm);
1282
1283         lightrec_free_reg(reg_cache, reg_imm);
1284 }
1285
1286 static void rec_store_memory(struct lightrec_cstate *cstate,
1287                              const struct block *block,
1288                              u16 offset, jit_code_t code,
1289                              jit_code_t swap_code, uintptr_t addr_offset,
1290                              u32 addr_mask, bool invalidate)
1291 {
1292         const struct lightrec_state *state = cstate->state;
1293         struct regcache *reg_cache = cstate->reg_cache;
1294         struct opcode *op = &block->opcode_list[offset];
1295         jit_state_t *_jit = block->_jit;
1296         union code c = op->c;
1297         u8 rs, rt, tmp = 0, tmp2 = 0, tmp3, addr_reg, addr_reg2;
1298         s16 imm = (s16)c.i.imm;
1299         s32 simm = (s32)imm << (1 - lut_is_32bit(state));
1300         s32 lut_offt = offsetof(struct lightrec_state, code_lut);
1301         bool no_mask = op_flag_no_mask(op->flags);
1302         bool add_imm = c.i.imm &&
1303                 (c.i.op == OP_META_SWU
1304                  || (!state->mirrors_mapped && !no_mask) || (invalidate &&
1305                 ((imm & 0x3) || simm + lut_offt != (s16)(simm + lut_offt))));
1306         bool need_tmp = !no_mask || add_imm || invalidate;
1307         bool swc2 = c.i.op == OP_SWC2;
1308         u8 in_reg = swc2 ? REG_TEMP : c.i.rt;
1309
1310         rs = lightrec_alloc_reg_in(reg_cache, _jit, c.i.rs, 0);
1311         if (need_tmp)
1312                 tmp = lightrec_alloc_reg_temp(reg_cache, _jit);
1313
1314         addr_reg = rs;
1315
1316         if (add_imm) {
1317                 jit_addi(tmp, addr_reg, (s16)c.i.imm);
1318                 lightrec_free_reg(reg_cache, rs);
1319                 addr_reg = tmp;
1320                 imm = 0;
1321         } else if (simm) {
1322                 lut_offt += simm;
1323         }
1324
1325         if (!no_mask) {
1326                 rec_and_mask(cstate, _jit, tmp, addr_reg, addr_mask);
1327                 addr_reg = tmp;
1328         }
1329
1330         if (addr_offset) {
1331                 tmp2 = lightrec_alloc_reg_temp(reg_cache, _jit);
1332                 rec_add_offset(cstate, _jit, tmp2, addr_reg, addr_offset);
1333                 addr_reg2 = tmp2;
1334         } else {
1335                 addr_reg2 = addr_reg;
1336         }
1337
1338         rt = lightrec_alloc_reg_in(reg_cache, _jit, in_reg, 0);
1339
1340         if (is_big_endian() && swap_code && in_reg) {
1341                 tmp3 = lightrec_alloc_reg_temp(reg_cache, _jit);
1342
1343                 jit_new_node_ww(swap_code, tmp3, rt);
1344
1345                 if (c.i.op == OP_META_SWU)
1346                         jit_unstr(addr_reg2, tmp3, LIGHTNING_UNALIGNED_32BIT);
1347                 else
1348                         jit_new_node_www(code, imm, addr_reg2, tmp3);
1349
1350                 lightrec_free_reg(reg_cache, tmp3);
1351         } else if (c.i.op == OP_META_SWU) {
1352                 jit_unstr(addr_reg2, rt, LIGHTNING_UNALIGNED_32BIT);
1353         } else {
1354                 jit_new_node_www(code, imm, addr_reg2, rt);
1355         }
1356
1357         lightrec_free_reg(reg_cache, rt);
1358
1359         if (invalidate) {
1360                 tmp3 = lightrec_alloc_reg_in(reg_cache, _jit, 0, 0);
1361
1362                 if (c.i.op != OP_SW) {
1363                         jit_andi(tmp, addr_reg, ~3);
1364                         addr_reg = tmp;
1365                 }
1366
1367                 if (!lut_is_32bit(state)) {
1368                         jit_lshi(tmp, addr_reg, 1);
1369                         addr_reg = tmp;
1370                 }
1371
1372                 if (addr_reg == rs && c.i.rs == 0) {
1373                         addr_reg = LIGHTREC_REG_STATE;
1374                 } else {
1375                         jit_add_state(tmp, addr_reg);
1376                         addr_reg = tmp;
1377                 }
1378
1379                 if (lut_is_32bit(state))
1380                         jit_stxi_i(lut_offt, addr_reg, tmp3);
1381                 else
1382                         jit_stxi(lut_offt, addr_reg, tmp3);
1383
1384                 lightrec_free_reg(reg_cache, tmp3);
1385         }
1386
1387         if (addr_offset)
1388                 lightrec_free_reg(reg_cache, tmp2);
1389         if (need_tmp)
1390                 lightrec_free_reg(reg_cache, tmp);
1391         lightrec_free_reg(reg_cache, rs);
1392 }
1393
1394 static void rec_store_ram(struct lightrec_cstate *cstate,
1395                           const struct block *block,
1396                           u16 offset, jit_code_t code,
1397                           jit_code_t swap_code, bool invalidate)
1398 {
1399         const struct lightrec_state *state = cstate->state;
1400
1401         _jit_note(block->_jit, __FILE__, __LINE__);
1402
1403         return rec_store_memory(cstate, block, offset, code, swap_code,
1404                                 state->offset_ram, rec_ram_mask(state),
1405                                 invalidate);
1406 }
1407
1408 static void rec_store_scratch(struct lightrec_cstate *cstate,
1409                               const struct block *block, u16 offset,
1410                               jit_code_t code, jit_code_t swap_code)
1411 {
1412         _jit_note(block->_jit, __FILE__, __LINE__);
1413
1414         return rec_store_memory(cstate, block, offset, code, swap_code,
1415                                 cstate->state->offset_scratch,
1416                                 0x1fffffff, false);
1417 }
1418
1419 static void rec_store_io(struct lightrec_cstate *cstate,
1420                          const struct block *block, u16 offset,
1421                          jit_code_t code, jit_code_t swap_code)
1422 {
1423         _jit_note(block->_jit, __FILE__, __LINE__);
1424
1425         return rec_store_memory(cstate, block, offset, code, swap_code,
1426                                 cstate->state->offset_io,
1427                                 rec_io_mask(cstate->state), false);
1428 }
1429
1430 static void rec_store_direct_no_invalidate(struct lightrec_cstate *cstate,
1431                                            const struct block *block,
1432                                            u16 offset, jit_code_t code,
1433                                            jit_code_t swap_code)
1434 {
1435         const struct lightrec_state *state = cstate->state;
1436         u32 ram_size = state->mirrors_mapped ? RAM_SIZE * 4 : RAM_SIZE;
1437         struct regcache *reg_cache = cstate->reg_cache;
1438         union code c = block->opcode_list[offset].c;
1439         jit_state_t *_jit = block->_jit;
1440         jit_node_t *to_not_ram, *to_end;
1441         bool swc2 = c.i.op == OP_SWC2;
1442         u8 addr_reg, tmp, tmp2 = 0, rs, rt, in_reg = swc2 ? REG_TEMP : c.i.rt;
1443         s16 imm;
1444
1445         jit_note(__FILE__, __LINE__);
1446         rs = lightrec_alloc_reg_in(reg_cache, _jit, c.i.rs, 0);
1447         tmp = lightrec_alloc_reg_temp(reg_cache, _jit);
1448
1449         /* Convert to KUNSEG and avoid RAM mirrors */
1450         if ((c.i.op == OP_META_SWU || !state->mirrors_mapped) && c.i.imm) {
1451                 imm = 0;
1452                 jit_addi(tmp, rs, (s16)c.i.imm);
1453                 addr_reg = tmp;
1454         } else {
1455                 imm = (s16)c.i.imm;
1456                 addr_reg = rs;
1457         }
1458
1459         rec_and_mask(cstate, _jit, tmp, addr_reg, 0x1f800000 | (ram_size - 1));
1460
1461         lightrec_free_reg(reg_cache, rs);
1462
1463         if (state->offset_ram != state->offset_scratch) {
1464                 tmp2 = lightrec_alloc_reg_temp(reg_cache, _jit);
1465
1466                 to_not_ram = jit_bmsi(tmp, BIT(28));
1467
1468                 jit_movi(tmp2, state->offset_ram);
1469
1470                 to_end = jit_b();
1471                 jit_patch(to_not_ram);
1472
1473                 jit_movi(tmp2, state->offset_scratch);
1474                 jit_patch(to_end);
1475         } else if (state->offset_ram) {
1476                 tmp2 = lightrec_alloc_reg_temp_with_value(reg_cache, _jit,
1477                                                           state->offset_ram);
1478         }
1479
1480         if (state->offset_ram || state->offset_scratch) {
1481                 jit_addr(tmp, tmp, tmp2);
1482                 lightrec_free_reg(reg_cache, tmp2);
1483         }
1484
1485         rt = lightrec_alloc_reg_in(reg_cache, _jit, in_reg, 0);
1486
1487         if (is_big_endian() && swap_code && in_reg) {
1488                 tmp2 = lightrec_alloc_reg_temp(reg_cache, _jit);
1489
1490                 jit_new_node_ww(swap_code, tmp2, rt);
1491
1492                 if (c.i.op == OP_META_SWU)
1493                         jit_unstr(tmp, tmp2, LIGHTNING_UNALIGNED_32BIT);
1494                 else
1495                         jit_new_node_www(code, imm, tmp, tmp2);
1496
1497                 lightrec_free_reg(reg_cache, tmp2);
1498         } else if (c.i.op == OP_META_SWU) {
1499                 jit_unstr(tmp, rt, LIGHTNING_UNALIGNED_32BIT);
1500         } else {
1501                 jit_new_node_www(code, imm, tmp, rt);
1502         }
1503
1504         lightrec_free_reg(reg_cache, rt);
1505         lightrec_free_reg(reg_cache, tmp);
1506 }
1507
1508 static void rec_store_direct(struct lightrec_cstate *cstate, const struct block *block,
1509                              u16 offset, jit_code_t code, jit_code_t swap_code)
1510 {
1511         const struct lightrec_state *state = cstate->state;
1512         u32 ram_size = state->mirrors_mapped ? RAM_SIZE * 4 : RAM_SIZE;
1513         struct regcache *reg_cache = cstate->reg_cache;
1514         union code c = block->opcode_list[offset].c;
1515         jit_state_t *_jit = block->_jit;
1516         jit_node_t *to_not_ram, *to_end;
1517         bool swc2 = c.i.op == OP_SWC2;
1518         u8 addr_reg, tmp, tmp2, tmp3, rs, rt, reg_imm;
1519         u8 in_reg = swc2 ? REG_TEMP : c.i.rt;
1520         u32 mask;
1521         bool different_offsets = state->offset_ram != state->offset_scratch;
1522
1523         jit_note(__FILE__, __LINE__);
1524
1525         rs = lightrec_alloc_reg_in(reg_cache, _jit, c.i.rs, 0);
1526         tmp2 = lightrec_alloc_reg_temp(reg_cache, _jit);
1527         tmp3 = lightrec_alloc_reg_in(reg_cache, _jit, 0, 0);
1528
1529         /* Convert to KUNSEG and avoid RAM mirrors */
1530         if (c.i.imm) {
1531                 jit_addi(tmp2, rs, (s16)c.i.imm);
1532                 addr_reg = tmp2;
1533         } else {
1534                 addr_reg = rs;
1535         }
1536
1537         rec_and_mask(cstate, _jit, tmp2, addr_reg, 0x1f800000 | (ram_size - 1));
1538
1539         lightrec_free_reg(reg_cache, rs);
1540         tmp = lightrec_alloc_reg_temp(reg_cache, _jit);
1541
1542         mask = c.i.op == OP_SW ? RAM_SIZE - 1 : (RAM_SIZE - 1) & ~3;
1543         reg_imm = lightrec_alloc_reg_temp_with_value(reg_cache, _jit, mask);
1544
1545         if (different_offsets) {
1546                 to_not_ram = jit_bgti(tmp2, ram_size);
1547                 addr_reg = tmp2;
1548         } else {
1549                 jit_lti_u(tmp, tmp2, ram_size);
1550                 jit_movnr(tmp, tmp2, tmp);
1551                 addr_reg = tmp;
1552         }
1553
1554         /* Compute the offset to the code LUT */
1555         jit_andr(tmp, addr_reg, reg_imm);
1556
1557         if (!lut_is_32bit(state))
1558                 jit_lshi(tmp, tmp, 1);
1559         jit_add_state(tmp, tmp);
1560
1561         /* Write NULL to the code LUT to invalidate any block that's there */
1562         if (lut_is_32bit(state))
1563                 jit_stxi_i(offsetof(struct lightrec_state, code_lut), tmp, tmp3);
1564         else
1565                 jit_stxi(offsetof(struct lightrec_state, code_lut), tmp, tmp3);
1566
1567         if (c.i.op == OP_META_SWU) {
1568                 /* With a SWU opcode, we might have touched the following 32-bit
1569                  * word, so invalidate it as well */
1570                 if (lut_is_32bit(state)) {
1571                         jit_stxi_i(offsetof(struct lightrec_state, code_lut) + 4,
1572                                    tmp, tmp3);
1573                 } else {
1574                         jit_stxi(offsetof(struct lightrec_state, code_lut)
1575                                  + sizeof(uintptr_t), tmp, tmp3);
1576                 }
1577         }
1578
1579         if (different_offsets) {
1580                 jit_movi(tmp, state->offset_ram);
1581
1582                 to_end = jit_b();
1583                 jit_patch(to_not_ram);
1584         }
1585
1586         if (state->offset_ram || state->offset_scratch)
1587                 jit_movi(tmp, state->offset_scratch);
1588
1589         if (different_offsets)
1590                 jit_patch(to_end);
1591
1592         if (state->offset_ram || state->offset_scratch)
1593                 jit_addr(tmp2, tmp2, tmp);
1594
1595         lightrec_free_reg(reg_cache, tmp);
1596         lightrec_free_reg(reg_cache, tmp3);
1597         lightrec_free_reg(reg_cache, reg_imm);
1598
1599         rt = lightrec_alloc_reg_in(reg_cache, _jit, in_reg, 0);
1600
1601         if (is_big_endian() && swap_code && in_reg) {
1602                 tmp = lightrec_alloc_reg_temp(reg_cache, _jit);
1603
1604                 jit_new_node_ww(swap_code, tmp, rt);
1605
1606                 if (c.i.op == OP_META_SWU)
1607                         jit_unstr(tmp2, tmp, LIGHTNING_UNALIGNED_32BIT);
1608                 else
1609                         jit_new_node_www(code, 0, tmp2, tmp);
1610
1611                 lightrec_free_reg(reg_cache, tmp);
1612         } else if (c.i.op == OP_META_SWU) {
1613                 jit_unstr(tmp2, rt, LIGHTNING_UNALIGNED_32BIT);
1614         } else {
1615                 jit_new_node_www(code, 0, tmp2, rt);
1616         }
1617
1618         lightrec_free_reg(reg_cache, rt);
1619         lightrec_free_reg(reg_cache, tmp2);
1620 }
1621
1622 static void rec_store(struct lightrec_cstate *state,
1623                       const struct block *block, u16 offset,
1624                       jit_code_t code, jit_code_t swap_code)
1625 {
1626         u32 flags = block->opcode_list[offset].flags;
1627         u32 mode = LIGHTREC_FLAGS_GET_IO_MODE(flags);
1628         bool no_invalidate = op_flag_no_invalidate(flags) ||
1629                 (state->state->opt_flags & LIGHTREC_OPT_INV_DMA_ONLY);
1630         union code c = block->opcode_list[offset].c;
1631         bool is_swc2 = c.i.op == OP_SWC2;
1632
1633         if (is_swc2) {
1634                 switch (mode) {
1635                 case LIGHTREC_IO_RAM:
1636                 case LIGHTREC_IO_SCRATCH:
1637                 case LIGHTREC_IO_DIRECT:
1638                 case LIGHTREC_IO_DIRECT_HW:
1639                         rec_cp2_do_mfc2(state, block, offset, c.i.rt, REG_TEMP);
1640                         break;
1641                 default:
1642                         break;
1643                 }
1644         }
1645
1646         switch (mode) {
1647         case LIGHTREC_IO_RAM:
1648                 rec_store_ram(state, block, offset, code,
1649                               swap_code, !no_invalidate);
1650                 break;
1651         case LIGHTREC_IO_SCRATCH:
1652                 rec_store_scratch(state, block, offset, code, swap_code);
1653                 break;
1654         case LIGHTREC_IO_DIRECT:
1655                 if (no_invalidate) {
1656                         rec_store_direct_no_invalidate(state, block, offset,
1657                                                        code, swap_code);
1658                 } else {
1659                         rec_store_direct(state, block, offset, code, swap_code);
1660                 }
1661                 break;
1662         case LIGHTREC_IO_DIRECT_HW:
1663                 rec_store_io(state, block, offset, code, swap_code);
1664                 break;
1665         default:
1666                 rec_io(state, block, offset, true, false);
1667                 return;
1668         }
1669
1670         if (is_swc2)
1671                 lightrec_discard_reg_if_loaded(state->reg_cache, REG_TEMP);
1672 }
1673
1674 static void rec_SB(struct lightrec_cstate *state,
1675                    const struct block *block, u16 offset)
1676 {
1677         _jit_name(block->_jit, __func__);
1678         rec_store(state, block, offset, jit_code_stxi_c, 0);
1679 }
1680
1681 static void rec_SH(struct lightrec_cstate *state,
1682                    const struct block *block, u16 offset)
1683 {
1684         _jit_name(block->_jit, __func__);
1685         rec_store(state, block, offset,
1686                   jit_code_stxi_s, jit_code_bswapr_us);
1687 }
1688
1689 static void rec_SW(struct lightrec_cstate *state,
1690                    const struct block *block, u16 offset)
1691
1692 {
1693         union code c = block->opcode_list[offset].c;
1694
1695         _jit_name(block->_jit, c.i.op == OP_SWC2 ? "rec_SWC2" : "rec_SW");
1696         rec_store(state, block, offset,
1697                   jit_code_stxi_i, jit_code_bswapr_ui);
1698 }
1699
1700 static void rec_SWL(struct lightrec_cstate *state,
1701                     const struct block *block, u16 offset)
1702 {
1703         _jit_name(block->_jit, __func__);
1704         rec_io(state, block, offset, true, false);
1705 }
1706
1707 static void rec_SWR(struct lightrec_cstate *state,
1708                     const struct block *block, u16 offset)
1709 {
1710         _jit_name(block->_jit, __func__);
1711         rec_io(state, block, offset, true, false);
1712 }
1713
1714 static void rec_load_memory(struct lightrec_cstate *cstate,
1715                             const struct block *block, u16 offset,
1716                             jit_code_t code, jit_code_t swap_code, bool is_unsigned,
1717                             uintptr_t addr_offset, u32 addr_mask)
1718 {
1719         struct lightrec_state *state = cstate->state;
1720         struct regcache *reg_cache = cstate->reg_cache;
1721         struct opcode *op = &block->opcode_list[offset];
1722         bool load_delay = op_flag_load_delay(op->flags) && !cstate->no_load_delay;
1723         jit_state_t *_jit = block->_jit;
1724         u8 rs, rt, out_reg, addr_reg, flags = REG_EXT;
1725         bool no_mask = op_flag_no_mask(op->flags);
1726         union code c = op->c;
1727         s16 imm;
1728
1729         if (load_delay || c.i.op == OP_LWC2)
1730                 out_reg = REG_TEMP;
1731         else if (c.i.rt)
1732                 out_reg = c.i.rt;
1733         else
1734                 return;
1735
1736         if (is_unsigned)
1737                 flags |= REG_ZEXT;
1738
1739         rs = lightrec_alloc_reg_in(reg_cache, _jit, c.i.rs, 0);
1740         rt = lightrec_alloc_reg_out(reg_cache, _jit, out_reg, flags);
1741
1742         if ((op->i.op == OP_META_LWU && c.i.imm)
1743             || (!state->mirrors_mapped && c.i.imm && !no_mask)) {
1744                 jit_addi(rt, rs, (s16)c.i.imm);
1745                 addr_reg = rt;
1746                 imm = 0;
1747         } else {
1748                 addr_reg = rs;
1749                 imm = (s16)c.i.imm;
1750         }
1751
1752         if (op->i.op == OP_META_LWU)
1753                 imm = LIGHTNING_UNALIGNED_32BIT;
1754
1755         if (!no_mask) {
1756                 rec_and_mask(cstate, _jit, rt, addr_reg, addr_mask);
1757                 addr_reg = rt;
1758         }
1759
1760         if (addr_offset) {
1761                 rec_add_offset(cstate, _jit, rt, addr_reg, addr_offset);
1762                 addr_reg = rt;
1763         }
1764
1765         jit_new_node_www(code, rt, addr_reg, imm);
1766
1767         if (is_big_endian() && swap_code) {
1768                 jit_new_node_ww(swap_code, rt, rt);
1769
1770                 if (c.i.op == OP_LH)
1771                         jit_extr_s(rt, rt);
1772                 else if (c.i.op == OP_LW && __WORDSIZE == 64)
1773                         jit_extr_i(rt, rt);
1774         }
1775
1776         lightrec_free_reg(reg_cache, rs);
1777         lightrec_free_reg(reg_cache, rt);
1778 }
1779
1780 static void rec_load_ram(struct lightrec_cstate *cstate,
1781                          const struct block *block, u16 offset,
1782                          jit_code_t code, jit_code_t swap_code, bool is_unsigned)
1783 {
1784         _jit_note(block->_jit, __FILE__, __LINE__);
1785
1786         rec_load_memory(cstate, block, offset, code, swap_code, is_unsigned,
1787                         cstate->state->offset_ram, rec_ram_mask(cstate->state));
1788 }
1789
1790 static void rec_load_bios(struct lightrec_cstate *cstate,
1791                           const struct block *block, u16 offset,
1792                           jit_code_t code, jit_code_t swap_code, bool is_unsigned)
1793 {
1794         _jit_note(block->_jit, __FILE__, __LINE__);
1795
1796         rec_load_memory(cstate, block, offset, code, swap_code, is_unsigned,
1797                         cstate->state->offset_bios, 0x1fffffff);
1798 }
1799
1800 static void rec_load_scratch(struct lightrec_cstate *cstate,
1801                              const struct block *block, u16 offset,
1802                              jit_code_t code, jit_code_t swap_code, bool is_unsigned)
1803 {
1804         _jit_note(block->_jit, __FILE__, __LINE__);
1805
1806         rec_load_memory(cstate, block, offset, code, swap_code, is_unsigned,
1807                         cstate->state->offset_scratch, 0x1fffffff);
1808 }
1809
1810 static void rec_load_io(struct lightrec_cstate *cstate,
1811                         const struct block *block, u16 offset,
1812                         jit_code_t code, jit_code_t swap_code, bool is_unsigned)
1813 {
1814         _jit_note(block->_jit, __FILE__, __LINE__);
1815
1816         rec_load_memory(cstate, block, offset, code, swap_code, is_unsigned,
1817                         cstate->state->offset_io, rec_io_mask(cstate->state));
1818 }
1819
1820 static void rec_load_direct(struct lightrec_cstate *cstate,
1821                             const struct block *block, u16 offset,
1822                             jit_code_t code, jit_code_t swap_code,
1823                             bool is_unsigned)
1824 {
1825         const struct lightrec_state *state = cstate->state;
1826         struct regcache *reg_cache = cstate->reg_cache;
1827         struct opcode *op = &block->opcode_list[offset];
1828         bool load_delay = op_flag_load_delay(op->flags) && !cstate->no_load_delay;
1829         jit_state_t *_jit = block->_jit;
1830         jit_node_t *to_not_ram, *to_not_bios, *to_end, *to_end2;
1831         u8 tmp, rs, rt, out_reg, addr_reg, flags = REG_EXT;
1832         bool different_offsets = state->offset_bios != state->offset_scratch;
1833         union code c = op->c;
1834         s32 addr_mask;
1835         u32 reg_imm;
1836         s8 offt_reg;
1837         s16 imm;
1838
1839         if (load_delay || c.i.op == OP_LWC2)
1840                 out_reg = REG_TEMP;
1841         else if (c.i.rt)
1842                 out_reg = c.i.rt;
1843         else
1844                 return;
1845
1846         if (is_unsigned)
1847                 flags |= REG_ZEXT;
1848
1849         jit_note(__FILE__, __LINE__);
1850         rs = lightrec_alloc_reg_in(reg_cache, _jit, c.i.rs, 0);
1851         rt = lightrec_alloc_reg_out(reg_cache, _jit, out_reg, flags);
1852
1853         if ((state->offset_ram == state->offset_bios &&
1854             state->offset_ram == state->offset_scratch &&
1855             state->mirrors_mapped && c.i.op != OP_META_LWU)
1856             || !c.i.imm) {
1857                 addr_reg = rs;
1858                 imm = (s16)c.i.imm;
1859         } else {
1860                 jit_addi(rt, rs, (s16)c.i.imm);
1861                 addr_reg = rt;
1862                 imm = 0;
1863
1864                 if (c.i.rs != c.i.rt)
1865                         lightrec_free_reg(reg_cache, rs);
1866         }
1867
1868         if (op->i.op == OP_META_LWU)
1869                 imm = LIGHTNING_UNALIGNED_32BIT;
1870
1871         tmp = lightrec_alloc_reg_temp(reg_cache, _jit);
1872
1873         if (state->offset_ram == state->offset_bios &&
1874             state->offset_ram == state->offset_scratch) {
1875                 if (!state->mirrors_mapped)
1876                         addr_mask = 0x1f800000 | (RAM_SIZE - 1);
1877                 else
1878                         addr_mask = 0x1fffffff;
1879
1880                 reg_imm = lightrec_alloc_reg_temp_with_value(reg_cache, _jit,
1881                                                              addr_mask);
1882                 if (!state->mirrors_mapped) {
1883                         jit_andi(tmp, addr_reg, BIT(28));
1884                         jit_rshi_u(tmp, tmp, 28 - 22);
1885                         jit_orr(tmp, tmp, reg_imm);
1886                         jit_andr(rt, addr_reg, tmp);
1887                 } else {
1888                         jit_andr(rt, addr_reg, reg_imm);
1889                 }
1890
1891                 lightrec_free_reg(reg_cache, reg_imm);
1892
1893                 if (state->offset_ram) {
1894                         offt_reg = lightrec_get_reg_with_value(reg_cache,
1895                                                                state->offset_ram);
1896                         if (offt_reg < 0) {
1897                                 jit_movi(tmp, state->offset_ram);
1898                                 lightrec_temp_set_value(reg_cache, tmp,
1899                                                         state->offset_ram);
1900                         } else {
1901                                 lightrec_free_reg(reg_cache, tmp);
1902                                 tmp = offt_reg;
1903                         }
1904                 }
1905         } else {
1906                 to_not_ram = jit_bmsi(addr_reg, BIT(28));
1907
1908                 /* Convert to KUNSEG and avoid RAM mirrors */
1909                 jit_andi(rt, addr_reg, RAM_SIZE - 1);
1910
1911                 if (state->offset_ram)
1912                         jit_movi(tmp, state->offset_ram);
1913
1914                 to_end = jit_b();
1915
1916                 jit_patch(to_not_ram);
1917
1918                 if (different_offsets)
1919                         to_not_bios = jit_bmci(addr_reg, BIT(22));
1920
1921                 /* Convert to KUNSEG */
1922                 jit_andi(rt, addr_reg, 0x1fc00000 | (BIOS_SIZE - 1));
1923
1924                 jit_movi(tmp, state->offset_bios);
1925
1926                 if (different_offsets) {
1927                         to_end2 = jit_b();
1928
1929                         jit_patch(to_not_bios);
1930
1931                         /* Convert to KUNSEG */
1932                         jit_andi(rt, addr_reg, 0x1f800fff);
1933
1934                         if (state->offset_scratch)
1935                                 jit_movi(tmp, state->offset_scratch);
1936
1937                         jit_patch(to_end2);
1938                 }
1939
1940                 jit_patch(to_end);
1941         }
1942
1943         if (state->offset_ram || state->offset_bios || state->offset_scratch)
1944                 jit_addr(rt, rt, tmp);
1945
1946         jit_new_node_www(code, rt, rt, imm);
1947
1948         if (is_big_endian() && swap_code) {
1949                 jit_new_node_ww(swap_code, rt, rt);
1950
1951                 if (c.i.op == OP_LH)
1952                         jit_extr_s(rt, rt);
1953                 else if (c.i.op == OP_LW && __WORDSIZE == 64)
1954                         jit_extr_i(rt, rt);
1955         }
1956
1957         lightrec_free_reg(reg_cache, addr_reg);
1958         lightrec_free_reg(reg_cache, rt);
1959         lightrec_free_reg(reg_cache, tmp);
1960 }
1961
1962 static void rec_load(struct lightrec_cstate *state, const struct block *block,
1963                      u16 offset, jit_code_t code, jit_code_t swap_code,
1964                      bool is_unsigned)
1965 {
1966         const struct opcode *op = &block->opcode_list[offset];
1967         u32 flags = op->flags;
1968
1969         switch (LIGHTREC_FLAGS_GET_IO_MODE(flags)) {
1970         case LIGHTREC_IO_RAM:
1971                 rec_load_ram(state, block, offset, code, swap_code, is_unsigned);
1972                 break;
1973         case LIGHTREC_IO_BIOS:
1974                 rec_load_bios(state, block, offset, code, swap_code, is_unsigned);
1975                 break;
1976         case LIGHTREC_IO_SCRATCH:
1977                 rec_load_scratch(state, block, offset, code, swap_code, is_unsigned);
1978                 break;
1979         case LIGHTREC_IO_DIRECT_HW:
1980                 rec_load_io(state, block, offset, code, swap_code, is_unsigned);
1981                 break;
1982         case LIGHTREC_IO_DIRECT:
1983                 rec_load_direct(state, block, offset, code, swap_code, is_unsigned);
1984                 break;
1985         default:
1986                 rec_io(state, block, offset, false, true);
1987                 return;
1988         }
1989
1990         if (op->i.op == OP_LWC2) {
1991                 rec_cp2_do_mtc2(state, block, offset, op->i.rt, REG_TEMP);
1992                 lightrec_discard_reg_if_loaded(state->reg_cache, REG_TEMP);
1993         }
1994 }
1995
1996 static void rec_LB(struct lightrec_cstate *state, const struct block *block, u16 offset)
1997 {
1998         _jit_name(block->_jit, __func__);
1999         rec_load(state, block, offset, jit_code_ldxi_c, 0, false);
2000 }
2001
2002 static void rec_LBU(struct lightrec_cstate *state, const struct block *block, u16 offset)
2003 {
2004         _jit_name(block->_jit, __func__);
2005         rec_load(state, block, offset, jit_code_ldxi_uc, 0, true);
2006 }
2007
2008 static void rec_LH(struct lightrec_cstate *state, const struct block *block, u16 offset)
2009 {
2010         jit_code_t code = is_big_endian() ? jit_code_ldxi_us : jit_code_ldxi_s;
2011
2012         _jit_name(block->_jit, __func__);
2013         rec_load(state, block, offset, code, jit_code_bswapr_us, false);
2014 }
2015
2016 static void rec_LHU(struct lightrec_cstate *state, const struct block *block, u16 offset)
2017 {
2018         _jit_name(block->_jit, __func__);
2019         rec_load(state, block, offset, jit_code_ldxi_us, jit_code_bswapr_us, true);
2020 }
2021
2022 static void rec_LWL(struct lightrec_cstate *state, const struct block *block, u16 offset)
2023 {
2024         _jit_name(block->_jit, __func__);
2025         rec_io(state, block, offset, true, true);
2026 }
2027
2028 static void rec_LWR(struct lightrec_cstate *state, const struct block *block, u16 offset)
2029 {
2030         _jit_name(block->_jit, __func__);
2031         rec_io(state, block, offset, true, true);
2032 }
2033
2034 static void rec_LW(struct lightrec_cstate *state, const struct block *block, u16 offset)
2035 {
2036         union code c = block->opcode_list[offset].c;
2037         jit_code_t code;
2038
2039         if (is_big_endian() && __WORDSIZE == 64)
2040                 code = jit_code_ldxi_ui;
2041         else
2042                 code = jit_code_ldxi_i;
2043
2044         _jit_name(block->_jit, c.i.op == OP_LWC2 ? "rec_LWC2" : "rec_LW");
2045         rec_load(state, block, offset, code, jit_code_bswapr_ui, false);
2046 }
2047
2048 static void rec_exit_early(struct lightrec_cstate *state,
2049                            const struct block *block, u16 offset,
2050                            u32 exit_code, u32 pc)
2051 {
2052         struct regcache *reg_cache = state->reg_cache;
2053         jit_state_t *_jit = block->_jit;
2054         u8 tmp;
2055
2056         _jit_note(block->_jit, __FILE__, __LINE__);
2057
2058         tmp = lightrec_alloc_reg_temp(reg_cache, _jit);
2059
2060         jit_movi(tmp, exit_code);
2061         jit_stxi_i(offsetof(struct lightrec_state, exit_flags),
2062                    LIGHTREC_REG_STATE, tmp);
2063
2064         jit_ldxi_i(tmp, LIGHTREC_REG_STATE,
2065                    offsetof(struct lightrec_state, target_cycle));
2066         jit_subr(tmp, tmp, LIGHTREC_REG_CYCLE);
2067         jit_movi(LIGHTREC_REG_CYCLE, 0);
2068         jit_stxi_i(offsetof(struct lightrec_state, target_cycle),
2069                    LIGHTREC_REG_STATE, tmp);
2070         jit_stxi_i(offsetof(struct lightrec_state, current_cycle),
2071                    LIGHTREC_REG_STATE, tmp);
2072
2073         lightrec_free_reg(reg_cache, tmp);
2074
2075         lightrec_emit_end_of_block(state, block, offset, -1, pc, 31, 0, true);
2076 }
2077
2078 static void rec_special_SYSCALL(struct lightrec_cstate *state,
2079                                 const struct block *block, u16 offset)
2080 {
2081         _jit_name(block->_jit, __func__);
2082
2083         /* TODO: the return address should be "pc - 4" if we're a delay slot */
2084         rec_exit_early(state, block, offset, LIGHTREC_EXIT_SYSCALL,
2085                        get_ds_pc(block, offset, 0));
2086 }
2087
2088 static void rec_special_BREAK(struct lightrec_cstate *state,
2089                               const struct block *block, u16 offset)
2090 {
2091         _jit_name(block->_jit, __func__);
2092         rec_exit_early(state, block, offset, LIGHTREC_EXIT_BREAK,
2093                        get_ds_pc(block, offset, 0));
2094 }
2095
2096 static void rec_mfc(struct lightrec_cstate *state, const struct block *block, u16 offset)
2097 {
2098         struct regcache *reg_cache = state->reg_cache;
2099         union code c = block->opcode_list[offset].c;
2100         jit_state_t *_jit = block->_jit;
2101
2102         jit_note(__FILE__, __LINE__);
2103
2104         if (c.i.op != OP_SWC2)
2105                 lightrec_clean_reg_if_loaded(reg_cache, _jit, c.i.rt, true);
2106
2107         call_to_c_wrapper(state, block, c.opcode, C_WRAPPER_MFC);
2108 }
2109
2110 static void rec_mtc(struct lightrec_cstate *state, const struct block *block, u16 offset)
2111 {
2112         struct regcache *reg_cache = state->reg_cache;
2113         union code c = block->opcode_list[offset].c;
2114         jit_state_t *_jit = block->_jit;
2115
2116         jit_note(__FILE__, __LINE__);
2117         lightrec_clean_reg_if_loaded(reg_cache, _jit, c.i.rs, false);
2118         lightrec_clean_reg_if_loaded(reg_cache, _jit, c.i.rt, false);
2119         lightrec_clean_reg_if_loaded(reg_cache, _jit, REG_TEMP, false);
2120
2121         call_to_c_wrapper(state, block, c.opcode, C_WRAPPER_MTC);
2122
2123         if (c.i.op == OP_CP0 &&
2124             !op_flag_no_ds(block->opcode_list[offset].flags) &&
2125             (c.r.rd == 12 || c.r.rd == 13))
2126                 lightrec_emit_end_of_block(state, block, offset, -1,
2127                                            get_ds_pc(block, offset, 1),
2128                                            0, 0, true);
2129 }
2130
2131 static void
2132 rec_mfc0(struct lightrec_cstate *state, const struct block *block, u16 offset)
2133 {
2134         struct regcache *reg_cache = state->reg_cache;
2135         union code c = block->opcode_list[offset].c;
2136         jit_state_t *_jit = block->_jit;
2137         u8 rt;
2138
2139         jit_note(__FILE__, __LINE__);
2140
2141         rt = lightrec_alloc_reg_out(reg_cache, _jit, c.i.rt, REG_EXT);
2142
2143         jit_ldxi_i(rt, LIGHTREC_REG_STATE,
2144                    offsetof(struct lightrec_state, regs.cp0[c.r.rd]));
2145
2146         lightrec_free_reg(reg_cache, rt);
2147 }
2148
2149 static bool block_uses_icache(const struct lightrec_cstate *state,
2150                               const struct block *block)
2151 {
2152         const struct lightrec_mem_map *map = &state->state->maps[PSX_MAP_KERNEL_USER_RAM];
2153         u32 pc = kunseg(block->pc);
2154
2155         if (pc < map->pc || pc >= map->pc + map->length)
2156                 return false;
2157
2158         return (block->pc >> 28) < 0xa;
2159 }
2160
2161 static void
2162 rec_mtc0(struct lightrec_cstate *state, const struct block *block, u16 offset)
2163 {
2164         struct regcache *reg_cache = state->reg_cache;
2165         const union code c = block->opcode_list[offset].c;
2166         jit_state_t *_jit = block->_jit;
2167         u8 rt, tmp = 0, tmp2, status;
2168         jit_node_t *to_end;
2169
2170         jit_note(__FILE__, __LINE__);
2171
2172         switch(c.r.rd) {
2173         case 1:
2174         case 4:
2175         case 8:
2176         case 14:
2177         case 15:
2178                 /* Those registers are read-only */
2179                 return;
2180         default:
2181                 break;
2182         }
2183
2184         if (!block_uses_icache(state, block) && c.r.rd == 12) {
2185                 /* If we are not running code from the RAM through kuseg or
2186                  * kseg0, handle writes to the Status register in C; as the
2187                  * code may toggle bit 16 which isolates the cache. Code
2188                  * running from kuseg or kseg0 in RAM cannot do that. */
2189                 rec_mtc(state, block, offset);
2190                 return;
2191         }
2192
2193         rt = lightrec_alloc_reg_in(reg_cache, _jit, c.i.rt, 0);
2194
2195         if (c.r.rd != 13) {
2196                 jit_stxi_i(offsetof(struct lightrec_state, regs.cp0[c.r.rd]),
2197                            LIGHTREC_REG_STATE, rt);
2198         }
2199
2200         if (c.r.rd == 12 || c.r.rd == 13) {
2201                 tmp = lightrec_alloc_reg_temp(reg_cache, _jit);
2202                 jit_ldxi_i(tmp, LIGHTREC_REG_STATE,
2203                            offsetof(struct lightrec_state, regs.cp0[13]));
2204
2205                 tmp2 = lightrec_alloc_reg_temp(reg_cache, _jit);
2206         }
2207
2208         if (c.r.rd == 12) {
2209                 status = rt;
2210         } else if (c.r.rd == 13) {
2211                 /* Cause = (Cause & ~0x0300) | (value & 0x0300) */
2212                 jit_andi(tmp2, rt, 0x0300);
2213                 jit_ori(tmp, tmp, 0x0300);
2214                 jit_xori(tmp, tmp, 0x0300);
2215                 jit_orr(tmp, tmp, tmp2);
2216                 jit_ldxi_i(tmp2, LIGHTREC_REG_STATE,
2217                            offsetof(struct lightrec_state, regs.cp0[12]));
2218                 jit_stxi_i(offsetof(struct lightrec_state, regs.cp0[13]),
2219                            LIGHTREC_REG_STATE, tmp);
2220                 status = tmp2;
2221         }
2222
2223         if (c.r.rd == 12 || c.r.rd == 13) {
2224                 /* Exit dynarec in case there's a software interrupt.
2225                  * exit_flags = !!(status & tmp & 0x0300) & status; */
2226                 jit_andr(tmp, tmp, status);
2227                 jit_andi(tmp, tmp, 0x0300);
2228                 jit_nei(tmp, tmp, 0);
2229                 jit_andr(tmp, tmp, status);
2230         }
2231
2232         if (c.r.rd == 12) {
2233                 /* Exit dynarec in case we unmask a hardware interrupt.
2234                  * exit_flags = !(~status & 0x401) */
2235
2236                 jit_comr(tmp2, status);
2237                 jit_andi(tmp2, tmp2, 0x401);
2238                 jit_eqi(tmp2, tmp2, 0);
2239                 jit_orr(tmp, tmp, tmp2);
2240         }
2241
2242         lightrec_free_reg(reg_cache, rt);
2243
2244         if (c.r.rd == 12 || c.r.rd == 13) {
2245                 to_end = jit_beqi(tmp, 0);
2246
2247                 jit_ldxi_i(tmp2, LIGHTREC_REG_STATE,
2248                            offsetof(struct lightrec_state, target_cycle));
2249                 jit_subr(tmp2, tmp2, LIGHTREC_REG_CYCLE);
2250                 jit_movi(LIGHTREC_REG_CYCLE, 0);
2251                 jit_stxi_i(offsetof(struct lightrec_state, target_cycle),
2252                            LIGHTREC_REG_STATE, tmp2);
2253                 jit_stxi_i(offsetof(struct lightrec_state, current_cycle),
2254                            LIGHTREC_REG_STATE, tmp2);
2255
2256
2257                 jit_patch(to_end);
2258         }
2259
2260         if (!op_flag_no_ds(block->opcode_list[offset].flags) &&
2261             (c.r.rd == 12 || c.r.rd == 13)) {
2262                 state->cycles += lightrec_cycles_of_opcode(state->state, c);
2263                 lightrec_emit_eob(state, block, offset + 1);
2264         }
2265 }
2266
2267 static void rec_cp0_MFC0(struct lightrec_cstate *state,
2268                          const struct block *block, u16 offset)
2269 {
2270         _jit_name(block->_jit, __func__);
2271         rec_mfc0(state, block, offset);
2272 }
2273
2274 static void rec_cp0_CFC0(struct lightrec_cstate *state,
2275                          const struct block *block, u16 offset)
2276 {
2277         _jit_name(block->_jit, __func__);
2278         rec_mfc0(state, block, offset);
2279 }
2280
2281 static void rec_cp0_MTC0(struct lightrec_cstate *state,
2282                          const struct block *block, u16 offset)
2283 {
2284         _jit_name(block->_jit, __func__);
2285         rec_mtc0(state, block, offset);
2286 }
2287
2288 static void rec_cp0_CTC0(struct lightrec_cstate *state,
2289                          const struct block *block, u16 offset)
2290 {
2291         _jit_name(block->_jit, __func__);
2292         rec_mtc0(state, block, offset);
2293 }
2294
2295 static unsigned int cp2d_i_offset(u8 reg)
2296 {
2297         return offsetof(struct lightrec_state, regs.cp2d[reg]);
2298 }
2299
2300 static unsigned int cp2d_s_offset(u8 reg)
2301 {
2302         return cp2d_i_offset(reg) + is_big_endian() * 2;
2303 }
2304
2305 static unsigned int cp2c_i_offset(u8 reg)
2306 {
2307         return offsetof(struct lightrec_state, regs.cp2c[reg]);
2308 }
2309
2310 static unsigned int cp2c_s_offset(u8 reg)
2311 {
2312         return cp2c_i_offset(reg) + is_big_endian() * 2;
2313 }
2314
2315 static void rec_cp2_do_mfc2(struct lightrec_cstate *state,
2316                             const struct block *block, u16 offset,
2317                             u8 reg, u8 out_reg)
2318 {
2319         struct regcache *reg_cache = state->reg_cache;
2320         jit_state_t *_jit = block->_jit;
2321         const u32 zext_regs = 0x300f0080;
2322         u8 rt, tmp, tmp2, tmp3, out, flags;
2323         unsigned int i;
2324
2325         _jit_name(block->_jit, __func__);
2326
2327         if (state->state->ops.cop2_notify) {
2328                 /* We must call cop2_notify, handle that in C. */
2329                 rec_mfc(state, block, offset);
2330                 return;
2331         }
2332
2333         flags = (zext_regs & BIT(reg)) ? REG_ZEXT : REG_EXT;
2334         rt = lightrec_alloc_reg_out(reg_cache, _jit, out_reg, flags);
2335
2336         if (reg == 15)
2337                 reg = 14;
2338
2339         switch (reg) {
2340         case 1:
2341         case 3:
2342         case 5:
2343         case 8:
2344         case 9:
2345         case 10:
2346         case 11:
2347                 jit_ldxi_s(rt, LIGHTREC_REG_STATE, cp2d_s_offset(reg));
2348                 break;
2349         case 7:
2350         case 16:
2351         case 17:
2352         case 18:
2353         case 19:
2354                 jit_ldxi_us(rt, LIGHTREC_REG_STATE, cp2d_s_offset(reg));
2355                 break;
2356         case 28:
2357         case 29:
2358                 tmp = lightrec_alloc_reg_temp(reg_cache, _jit);
2359                 tmp2 = lightrec_alloc_reg_temp(reg_cache, _jit);
2360                 tmp3 = lightrec_alloc_reg_temp(reg_cache, _jit);
2361
2362                 for (i = 0; i < 3; i++) {
2363                         out = i == 0 ? rt : tmp;
2364
2365                         jit_ldxi_s(tmp, LIGHTREC_REG_STATE, cp2d_s_offset(9 + i));
2366                         jit_movi(tmp2, 0x1f);
2367                         jit_rshi(out, tmp, 7);
2368
2369                         jit_ltr(tmp3, tmp2, out);
2370                         jit_movnr(out, tmp2, tmp3);
2371
2372                         jit_gei(tmp2, out, 0);
2373                         jit_movzr(out, tmp2, tmp2);
2374
2375                         if (i > 0) {
2376                                 jit_lshi(tmp, tmp, 5 * i);
2377                                 jit_orr(rt, rt, tmp);
2378                         }
2379                 }
2380
2381
2382                 lightrec_free_reg(reg_cache, tmp);
2383                 lightrec_free_reg(reg_cache, tmp2);
2384                 lightrec_free_reg(reg_cache, tmp3);
2385                 break;
2386         default:
2387                 jit_ldxi_i(rt, LIGHTREC_REG_STATE, cp2d_i_offset(reg));
2388                 break;
2389         }
2390
2391         lightrec_free_reg(reg_cache, rt);
2392 }
2393
2394 static void rec_cp2_basic_MFC2(struct lightrec_cstate *state,
2395                                const struct block *block, u16 offset)
2396 {
2397         const union code c = block->opcode_list[offset].c;
2398
2399         rec_cp2_do_mfc2(state, block, offset, c.r.rd, c.r.rt);
2400 }
2401
2402 static void rec_cp2_basic_CFC2(struct lightrec_cstate *state,
2403                                const struct block *block, u16 offset)
2404 {
2405         struct regcache *reg_cache = state->reg_cache;
2406         const union code c = block->opcode_list[offset].c;
2407         jit_state_t *_jit = block->_jit;
2408         u8 rt;
2409
2410         _jit_name(block->_jit, __func__);
2411
2412         if (state->state->ops.cop2_notify) {
2413                 /* We must call cop2_notify, handle that in C. */
2414                 rec_mfc(state, block, offset);
2415                 return;
2416         }
2417
2418         switch (c.r.rd) {
2419         case 4:
2420         case 12:
2421         case 20:
2422         case 26:
2423         case 27:
2424         case 29:
2425         case 30:
2426                 rt = lightrec_alloc_reg_out(reg_cache, _jit, c.r.rt, REG_EXT);
2427                 jit_ldxi_s(rt, LIGHTREC_REG_STATE, cp2c_s_offset(c.r.rd));
2428                 break;
2429         default:
2430                 rt = lightrec_alloc_reg_out(reg_cache, _jit, c.r.rt, REG_ZEXT);
2431                 jit_ldxi_ui(rt, LIGHTREC_REG_STATE, cp2c_i_offset(c.r.rd));
2432                 break;
2433         }
2434
2435         lightrec_free_reg(reg_cache, rt);
2436 }
2437
2438 static void rec_cp2_do_mtc2(struct lightrec_cstate *state,
2439                             const struct block *block, u16 offset,
2440                             u8 reg, u8 in_reg)
2441 {
2442         struct regcache *reg_cache = state->reg_cache;
2443         jit_state_t *_jit = block->_jit;
2444         u8 rt, tmp, tmp2, flags = 0;
2445
2446         _jit_name(block->_jit, __func__);
2447
2448         if (state->state->ops.cop2_notify) {
2449                 /* We must call cop2_notify, handle that in C. */
2450                 rec_mtc(state, block, offset);
2451                 return;
2452         }
2453
2454         if (reg == 31)
2455                 return;
2456
2457         if (reg == 30)
2458                 flags |= REG_EXT;
2459
2460         rt = lightrec_alloc_reg_in(reg_cache, _jit, in_reg, flags);
2461
2462         switch (reg) {
2463         case 15:
2464                 tmp = lightrec_alloc_reg_temp(reg_cache, _jit);
2465                 jit_ldxi_i(tmp, LIGHTREC_REG_STATE, cp2d_i_offset(13));
2466
2467                 tmp2 = lightrec_alloc_reg_temp(reg_cache, _jit);
2468                 jit_ldxi_i(tmp2, LIGHTREC_REG_STATE, cp2d_i_offset(14));
2469
2470                 jit_stxi_i(cp2d_i_offset(12), LIGHTREC_REG_STATE, tmp);
2471                 jit_stxi_i(cp2d_i_offset(13), LIGHTREC_REG_STATE, tmp2);
2472                 jit_stxi_i(cp2d_i_offset(14), LIGHTREC_REG_STATE, rt);
2473
2474                 lightrec_free_reg(reg_cache, tmp);
2475                 lightrec_free_reg(reg_cache, tmp2);
2476                 break;
2477         case 28:
2478                 tmp = lightrec_alloc_reg_temp(reg_cache, _jit);
2479
2480                 jit_lshi(tmp, rt, 7);
2481                 jit_andi(tmp, tmp, 0xf80);
2482                 jit_stxi_s(cp2d_s_offset(9), LIGHTREC_REG_STATE, tmp);
2483
2484                 jit_lshi(tmp, rt, 2);
2485                 jit_andi(tmp, tmp, 0xf80);
2486                 jit_stxi_s(cp2d_s_offset(10), LIGHTREC_REG_STATE, tmp);
2487
2488                 jit_rshi(tmp, rt, 3);
2489                 jit_andi(tmp, tmp, 0xf80);
2490                 jit_stxi_s(cp2d_s_offset(11), LIGHTREC_REG_STATE, tmp);
2491
2492                 lightrec_free_reg(reg_cache, tmp);
2493                 break;
2494         case 30:
2495                 tmp = lightrec_alloc_reg_temp(reg_cache, _jit);
2496
2497                 /* if (rt < 0) rt = ~rt; */
2498                 jit_rshi(tmp, rt, 31);
2499                 jit_xorr(tmp, rt, tmp);
2500
2501                 /* Count leading zeros */
2502                 jit_clzr(tmp, tmp);
2503                 if (__WORDSIZE != 32)
2504                         jit_subi(tmp, tmp, __WORDSIZE - 32);
2505
2506                 jit_stxi_i(cp2d_i_offset(31), LIGHTREC_REG_STATE, tmp);
2507
2508                 lightrec_free_reg(reg_cache, tmp);
2509                 fallthrough;
2510         default:
2511                 jit_stxi_i(cp2d_i_offset(reg), LIGHTREC_REG_STATE, rt);
2512                 break;
2513         }
2514
2515         lightrec_free_reg(reg_cache, rt);
2516 }
2517
2518 static void rec_cp2_basic_MTC2(struct lightrec_cstate *state,
2519                                const struct block *block, u16 offset)
2520 {
2521         const union code c = block->opcode_list[offset].c;
2522
2523         rec_cp2_do_mtc2(state, block, offset, c.r.rd, c.r.rt);
2524 }
2525
2526 static void rec_cp2_basic_CTC2(struct lightrec_cstate *state,
2527                                const struct block *block, u16 offset)
2528 {
2529         struct regcache *reg_cache = state->reg_cache;
2530         const union code c = block->opcode_list[offset].c;
2531         jit_state_t *_jit = block->_jit;
2532         u8 rt, tmp, tmp2;
2533
2534         _jit_name(block->_jit, __func__);
2535
2536         if (state->state->ops.cop2_notify) {
2537                 /* We must call cop2_notify, handle that in C. */
2538                 rec_mtc(state, block, offset);
2539                 return;
2540         }
2541
2542         rt = lightrec_alloc_reg_in(reg_cache, _jit, c.r.rt, 0);
2543
2544         switch (c.r.rd) {
2545         case 4:
2546         case 12:
2547         case 20:
2548         case 26:
2549         case 27:
2550         case 29:
2551         case 30:
2552                 jit_stxi_s(cp2c_s_offset(c.r.rd), LIGHTREC_REG_STATE, rt);
2553                 break;
2554         case 31:
2555                 tmp = lightrec_alloc_reg_temp(reg_cache, _jit);
2556                 tmp2 = lightrec_alloc_reg_temp(reg_cache, _jit);
2557
2558                 jit_andi(tmp, rt, 0x7f87e000);
2559                 jit_nei(tmp, tmp, 0);
2560                 jit_lshi(tmp, tmp, 31);
2561
2562                 jit_andi(tmp2, rt, 0x7ffff000);
2563                 jit_orr(tmp, tmp2, tmp);
2564
2565                 jit_stxi_i(cp2c_i_offset(31), LIGHTREC_REG_STATE, tmp);
2566
2567                 lightrec_free_reg(reg_cache, tmp);
2568                 lightrec_free_reg(reg_cache, tmp2);
2569                 break;
2570
2571         default:
2572                 jit_stxi_i(cp2c_i_offset(c.r.rd), LIGHTREC_REG_STATE, rt);
2573         }
2574
2575         lightrec_free_reg(reg_cache, rt);
2576 }
2577
2578 static void rec_cp0_RFE(struct lightrec_cstate *state,
2579                         const struct block *block, u16 offset)
2580 {
2581         struct regcache *reg_cache = state->reg_cache;
2582         jit_state_t *_jit = block->_jit;
2583         u8 status, tmp;
2584
2585         jit_name(__func__);
2586         jit_note(__FILE__, __LINE__);
2587
2588         status = lightrec_alloc_reg_temp(reg_cache, _jit);
2589         jit_ldxi_i(status, LIGHTREC_REG_STATE,
2590                    offsetof(struct lightrec_state, regs.cp0[12]));
2591
2592         tmp = lightrec_alloc_reg_temp(reg_cache, _jit);
2593
2594         /* status = ((status >> 2) & 0xf) | status & ~0xf; */
2595         jit_rshi(tmp, status, 2);
2596         jit_andi(tmp, tmp, 0xf);
2597         jit_andi(status, status, ~0xful);
2598         jit_orr(status, status, tmp);
2599
2600         jit_ldxi_i(tmp, LIGHTREC_REG_STATE,
2601                    offsetof(struct lightrec_state, regs.cp0[13]));
2602         jit_stxi_i(offsetof(struct lightrec_state, regs.cp0[12]),
2603                    LIGHTREC_REG_STATE, status);
2604
2605         /* Exit dynarec in case there's a software interrupt.
2606          * exit_flags = !!(status & cause & 0x0300) & status; */
2607         jit_andr(tmp, tmp, status);
2608         jit_andi(tmp, tmp, 0x0300);
2609         jit_nei(tmp, tmp, 0);
2610         jit_andr(tmp, tmp, status);
2611         jit_stxi_i(offsetof(struct lightrec_state, exit_flags),
2612                    LIGHTREC_REG_STATE, tmp);
2613
2614         lightrec_free_reg(reg_cache, status);
2615         lightrec_free_reg(reg_cache, tmp);
2616 }
2617
2618 static void rec_CP(struct lightrec_cstate *state,
2619                    const struct block *block, u16 offset)
2620 {
2621         union code c = block->opcode_list[offset].c;
2622         jit_state_t *_jit = block->_jit;
2623
2624         jit_name(__func__);
2625         jit_note(__FILE__, __LINE__);
2626
2627         call_to_c_wrapper(state, block, c.opcode, C_WRAPPER_CP);
2628 }
2629
2630 static void rec_meta_MOV(struct lightrec_cstate *state,
2631                          const struct block *block, u16 offset)
2632 {
2633         struct regcache *reg_cache = state->reg_cache;
2634         const struct opcode *op = &block->opcode_list[offset];
2635         union code c = op->c;
2636         jit_state_t *_jit = block->_jit;
2637         bool unload_rd;
2638         bool unload_rs, discard_rs;
2639         u8 rs, rd;
2640
2641         _jit_name(block->_jit, __func__);
2642         jit_note(__FILE__, __LINE__);
2643
2644         unload_rs = OPT_EARLY_UNLOAD
2645                 && LIGHTREC_FLAGS_GET_RS(op->flags) == LIGHTREC_REG_UNLOAD;
2646         discard_rs = OPT_EARLY_UNLOAD
2647                 && LIGHTREC_FLAGS_GET_RS(op->flags) == LIGHTREC_REG_DISCARD;
2648
2649         if ((unload_rs || discard_rs) && c.m.rs) {
2650                 /* If the source register is going to be unloaded or discarded,
2651                  * then we can simply mark its host register as now pointing to
2652                  * the destination register. */
2653                 pr_debug("Remap %s to %s at offset 0x%x\n",
2654                          lightrec_reg_name(c.m.rs), lightrec_reg_name(c.m.rd),
2655                          offset << 2);
2656                 rs = lightrec_alloc_reg_in(reg_cache, _jit, c.m.rs, 0);
2657                 lightrec_remap_reg(reg_cache, _jit, rs, c.m.rd, discard_rs);
2658                 lightrec_free_reg(reg_cache, rs);
2659                 return;
2660         }
2661
2662         unload_rd = OPT_EARLY_UNLOAD
2663                 && LIGHTREC_FLAGS_GET_RD(op->flags) == LIGHTREC_REG_UNLOAD;
2664
2665         if (c.m.rs && !lightrec_reg_is_loaded(reg_cache, c.m.rs)) {
2666                 /* The source register is not yet loaded - we can load its value
2667                  * from the register cache directly into the target register. */
2668                 rd = lightrec_alloc_reg_out(reg_cache, _jit, c.m.rd, REG_EXT);
2669
2670                 jit_ldxi_i(rd, LIGHTREC_REG_STATE,
2671                            offsetof(struct lightrec_state, regs.gpr) + (c.m.rs << 2));
2672
2673                 lightrec_free_reg(reg_cache, rd);
2674         } else if (unload_rd) {
2675                 /* If the destination register will be unloaded right after the
2676                  * MOV meta-opcode, we don't actually need to write any host
2677                  * register - we can just store the source register directly to
2678                  * the register cache, at the offset corresponding to the
2679                  * destination register. */
2680                 lightrec_discard_reg_if_loaded(reg_cache, c.m.rd);
2681
2682                 rs = lightrec_alloc_reg_in(reg_cache, _jit, c.m.rs, 0);
2683
2684                 jit_stxi_i(offsetof(struct lightrec_state, regs.gpr)
2685                            + (c.m.rd << 2), LIGHTREC_REG_STATE, rs);
2686
2687                 lightrec_free_reg(reg_cache, rs);
2688         } else {
2689                 if (c.m.rs)
2690                         rs = lightrec_alloc_reg_in(reg_cache, _jit, c.m.rs, 0);
2691
2692                 rd = lightrec_alloc_reg_out(reg_cache, _jit, c.m.rd, REG_EXT);
2693
2694                 if (c.m.rs == 0) {
2695                         jit_movi(rd, 0);
2696                 } else {
2697                         jit_extr_i(rd, rs);
2698                         lightrec_free_reg(reg_cache, rs);
2699                 }
2700
2701                 lightrec_free_reg(reg_cache, rd);
2702         }
2703 }
2704
2705 static void rec_meta_EXTC_EXTS(struct lightrec_cstate *state,
2706                                const struct block *block,
2707                                u16 offset)
2708 {
2709         struct regcache *reg_cache = state->reg_cache;
2710         union code c = block->opcode_list[offset].c;
2711         jit_state_t *_jit = block->_jit;
2712         u8 rs, rd;
2713
2714         _jit_name(block->_jit, __func__);
2715         jit_note(__FILE__, __LINE__);
2716
2717         rec_alloc_rs_rd(reg_cache, _jit, &block->opcode_list[offset],
2718                         c.m.rs, c.m.rd, 0, REG_EXT, &rs, &rd);
2719
2720         if (c.m.op == OP_META_EXTC)
2721                 jit_extr_c(rd, rs);
2722         else
2723                 jit_extr_s(rd, rs);
2724
2725         lightrec_free_reg(reg_cache, rs);
2726         lightrec_free_reg(reg_cache, rd);
2727 }
2728
2729 static void rec_meta_MULT2(struct lightrec_cstate *state,
2730                            const struct block *block,
2731                            u16 offset)
2732 {
2733         struct regcache *reg_cache = state->reg_cache;
2734         union code c = block->opcode_list[offset].c;
2735         jit_state_t *_jit = block->_jit;
2736         u8 reg_lo = get_mult_div_lo(c);
2737         u8 reg_hi = get_mult_div_hi(c);
2738         u32 flags = block->opcode_list[offset].flags;
2739         bool is_signed = c.i.op == OP_META_MULT2;
2740         u8 rs, lo, hi, rflags = 0, hiflags = 0;
2741         unsigned int i;
2742
2743         if (!op_flag_no_hi(flags) && c.r.op < 32) {
2744                 rflags = is_signed ? REG_EXT : REG_ZEXT;
2745                 hiflags = is_signed ? REG_EXT : (REG_EXT | REG_ZEXT);
2746         }
2747
2748         _jit_name(block->_jit, __func__);
2749         jit_note(__FILE__, __LINE__);
2750
2751         rs = lightrec_alloc_reg_in(reg_cache, _jit, c.i.rs, rflags);
2752
2753         /*
2754          * We must handle the case where one of the output registers is our rs
2755          * input register. Thanksfully, computing LO/HI can be done in any
2756          * order. Here, we make sure that the computation that overwrites the
2757          * input register is always performed last.
2758          */
2759         for (i = 0; i < 2; i++) {
2760                 if ((!i ^ (reg_lo == c.i.rs)) && !op_flag_no_lo(flags)) {
2761                         lo = lightrec_alloc_reg_out(reg_cache, _jit, reg_lo, 0);
2762
2763                         if (c.r.op < 32)
2764                                 jit_lshi(lo, rs, c.r.op);
2765                         else
2766                                 jit_movi(lo, 0);
2767
2768                         lightrec_free_reg(reg_cache, lo);
2769                         continue;
2770                 }
2771
2772                 if ((!!i ^ (reg_lo == c.i.rs)) && !op_flag_no_hi(flags)) {
2773                         hi = lightrec_alloc_reg_out(reg_cache, _jit,
2774                                                     reg_hi, hiflags);
2775
2776                         if (c.r.op >= 32) {
2777                                 jit_lshi(hi, rs, c.r.op - 32);
2778                         } else if (is_signed) {
2779                                 if (c.r.op)
2780                                         jit_rshi(hi, rs, 32 - c.r.op);
2781                                 else
2782                                         jit_rshi(hi, rs, 31);
2783                         } else {
2784                                 if (c.r.op)
2785                                         jit_rshi_u(hi, rs, 32 - c.r.op);
2786                                 else
2787                                         jit_movi(hi, 0);
2788                         }
2789
2790                         lightrec_free_reg(reg_cache, hi);
2791                 }
2792         }
2793
2794         lightrec_free_reg(reg_cache, rs);
2795
2796         _jit_name(block->_jit, __func__);
2797         jit_note(__FILE__, __LINE__);
2798 }
2799
2800 static void rec_meta_COM(struct lightrec_cstate *state,
2801                          const struct block *block, u16 offset)
2802 {
2803         struct regcache *reg_cache = state->reg_cache;
2804         union code c = block->opcode_list[offset].c;
2805         jit_state_t *_jit = block->_jit;
2806         u8 rd, rs, flags;
2807
2808         jit_note(__FILE__, __LINE__);
2809
2810         rec_alloc_rs_rd(reg_cache, _jit, &block->opcode_list[offset],
2811                         c.m.rs, c.m.rd, 0, 0, &rs, &rd);
2812
2813         flags = lightrec_get_reg_in_flags(reg_cache, rs);
2814
2815         lightrec_set_reg_out_flags(reg_cache, rd,
2816                                    flags & REG_EXT);
2817
2818         jit_comr(rd, rs);
2819
2820         lightrec_free_reg(reg_cache, rs);
2821         lightrec_free_reg(reg_cache, rd);
2822 }
2823
2824 static void rec_meta_LWU(struct lightrec_cstate *state,
2825                          const struct block *block,
2826                          u16 offset)
2827 {
2828         jit_code_t code;
2829
2830         if (is_big_endian() && __WORDSIZE == 64)
2831                 code = jit_code_unldr_u;
2832         else
2833                 code = jit_code_unldr;
2834
2835         _jit_name(block->_jit, __func__);
2836         rec_load(state, block, offset, code, jit_code_bswapr_ui, false);
2837 }
2838
2839 static void rec_meta_SWU(struct lightrec_cstate *state,
2840                          const struct block *block,
2841                          u16 offset)
2842 {
2843         _jit_name(block->_jit, __func__);
2844         rec_store(state, block, offset, jit_code_unstr, jit_code_bswapr_ui);
2845 }
2846
2847 static void unknown_opcode(struct lightrec_cstate *state,
2848                            const struct block *block, u16 offset)
2849 {
2850         rec_exit_early(state, block, offset, LIGHTREC_EXIT_UNKNOWN_OP,
2851                        block->pc + (offset << 2));
2852 }
2853
2854 static const lightrec_rec_func_t rec_standard[64] = {
2855         SET_DEFAULT_ELM(rec_standard, unknown_opcode),
2856         [OP_SPECIAL]            = rec_SPECIAL,
2857         [OP_REGIMM]             = rec_REGIMM,
2858         [OP_J]                  = rec_J,
2859         [OP_JAL]                = rec_JAL,
2860         [OP_BEQ]                = rec_BEQ,
2861         [OP_BNE]                = rec_BNE,
2862         [OP_BLEZ]               = rec_BLEZ,
2863         [OP_BGTZ]               = rec_BGTZ,
2864         [OP_ADDI]               = rec_ADDI,
2865         [OP_ADDIU]              = rec_ADDIU,
2866         [OP_SLTI]               = rec_SLTI,
2867         [OP_SLTIU]              = rec_SLTIU,
2868         [OP_ANDI]               = rec_ANDI,
2869         [OP_ORI]                = rec_ORI,
2870         [OP_XORI]               = rec_XORI,
2871         [OP_LUI]                = rec_LUI,
2872         [OP_CP0]                = rec_CP0,
2873         [OP_CP2]                = rec_CP2,
2874         [OP_LB]                 = rec_LB,
2875         [OP_LH]                 = rec_LH,
2876         [OP_LWL]                = rec_LWL,
2877         [OP_LW]                 = rec_LW,
2878         [OP_LBU]                = rec_LBU,
2879         [OP_LHU]                = rec_LHU,
2880         [OP_LWR]                = rec_LWR,
2881         [OP_SB]                 = rec_SB,
2882         [OP_SH]                 = rec_SH,
2883         [OP_SWL]                = rec_SWL,
2884         [OP_SW]                 = rec_SW,
2885         [OP_SWR]                = rec_SWR,
2886         [OP_LWC2]               = rec_LW,
2887         [OP_SWC2]               = rec_SW,
2888
2889         [OP_META]               = rec_META,
2890         [OP_META_MULT2]         = rec_meta_MULT2,
2891         [OP_META_MULTU2]        = rec_meta_MULT2,
2892         [OP_META_LWU]           = rec_meta_LWU,
2893         [OP_META_SWU]           = rec_meta_SWU,
2894 };
2895
2896 static const lightrec_rec_func_t rec_special[64] = {
2897         SET_DEFAULT_ELM(rec_special, unknown_opcode),
2898         [OP_SPECIAL_SLL]        = rec_special_SLL,
2899         [OP_SPECIAL_SRL]        = rec_special_SRL,
2900         [OP_SPECIAL_SRA]        = rec_special_SRA,
2901         [OP_SPECIAL_SLLV]       = rec_special_SLLV,
2902         [OP_SPECIAL_SRLV]       = rec_special_SRLV,
2903         [OP_SPECIAL_SRAV]       = rec_special_SRAV,
2904         [OP_SPECIAL_JR]         = rec_special_JR,
2905         [OP_SPECIAL_JALR]       = rec_special_JALR,
2906         [OP_SPECIAL_SYSCALL]    = rec_special_SYSCALL,
2907         [OP_SPECIAL_BREAK]      = rec_special_BREAK,
2908         [OP_SPECIAL_MFHI]       = rec_special_MFHI,
2909         [OP_SPECIAL_MTHI]       = rec_special_MTHI,
2910         [OP_SPECIAL_MFLO]       = rec_special_MFLO,
2911         [OP_SPECIAL_MTLO]       = rec_special_MTLO,
2912         [OP_SPECIAL_MULT]       = rec_special_MULT,
2913         [OP_SPECIAL_MULTU]      = rec_special_MULTU,
2914         [OP_SPECIAL_DIV]        = rec_special_DIV,
2915         [OP_SPECIAL_DIVU]       = rec_special_DIVU,
2916         [OP_SPECIAL_ADD]        = rec_special_ADD,
2917         [OP_SPECIAL_ADDU]       = rec_special_ADDU,
2918         [OP_SPECIAL_SUB]        = rec_special_SUB,
2919         [OP_SPECIAL_SUBU]       = rec_special_SUBU,
2920         [OP_SPECIAL_AND]        = rec_special_AND,
2921         [OP_SPECIAL_OR]         = rec_special_OR,
2922         [OP_SPECIAL_XOR]        = rec_special_XOR,
2923         [OP_SPECIAL_NOR]        = rec_special_NOR,
2924         [OP_SPECIAL_SLT]        = rec_special_SLT,
2925         [OP_SPECIAL_SLTU]       = rec_special_SLTU,
2926 };
2927
2928 static const lightrec_rec_func_t rec_regimm[64] = {
2929         SET_DEFAULT_ELM(rec_regimm, unknown_opcode),
2930         [OP_REGIMM_BLTZ]        = rec_regimm_BLTZ,
2931         [OP_REGIMM_BGEZ]        = rec_regimm_BGEZ,
2932         [OP_REGIMM_BLTZAL]      = rec_regimm_BLTZAL,
2933         [OP_REGIMM_BGEZAL]      = rec_regimm_BGEZAL,
2934 };
2935
2936 static const lightrec_rec_func_t rec_cp0[64] = {
2937         SET_DEFAULT_ELM(rec_cp0, rec_CP),
2938         [OP_CP0_MFC0]           = rec_cp0_MFC0,
2939         [OP_CP0_CFC0]           = rec_cp0_CFC0,
2940         [OP_CP0_MTC0]           = rec_cp0_MTC0,
2941         [OP_CP0_CTC0]           = rec_cp0_CTC0,
2942         [OP_CP0_RFE]            = rec_cp0_RFE,
2943 };
2944
2945 static const lightrec_rec_func_t rec_cp2_basic[64] = {
2946         SET_DEFAULT_ELM(rec_cp2_basic, rec_CP),
2947         [OP_CP2_BASIC_MFC2]     = rec_cp2_basic_MFC2,
2948         [OP_CP2_BASIC_CFC2]     = rec_cp2_basic_CFC2,
2949         [OP_CP2_BASIC_MTC2]     = rec_cp2_basic_MTC2,
2950         [OP_CP2_BASIC_CTC2]     = rec_cp2_basic_CTC2,
2951 };
2952
2953 static const lightrec_rec_func_t rec_meta[64] = {
2954         SET_DEFAULT_ELM(rec_meta, unknown_opcode),
2955         [OP_META_MOV]           = rec_meta_MOV,
2956         [OP_META_EXTC]          = rec_meta_EXTC_EXTS,
2957         [OP_META_EXTS]          = rec_meta_EXTC_EXTS,
2958         [OP_META_COM]           = rec_meta_COM,
2959 };
2960
2961 static void rec_SPECIAL(struct lightrec_cstate *state,
2962                         const struct block *block, u16 offset)
2963 {
2964         union code c = block->opcode_list[offset].c;
2965         lightrec_rec_func_t f = rec_special[c.r.op];
2966
2967         if (!HAS_DEFAULT_ELM && unlikely(!f))
2968                 unknown_opcode(state, block, offset);
2969         else
2970                 (*f)(state, block, offset);
2971 }
2972
2973 static void rec_REGIMM(struct lightrec_cstate *state,
2974                        const struct block *block, u16 offset)
2975 {
2976         union code c = block->opcode_list[offset].c;
2977         lightrec_rec_func_t f = rec_regimm[c.r.rt];
2978
2979         if (!HAS_DEFAULT_ELM && unlikely(!f))
2980                 unknown_opcode(state, block, offset);
2981         else
2982                 (*f)(state, block, offset);
2983 }
2984
2985 static void rec_CP0(struct lightrec_cstate *state,
2986                     const struct block *block, u16 offset)
2987 {
2988         union code c = block->opcode_list[offset].c;
2989         lightrec_rec_func_t f = rec_cp0[c.r.rs];
2990
2991         if (!HAS_DEFAULT_ELM && unlikely(!f))
2992                 rec_CP(state, block, offset);
2993         else
2994                 (*f)(state, block, offset);
2995 }
2996
2997 static void rec_CP2(struct lightrec_cstate *state,
2998                     const struct block *block, u16 offset)
2999 {
3000         union code c = block->opcode_list[offset].c;
3001
3002         if (c.r.op == OP_CP2_BASIC) {
3003                 lightrec_rec_func_t f = rec_cp2_basic[c.r.rs];
3004
3005                 if (HAS_DEFAULT_ELM || likely(f)) {
3006                         (*f)(state, block, offset);
3007                         return;
3008                 }
3009         }
3010
3011         rec_CP(state, block, offset);
3012 }
3013
3014 static void rec_META(struct lightrec_cstate *state,
3015                      const struct block *block, u16 offset)
3016 {
3017         union code c = block->opcode_list[offset].c;
3018         lightrec_rec_func_t f = rec_meta[c.m.op];
3019
3020         if (!HAS_DEFAULT_ELM && unlikely(!f))
3021                 unknown_opcode(state, block, offset);
3022         else
3023                 (*f)(state, block, offset);
3024 }
3025
3026 void lightrec_rec_opcode(struct lightrec_cstate *state,
3027                          const struct block *block, u16 offset)
3028 {
3029         struct regcache *reg_cache = state->reg_cache;
3030         struct lightrec_branch_target *target;
3031         const struct opcode *op = &block->opcode_list[offset];
3032         jit_state_t *_jit = block->_jit;
3033         lightrec_rec_func_t f;
3034         u16 unload_offset;
3035
3036         if (op_flag_sync(op->flags)) {
3037                 if (state->cycles)
3038                         jit_subi(LIGHTREC_REG_CYCLE, LIGHTREC_REG_CYCLE, state->cycles);
3039                 state->cycles = 0;
3040
3041                 lightrec_storeback_regs(reg_cache, _jit);
3042                 lightrec_regcache_reset(reg_cache);
3043
3044                 pr_debug("Adding branch target at offset 0x%x\n", offset << 2);
3045                 target = &state->targets[state->nb_targets++];
3046                 target->offset = offset;
3047                 target->label = jit_indirect();
3048         }
3049
3050         if (likely(op->opcode)) {
3051                 f = rec_standard[op->i.op];
3052
3053                 if (!HAS_DEFAULT_ELM && unlikely(!f))
3054                         unknown_opcode(state, block, offset);
3055                 else
3056                         (*f)(state, block, offset);
3057         }
3058
3059         if (OPT_EARLY_UNLOAD) {
3060                 unload_offset = offset +
3061                         (has_delay_slot(op->c) && !op_flag_no_ds(op->flags));
3062
3063                 lightrec_do_early_unload(state, block, unload_offset);
3064         }
3065
3066         state->no_load_delay = false;
3067 }