deps/lightrec/emitter.c

   1 // SPDX-License-Identifier: LGPL-2.1-or-later
   2 /*
   3  * Copyright (C) 2014-2021 Paul Cercueil <paul@crapouillou.net>
   4  */
   5
   6 #include "blockcache.h"
   7 #include "debug.h"
   8 #include "disassembler.h"
   9 #include "emitter.h"
  10 #include "lightning-wrapper.h"
  11 #include "optimizer.h"
  12 #include "regcache.h"
  13
  14 #include <stdbool.h>
  15 #include <stddef.h>
  16
  17 typedef void (*lightrec_rec_func_t)(struct lightrec_cstate *, const struct block *, u16);
  18
  19 /* Forward declarations */
  20 static void rec_SPECIAL(struct lightrec_cstate *state, const struct block *block, u16 offset);
  21 static void rec_REGIMM(struct lightrec_cstate *state, const struct block *block, u16 offset);
  22 static void rec_CP0(struct lightrec_cstate *state, const struct block *block, u16 offset);
  23 static void rec_CP2(struct lightrec_cstate *state, const struct block *block, u16 offset);
  24
  25 static void unknown_opcode(struct lightrec_cstate *state, const struct block *block, u16 offset)
  26 {
  27         pr_warn("Unknown opcode: 0x%08x at PC 0x%08x\n",
  28                 block->opcode_list[offset].c.opcode,
  29                 block->pc + (offset << 2));
  30 }
  31
  32 static void
  33 lightrec_jump_to_eob(struct lightrec_cstate *state, jit_state_t *_jit)
  34 {
  35         /* Prevent jit_jmpi() from using our cycles register as a temporary */
  36         jit_live(LIGHTREC_REG_CYCLE);
  37
  38         jit_patch_abs(jit_jmpi(), state->state->eob_wrapper_func);
  39 }
  40
  41 static void lightrec_emit_end_of_block(struct lightrec_cstate *state,
  42                                        const struct block *block, u16 offset,
  43                                        s8 reg_new_pc, u32 imm, u8 ra_reg,
  44                                        u32 link, bool update_cycles)
  45 {
  46         struct regcache *reg_cache = state->reg_cache;
  47         jit_state_t *_jit = block->_jit;
  48         const struct opcode *op = &block->opcode_list[offset],
  49                             *next = &block->opcode_list[offset + 1];
  50         u32 cycles = state->cycles + lightrec_cycles_of_opcode(op->c);
  51
  52         jit_note(__FILE__, __LINE__);
  53
  54         if (link) {
  55                 /* Update the $ra register */
  56                 u8 link_reg = lightrec_alloc_reg_out(reg_cache, _jit, ra_reg, 0);
  57                 jit_movi(link_reg, link);
  58                 lightrec_free_reg(reg_cache, link_reg);
  59         }
  60
  61         if (reg_new_pc < 0) {
  62                 reg_new_pc = lightrec_alloc_reg(reg_cache, _jit, JIT_V0);
  63                 lightrec_lock_reg(reg_cache, _jit, reg_new_pc);
  64
  65                 jit_movi(reg_new_pc, imm);
  66         }
  67
  68         if (has_delay_slot(op->c) &&
  69             !op_flag_no_ds(op->flags) && !op_flag_local_branch(op->flags)) {
  70                 cycles += lightrec_cycles_of_opcode(next->c);
  71
  72                 /* Recompile the delay slot */
  73                 if (next->c.opcode)
  74                         lightrec_rec_opcode(state, block, offset + 1);
  75         }
  76
  77         /* Clean the remaining registers */
  78         lightrec_clean_regs(reg_cache, _jit);
  79
  80         jit_movr(JIT_V0, reg_new_pc);
  81
  82         if (cycles && update_cycles) {
  83                 jit_subi(LIGHTREC_REG_CYCLE, LIGHTREC_REG_CYCLE, cycles);
  84                 pr_debug("EOB: %u cycles\n", cycles);
  85         }
  86
  87         lightrec_jump_to_eob(state, _jit);
  88 }
  89
  90 void lightrec_emit_eob(struct lightrec_cstate *state, const struct block *block,
  91                        u16 offset, bool after_op)
  92 {
  93         struct regcache *reg_cache = state->reg_cache;
  94         jit_state_t *_jit = block->_jit;
  95         union code c = block->opcode_list[offset].c;
  96         u32 cycles = state->cycles;
  97
  98         if (after_op)
  99                 cycles += lightrec_cycles_of_opcode(c);
 100
 101         lightrec_clean_regs(reg_cache, _jit);
 102
 103         jit_movi(JIT_V0, block->pc + (offset << 2));
 104         jit_subi(LIGHTREC_REG_CYCLE, LIGHTREC_REG_CYCLE, cycles);
 105
 106         lightrec_jump_to_eob(state, _jit);
 107 }
 108
 109 static u8 get_jr_jalr_reg(struct lightrec_cstate *state, const struct block *block, u16 offset)
 110 {
 111         struct regcache *reg_cache = state->reg_cache;
 112         jit_state_t *_jit = block->_jit;
 113         const struct opcode *op = &block->opcode_list[offset];
 114         u8 rs;
 115
 116         rs = lightrec_request_reg_in(reg_cache, _jit, op->r.rs, JIT_V0);
 117         lightrec_lock_reg(reg_cache, _jit, rs);
 118
 119         return rs;
 120 }
 121
 122 static void rec_special_JR(struct lightrec_cstate *state, const struct block *block, u16 offset)
 123 {
 124         u8 rs = get_jr_jalr_reg(state, block, offset);
 125
 126         _jit_name(block->_jit, __func__);
 127         lightrec_emit_end_of_block(state, block, offset, rs, 0, 31, 0, true);
 128 }
 129
 130 static void rec_special_JALR(struct lightrec_cstate *state, const struct block *block, u16 offset)
 131 {
 132         u8 rs = get_jr_jalr_reg(state, block, offset);
 133         union code c = block->opcode_list[offset].c;
 134
 135         _jit_name(block->_jit, __func__);
 136         lightrec_emit_end_of_block(state, block, offset, rs, 0, c.r.rd,
 137                                    get_branch_pc(block, offset, 2), true);
 138 }
 139
 140 static void rec_J(struct lightrec_cstate *state, const struct block *block, u16 offset)
 141 {
 142         union code c = block->opcode_list[offset].c;
 143
 144         _jit_name(block->_jit, __func__);
 145         lightrec_emit_end_of_block(state, block, offset, -1,
 146                                    (block->pc & 0xf0000000) | (c.j.imm << 2),
 147                                    31, 0, true);
 148 }
 149
 150 static void rec_JAL(struct lightrec_cstate *state, const struct block *block, u16 offset)
 151 {
 152         union code c = block->opcode_list[offset].c;
 153
 154         _jit_name(block->_jit, __func__);
 155         lightrec_emit_end_of_block(state, block, offset, -1,
 156                                    (block->pc & 0xf0000000) | (c.j.imm << 2),
 157                                    31, get_branch_pc(block, offset, 2), true);
 158 }
 159
 160 static void lightrec_do_early_unload(struct lightrec_cstate *state,
 161                                      const struct block *block, u16 offset)
 162 {
 163         struct regcache *reg_cache = state->reg_cache;
 164         const struct opcode *op = &block->opcode_list[offset];
 165         jit_state_t *_jit = block->_jit;
 166         unsigned int i;
 167         u8 reg;
 168         struct {
 169                 u8 reg, op;
 170         } reg_ops[3] = {
 171                 { op->r.rd, LIGHTREC_FLAGS_GET_RD(op->flags), },
 172                 { op->i.rt, LIGHTREC_FLAGS_GET_RT(op->flags), },
 173                 { op->i.rs, LIGHTREC_FLAGS_GET_RS(op->flags), },
 174         };
 175
 176         for (i = 0; i < ARRAY_SIZE(reg_ops); i++) {
 177                 reg = reg_ops[i].reg;
 178
 179                 switch (reg_ops[i].op) {
 180                 case LIGHTREC_REG_UNLOAD:
 181                         lightrec_clean_reg_if_loaded(reg_cache, _jit, reg, true);
 182                         break;
 183
 184                 case LIGHTREC_REG_DISCARD:
 185                         lightrec_discard_reg_if_loaded(reg_cache, reg);
 186                         break;
 187
 188                 case LIGHTREC_REG_CLEAN:
 189                         lightrec_clean_reg_if_loaded(reg_cache, _jit, reg, false);
 190                         break;
 191                 default:
 192                         break;
 193                 };
 194         }
 195 }
 196
 197 static void rec_b(struct lightrec_cstate *state, const struct block *block, u16 offset,
 198                   jit_code_t code, jit_code_t code2, u32 link, bool unconditional, bool bz)
 199 {
 200         struct regcache *reg_cache = state->reg_cache;
 201         struct native_register *regs_backup;
 202         jit_state_t *_jit = block->_jit;
 203         struct lightrec_branch *branch;
 204         const struct opcode *op = &block->opcode_list[offset],
 205                             *next = &block->opcode_list[offset + 1];
 206         jit_node_t *addr;
 207         u8 link_reg, rs, rt;
 208         bool is_forward = (s16)op->i.imm >= -1;
 209         int op_cycles = lightrec_cycles_of_opcode(op->c);
 210         u32 target_offset, cycles = state->cycles + op_cycles;
 211         bool no_indirection = false;
 212         u32 next_pc;
 213
 214         jit_note(__FILE__, __LINE__);
 215
 216         if (!op_flag_no_ds(op->flags))
 217                 cycles += lightrec_cycles_of_opcode(next->c);
 218
 219         state->cycles = -op_cycles;
 220
 221         if (!unconditional) {
 222                 rs = lightrec_alloc_reg_in(reg_cache, _jit, op->i.rs, REG_EXT);
 223                 rt = bz ? 0 : lightrec_alloc_reg_in(reg_cache,
 224                                                     _jit, op->i.rt, REG_EXT);
 225
 226                 /* Unload dead registers before evaluating the branch */
 227                 if (OPT_EARLY_UNLOAD)
 228                         lightrec_do_early_unload(state, block, offset);
 229
 230                 if (op_flag_local_branch(op->flags) &&
 231                     (op_flag_no_ds(op->flags) || !next->opcode) &&
 232                     is_forward && !lightrec_has_dirty_regs(reg_cache))
 233                         no_indirection = true;
 234
 235                 if (no_indirection)
 236                         pr_debug("Using no indirection for branch at offset 0x%hx\n", offset << 2);
 237         }
 238
 239         if (cycles)
 240                 jit_subi(LIGHTREC_REG_CYCLE, LIGHTREC_REG_CYCLE, cycles);
 241
 242         if (!unconditional) {
 243                 /* Generate the branch opcode */
 244                 if (!no_indirection)
 245                         addr = jit_new_node_pww(code, NULL, rs, rt);
 246
 247                 lightrec_free_regs(reg_cache);
 248                 regs_backup = lightrec_regcache_enter_branch(reg_cache);
 249         }
 250
 251         if (op_flag_local_branch(op->flags)) {
 252                 /* Recompile the delay slot */
 253                 if (next && next->opcode && !op_flag_no_ds(op->flags))
 254                         lightrec_rec_opcode(state, block, offset + 1);
 255
 256                 if (link) {
 257                         /* Update the $ra register */
 258                         link_reg = lightrec_alloc_reg_out(reg_cache, _jit, 31, 0);
 259                         jit_movi(link_reg, link);
 260                         lightrec_free_reg(reg_cache, link_reg);
 261                 }
 262
 263                 /* Clean remaining registers */
 264                 lightrec_clean_regs(reg_cache, _jit);
 265
 266                 target_offset = offset + 1 + (s16)op->i.imm
 267                         - !!op_flag_no_ds(op->flags);
 268                 pr_debug("Adding local branch to offset 0x%x\n",
 269                          target_offset << 2);
 270                 branch = &state->local_branches[
 271                         state->nb_local_branches++];
 272
 273                 branch->target = target_offset;
 274
 275                 if (no_indirection)
 276                         branch->branch = jit_new_node_pww(code2, NULL, rs, rt);
 277                 else if (is_forward)
 278                         branch->branch = jit_b();
 279                 else
 280                         branch->branch = jit_bgti(LIGHTREC_REG_CYCLE, 0);
 281         }
 282
 283         if (!op_flag_local_branch(op->flags) || !is_forward) {
 284                 next_pc = get_branch_pc(block, offset, 1 + (s16)op->i.imm);
 285                 lightrec_emit_end_of_block(state, block, offset, -1, next_pc,
 286                                            31, link, false);
 287         }
 288
 289         if (!unconditional) {
 290                 if (!no_indirection)
 291                         jit_patch(addr);
 292
 293                 lightrec_regcache_leave_branch(reg_cache, regs_backup);
 294
 295                 if (bz && link) {
 296                         /* Update the $ra register */
 297                         link_reg = lightrec_alloc_reg_out(reg_cache, _jit,
 298                                                           31, REG_EXT);
 299                         jit_movi(link_reg, (s32)link);
 300                         lightrec_free_reg(reg_cache, link_reg);
 301                 }
 302
 303                 if (!op_flag_no_ds(op->flags) && next->opcode)
 304                         lightrec_rec_opcode(state, block, offset + 1);
 305         }
 306 }
 307
 308 static void rec_BNE(struct lightrec_cstate *state,
 309                     const struct block *block, u16 offset)
 310 {
 311         union code c = block->opcode_list[offset].c;
 312
 313         _jit_name(block->_jit, __func__);
 314
 315         if (c.i.rt == 0)
 316                 rec_b(state, block, offset, jit_code_beqi, jit_code_bnei, 0, false, true);
 317         else
 318                 rec_b(state, block, offset, jit_code_beqr, jit_code_bner, 0, false, false);
 319 }
 320
 321 static void rec_BEQ(struct lightrec_cstate *state,
 322                     const struct block *block, u16 offset)
 323 {
 324         union code c = block->opcode_list[offset].c;
 325
 326         _jit_name(block->_jit, __func__);
 327
 328         if (c.i.rt == 0)
 329                 rec_b(state, block, offset, jit_code_bnei, jit_code_beqi, 0, c.i.rs == 0, true);
 330         else
 331                 rec_b(state, block, offset, jit_code_bner, jit_code_beqr, 0, c.i.rs == c.i.rt, false);
 332 }
 333
 334 static void rec_BLEZ(struct lightrec_cstate *state,
 335                      const struct block *block, u16 offset)
 336 {
 337         union code c = block->opcode_list[offset].c;
 338
 339         _jit_name(block->_jit, __func__);
 340         rec_b(state, block, offset, jit_code_bgti, jit_code_blei, 0, c.i.rs == 0, true);
 341 }
 342
 343 static void rec_BGTZ(struct lightrec_cstate *state,
 344                      const struct block *block, u16 offset)
 345 {
 346         _jit_name(block->_jit, __func__);
 347         rec_b(state, block, offset, jit_code_blei, jit_code_bgti, 0, false, true);
 348 }
 349
 350 static void rec_regimm_BLTZ(struct lightrec_cstate *state,
 351                             const struct block *block, u16 offset)
 352 {
 353         _jit_name(block->_jit, __func__);
 354         rec_b(state, block, offset, jit_code_bgei, jit_code_blti, 0, false, true);
 355 }
 356
 357 static void rec_regimm_BLTZAL(struct lightrec_cstate *state,
 358                               const struct block *block, u16 offset)
 359 {
 360         _jit_name(block->_jit, __func__);
 361         rec_b(state, block, offset, jit_code_bgei, jit_code_blti,
 362               get_branch_pc(block, offset, 2), false, true);
 363 }
 364
 365 static void rec_regimm_BGEZ(struct lightrec_cstate *state,
 366                             const struct block *block, u16 offset)
 367 {
 368         union code c = block->opcode_list[offset].c;
 369
 370         _jit_name(block->_jit, __func__);
 371         rec_b(state, block, offset, jit_code_blti, jit_code_bgei, 0, !c.i.rs, true);
 372 }
 373
 374 static void rec_regimm_BGEZAL(struct lightrec_cstate *state,
 375                               const struct block *block, u16 offset)
 376 {
 377         const struct opcode *op = &block->opcode_list[offset];
 378         _jit_name(block->_jit, __func__);
 379         rec_b(state, block, offset, jit_code_blti, jit_code_bgei,
 380               get_branch_pc(block, offset, 2),
 381               !op->i.rs, true);
 382 }
 383
 384 static void rec_alu_imm(struct lightrec_cstate *state, const struct block *block,
 385                         u16 offset, jit_code_t code, bool slti)
 386 {
 387         struct regcache *reg_cache = state->reg_cache;
 388         union code c = block->opcode_list[offset].c;
 389         jit_state_t *_jit = block->_jit;
 390         u8 rs, rt, out_flags = REG_EXT;
 391
 392         if (slti)
 393                 out_flags |= REG_ZEXT;
 394
 395         jit_note(__FILE__, __LINE__);
 396         rs = lightrec_alloc_reg_in(reg_cache, _jit, c.i.rs, REG_EXT);
 397         rt = lightrec_alloc_reg_out(reg_cache, _jit, c.i.rt, out_flags);
 398
 399         jit_new_node_www(code, rt, rs, (s32)(s16) c.i.imm);
 400
 401         lightrec_free_reg(reg_cache, rs);
 402         lightrec_free_reg(reg_cache, rt);
 403 }
 404
 405 static void rec_alu_special(struct lightrec_cstate *state, const struct block *block,
 406                             u16 offset, jit_code_t code, bool out_ext)
 407 {
 408         struct regcache *reg_cache = state->reg_cache;
 409         union code c = block->opcode_list[offset].c;
 410         jit_state_t *_jit = block->_jit;
 411         u8 rd, rt, rs;
 412
 413         jit_note(__FILE__, __LINE__);
 414         rs = lightrec_alloc_reg_in(reg_cache, _jit, c.r.rs, REG_EXT);
 415         rt = lightrec_alloc_reg_in(reg_cache, _jit, c.r.rt, REG_EXT);
 416         rd = lightrec_alloc_reg_out(reg_cache, _jit, c.r.rd,
 417                                     out_ext ? REG_EXT | REG_ZEXT : 0);
 418
 419         jit_new_node_www(code, rd, rs, rt);
 420
 421         lightrec_free_reg(reg_cache, rs);
 422         lightrec_free_reg(reg_cache, rt);
 423         lightrec_free_reg(reg_cache, rd);
 424 }
 425
 426 static void rec_alu_shiftv(struct lightrec_cstate *state, const struct block *block,
 427                            u16 offset, jit_code_t code)
 428 {
 429         struct regcache *reg_cache = state->reg_cache;
 430         union code c = block->opcode_list[offset].c;
 431         jit_state_t *_jit = block->_jit;
 432         u8 rd, rt, rs, temp, flags = 0;
 433
 434         jit_note(__FILE__, __LINE__);
 435         rs = lightrec_alloc_reg_in(reg_cache, _jit, c.r.rs, 0);
 436
 437         if (code == jit_code_rshr)
 438                 flags = REG_EXT;
 439         else if (code == jit_code_rshr_u)
 440                 flags = REG_ZEXT;
 441
 442         rt = lightrec_alloc_reg_in(reg_cache, _jit, c.r.rt, flags);
 443         rd = lightrec_alloc_reg_out(reg_cache, _jit, c.r.rd, flags);
 444
 445         if (rs != rd && rt != rd) {
 446                 jit_andi(rd, rs, 0x1f);
 447                 jit_new_node_www(code, rd, rt, rd);
 448         } else {
 449                 temp = lightrec_alloc_reg_temp(reg_cache, _jit);
 450                 jit_andi(temp, rs, 0x1f);
 451                 jit_new_node_www(code, rd, rt, temp);
 452                 lightrec_free_reg(reg_cache, temp);
 453         }
 454
 455         lightrec_free_reg(reg_cache, rs);
 456         lightrec_free_reg(reg_cache, rt);
 457         lightrec_free_reg(reg_cache, rd);
 458 }
 459
 460 static void rec_movi(struct lightrec_cstate *state,
 461                      const struct block *block, u16 offset)
 462 {
 463         struct regcache *reg_cache = state->reg_cache;
 464         union code c = block->opcode_list[offset].c;
 465         jit_state_t *_jit = block->_jit;
 466         u16 flags = REG_EXT;
 467         u8 rt;
 468
 469         if (!(c.i.imm & 0x8000))
 470                 flags |= REG_ZEXT;
 471
 472         rt = lightrec_alloc_reg_out(reg_cache, _jit, c.i.rt, flags);
 473
 474         jit_movi(rt, (s32)(s16) c.i.imm);
 475
 476         lightrec_free_reg(reg_cache, rt);
 477 }
 478
 479 static void rec_ADDIU(struct lightrec_cstate *state,
 480                       const struct block *block, u16 offset)
 481 {
 482         _jit_name(block->_jit, __func__);
 483
 484         if (block->opcode_list[offset].c.i.rs)
 485                 rec_alu_imm(state, block, offset, jit_code_addi, false);
 486         else
 487                 rec_movi(state, block, offset);
 488 }
 489
 490 static void rec_ADDI(struct lightrec_cstate *state,
 491                      const struct block *block, u16 offset)
 492 {
 493         /* TODO: Handle the exception? */
 494         _jit_name(block->_jit, __func__);
 495         rec_ADDIU(state, block, offset);
 496 }
 497
 498 static void rec_SLTIU(struct lightrec_cstate *state,
 499                       const struct block *block, u16 offset)
 500 {
 501         _jit_name(block->_jit, __func__);
 502         rec_alu_imm(state, block, offset, jit_code_lti_u, true);
 503 }
 504
 505 static void rec_SLTI(struct lightrec_cstate *state,
 506                      const struct block *block, u16 offset)
 507 {
 508         _jit_name(block->_jit, __func__);
 509         rec_alu_imm(state, block, offset, jit_code_lti, true);
 510 }
 511
 512 static void rec_ANDI(struct lightrec_cstate *state,
 513                      const struct block *block, u16 offset)
 514 {
 515         struct regcache *reg_cache = state->reg_cache;
 516         union code c = block->opcode_list[offset].c;
 517         jit_state_t *_jit = block->_jit;
 518         u8 rs, rt;
 519
 520         _jit_name(block->_jit, __func__);
 521         jit_note(__FILE__, __LINE__);
 522         rs = lightrec_alloc_reg_in(reg_cache, _jit, c.i.rs, 0);
 523         rt = lightrec_alloc_reg_out(reg_cache, _jit, c.i.rt,
 524                                     REG_EXT | REG_ZEXT);
 525
 526         /* PSX code uses ANDI 0xff / ANDI 0xffff a lot, which are basically
 527          * casts to uint8_t / uint16_t. */
 528         if (c.i.imm == 0xff)
 529                 jit_extr_uc(rt, rs);
 530         else if (c.i.imm == 0xffff)
 531                 jit_extr_us(rt, rs);
 532         else
 533                 jit_andi(rt, rs, (u32)(u16) c.i.imm);
 534
 535         lightrec_free_reg(reg_cache, rs);
 536         lightrec_free_reg(reg_cache, rt);
 537 }
 538
 539 static void rec_alu_or_xor(struct lightrec_cstate *state, const struct block *block,
 540                            u16 offset, jit_code_t code)
 541 {
 542         struct regcache *reg_cache = state->reg_cache;
 543         union code c = block->opcode_list[offset].c;
 544         jit_state_t *_jit = block->_jit;
 545         u8 rs, rt, flags;
 546
 547         jit_note(__FILE__, __LINE__);
 548         rs = lightrec_alloc_reg_in(reg_cache, _jit, c.i.rs, 0);
 549         rt = lightrec_alloc_reg_out(reg_cache, _jit, c.i.rt, 0);
 550
 551         flags = lightrec_get_reg_in_flags(reg_cache, rs);
 552         lightrec_set_reg_out_flags(reg_cache, rt, flags);
 553
 554         jit_new_node_www(code, rt, rs, (u32)(u16) c.i.imm);
 555
 556         lightrec_free_reg(reg_cache, rs);
 557         lightrec_free_reg(reg_cache, rt);
 558 }
 559
 560
 561 static void rec_ORI(struct lightrec_cstate *state,
 562                     const struct block *block, u16 offset)
 563 {
 564         _jit_name(block->_jit, __func__);
 565         rec_alu_or_xor(state, block, offset, jit_code_ori);
 566 }
 567
 568 static void rec_XORI(struct lightrec_cstate *state,
 569                      const struct block *block, u16 offset)
 570 {
 571         _jit_name(block->_jit, __func__);
 572         rec_alu_or_xor(state, block, offset, jit_code_xori);
 573 }
 574
 575 static void rec_LUI(struct lightrec_cstate *state,
 576                     const struct block *block, u16 offset)
 577 {
 578         struct regcache *reg_cache = state->reg_cache;
 579         union code c = block->opcode_list[offset].c;
 580         jit_state_t *_jit = block->_jit;
 581         u8 rt, flags = REG_EXT;
 582
 583         jit_name(__func__);
 584         jit_note(__FILE__, __LINE__);
 585
 586         if (!(c.i.imm & BIT(15)))
 587                 flags |= REG_ZEXT;
 588
 589         rt = lightrec_alloc_reg_out(reg_cache, _jit, c.i.rt, flags);
 590
 591         jit_movi(rt, (s32)(c.i.imm << 16));
 592
 593         lightrec_free_reg(reg_cache, rt);
 594 }
 595
 596 static void rec_special_ADDU(struct lightrec_cstate *state,
 597                              const struct block *block, u16 offset)
 598 {
 599         _jit_name(block->_jit, __func__);
 600         rec_alu_special(state, block, offset, jit_code_addr, false);
 601 }
 602
 603 static void rec_special_ADD(struct lightrec_cstate *state,
 604                             const struct block *block, u16 offset)
 605 {
 606         /* TODO: Handle the exception? */
 607         _jit_name(block->_jit, __func__);
 608         rec_alu_special(state, block, offset, jit_code_addr, false);
 609 }
 610
 611 static void rec_special_SUBU(struct lightrec_cstate *state,
 612                              const struct block *block, u16 offset)
 613 {
 614         _jit_name(block->_jit, __func__);
 615         rec_alu_special(state, block, offset, jit_code_subr, false);
 616 }
 617
 618 static void rec_special_SUB(struct lightrec_cstate *state,
 619                             const struct block *block, u16 offset)
 620 {
 621         /* TODO: Handle the exception? */
 622         _jit_name(block->_jit, __func__);
 623         rec_alu_special(state, block, offset, jit_code_subr, false);
 624 }
 625
 626 static void rec_special_AND(struct lightrec_cstate *state,
 627                             const struct block *block, u16 offset)
 628 {
 629         struct regcache *reg_cache = state->reg_cache;
 630         union code c = block->opcode_list[offset].c;
 631         jit_state_t *_jit = block->_jit;
 632         u8 rd, rt, rs, flags_rs, flags_rt, flags_rd;
 633
 634         _jit_name(block->_jit, __func__);
 635         jit_note(__FILE__, __LINE__);
 636         rs = lightrec_alloc_reg_in(reg_cache, _jit, c.r.rs, 0);
 637         rt = lightrec_alloc_reg_in(reg_cache, _jit, c.r.rt, 0);
 638         rd = lightrec_alloc_reg_out(reg_cache, _jit, c.r.rd, 0);
 639
 640         flags_rs = lightrec_get_reg_in_flags(reg_cache, rs);
 641         flags_rt = lightrec_get_reg_in_flags(reg_cache, rt);
 642
 643         /* Z(rd) = Z(rs) | Z(rt) */
 644         flags_rd = REG_ZEXT & (flags_rs | flags_rt);
 645
 646         /* E(rd) = (E(rt) & Z(rt)) | (E(rs) & Z(rs)) | (E(rs) & E(rt)) */
 647         if (((flags_rs & REG_EXT) && (flags_rt & REG_ZEXT)) ||
 648             ((flags_rt & REG_EXT) && (flags_rs & REG_ZEXT)) ||
 649             (REG_EXT & flags_rs & flags_rt))
 650                 flags_rd |= REG_EXT;
 651
 652         lightrec_set_reg_out_flags(reg_cache, rd, flags_rd);
 653
 654         jit_andr(rd, rs, rt);
 655
 656         lightrec_free_reg(reg_cache, rs);
 657         lightrec_free_reg(reg_cache, rt);
 658         lightrec_free_reg(reg_cache, rd);
 659 }
 660
 661 static void rec_special_or_nor(struct lightrec_cstate *state,
 662                                const struct block *block, u16 offset, bool nor)
 663 {
 664         struct regcache *reg_cache = state->reg_cache;
 665         union code c = block->opcode_list[offset].c;
 666         jit_state_t *_jit = block->_jit;
 667         u8 rd, rt, rs, flags_rs, flags_rt, flags_rd = 0;
 668
 669         jit_note(__FILE__, __LINE__);
 670         rs = lightrec_alloc_reg_in(reg_cache, _jit, c.r.rs, 0);
 671         rt = lightrec_alloc_reg_in(reg_cache, _jit, c.r.rt, 0);
 672         rd = lightrec_alloc_reg_out(reg_cache, _jit, c.r.rd, 0);
 673
 674         flags_rs = lightrec_get_reg_in_flags(reg_cache, rs);
 675         flags_rt = lightrec_get_reg_in_flags(reg_cache, rt);
 676
 677         /* or: Z(rd) = Z(rs) & Z(rt)
 678          * nor: Z(rd) = 0 */
 679         if (!nor)
 680                 flags_rd = REG_ZEXT & flags_rs & flags_rt;
 681
 682         /* E(rd) = E(rs) & E(rt) */
 683         if (REG_EXT & flags_rs & flags_rt)
 684                 flags_rd |= REG_EXT;
 685
 686         lightrec_set_reg_out_flags(reg_cache, rd, flags_rd);
 687
 688         jit_orr(rd, rs, rt);
 689
 690         if (nor)
 691                 jit_comr(rd, rd);
 692
 693         lightrec_free_reg(reg_cache, rs);
 694         lightrec_free_reg(reg_cache, rt);
 695         lightrec_free_reg(reg_cache, rd);
 696 }
 697
 698 static void rec_special_OR(struct lightrec_cstate *state,
 699                            const struct block *block, u16 offset)
 700 {
 701         _jit_name(block->_jit, __func__);
 702         rec_special_or_nor(state, block, offset, false);
 703 }
 704
 705 static void rec_special_NOR(struct lightrec_cstate *state,
 706                             const struct block *block, u16 offset)
 707 {
 708         _jit_name(block->_jit, __func__);
 709         rec_special_or_nor(state, block, offset, true);
 710 }
 711
 712 static void rec_special_XOR(struct lightrec_cstate *state,
 713                             const struct block *block, u16 offset)
 714 {
 715         struct regcache *reg_cache = state->reg_cache;
 716         union code c = block->opcode_list[offset].c;
 717         jit_state_t *_jit = block->_jit;
 718         u8 rd, rt, rs, flags_rs, flags_rt, flags_rd;
 719
 720         _jit_name(block->_jit, __func__);
 721
 722         jit_note(__FILE__, __LINE__);
 723         rs = lightrec_alloc_reg_in(reg_cache, _jit, c.r.rs, 0);
 724         rt = lightrec_alloc_reg_in(reg_cache, _jit, c.r.rt, 0);
 725         rd = lightrec_alloc_reg_out(reg_cache, _jit, c.r.rd, 0);
 726
 727         flags_rs = lightrec_get_reg_in_flags(reg_cache, rs);
 728         flags_rt = lightrec_get_reg_in_flags(reg_cache, rt);
 729
 730         /* Z(rd) = Z(rs) & Z(rt) */
 731         flags_rd = REG_ZEXT & flags_rs & flags_rt;
 732
 733         /* E(rd) = E(rs) & E(rt) */
 734         flags_rd |= REG_EXT & flags_rs & flags_rt;
 735
 736         lightrec_set_reg_out_flags(reg_cache, rd, flags_rd);
 737
 738         jit_xorr(rd, rs, rt);
 739
 740         lightrec_free_reg(reg_cache, rs);
 741         lightrec_free_reg(reg_cache, rt);
 742         lightrec_free_reg(reg_cache, rd);
 743 }
 744
 745 static void rec_special_SLTU(struct lightrec_cstate *state,
 746                              const struct block *block, u16 offset)
 747 {
 748         _jit_name(block->_jit, __func__);
 749         rec_alu_special(state, block, offset, jit_code_ltr_u, true);
 750 }
 751
 752 static void rec_special_SLT(struct lightrec_cstate *state,
 753                             const struct block *block, u16 offset)
 754 {
 755         _jit_name(block->_jit, __func__);
 756         rec_alu_special(state, block, offset, jit_code_ltr, true);
 757 }
 758
 759 static void rec_special_SLLV(struct lightrec_cstate *state,
 760                              const struct block *block, u16 offset)
 761 {
 762         _jit_name(block->_jit, __func__);
 763         rec_alu_shiftv(state, block, offset, jit_code_lshr);
 764 }
 765
 766 static void rec_special_SRLV(struct lightrec_cstate *state,
 767                              const struct block *block, u16 offset)
 768 {
 769         _jit_name(block->_jit, __func__);
 770         rec_alu_shiftv(state, block, offset, jit_code_rshr_u);
 771 }
 772
 773 static void rec_special_SRAV(struct lightrec_cstate *state,
 774                              const struct block *block, u16 offset)
 775 {
 776         _jit_name(block->_jit, __func__);
 777         rec_alu_shiftv(state, block, offset, jit_code_rshr);
 778 }
 779
 780 static void rec_alu_shift(struct lightrec_cstate *state, const struct block *block,
 781                           u16 offset, jit_code_t code)
 782 {
 783         struct regcache *reg_cache = state->reg_cache;
 784         union code c = block->opcode_list[offset].c;
 785         jit_state_t *_jit = block->_jit;
 786         u8 rd, rt, flags = 0;
 787
 788         jit_note(__FILE__, __LINE__);
 789
 790         if (code == jit_code_rshi)
 791                 flags = REG_EXT;
 792         else if (code == jit_code_rshi_u)
 793                 flags = REG_ZEXT;
 794
 795         rt = lightrec_alloc_reg_in(reg_cache, _jit, c.r.rt, flags);
 796
 797         /* Input reg is zero-extended, if we SRL at least by one bit, we know
 798          * the output reg will be both zero-extended and sign-extended. */
 799         if (code == jit_code_rshi_u && c.r.imm)
 800                 flags |= REG_EXT;
 801         rd = lightrec_alloc_reg_out(reg_cache, _jit, c.r.rd, flags);
 802
 803         jit_new_node_www(code, rd, rt, c.r.imm);
 804
 805         lightrec_free_reg(reg_cache, rt);
 806         lightrec_free_reg(reg_cache, rd);
 807 }
 808
 809 static void rec_special_SLL(struct lightrec_cstate *state,
 810                             const struct block *block, u16 offset)
 811 {
 812         _jit_name(block->_jit, __func__);
 813         rec_alu_shift(state, block, offset, jit_code_lshi);
 814 }
 815
 816 static void rec_special_SRL(struct lightrec_cstate *state,
 817                             const struct block *block, u16 offset)
 818 {
 819         _jit_name(block->_jit, __func__);
 820         rec_alu_shift(state, block, offset, jit_code_rshi_u);
 821 }
 822
 823 static void rec_special_SRA(struct lightrec_cstate *state,
 824                             const struct block *block, u16 offset)
 825 {
 826         _jit_name(block->_jit, __func__);
 827         rec_alu_shift(state, block, offset, jit_code_rshi);
 828 }
 829
 830 static void rec_alu_mult(struct lightrec_cstate *state,
 831                          const struct block *block, u16 offset, bool is_signed)
 832 {
 833         struct regcache *reg_cache = state->reg_cache;
 834         union code c = block->opcode_list[offset].c;
 835         u32 flags = block->opcode_list[offset].flags;
 836         u8 reg_lo = get_mult_div_lo(c);
 837         u8 reg_hi = get_mult_div_hi(c);
 838         jit_state_t *_jit = block->_jit;
 839         u8 lo, hi, rs, rt, rflags = 0;
 840
 841         jit_note(__FILE__, __LINE__);
 842
 843         if (is_signed)
 844                 rflags = REG_EXT;
 845         else
 846                 rflags = REG_ZEXT;
 847
 848         rs = lightrec_alloc_reg_in(reg_cache, _jit, c.r.rs, rflags);
 849         rt = lightrec_alloc_reg_in(reg_cache, _jit, c.r.rt, rflags);
 850
 851         if (!op_flag_no_lo(flags))
 852                 lo = lightrec_alloc_reg_out(reg_cache, _jit, reg_lo, 0);
 853         else if (__WORDSIZE == 32)
 854                 lo = lightrec_alloc_reg_temp(reg_cache, _jit);
 855
 856         if (!op_flag_no_hi(flags))
 857                 hi = lightrec_alloc_reg_out(reg_cache, _jit, reg_hi, REG_EXT);
 858
 859         if (__WORDSIZE == 32) {
 860                 /* On 32-bit systems, do a 32*32->64 bit operation, or a 32*32->32 bit
 861                  * operation if the MULT was detected a 32-bit only. */
 862                 if (!op_flag_no_hi(flags)) {
 863                         if (is_signed)
 864                                 jit_qmulr(lo, hi, rs, rt);
 865                         else
 866                                 jit_qmulr_u(lo, hi, rs, rt);
 867                 } else {
 868                         jit_mulr(lo, rs, rt);
 869                 }
 870         } else {
 871                 /* On 64-bit systems, do a 64*64->64 bit operation. */
 872                 if (op_flag_no_lo(flags)) {
 873                         jit_mulr(hi, rs, rt);
 874                         jit_rshi(hi, hi, 32);
 875                 } else {
 876                         jit_mulr(lo, rs, rt);
 877
 878                         /* The 64-bit output value is in $lo, store the upper 32 bits in $hi */
 879                         if (!op_flag_no_hi(flags))
 880                                 jit_rshi(hi, lo, 32);
 881                 }
 882         }
 883
 884         lightrec_free_reg(reg_cache, rs);
 885         lightrec_free_reg(reg_cache, rt);
 886         if (!op_flag_no_lo(flags) || __WORDSIZE == 32)
 887                 lightrec_free_reg(reg_cache, lo);
 888         if (!op_flag_no_hi(flags))
 889                 lightrec_free_reg(reg_cache, hi);
 890 }
 891
 892 static void rec_alu_div(struct lightrec_cstate *state,
 893                         const struct block *block, u16 offset, bool is_signed)
 894 {
 895         struct regcache *reg_cache = state->reg_cache;
 896         union code c = block->opcode_list[offset].c;
 897         u32 flags = block->opcode_list[offset].flags;
 898         bool no_check = op_flag_no_div_check(flags);
 899         u8 reg_lo = get_mult_div_lo(c);
 900         u8 reg_hi = get_mult_div_hi(c);
 901         jit_state_t *_jit = block->_jit;
 902         jit_node_t *branch, *to_end;
 903         u8 lo = 0, hi = 0, rs, rt, rflags = 0;
 904
 905         jit_note(__FILE__, __LINE__);
 906
 907         if (is_signed)
 908                 rflags = REG_EXT;
 909         else
 910                 rflags = REG_ZEXT;
 911
 912         rs = lightrec_alloc_reg_in(reg_cache, _jit, c.r.rs, rflags);
 913         rt = lightrec_alloc_reg_in(reg_cache, _jit, c.r.rt, rflags);
 914
 915         if (!op_flag_no_lo(flags))
 916                 lo = lightrec_alloc_reg_out(reg_cache, _jit, reg_lo, 0);
 917
 918         if (!op_flag_no_hi(flags))
 919                 hi = lightrec_alloc_reg_out(reg_cache, _jit, reg_hi, 0);
 920
 921         /* Jump to special handler if dividing by zero  */
 922         if (!no_check)
 923                 branch = jit_beqi(rt, 0);
 924
 925         if (op_flag_no_lo(flags)) {
 926                 if (is_signed)
 927                         jit_remr(hi, rs, rt);
 928                 else
 929                         jit_remr_u(hi, rs, rt);
 930         } else if (op_flag_no_hi(flags)) {
 931                 if (is_signed)
 932                         jit_divr(lo, rs, rt);
 933                 else
 934                         jit_divr_u(lo, rs, rt);
 935         } else {
 936                 if (is_signed)
 937                         jit_qdivr(lo, hi, rs, rt);
 938                 else
 939                         jit_qdivr_u(lo, hi, rs, rt);
 940         }
 941
 942         if (!no_check) {
 943                 /* Jump above the div-by-zero handler */
 944                 to_end = jit_b();
 945
 946                 jit_patch(branch);
 947
 948                 if (!op_flag_no_lo(flags)) {
 949                         if (is_signed) {
 950                                 jit_lti(lo, rs, 0);
 951                                 jit_lshi(lo, lo, 1);
 952                                 jit_subi(lo, lo, 1);
 953                         } else {
 954                                 jit_movi(lo, 0xffffffff);
 955                         }
 956                 }
 957
 958                 if (!op_flag_no_hi(flags))
 959                         jit_movr(hi, rs);
 960
 961                 jit_patch(to_end);
 962         }
 963
 964         lightrec_free_reg(reg_cache, rs);
 965         lightrec_free_reg(reg_cache, rt);
 966
 967         if (!op_flag_no_lo(flags))
 968                 lightrec_free_reg(reg_cache, lo);
 969
 970         if (!op_flag_no_hi(flags))
 971                 lightrec_free_reg(reg_cache, hi);
 972 }
 973
 974 static void rec_special_MULT(struct lightrec_cstate *state,
 975                              const struct block *block, u16 offset)
 976 {
 977         _jit_name(block->_jit, __func__);
 978         rec_alu_mult(state, block, offset, true);
 979 }
 980
 981 static void rec_special_MULTU(struct lightrec_cstate *state,
 982                               const struct block *block, u16 offset)
 983 {
 984         _jit_name(block->_jit, __func__);
 985         rec_alu_mult(state, block, offset, false);
 986 }
 987
 988 static void rec_special_DIV(struct lightrec_cstate *state,
 989                             const struct block *block, u16 offset)
 990 {
 991         _jit_name(block->_jit, __func__);
 992         rec_alu_div(state, block, offset, true);
 993 }
 994
 995 static void rec_special_DIVU(struct lightrec_cstate *state,
 996                              const struct block *block, u16 offset)
 997 {
 998         _jit_name(block->_jit, __func__);
 999         rec_alu_div(state, block, offset, false);
1000 }
1001
1002 static void rec_alu_mv_lo_hi(struct lightrec_cstate *state,
1003                              const struct block *block, u8 dst, u8 src)
1004 {
1005         struct regcache *reg_cache = state->reg_cache;
1006         jit_state_t *_jit = block->_jit;
1007
1008         jit_note(__FILE__, __LINE__);
1009         src = lightrec_alloc_reg_in(reg_cache, _jit, src, 0);
1010         dst = lightrec_alloc_reg_out(reg_cache, _jit, dst, REG_EXT);
1011
1012         jit_extr_i(dst, src);
1013
1014         lightrec_free_reg(reg_cache, src);
1015         lightrec_free_reg(reg_cache, dst);
1016 }
1017
1018 static void rec_special_MFHI(struct lightrec_cstate *state,
1019                              const struct block *block, u16 offset)
1020 {
1021         union code c = block->opcode_list[offset].c;
1022
1023         _jit_name(block->_jit, __func__);
1024         rec_alu_mv_lo_hi(state, block, c.r.rd, REG_HI);
1025 }
1026
1027 static void rec_special_MTHI(struct lightrec_cstate *state,
1028                              const struct block *block, u16 offset)
1029 {
1030         union code c = block->opcode_list[offset].c;
1031
1032         _jit_name(block->_jit, __func__);
1033         rec_alu_mv_lo_hi(state, block, REG_HI, c.r.rs);
1034 }
1035
1036 static void rec_special_MFLO(struct lightrec_cstate *state,
1037                              const struct block *block, u16 offset)
1038 {
1039         union code c = block->opcode_list[offset].c;
1040
1041         _jit_name(block->_jit, __func__);
1042         rec_alu_mv_lo_hi(state, block, c.r.rd, REG_LO);
1043 }
1044
1045 static void rec_special_MTLO(struct lightrec_cstate *state,
1046                              const struct block *block, u16 offset)
1047 {
1048         union code c = block->opcode_list[offset].c;
1049
1050         _jit_name(block->_jit, __func__);
1051         rec_alu_mv_lo_hi(state, block, REG_LO, c.r.rs);
1052 }
1053
1054 static void call_to_c_wrapper(struct lightrec_cstate *state,
1055                               const struct block *block, u32 arg,
1056                               enum c_wrappers wrapper)
1057 {
1058         struct regcache *reg_cache = state->reg_cache;
1059         jit_state_t *_jit = block->_jit;
1060         s8 tmp, tmp2;
1061
1062         /* Make sure JIT_R1 is not mapped; it will be used in the C wrapper. */
1063         tmp2 = lightrec_alloc_reg(reg_cache, _jit, JIT_R1);
1064
1065         tmp = lightrec_get_reg_with_value(reg_cache,
1066                                           (intptr_t) state->state->wrappers_eps[wrapper]);
1067         if (tmp < 0) {
1068                 tmp = lightrec_alloc_reg_temp(reg_cache, _jit);
1069                 jit_ldxi(tmp, LIGHTREC_REG_STATE,
1070                          offsetof(struct lightrec_state, wrappers_eps[wrapper]));
1071
1072                 lightrec_temp_set_value(reg_cache, tmp,
1073                                         (intptr_t) state->state->wrappers_eps[wrapper]);
1074         }
1075
1076         lightrec_free_reg(reg_cache, tmp2);
1077
1078 #ifdef __mips__
1079         /* On MIPS, register t9 is always used as the target register for JALR.
1080          * Therefore if it does not contain the target address we must
1081          * invalidate it. */
1082         if (tmp != _T9)
1083                 lightrec_unload_reg(reg_cache, _jit, _T9);
1084 #endif
1085
1086         jit_prepare();
1087         jit_pushargi(arg);
1088
1089         lightrec_regcache_mark_live(reg_cache, _jit);
1090         jit_callr(tmp);
1091
1092         lightrec_free_reg(reg_cache, tmp);
1093         lightrec_regcache_mark_live(reg_cache, _jit);
1094 }
1095
1096 static void rec_io(struct lightrec_cstate *state,
1097                    const struct block *block, u16 offset,
1098                    bool load_rt, bool read_rt)
1099 {
1100         struct regcache *reg_cache = state->reg_cache;
1101         jit_state_t *_jit = block->_jit;
1102         union code c = block->opcode_list[offset].c;
1103         u32 flags = block->opcode_list[offset].flags;
1104         bool is_tagged = LIGHTREC_FLAGS_GET_IO_MODE(flags);
1105         u32 lut_entry;
1106
1107         jit_note(__FILE__, __LINE__);
1108
1109         lightrec_clean_reg_if_loaded(reg_cache, _jit, c.i.rs, false);
1110
1111         if (read_rt && likely(c.i.rt))
1112                 lightrec_clean_reg_if_loaded(reg_cache, _jit, c.i.rt, true);
1113         else if (load_rt)
1114                 lightrec_clean_reg_if_loaded(reg_cache, _jit, c.i.rt, false);
1115
1116         if (is_tagged) {
1117                 call_to_c_wrapper(state, block, c.opcode, C_WRAPPER_RW);
1118         } else {
1119                 lut_entry = lightrec_get_lut_entry(block);
1120                 call_to_c_wrapper(state, block, (lut_entry << 16) | offset,
1121                                   C_WRAPPER_RW_GENERIC);
1122         }
1123 }
1124
1125 static u32 rec_ram_mask(struct lightrec_state *state)
1126 {
1127         return (RAM_SIZE << (state->mirrors_mapped * 2)) - 1;
1128 }
1129
1130 static void rec_store_memory(struct lightrec_cstate *cstate,
1131                              const struct block *block,
1132                              u16 offset, jit_code_t code,
1133                              jit_code_t swap_code,
1134                              uintptr_t addr_offset, u32 addr_mask,
1135                              bool invalidate)
1136 {
1137         const struct lightrec_state *state = cstate->state;
1138         struct regcache *reg_cache = cstate->reg_cache;
1139         struct opcode *op = &block->opcode_list[offset];
1140         jit_state_t *_jit = block->_jit;
1141         union code c = op->c;
1142         u8 rs, rt, tmp, tmp2, tmp3, addr_reg, addr_reg2;
1143         s16 imm = (s16)c.i.imm;
1144         s32 simm = (s32)imm << (1 - lut_is_32bit(state));
1145         s32 lut_offt = offsetof(struct lightrec_state, code_lut);
1146         bool no_mask = op_flag_no_mask(op->flags);
1147         bool add_imm = c.i.imm &&
1148                 ((!state->mirrors_mapped && !no_mask) || (invalidate &&
1149                 ((imm & 0x3) || simm + lut_offt != (s16)(simm + lut_offt))));
1150         bool need_tmp = !no_mask || addr_offset || add_imm || invalidate;
1151
1152         rt = lightrec_alloc_reg_in(reg_cache, _jit, c.i.rt, 0);
1153         rs = lightrec_alloc_reg_in(reg_cache, _jit, c.i.rs, 0);
1154         if (need_tmp)
1155                 tmp = lightrec_alloc_reg_temp(reg_cache, _jit);
1156
1157         addr_reg = rs;
1158
1159         if (add_imm) {
1160                 jit_addi(tmp, addr_reg, (s16)c.i.imm);
1161                 addr_reg = tmp;
1162                 imm = 0;
1163         } else if (simm) {
1164                 lut_offt += simm;
1165         }
1166
1167         if (!no_mask) {
1168                 jit_andi(tmp, addr_reg, addr_mask);
1169                 addr_reg = tmp;
1170         }
1171
1172         if (addr_offset) {
1173                 tmp2 = lightrec_alloc_reg_temp(reg_cache, _jit);
1174                 jit_addi(tmp2, addr_reg, addr_offset);
1175                 addr_reg2 = tmp2;
1176         } else {
1177                 addr_reg2 = addr_reg;
1178         }
1179
1180         if (is_big_endian() && swap_code && c.i.rt) {
1181                 tmp3 = lightrec_alloc_reg_temp(reg_cache, _jit);
1182
1183                 jit_new_node_ww(swap_code, tmp3, rt);
1184                 jit_new_node_www(code, imm, addr_reg2, tmp3);
1185
1186                 lightrec_free_reg(reg_cache, tmp3);
1187         } else {
1188                 jit_new_node_www(code, imm, addr_reg2, rt);
1189         }
1190
1191         lightrec_free_reg(reg_cache, rt);
1192
1193         if (invalidate) {
1194                 tmp3 = lightrec_alloc_reg_in(reg_cache, _jit, 0, 0);
1195
1196                 if (c.i.op != OP_SW) {
1197                         jit_andi(tmp, addr_reg, ~3);
1198                         addr_reg = tmp;
1199                 }
1200
1201                 if (!lut_is_32bit(state)) {
1202                         jit_lshi(tmp, addr_reg, 1);
1203                         addr_reg = tmp;
1204                 }
1205
1206                 if (addr_reg == rs && c.i.rs == 0) {
1207                         addr_reg = LIGHTREC_REG_STATE;
1208                 } else {
1209                         jit_addr(tmp, addr_reg, LIGHTREC_REG_STATE);
1210                         addr_reg = tmp;
1211                 }
1212
1213                 if (lut_is_32bit(state))
1214                         jit_stxi_i(lut_offt, addr_reg, tmp3);
1215                 else
1216                         jit_stxi(lut_offt, addr_reg, tmp3);
1217
1218                 lightrec_free_reg(reg_cache, tmp3);
1219         }
1220
1221         if (addr_offset)
1222                 lightrec_free_reg(reg_cache, tmp2);
1223         if (need_tmp)
1224                 lightrec_free_reg(reg_cache, tmp);
1225         lightrec_free_reg(reg_cache, rs);
1226 }
1227
1228 static void rec_store_ram(struct lightrec_cstate *cstate,
1229                           const struct block *block,
1230                           u16 offset, jit_code_t code,
1231                           jit_code_t swap_code, bool invalidate)
1232 {
1233         struct lightrec_state *state = cstate->state;
1234
1235         _jit_note(block->_jit, __FILE__, __LINE__);
1236
1237         return rec_store_memory(cstate, block, offset, code, swap_code,
1238                                 state->offset_ram, rec_ram_mask(state),
1239                                 invalidate);
1240 }
1241
1242 static void rec_store_scratch(struct lightrec_cstate *cstate,
1243                               const struct block *block, u16 offset,
1244                               jit_code_t code, jit_code_t swap_code)
1245 {
1246         _jit_note(block->_jit, __FILE__, __LINE__);
1247
1248         return rec_store_memory(cstate, block, offset, code, swap_code,
1249                                 cstate->state->offset_scratch,
1250                                 0x1fffffff, false);
1251 }
1252
1253 static void rec_store_io(struct lightrec_cstate *cstate,
1254                          const struct block *block, u16 offset,
1255                          jit_code_t code, jit_code_t swap_code)
1256 {
1257         _jit_note(block->_jit, __FILE__, __LINE__);
1258
1259         return rec_store_memory(cstate, block, offset, code, swap_code,
1260                                 cstate->state->offset_io,
1261                                 0x1fffffff, false);
1262 }
1263
1264 static void rec_store_direct_no_invalidate(struct lightrec_cstate *cstate,
1265                                            const struct block *block,
1266                                            u16 offset, jit_code_t code,
1267                                            jit_code_t swap_code)
1268 {
1269         struct lightrec_state *state = cstate->state;
1270         struct regcache *reg_cache = cstate->reg_cache;
1271         union code c = block->opcode_list[offset].c;
1272         jit_state_t *_jit = block->_jit;
1273         jit_node_t *to_not_ram, *to_end;
1274         u8 tmp, tmp2, rs, rt;
1275         s16 imm;
1276
1277         jit_note(__FILE__, __LINE__);
1278         rs = lightrec_alloc_reg_in(reg_cache, _jit, c.i.rs, 0);
1279         tmp = lightrec_alloc_reg_temp(reg_cache, _jit);
1280
1281         if (state->offset_ram || state->offset_scratch)
1282                 tmp2 = lightrec_alloc_reg_temp(reg_cache, _jit);
1283
1284         /* Convert to KUNSEG and avoid RAM mirrors */
1285         if (state->mirrors_mapped) {
1286                 imm = (s16)c.i.imm;
1287                 jit_andi(tmp, rs, 0x1f800000 | (4 * RAM_SIZE - 1));
1288         } else if (c.i.imm) {
1289                 imm = 0;
1290                 jit_addi(tmp, rs, (s16)c.i.imm);
1291                 jit_andi(tmp, tmp, 0x1f800000 | (RAM_SIZE - 1));
1292         } else {
1293                 imm = 0;
1294                 jit_andi(tmp, rs, 0x1f800000 | (RAM_SIZE - 1));
1295         }
1296
1297         lightrec_free_reg(reg_cache, rs);
1298
1299         if (state->offset_ram != state->offset_scratch) {
1300                 to_not_ram = jit_bmsi(tmp, BIT(28));
1301
1302                 jit_movi(tmp2, state->offset_ram);
1303
1304                 to_end = jit_b();
1305                 jit_patch(to_not_ram);
1306
1307                 jit_movi(tmp2, state->offset_scratch);
1308                 jit_patch(to_end);
1309         } else if (state->offset_ram) {
1310                 jit_movi(tmp2, state->offset_ram);
1311         }
1312
1313         if (state->offset_ram || state->offset_scratch) {
1314                 jit_addr(tmp, tmp, tmp2);
1315                 lightrec_free_reg(reg_cache, tmp2);
1316         }
1317
1318         rt = lightrec_alloc_reg_in(reg_cache, _jit, c.i.rt, 0);
1319
1320         if (is_big_endian() && swap_code && c.i.rt) {
1321                 tmp2 = lightrec_alloc_reg_temp(reg_cache, _jit);
1322
1323                 jit_new_node_ww(swap_code, tmp2, rt);
1324                 jit_new_node_www(code, imm, tmp, tmp2);
1325
1326                 lightrec_free_reg(reg_cache, tmp2);
1327         } else {
1328                 jit_new_node_www(code, imm, tmp, rt);
1329         }
1330
1331         lightrec_free_reg(reg_cache, rt);
1332         lightrec_free_reg(reg_cache, tmp);
1333 }
1334
1335 static void rec_store_direct(struct lightrec_cstate *cstate, const struct block *block,
1336                              u16 offset, jit_code_t code, jit_code_t swap_code)
1337 {
1338         struct lightrec_state *state = cstate->state;
1339         u32 ram_size = state->mirrors_mapped ? RAM_SIZE * 4 : RAM_SIZE;
1340         struct regcache *reg_cache = cstate->reg_cache;
1341         union code c = block->opcode_list[offset].c;
1342         jit_state_t *_jit = block->_jit;
1343         jit_node_t *to_not_ram, *to_end;
1344         u8 tmp, tmp2, tmp3, rs, rt;
1345
1346         jit_note(__FILE__, __LINE__);
1347
1348         rs = lightrec_alloc_reg_in(reg_cache, _jit, c.i.rs, 0);
1349         tmp2 = lightrec_alloc_reg_temp(reg_cache, _jit);
1350         tmp3 = lightrec_alloc_reg_in(reg_cache, _jit, 0, 0);
1351
1352         /* Convert to KUNSEG and avoid RAM mirrors */
1353         if (c.i.imm) {
1354                 jit_addi(tmp2, rs, (s16)c.i.imm);
1355                 jit_andi(tmp2, tmp2, 0x1f800000 | (ram_size - 1));
1356         } else {
1357                 jit_andi(tmp2, rs, 0x1f800000 | (ram_size - 1));
1358         }
1359
1360         lightrec_free_reg(reg_cache, rs);
1361         tmp = lightrec_alloc_reg_temp(reg_cache, _jit);
1362
1363         to_not_ram = jit_bgti(tmp2, ram_size);
1364
1365         /* Compute the offset to the code LUT */
1366         jit_andi(tmp, tmp2, (RAM_SIZE - 1) & ~3);
1367         if (!lut_is_32bit(state))
1368                 jit_lshi(tmp, tmp, 1);
1369         jit_addr(tmp, LIGHTREC_REG_STATE, tmp);
1370
1371         /* Write NULL to the code LUT to invalidate any block that's there */
1372         if (lut_is_32bit(state))
1373                 jit_stxi_i(offsetof(struct lightrec_state, code_lut), tmp, tmp3);
1374         else
1375                 jit_stxi(offsetof(struct lightrec_state, code_lut), tmp, tmp3);
1376
1377         if (state->offset_ram != state->offset_scratch) {
1378                 jit_movi(tmp, state->offset_ram);
1379
1380                 to_end = jit_b();
1381         }
1382
1383         jit_patch(to_not_ram);
1384
1385         if (state->offset_ram || state->offset_scratch)
1386                 jit_movi(tmp, state->offset_scratch);
1387
1388         if (state->offset_ram != state->offset_scratch)
1389                 jit_patch(to_end);
1390
1391         if (state->offset_ram || state->offset_scratch)
1392                 jit_addr(tmp2, tmp2, tmp);
1393
1394         lightrec_free_reg(reg_cache, tmp);
1395         lightrec_free_reg(reg_cache, tmp3);
1396
1397         rt = lightrec_alloc_reg_in(reg_cache, _jit, c.i.rt, 0);
1398
1399         if (is_big_endian() && swap_code && c.i.rt) {
1400                 tmp = lightrec_alloc_reg_temp(reg_cache, _jit);
1401
1402                 jit_new_node_ww(swap_code, tmp, rt);
1403                 jit_new_node_www(code, 0, tmp2, tmp);
1404
1405                 lightrec_free_reg(reg_cache, tmp);
1406         } else {
1407                 jit_new_node_www(code, 0, tmp2, rt);
1408         }
1409
1410         lightrec_free_reg(reg_cache, rt);
1411         lightrec_free_reg(reg_cache, tmp2);
1412 }
1413
1414 static void rec_store(struct lightrec_cstate *state,
1415                       const struct block *block, u16 offset,
1416                       jit_code_t code, jit_code_t swap_code)
1417 {
1418         u32 flags = block->opcode_list[offset].flags;
1419         bool no_invalidate = op_flag_no_invalidate(flags) ||
1420                 state->state->invalidate_from_dma_only;
1421
1422         switch (LIGHTREC_FLAGS_GET_IO_MODE(flags)) {
1423         case LIGHTREC_IO_RAM:
1424                 rec_store_ram(state, block, offset, code,
1425                               swap_code, !no_invalidate);
1426                 break;
1427         case LIGHTREC_IO_SCRATCH:
1428                 rec_store_scratch(state, block, offset, code, swap_code);
1429                 break;
1430         case LIGHTREC_IO_DIRECT:
1431                 if (no_invalidate) {
1432                         rec_store_direct_no_invalidate(state, block, offset,
1433                                                        code, swap_code);
1434                 } else {
1435                         rec_store_direct(state, block, offset, code, swap_code);
1436                 }
1437                 break;
1438         case LIGHTREC_IO_DIRECT_HW:
1439                 rec_store_io(state, block, offset, code, swap_code);
1440                 break;
1441         default:
1442                 rec_io(state, block, offset, true, false);
1443                 break;
1444         }
1445 }
1446
1447 static void rec_SB(struct lightrec_cstate *state,
1448                    const struct block *block, u16 offset)
1449 {
1450         _jit_name(block->_jit, __func__);
1451         rec_store(state, block, offset, jit_code_stxi_c, 0);
1452 }
1453
1454 static void rec_SH(struct lightrec_cstate *state,
1455                    const struct block *block, u16 offset)
1456 {
1457         _jit_name(block->_jit, __func__);
1458         rec_store(state, block, offset,
1459                   jit_code_stxi_s, jit_code_bswapr_us);
1460 }
1461
1462 static void rec_SW(struct lightrec_cstate *state,
1463                    const struct block *block, u16 offset)
1464
1465 {
1466         _jit_name(block->_jit, __func__);
1467         rec_store(state, block, offset,
1468                   jit_code_stxi_i, jit_code_bswapr_ui);
1469 }
1470
1471 static void rec_SWL(struct lightrec_cstate *state,
1472                     const struct block *block, u16 offset)
1473 {
1474         _jit_name(block->_jit, __func__);
1475         rec_io(state, block, offset, true, false);
1476 }
1477
1478 static void rec_SWR(struct lightrec_cstate *state,
1479                     const struct block *block, u16 offset)
1480 {
1481         _jit_name(block->_jit, __func__);
1482         rec_io(state, block, offset, true, false);
1483 }
1484
1485 static void rec_SWC2(struct lightrec_cstate *state,
1486                      const struct block *block, u16 offset)
1487 {
1488         _jit_name(block->_jit, __func__);
1489         rec_io(state, block, offset, false, false);
1490 }
1491
1492 static void rec_load_memory(struct lightrec_cstate *cstate,
1493                             const struct block *block, u16 offset,
1494                             jit_code_t code, jit_code_t swap_code, bool is_unsigned,
1495                             uintptr_t addr_offset, u32 addr_mask)
1496 {
1497         struct regcache *reg_cache = cstate->reg_cache;
1498         struct opcode *op = &block->opcode_list[offset];
1499         jit_state_t *_jit = block->_jit;
1500         u8 rs, rt, addr_reg, flags = REG_EXT;
1501         bool no_mask = op_flag_no_mask(op->flags);
1502         union code c = op->c;
1503         s16 imm;
1504
1505         if (!c.i.rt)
1506                 return;
1507
1508         if (is_unsigned)
1509                 flags |= REG_ZEXT;
1510
1511         rs = lightrec_alloc_reg_in(reg_cache, _jit, c.i.rs, 0);
1512         rt = lightrec_alloc_reg_out(reg_cache, _jit, c.i.rt, flags);
1513
1514         if (!cstate->state->mirrors_mapped && c.i.imm && !no_mask) {
1515                 jit_addi(rt, rs, (s16)c.i.imm);
1516                 addr_reg = rt;
1517                 imm = 0;
1518         } else {
1519                 addr_reg = rs;
1520                 imm = (s16)c.i.imm;
1521         }
1522
1523         if (!no_mask) {
1524                 jit_andi(rt, addr_reg, addr_mask);
1525                 addr_reg = rt;
1526         }
1527
1528         if (addr_offset) {
1529                 jit_addi(rt, addr_reg, addr_offset);
1530                 addr_reg = rt;
1531         }
1532
1533         jit_new_node_www(code, rt, addr_reg, imm);
1534
1535         if (is_big_endian() && swap_code) {
1536                 jit_new_node_ww(swap_code, rt, rt);
1537
1538                 if (c.i.op == OP_LH)
1539                         jit_extr_s(rt, rt);
1540                 else if (c.i.op == OP_LW && __WORDSIZE == 64)
1541                         jit_extr_i(rt, rt);
1542         }
1543
1544         lightrec_free_reg(reg_cache, rs);
1545         lightrec_free_reg(reg_cache, rt);
1546 }
1547
1548 static void rec_load_ram(struct lightrec_cstate *cstate,
1549                          const struct block *block, u16 offset,
1550                          jit_code_t code, jit_code_t swap_code, bool is_unsigned)
1551 {
1552         _jit_note(block->_jit, __FILE__, __LINE__);
1553
1554         rec_load_memory(cstate, block, offset, code, swap_code, is_unsigned,
1555                         cstate->state->offset_ram, rec_ram_mask(cstate->state));
1556 }
1557
1558 static void rec_load_bios(struct lightrec_cstate *cstate,
1559                           const struct block *block, u16 offset,
1560                           jit_code_t code, jit_code_t swap_code, bool is_unsigned)
1561 {
1562         _jit_note(block->_jit, __FILE__, __LINE__);
1563
1564         rec_load_memory(cstate, block, offset, code, swap_code, is_unsigned,
1565                         cstate->state->offset_bios, 0x1fffffff);
1566 }
1567
1568 static void rec_load_scratch(struct lightrec_cstate *cstate,
1569                              const struct block *block, u16 offset,
1570                              jit_code_t code, jit_code_t swap_code, bool is_unsigned)
1571 {
1572         _jit_note(block->_jit, __FILE__, __LINE__);
1573
1574         rec_load_memory(cstate, block, offset, code, swap_code, is_unsigned,
1575                         cstate->state->offset_scratch, 0x1fffffff);
1576 }
1577
1578 static void rec_load_io(struct lightrec_cstate *cstate,
1579                         const struct block *block, u16 offset,
1580                         jit_code_t code, jit_code_t swap_code, bool is_unsigned)
1581 {
1582         _jit_note(block->_jit, __FILE__, __LINE__);
1583
1584         rec_load_memory(cstate, block, offset, code, swap_code, is_unsigned,
1585                         cstate->state->offset_io, 0x1fffffff);
1586 }
1587
1588 static void rec_load_direct(struct lightrec_cstate *cstate,
1589                             const struct block *block, u16 offset,
1590                             jit_code_t code, jit_code_t swap_code,
1591                             bool is_unsigned)
1592 {
1593         struct lightrec_state *state = cstate->state;
1594         struct regcache *reg_cache = cstate->reg_cache;
1595         union code c = block->opcode_list[offset].c;
1596         jit_state_t *_jit = block->_jit;
1597         jit_node_t *to_not_ram, *to_not_bios, *to_end, *to_end2;
1598         u8 tmp, rs, rt, addr_reg, flags = REG_EXT;
1599         s16 imm;
1600
1601         if (!c.i.rt)
1602                 return;
1603
1604         if (is_unsigned)
1605                 flags |= REG_ZEXT;
1606
1607         jit_note(__FILE__, __LINE__);
1608         rs = lightrec_alloc_reg_in(reg_cache, _jit, c.i.rs, 0);
1609         rt = lightrec_alloc_reg_out(reg_cache, _jit, c.i.rt, flags);
1610
1611         if ((state->offset_ram == state->offset_bios &&
1612             state->offset_ram == state->offset_scratch &&
1613             state->mirrors_mapped) || !c.i.imm) {
1614                 addr_reg = rs;
1615                 imm = (s16)c.i.imm;
1616         } else {
1617                 jit_addi(rt, rs, (s16)c.i.imm);
1618                 addr_reg = rt;
1619                 imm = 0;
1620
1621                 if (c.i.rs != c.i.rt)
1622                         lightrec_free_reg(reg_cache, rs);
1623         }
1624
1625         tmp = lightrec_alloc_reg_temp(reg_cache, _jit);
1626
1627         if (state->offset_ram == state->offset_bios &&
1628             state->offset_ram == state->offset_scratch) {
1629                 if (!state->mirrors_mapped) {
1630                         jit_andi(tmp, addr_reg, BIT(28));
1631                         jit_rshi_u(tmp, tmp, 28 - 22);
1632                         jit_ori(tmp, tmp, 0x1f800000 | (RAM_SIZE - 1));
1633                         jit_andr(rt, addr_reg, tmp);
1634                 } else {
1635                         jit_andi(rt, addr_reg, 0x1fffffff);
1636                 }
1637
1638                 if (state->offset_ram)
1639                         jit_movi(tmp, state->offset_ram);
1640         } else {
1641                 to_not_ram = jit_bmsi(addr_reg, BIT(28));
1642
1643                 /* Convert to KUNSEG and avoid RAM mirrors */
1644                 jit_andi(rt, addr_reg, RAM_SIZE - 1);
1645
1646                 if (state->offset_ram)
1647                         jit_movi(tmp, state->offset_ram);
1648
1649                 to_end = jit_b();
1650
1651                 jit_patch(to_not_ram);
1652
1653                 if (state->offset_bios != state->offset_scratch)
1654                         to_not_bios = jit_bmci(addr_reg, BIT(22));
1655
1656                 /* Convert to KUNSEG */
1657                 jit_andi(rt, addr_reg, 0x1fc00000 | (BIOS_SIZE - 1));
1658
1659                 jit_movi(tmp, state->offset_bios);
1660
1661                 if (state->offset_bios != state->offset_scratch) {
1662                         to_end2 = jit_b();
1663
1664                         jit_patch(to_not_bios);
1665
1666                         /* Convert to KUNSEG */
1667                         jit_andi(rt, addr_reg, 0x1f800fff);
1668
1669                         if (state->offset_scratch)
1670                                 jit_movi(tmp, state->offset_scratch);
1671
1672                         jit_patch(to_end2);
1673                 }
1674
1675                 jit_patch(to_end);
1676         }
1677
1678         if (state->offset_ram || state->offset_bios || state->offset_scratch)
1679                 jit_addr(rt, rt, tmp);
1680
1681         jit_new_node_www(code, rt, rt, imm);
1682
1683         if (is_big_endian() && swap_code) {
1684                 jit_new_node_ww(swap_code, rt, rt);
1685
1686                 if (c.i.op == OP_LH)
1687                         jit_extr_s(rt, rt);
1688                 else if (c.i.op == OP_LW && __WORDSIZE == 64)
1689                         jit_extr_i(rt, rt);
1690         }
1691
1692         lightrec_free_reg(reg_cache, addr_reg);
1693         lightrec_free_reg(reg_cache, rt);
1694         lightrec_free_reg(reg_cache, tmp);
1695 }
1696
1697 static void rec_load(struct lightrec_cstate *state, const struct block *block,
1698                      u16 offset, jit_code_t code, jit_code_t swap_code,
1699                      bool is_unsigned)
1700 {
1701         u32 flags = block->opcode_list[offset].flags;
1702
1703         switch (LIGHTREC_FLAGS_GET_IO_MODE(flags)) {
1704         case LIGHTREC_IO_RAM:
1705                 rec_load_ram(state, block, offset, code, swap_code, is_unsigned);
1706                 break;
1707         case LIGHTREC_IO_BIOS:
1708                 rec_load_bios(state, block, offset, code, swap_code, is_unsigned);
1709                 break;
1710         case LIGHTREC_IO_SCRATCH:
1711                 rec_load_scratch(state, block, offset, code, swap_code, is_unsigned);
1712                 break;
1713         case LIGHTREC_IO_DIRECT_HW:
1714                 rec_load_io(state, block, offset, code, swap_code, is_unsigned);
1715                 break;
1716         case LIGHTREC_IO_DIRECT:
1717                 rec_load_direct(state, block, offset, code, swap_code, is_unsigned);
1718                 break;
1719         default:
1720                 rec_io(state, block, offset, false, true);
1721                 break;
1722         }
1723 }
1724
1725 static void rec_LB(struct lightrec_cstate *state, const struct block *block, u16 offset)
1726 {
1727         _jit_name(block->_jit, __func__);
1728         rec_load(state, block, offset, jit_code_ldxi_c, 0, false);
1729 }
1730
1731 static void rec_LBU(struct lightrec_cstate *state, const struct block *block, u16 offset)
1732 {
1733         _jit_name(block->_jit, __func__);
1734         rec_load(state, block, offset, jit_code_ldxi_uc, 0, true);
1735 }
1736
1737 static void rec_LH(struct lightrec_cstate *state, const struct block *block, u16 offset)
1738 {
1739         jit_code_t code = is_big_endian() ? jit_code_ldxi_us : jit_code_ldxi_s;
1740
1741         _jit_name(block->_jit, __func__);
1742         rec_load(state, block, offset, code, jit_code_bswapr_us, false);
1743 }
1744
1745 static void rec_LHU(struct lightrec_cstate *state, const struct block *block, u16 offset)
1746 {
1747         _jit_name(block->_jit, __func__);
1748         rec_load(state, block, offset, jit_code_ldxi_us, jit_code_bswapr_us, true);
1749 }
1750
1751 static void rec_LWL(struct lightrec_cstate *state, const struct block *block, u16 offset)
1752 {
1753         _jit_name(block->_jit, __func__);
1754         rec_io(state, block, offset, true, true);
1755 }
1756
1757 static void rec_LWR(struct lightrec_cstate *state, const struct block *block, u16 offset)
1758 {
1759         _jit_name(block->_jit, __func__);
1760         rec_io(state, block, offset, true, true);
1761 }
1762
1763 static void rec_LW(struct lightrec_cstate *state, const struct block *block, u16 offset)
1764 {
1765         jit_code_t code;
1766
1767         if (is_big_endian() && __WORDSIZE == 64)
1768                 code = jit_code_ldxi_ui;
1769         else
1770                 code = jit_code_ldxi_i;
1771
1772         _jit_name(block->_jit, __func__);
1773         rec_load(state, block, offset, code, jit_code_bswapr_ui, false);
1774 }
1775
1776 static void rec_LWC2(struct lightrec_cstate *state, const struct block *block, u16 offset)
1777 {
1778         _jit_name(block->_jit, __func__);
1779         rec_io(state, block, offset, false, false);
1780 }
1781
1782 static void rec_break_syscall(struct lightrec_cstate *state,
1783                               const struct block *block, u16 offset,
1784                               u32 exit_code)
1785 {
1786         struct regcache *reg_cache = state->reg_cache;
1787         jit_state_t *_jit = block->_jit;
1788         u8 tmp;
1789
1790         _jit_note(block->_jit, __FILE__, __LINE__);
1791
1792         tmp = lightrec_alloc_reg_temp(reg_cache, _jit);
1793
1794         jit_movi(tmp, exit_code);
1795         jit_stxi_i(offsetof(struct lightrec_state, exit_flags),
1796                    LIGHTREC_REG_STATE, tmp);
1797
1798         lightrec_free_reg(reg_cache, tmp);
1799
1800         /* TODO: the return address should be "pc - 4" if we're a delay slot */
1801         lightrec_emit_end_of_block(state, block, offset, -1,
1802                                    get_ds_pc(block, offset, 0),
1803                                    31, 0, true);
1804 }
1805
1806 static void rec_special_SYSCALL(struct lightrec_cstate *state,
1807                                 const struct block *block, u16 offset)
1808 {
1809         _jit_name(block->_jit, __func__);
1810         rec_break_syscall(state, block, offset, LIGHTREC_EXIT_SYSCALL);
1811 }
1812
1813 static void rec_special_BREAK(struct lightrec_cstate *state,
1814                               const struct block *block, u16 offset)
1815 {
1816         _jit_name(block->_jit, __func__);
1817         rec_break_syscall(state, block, offset, LIGHTREC_EXIT_BREAK);
1818 }
1819
1820 static void rec_mfc(struct lightrec_cstate *state, const struct block *block, u16 offset)
1821 {
1822         struct regcache *reg_cache = state->reg_cache;
1823         union code c = block->opcode_list[offset].c;
1824         jit_state_t *_jit = block->_jit;
1825
1826         jit_note(__FILE__, __LINE__);
1827         lightrec_clean_reg_if_loaded(reg_cache, _jit, c.i.rt, true);
1828
1829         call_to_c_wrapper(state, block, c.opcode, C_WRAPPER_MFC);
1830 }
1831
1832 static void rec_mtc(struct lightrec_cstate *state, const struct block *block, u16 offset)
1833 {
1834         struct regcache *reg_cache = state->reg_cache;
1835         union code c = block->opcode_list[offset].c;
1836         jit_state_t *_jit = block->_jit;
1837
1838         jit_note(__FILE__, __LINE__);
1839         lightrec_clean_reg_if_loaded(reg_cache, _jit, c.i.rs, false);
1840         lightrec_clean_reg_if_loaded(reg_cache, _jit, c.i.rt, false);
1841
1842         call_to_c_wrapper(state, block, c.opcode, C_WRAPPER_MTC);
1843
1844         if (c.i.op == OP_CP0 &&
1845             !op_flag_no_ds(block->opcode_list[offset].flags) &&
1846             (c.r.rd == 12 || c.r.rd == 13))
1847                 lightrec_emit_end_of_block(state, block, offset, -1,
1848                                            get_ds_pc(block, offset, 1),
1849                                            0, 0, true);
1850 }
1851
1852 static void
1853 rec_mfc0(struct lightrec_cstate *state, const struct block *block, u16 offset)
1854 {
1855         struct regcache *reg_cache = state->reg_cache;
1856         union code c = block->opcode_list[offset].c;
1857         jit_state_t *_jit = block->_jit;
1858         u8 rt;
1859
1860         jit_note(__FILE__, __LINE__);
1861
1862         rt = lightrec_alloc_reg_out(reg_cache, _jit, c.i.rt, REG_EXT);
1863
1864         jit_ldxi_i(rt, LIGHTREC_REG_STATE,
1865                    offsetof(struct lightrec_state, regs.cp0[c.r.rd]));
1866
1867         lightrec_free_reg(reg_cache, rt);
1868 }
1869
1870 static bool block_in_bios(const struct lightrec_cstate *state,
1871                           const struct block *block)
1872 {
1873         const struct lightrec_mem_map *bios = &state->state->maps[PSX_MAP_BIOS];
1874         u32 pc = kunseg(block->pc);
1875
1876         return pc >= bios->pc && pc < bios->pc + bios->length;
1877 }
1878
1879 static void
1880 rec_mtc0(struct lightrec_cstate *state, const struct block *block, u16 offset)
1881 {
1882         struct regcache *reg_cache = state->reg_cache;
1883         const union code c = block->opcode_list[offset].c;
1884         jit_state_t *_jit = block->_jit;
1885         u8 rt, tmp = 0, tmp2, status;
1886
1887         jit_note(__FILE__, __LINE__);
1888
1889         switch(c.r.rd) {
1890         case 1:
1891         case 4:
1892         case 8:
1893         case 14:
1894         case 15:
1895                 /* Those registers are read-only */
1896                 return;
1897         default:
1898                 break;
1899         }
1900
1901         if (block_in_bios(state, block) && c.r.rd == 12) {
1902                 /* If we are running code from the BIOS, handle writes to the
1903                  * Status register in C. BIOS code may toggle bit 16 which will
1904                  * map/unmap the RAM, while game code cannot do that. */
1905                 rec_mtc(state, block, offset);
1906                 return;
1907         }
1908
1909         rt = lightrec_alloc_reg_in(reg_cache, _jit, c.i.rt, 0);
1910
1911         if (c.r.rd != 13) {
1912                 jit_stxi_i(offsetof(struct lightrec_state, regs.cp0[c.r.rd]),
1913                            LIGHTREC_REG_STATE, rt);
1914         }
1915
1916         if (c.r.rd == 12 || c.r.rd == 13) {
1917                 tmp = lightrec_alloc_reg_temp(reg_cache, _jit);
1918                 jit_ldxi_i(tmp, LIGHTREC_REG_STATE,
1919                            offsetof(struct lightrec_state, regs.cp0[13]));
1920
1921                 tmp2 = lightrec_alloc_reg_temp(reg_cache, _jit);
1922         }
1923
1924         if (c.r.rd == 12) {
1925                 status = rt;
1926         } else if (c.r.rd == 13) {
1927                 /* Cause = (Cause & ~0x0300) | (value & 0x0300) */
1928                 jit_andi(tmp2, rt, 0x0300);
1929                 jit_ori(tmp, tmp, 0x0300);
1930                 jit_xori(tmp, tmp, 0x0300);
1931                 jit_orr(tmp, tmp, tmp2);
1932                 jit_ldxi_i(tmp2, LIGHTREC_REG_STATE,
1933                            offsetof(struct lightrec_state, regs.cp0[12]));
1934                 jit_stxi_i(offsetof(struct lightrec_state, regs.cp0[13]),
1935                            LIGHTREC_REG_STATE, tmp);
1936                 status = tmp2;
1937         }
1938
1939         if (c.r.rd == 12 || c.r.rd == 13) {
1940                 /* Exit dynarec in case there's a software interrupt.
1941                  * exit_flags = !!(status & tmp & 0x0300) & status; */
1942                 jit_andr(tmp, tmp, status);
1943                 jit_andi(tmp, tmp, 0x0300);
1944                 jit_nei(tmp, tmp, 0);
1945                 jit_andr(tmp, tmp, status);
1946         }
1947
1948         if (c.r.rd == 12) {
1949                 /* Exit dynarec in case we unmask a hardware interrupt.
1950                  * exit_flags = !(~status & 0x401) */
1951
1952                 jit_comr(tmp2, status);
1953                 jit_andi(tmp2, tmp2, 0x401);
1954                 jit_eqi(tmp2, tmp2, 0);
1955                 jit_orr(tmp, tmp, tmp2);
1956         }
1957
1958         if (c.r.rd == 12 || c.r.rd == 13) {
1959                 jit_stxi_i(offsetof(struct lightrec_state, exit_flags),
1960                            LIGHTREC_REG_STATE, tmp);
1961
1962                 lightrec_free_reg(reg_cache, tmp);
1963                 lightrec_free_reg(reg_cache, tmp2);
1964         }
1965
1966         lightrec_free_reg(reg_cache, rt);
1967
1968         if (!op_flag_no_ds(block->opcode_list[offset].flags) &&
1969             (c.r.rd == 12 || c.r.rd == 13))
1970                 lightrec_emit_eob(state, block, offset + 1, true);
1971 }
1972
1973 static void rec_cp0_MFC0(struct lightrec_cstate *state,
1974                          const struct block *block, u16 offset)
1975 {
1976         _jit_name(block->_jit, __func__);
1977         rec_mfc0(state, block, offset);
1978 }
1979
1980 static void rec_cp0_CFC0(struct lightrec_cstate *state,
1981                          const struct block *block, u16 offset)
1982 {
1983         _jit_name(block->_jit, __func__);
1984         rec_mfc0(state, block, offset);
1985 }
1986
1987 static void rec_cp0_MTC0(struct lightrec_cstate *state,
1988                          const struct block *block, u16 offset)
1989 {
1990         _jit_name(block->_jit, __func__);
1991         rec_mtc0(state, block, offset);
1992 }
1993
1994 static void rec_cp0_CTC0(struct lightrec_cstate *state,
1995                          const struct block *block, u16 offset)
1996 {
1997         _jit_name(block->_jit, __func__);
1998         rec_mtc0(state, block, offset);
1999 }
2000
2001 static unsigned int cp2d_i_offset(u8 reg)
2002 {
2003         return offsetof(struct lightrec_state, regs.cp2d[reg]);
2004 }
2005
2006 static unsigned int cp2d_s_offset(u8 reg)
2007 {
2008         return cp2d_i_offset(reg) + is_big_endian() * 2;
2009 }
2010
2011 static unsigned int cp2c_i_offset(u8 reg)
2012 {
2013         return offsetof(struct lightrec_state, regs.cp2c[reg]);
2014 }
2015
2016 static unsigned int cp2c_s_offset(u8 reg)
2017 {
2018         return cp2c_i_offset(reg) + is_big_endian() * 2;
2019 }
2020
2021 static void rec_cp2_basic_MFC2(struct lightrec_cstate *state,
2022                                const struct block *block, u16 offset)
2023 {
2024         struct regcache *reg_cache = state->reg_cache;
2025         const union code c = block->opcode_list[offset].c;
2026         jit_state_t *_jit = block->_jit;
2027         const u32 zext_regs = 0x300f0080;
2028         u8 rt, tmp, tmp2, tmp3, out, flags;
2029         u8 reg = c.r.rd == 15 ? 14 : c.r.rd;
2030         unsigned int i;
2031
2032         _jit_name(block->_jit, __func__);
2033
2034         if (state->state->ops.cop2_notify) {
2035                 /* We must call cop2_notify, handle that in C. */
2036                 rec_mfc(state, block, offset);
2037                 return;
2038         }
2039
2040         flags = (zext_regs & BIT(reg)) ? REG_ZEXT : REG_EXT;
2041         rt = lightrec_alloc_reg_out(reg_cache, _jit, c.r.rt, flags);
2042
2043         switch (reg) {
2044         case 1:
2045         case 3:
2046         case 5:
2047         case 8:
2048         case 9:
2049         case 10:
2050         case 11:
2051                 jit_ldxi_s(rt, LIGHTREC_REG_STATE, cp2d_s_offset(reg));
2052                 break;
2053         case 7:
2054         case 16:
2055         case 17:
2056         case 18:
2057         case 19:
2058                 jit_ldxi_us(rt, LIGHTREC_REG_STATE, cp2d_s_offset(reg));
2059                 break;
2060         case 28:
2061         case 29:
2062                 tmp = lightrec_alloc_reg_temp(reg_cache, _jit);
2063                 tmp2 = lightrec_alloc_reg_temp(reg_cache, _jit);
2064                 tmp3 = lightrec_alloc_reg_temp(reg_cache, _jit);
2065
2066                 for (i = 0; i < 3; i++) {
2067                         out = i == 0 ? rt : tmp;
2068
2069                         jit_ldxi_s(tmp, LIGHTREC_REG_STATE, cp2d_s_offset(9 + i));
2070                         jit_movi(tmp2, 0x1f);
2071                         jit_rshi(out, tmp, 7);
2072
2073                         jit_ltr(tmp3, tmp2, out);
2074                         jit_movnr(out, tmp2, tmp3);
2075
2076                         jit_gei(tmp2, out, 0);
2077                         jit_movzr(out, tmp2, tmp2);
2078
2079                         if (i > 0) {
2080                                 jit_lshi(tmp, tmp, 5 * i);
2081                                 jit_orr(rt, rt, tmp);
2082                         }
2083                 }
2084
2085
2086                 lightrec_free_reg(reg_cache, tmp);
2087                 lightrec_free_reg(reg_cache, tmp2);
2088                 lightrec_free_reg(reg_cache, tmp3);
2089                 break;
2090         default:
2091                 jit_ldxi_i(rt, LIGHTREC_REG_STATE, cp2d_i_offset(reg));
2092                 break;
2093         }
2094
2095         lightrec_free_reg(reg_cache, rt);
2096 }
2097
2098 static void rec_cp2_basic_CFC2(struct lightrec_cstate *state,
2099                                const struct block *block, u16 offset)
2100 {
2101         struct regcache *reg_cache = state->reg_cache;
2102         const union code c = block->opcode_list[offset].c;
2103         jit_state_t *_jit = block->_jit;
2104         u8 rt;
2105
2106         _jit_name(block->_jit, __func__);
2107
2108         if (state->state->ops.cop2_notify) {
2109                 /* We must call cop2_notify, handle that in C. */
2110                 rec_mfc(state, block, offset);
2111                 return;
2112         }
2113
2114         switch (c.r.rd) {
2115         case 4:
2116         case 12:
2117         case 20:
2118         case 26:
2119         case 27:
2120         case 29:
2121         case 30:
2122                 rt = lightrec_alloc_reg_out(reg_cache, _jit, c.r.rt, REG_EXT);
2123                 jit_ldxi_s(rt, LIGHTREC_REG_STATE, cp2c_s_offset(c.r.rd));
2124                 break;
2125         default:
2126                 rt = lightrec_alloc_reg_out(reg_cache, _jit, c.r.rt, REG_ZEXT);
2127                 jit_ldxi_i(rt, LIGHTREC_REG_STATE, cp2c_i_offset(c.r.rd));
2128                 break;
2129         }
2130
2131         lightrec_free_reg(reg_cache, rt);
2132 }
2133
2134 static void rec_cp2_basic_MTC2(struct lightrec_cstate *state,
2135                                const struct block *block, u16 offset)
2136 {
2137         struct regcache *reg_cache = state->reg_cache;
2138         const union code c = block->opcode_list[offset].c;
2139         jit_state_t *_jit = block->_jit;
2140         jit_node_t *loop, *to_loop;
2141         u8 rt, tmp, tmp2, flags = 0;
2142
2143         _jit_name(block->_jit, __func__);
2144
2145         if (state->state->ops.cop2_notify) {
2146                 /* We must call cop2_notify, handle that in C. */
2147                 rec_mtc(state, block, offset);
2148                 return;
2149         }
2150
2151         if (c.r.rd == 31)
2152                 return;
2153
2154         if (c.r.rd == 30)
2155                 flags |= REG_EXT;
2156
2157         rt = lightrec_alloc_reg_in(reg_cache, _jit, c.r.rt, flags);
2158
2159         switch (c.r.rd) {
2160         case 15:
2161                 tmp = lightrec_alloc_reg_temp(reg_cache, _jit);
2162                 jit_ldxi_i(tmp, LIGHTREC_REG_STATE, cp2d_i_offset(13));
2163
2164                 tmp2 = lightrec_alloc_reg_temp(reg_cache, _jit);
2165                 jit_ldxi_i(tmp2, LIGHTREC_REG_STATE, cp2d_i_offset(14));
2166
2167                 jit_stxi_i(cp2d_i_offset(12), LIGHTREC_REG_STATE, tmp);
2168                 jit_stxi_i(cp2d_i_offset(13), LIGHTREC_REG_STATE, tmp2);
2169                 jit_stxi_i(cp2d_i_offset(14), LIGHTREC_REG_STATE, rt);
2170
2171                 lightrec_free_reg(reg_cache, tmp);
2172                 lightrec_free_reg(reg_cache, tmp2);
2173                 break;
2174         case 28:
2175                 tmp = lightrec_alloc_reg_temp(reg_cache, _jit);
2176
2177                 jit_lshi(tmp, rt, 7);
2178                 jit_andi(tmp, tmp, 0xf80);
2179                 jit_stxi_s(cp2d_s_offset(9), LIGHTREC_REG_STATE, tmp);
2180
2181                 jit_lshi(tmp, rt, 2);
2182                 jit_andi(tmp, tmp, 0xf80);
2183                 jit_stxi_s(cp2d_s_offset(10), LIGHTREC_REG_STATE, tmp);
2184
2185                 jit_rshi(tmp, rt, 3);
2186                 jit_andi(tmp, tmp, 0xf80);
2187                 jit_stxi_s(cp2d_s_offset(11), LIGHTREC_REG_STATE, tmp);
2188
2189                 lightrec_free_reg(reg_cache, tmp);
2190                 break;
2191         case 30:
2192                 tmp = lightrec_alloc_reg_temp(reg_cache, _jit);
2193                 tmp2 = lightrec_alloc_reg_temp(reg_cache, _jit);
2194
2195                 /* if (rt < 0) rt = ~rt; */
2196                 jit_rshi(tmp, rt, 31);
2197                 jit_xorr(tmp, rt, tmp);
2198
2199                 /* We know the sign bit is 0. Left-shift by 1 to start the algorithm */
2200                 jit_lshi(tmp, tmp, 1);
2201                 jit_movi(tmp2, 33);
2202
2203                 /* Decrement tmp2 and right-shift the value by 1 until it equals zero */
2204                 loop = jit_label();
2205                 jit_subi(tmp2, tmp2, 1);
2206                 jit_rshi_u(tmp, tmp, 1);
2207                 to_loop = jit_bnei(tmp, 0);
2208
2209                 jit_patch_at(to_loop, loop);
2210
2211                 jit_stxi_i(cp2d_i_offset(31), LIGHTREC_REG_STATE, tmp2);
2212                 jit_stxi_i(cp2d_i_offset(30), LIGHTREC_REG_STATE, rt);
2213
2214                 lightrec_free_reg(reg_cache, tmp);
2215                 lightrec_free_reg(reg_cache, tmp2);
2216                 break;
2217         default:
2218                 jit_stxi_i(cp2d_i_offset(c.r.rd), LIGHTREC_REG_STATE, rt);
2219                 break;
2220         }
2221
2222         lightrec_free_reg(reg_cache, rt);
2223 }
2224
2225 static void rec_cp2_basic_CTC2(struct lightrec_cstate *state,
2226                                const struct block *block, u16 offset)
2227 {
2228         struct regcache *reg_cache = state->reg_cache;
2229         const union code c = block->opcode_list[offset].c;
2230         jit_state_t *_jit = block->_jit;
2231         u8 rt, tmp, tmp2;
2232
2233         _jit_name(block->_jit, __func__);
2234
2235         if (state->state->ops.cop2_notify) {
2236                 /* We must call cop2_notify, handle that in C. */
2237                 rec_mtc(state, block, offset);
2238                 return;
2239         }
2240
2241         rt = lightrec_alloc_reg_in(reg_cache, _jit, c.r.rt, 0);
2242
2243         switch (c.r.rd) {
2244         case 4:
2245         case 12:
2246         case 20:
2247         case 26:
2248         case 27:
2249         case 29:
2250         case 30:
2251                 jit_stxi_s(cp2c_s_offset(c.r.rd), LIGHTREC_REG_STATE, rt);
2252                 break;
2253         case 31:
2254                 tmp = lightrec_alloc_reg_temp(reg_cache, _jit);
2255                 tmp2 = lightrec_alloc_reg_temp(reg_cache, _jit);
2256
2257                 jit_andi(tmp, rt, 0x7f87e000);
2258                 jit_nei(tmp, tmp, 0);
2259                 jit_lshi(tmp, tmp, 31);
2260
2261                 jit_andi(tmp2, rt, 0x7ffff000);
2262                 jit_orr(tmp, tmp2, tmp);
2263
2264                 jit_stxi_i(cp2c_i_offset(31), LIGHTREC_REG_STATE, tmp);
2265
2266                 lightrec_free_reg(reg_cache, tmp);
2267                 lightrec_free_reg(reg_cache, tmp2);
2268                 break;
2269
2270         default:
2271                 jit_stxi_i(cp2c_i_offset(c.r.rd), LIGHTREC_REG_STATE, rt);
2272         }
2273
2274         lightrec_free_reg(reg_cache, rt);
2275 }
2276
2277 static void rec_cp0_RFE(struct lightrec_cstate *state,
2278                         const struct block *block, u16 offset)
2279 {
2280         struct regcache *reg_cache = state->reg_cache;
2281         jit_state_t *_jit = block->_jit;
2282         u8 status, tmp;
2283
2284         jit_name(__func__);
2285         jit_note(__FILE__, __LINE__);
2286
2287         status = lightrec_alloc_reg_temp(reg_cache, _jit);
2288         jit_ldxi_i(status, LIGHTREC_REG_STATE,
2289                    offsetof(struct lightrec_state, regs.cp0[12]));
2290
2291         tmp = lightrec_alloc_reg_temp(reg_cache, _jit);
2292
2293         /* status = ((status >> 2) & 0xf) | status & ~0xf; */
2294         jit_rshi(tmp, status, 2);
2295         jit_andi(tmp, tmp, 0xf);
2296         jit_andi(status, status, ~0xful);
2297         jit_orr(status, status, tmp);
2298
2299         jit_ldxi_i(tmp, LIGHTREC_REG_STATE,
2300                    offsetof(struct lightrec_state, regs.cp0[13]));
2301         jit_stxi_i(offsetof(struct lightrec_state, regs.cp0[12]),
2302                    LIGHTREC_REG_STATE, status);
2303
2304         /* Exit dynarec in case there's a software interrupt.
2305          * exit_flags = !!(status & cause & 0x0300) & status; */
2306         jit_andr(tmp, tmp, status);
2307         jit_andi(tmp, tmp, 0x0300);
2308         jit_nei(tmp, tmp, 0);
2309         jit_andr(tmp, tmp, status);
2310         jit_stxi_i(offsetof(struct lightrec_state, exit_flags),
2311                    LIGHTREC_REG_STATE, tmp);
2312
2313         lightrec_free_reg(reg_cache, status);
2314         lightrec_free_reg(reg_cache, tmp);
2315 }
2316
2317 static void rec_CP(struct lightrec_cstate *state,
2318                    const struct block *block, u16 offset)
2319 {
2320         union code c = block->opcode_list[offset].c;
2321         jit_state_t *_jit = block->_jit;
2322
2323         jit_name(__func__);
2324         jit_note(__FILE__, __LINE__);
2325
2326         call_to_c_wrapper(state, block, c.opcode, C_WRAPPER_CP);
2327 }
2328
2329 static void rec_meta_MOV(struct lightrec_cstate *state,
2330                          const struct block *block, u16 offset)
2331 {
2332         struct regcache *reg_cache = state->reg_cache;
2333         union code c = block->opcode_list[offset].c;
2334         jit_state_t *_jit = block->_jit;
2335         u8 rs, rd;
2336
2337         _jit_name(block->_jit, __func__);
2338         jit_note(__FILE__, __LINE__);
2339         if (c.r.rs)
2340                 rs = lightrec_alloc_reg_in(reg_cache, _jit, c.r.rs, 0);
2341         rd = lightrec_alloc_reg_out(reg_cache, _jit, c.r.rd, REG_EXT);
2342
2343         if (c.r.rs == 0)
2344                 jit_movi(rd, 0);
2345         else
2346                 jit_extr_i(rd, rs);
2347
2348         if (c.r.rs)
2349                 lightrec_free_reg(reg_cache, rs);
2350         lightrec_free_reg(reg_cache, rd);
2351 }
2352
2353 static void rec_meta_EXTC_EXTS(struct lightrec_cstate *state,
2354                                const struct block *block,
2355                                u16 offset)
2356 {
2357         struct regcache *reg_cache = state->reg_cache;
2358         union code c = block->opcode_list[offset].c;
2359         jit_state_t *_jit = block->_jit;
2360         u8 rs, rt;
2361
2362         _jit_name(block->_jit, __func__);
2363         jit_note(__FILE__, __LINE__);
2364
2365         rs = lightrec_alloc_reg_in(reg_cache, _jit, c.i.rs, 0);
2366         rt = lightrec_alloc_reg_out(reg_cache, _jit, c.i.rt, REG_EXT);
2367
2368         if (c.i.op == OP_META_EXTC)
2369                 jit_extr_c(rt, rs);
2370         else
2371                 jit_extr_s(rt, rs);
2372
2373         lightrec_free_reg(reg_cache, rs);
2374         lightrec_free_reg(reg_cache, rt);
2375 }
2376
2377 static void rec_meta_MULT2(struct lightrec_cstate *state,
2378                            const struct block *block,
2379                            u16 offset)
2380 {
2381         struct regcache *reg_cache = state->reg_cache;
2382         union code c = block->opcode_list[offset].c;
2383         jit_state_t *_jit = block->_jit;
2384         u8 reg_lo = get_mult_div_lo(c);
2385         u8 reg_hi = get_mult_div_hi(c);
2386         u32 flags = block->opcode_list[offset].flags;
2387         bool is_signed = c.i.op == OP_META_MULT2;
2388         u8 rs, lo, hi, rflags = 0, hiflags = 0;
2389
2390         if (!op_flag_no_hi(flags) && c.r.op < 32) {
2391                 rflags = is_signed ? REG_EXT : REG_ZEXT;
2392                 hiflags = is_signed ? REG_EXT : (REG_EXT | REG_ZEXT);
2393         }
2394
2395         _jit_name(block->_jit, __func__);
2396         jit_note(__FILE__, __LINE__);
2397
2398         rs = lightrec_alloc_reg_in(reg_cache, _jit, c.i.rs, rflags);
2399
2400         if (!op_flag_no_lo(flags)) {
2401                 lo = lightrec_alloc_reg_out(reg_cache, _jit, reg_lo, 0);
2402
2403                 if (c.r.op < 32)
2404                         jit_lshi(lo, rs, c.r.op);
2405                 else
2406                         jit_movi(lo, 0);
2407
2408                 lightrec_free_reg(reg_cache, lo);
2409         }
2410
2411         if (!op_flag_no_hi(flags)) {
2412                 hi = lightrec_alloc_reg_out(reg_cache, _jit, reg_hi, hiflags);
2413
2414                 if (c.r.op >= 32)
2415                         jit_lshi(hi, rs, c.r.op - 32);
2416                 else if (is_signed)
2417                         jit_rshi(hi, rs, 32 - c.r.op);
2418                 else
2419                         jit_rshi_u(hi, rs, 32 - c.r.op);
2420
2421                 lightrec_free_reg(reg_cache, hi);
2422         }
2423
2424         lightrec_free_reg(reg_cache, rs);
2425
2426         _jit_name(block->_jit, __func__);
2427         jit_note(__FILE__, __LINE__);
2428 }
2429
2430 static const lightrec_rec_func_t rec_standard[64] = {
2431         SET_DEFAULT_ELM(rec_standard, unknown_opcode),
2432         [OP_SPECIAL]            = rec_SPECIAL,
2433         [OP_REGIMM]             = rec_REGIMM,
2434         [OP_J]                  = rec_J,
2435         [OP_JAL]                = rec_JAL,
2436         [OP_BEQ]                = rec_BEQ,
2437         [OP_BNE]                = rec_BNE,
2438         [OP_BLEZ]               = rec_BLEZ,
2439         [OP_BGTZ]               = rec_BGTZ,
2440         [OP_ADDI]               = rec_ADDI,
2441         [OP_ADDIU]              = rec_ADDIU,
2442         [OP_SLTI]               = rec_SLTI,
2443         [OP_SLTIU]              = rec_SLTIU,
2444         [OP_ANDI]               = rec_ANDI,
2445         [OP_ORI]                = rec_ORI,
2446         [OP_XORI]               = rec_XORI,
2447         [OP_LUI]                = rec_LUI,
2448         [OP_CP0]                = rec_CP0,
2449         [OP_CP2]                = rec_CP2,
2450         [OP_LB]                 = rec_LB,
2451         [OP_LH]                 = rec_LH,
2452         [OP_LWL]                = rec_LWL,
2453         [OP_LW]                 = rec_LW,
2454         [OP_LBU]                = rec_LBU,
2455         [OP_LHU]                = rec_LHU,
2456         [OP_LWR]                = rec_LWR,
2457         [OP_SB]                 = rec_SB,
2458         [OP_SH]                 = rec_SH,
2459         [OP_SWL]                = rec_SWL,
2460         [OP_SW]                 = rec_SW,
2461         [OP_SWR]                = rec_SWR,
2462         [OP_LWC2]               = rec_LWC2,
2463         [OP_SWC2]               = rec_SWC2,
2464
2465         [OP_META_MOV]           = rec_meta_MOV,
2466         [OP_META_EXTC]          = rec_meta_EXTC_EXTS,
2467         [OP_META_EXTS]          = rec_meta_EXTC_EXTS,
2468         [OP_META_MULT2]         = rec_meta_MULT2,
2469         [OP_META_MULTU2]        = rec_meta_MULT2,
2470 };
2471
2472 static const lightrec_rec_func_t rec_special[64] = {
2473         SET_DEFAULT_ELM(rec_special, unknown_opcode),
2474         [OP_SPECIAL_SLL]        = rec_special_SLL,
2475         [OP_SPECIAL_SRL]        = rec_special_SRL,
2476         [OP_SPECIAL_SRA]        = rec_special_SRA,
2477         [OP_SPECIAL_SLLV]       = rec_special_SLLV,
2478         [OP_SPECIAL_SRLV]       = rec_special_SRLV,
2479         [OP_SPECIAL_SRAV]       = rec_special_SRAV,
2480         [OP_SPECIAL_JR]         = rec_special_JR,
2481         [OP_SPECIAL_JALR]       = rec_special_JALR,
2482         [OP_SPECIAL_SYSCALL]    = rec_special_SYSCALL,
2483         [OP_SPECIAL_BREAK]      = rec_special_BREAK,
2484         [OP_SPECIAL_MFHI]       = rec_special_MFHI,
2485         [OP_SPECIAL_MTHI]       = rec_special_MTHI,
2486         [OP_SPECIAL_MFLO]       = rec_special_MFLO,
2487         [OP_SPECIAL_MTLO]       = rec_special_MTLO,
2488         [OP_SPECIAL_MULT]       = rec_special_MULT,
2489         [OP_SPECIAL_MULTU]      = rec_special_MULTU,
2490         [OP_SPECIAL_DIV]        = rec_special_DIV,
2491         [OP_SPECIAL_DIVU]       = rec_special_DIVU,
2492         [OP_SPECIAL_ADD]        = rec_special_ADD,
2493         [OP_SPECIAL_ADDU]       = rec_special_ADDU,
2494         [OP_SPECIAL_SUB]        = rec_special_SUB,
2495         [OP_SPECIAL_SUBU]       = rec_special_SUBU,
2496         [OP_SPECIAL_AND]        = rec_special_AND,
2497         [OP_SPECIAL_OR]         = rec_special_OR,
2498         [OP_SPECIAL_XOR]        = rec_special_XOR,
2499         [OP_SPECIAL_NOR]        = rec_special_NOR,
2500         [OP_SPECIAL_SLT]        = rec_special_SLT,
2501         [OP_SPECIAL_SLTU]       = rec_special_SLTU,
2502 };
2503
2504 static const lightrec_rec_func_t rec_regimm[64] = {
2505         SET_DEFAULT_ELM(rec_regimm, unknown_opcode),
2506         [OP_REGIMM_BLTZ]        = rec_regimm_BLTZ,
2507         [OP_REGIMM_BGEZ]        = rec_regimm_BGEZ,
2508         [OP_REGIMM_BLTZAL]      = rec_regimm_BLTZAL,
2509         [OP_REGIMM_BGEZAL]      = rec_regimm_BGEZAL,
2510 };
2511
2512 static const lightrec_rec_func_t rec_cp0[64] = {
2513         SET_DEFAULT_ELM(rec_cp0, rec_CP),
2514         [OP_CP0_MFC0]           = rec_cp0_MFC0,
2515         [OP_CP0_CFC0]           = rec_cp0_CFC0,
2516         [OP_CP0_MTC0]           = rec_cp0_MTC0,
2517         [OP_CP0_CTC0]           = rec_cp0_CTC0,
2518         [OP_CP0_RFE]            = rec_cp0_RFE,
2519 };
2520
2521 static const lightrec_rec_func_t rec_cp2_basic[64] = {
2522         SET_DEFAULT_ELM(rec_cp2_basic, rec_CP),
2523         [OP_CP2_BASIC_MFC2]     = rec_cp2_basic_MFC2,
2524         [OP_CP2_BASIC_CFC2]     = rec_cp2_basic_CFC2,
2525         [OP_CP2_BASIC_MTC2]     = rec_cp2_basic_MTC2,
2526         [OP_CP2_BASIC_CTC2]     = rec_cp2_basic_CTC2,
2527 };
2528
2529 static void rec_SPECIAL(struct lightrec_cstate *state,
2530                         const struct block *block, u16 offset)
2531 {
2532         union code c = block->opcode_list[offset].c;
2533         lightrec_rec_func_t f = rec_special[c.r.op];
2534
2535         if (!HAS_DEFAULT_ELM && unlikely(!f))
2536                 unknown_opcode(state, block, offset);
2537         else
2538                 (*f)(state, block, offset);
2539 }
2540
2541 static void rec_REGIMM(struct lightrec_cstate *state,
2542                        const struct block *block, u16 offset)
2543 {
2544         union code c = block->opcode_list[offset].c;
2545         lightrec_rec_func_t f = rec_regimm[c.r.rt];
2546
2547         if (!HAS_DEFAULT_ELM && unlikely(!f))
2548                 unknown_opcode(state, block, offset);
2549         else
2550                 (*f)(state, block, offset);
2551 }
2552
2553 static void rec_CP0(struct lightrec_cstate *state,
2554                     const struct block *block, u16 offset)
2555 {
2556         union code c = block->opcode_list[offset].c;
2557         lightrec_rec_func_t f = rec_cp0[c.r.rs];
2558
2559         if (!HAS_DEFAULT_ELM && unlikely(!f))
2560                 rec_CP(state, block, offset);
2561         else
2562                 (*f)(state, block, offset);
2563 }
2564
2565 static void rec_CP2(struct lightrec_cstate *state,
2566                     const struct block *block, u16 offset)
2567 {
2568         union code c = block->opcode_list[offset].c;
2569
2570         if (c.r.op == OP_CP2_BASIC) {
2571                 lightrec_rec_func_t f = rec_cp2_basic[c.r.rs];
2572
2573                 if (HAS_DEFAULT_ELM || likely(f)) {
2574                         (*f)(state, block, offset);
2575                         return;
2576                 }
2577         }
2578
2579         rec_CP(state, block, offset);
2580 }
2581
2582 void lightrec_rec_opcode(struct lightrec_cstate *state,
2583                          const struct block *block, u16 offset)
2584 {
2585         struct regcache *reg_cache = state->reg_cache;
2586         struct lightrec_branch_target *target;
2587         const struct opcode *op = &block->opcode_list[offset];
2588         jit_state_t *_jit = block->_jit;
2589         lightrec_rec_func_t f;
2590         u16 unload_offset;
2591
2592         if (op_flag_sync(op->flags)) {
2593                 if (state->cycles)
2594                         jit_subi(LIGHTREC_REG_CYCLE, LIGHTREC_REG_CYCLE, state->cycles);
2595                 state->cycles = 0;
2596
2597                 lightrec_storeback_regs(reg_cache, _jit);
2598                 lightrec_regcache_reset(reg_cache);
2599
2600                 pr_debug("Adding branch target at offset 0x%x\n", offset << 2);
2601                 target = &state->targets[state->nb_targets++];
2602                 target->offset = offset;
2603                 target->label = jit_indirect();
2604         }
2605
2606         if (likely(op->opcode)) {
2607                 f = rec_standard[op->i.op];
2608
2609                 if (!HAS_DEFAULT_ELM && unlikely(!f))
2610                         unknown_opcode(state, block, offset);
2611                 else
2612                         (*f)(state, block, offset);
2613         }
2614
2615         if (OPT_EARLY_UNLOAD) {
2616                 unload_offset = offset +
2617                         (has_delay_slot(op->c) && !op_flag_no_ds(op->flags));
2618
2619                 lightrec_do_early_unload(state, block, unload_offset);
2620         }
2621 }