deps/lightrec/emitter.c

   1 // SPDX-License-Identifier: LGPL-2.1-or-later
   2 /*
   3  * Copyright (C) 2014-2021 Paul Cercueil <paul@crapouillou.net>
   4  */
   5
   6 #include "blockcache.h"
   7 #include "debug.h"
   8 #include "disassembler.h"
   9 #include "emitter.h"
  10 #include "lightning-wrapper.h"
  11 #include "optimizer.h"
  12 #include "regcache.h"
  13
  14 #include <stdbool.h>
  15 #include <stddef.h>
  16
  17 typedef void (*lightrec_rec_func_t)(struct lightrec_cstate *, const struct block *, u16);
  18
  19 /* Forward declarations */
  20 static void rec_SPECIAL(struct lightrec_cstate *state, const struct block *block, u16 offset);
  21 static void rec_REGIMM(struct lightrec_cstate *state, const struct block *block, u16 offset);
  22 static void rec_CP0(struct lightrec_cstate *state, const struct block *block, u16 offset);
  23 static void rec_CP2(struct lightrec_cstate *state, const struct block *block, u16 offset);
  24 static void rec_cp2_do_mtc2(struct lightrec_cstate *state,
  25                             const struct block *block, u16 offset, u8 reg, u8 in_reg);
  26 static void rec_cp2_do_mfc2(struct lightrec_cstate *state,
  27                             const struct block *block, u16 offset,
  28                             u8 reg, u8 out_reg);
  29
  30 static void unknown_opcode(struct lightrec_cstate *state, const struct block *block, u16 offset)
  31 {
  32         pr_warn("Unknown opcode: 0x%08x at PC 0x%08x\n",
  33                 block->opcode_list[offset].c.opcode,
  34                 block->pc + (offset << 2));
  35 }
  36
  37 static void
  38 lightrec_jump_to_eob(struct lightrec_cstate *state, jit_state_t *_jit)
  39 {
  40         /* Prevent jit_jmpi() from using our cycles register as a temporary */
  41         jit_live(LIGHTREC_REG_CYCLE);
  42
  43         jit_patch_abs(jit_jmpi(), state->state->eob_wrapper_func);
  44 }
  45
  46 static void update_ra_register(struct regcache *reg_cache, jit_state_t *_jit,
  47                                u8 ra_reg, u32 pc, u32 link)
  48 {
  49         u8 link_reg;
  50
  51         link_reg = lightrec_alloc_reg_out(reg_cache, _jit, ra_reg, 0);
  52         lightrec_load_imm(reg_cache, _jit, link_reg, pc, link);
  53         lightrec_free_reg(reg_cache, link_reg);
  54 }
  55
  56 static void lightrec_emit_end_of_block(struct lightrec_cstate *state,
  57                                        const struct block *block, u16 offset,
  58                                        s8 reg_new_pc, u32 imm, u8 ra_reg,
  59                                        u32 link, bool update_cycles)
  60 {
  61         struct regcache *reg_cache = state->reg_cache;
  62         jit_state_t *_jit = block->_jit;
  63         const struct opcode *op = &block->opcode_list[offset],
  64                             *next = &block->opcode_list[offset + 1];
  65         u32 cycles = state->cycles + lightrec_cycles_of_opcode(op->c);
  66
  67         jit_note(__FILE__, __LINE__);
  68
  69         if (link && ra_reg != reg_new_pc)
  70                 update_ra_register(reg_cache, _jit, ra_reg, block->pc, link);
  71
  72         if (reg_new_pc < 0)
  73                 lightrec_load_next_pc_imm(reg_cache, _jit, block->pc, imm);
  74         else
  75                 lightrec_load_next_pc(reg_cache, _jit, reg_new_pc);
  76
  77         if (link && ra_reg == reg_new_pc) {
  78                 /* Handle the special case: JALR $r0, $r0
  79                  * In that case the target PC should be the old value of the
  80                  * register. */
  81                 update_ra_register(reg_cache, _jit, ra_reg, block->pc, link);
  82         }
  83
  84         if (has_delay_slot(op->c) &&
  85             !op_flag_no_ds(op->flags) && !op_flag_local_branch(op->flags)) {
  86                 cycles += lightrec_cycles_of_opcode(next->c);
  87
  88                 /* Recompile the delay slot */
  89                 if (next->c.opcode)
  90                         lightrec_rec_opcode(state, block, offset + 1);
  91         }
  92
  93         /* Clean the remaining registers */
  94         lightrec_clean_regs(reg_cache, _jit);
  95
  96         if (cycles && update_cycles) {
  97                 jit_subi(LIGHTREC_REG_CYCLE, LIGHTREC_REG_CYCLE, cycles);
  98                 pr_debug("EOB: %u cycles\n", cycles);
  99         }
 100
 101         lightrec_jump_to_eob(state, _jit);
 102 }
 103
 104 void lightrec_emit_eob(struct lightrec_cstate *state,
 105                        const struct block *block, u16 offset)
 106 {
 107         struct regcache *reg_cache = state->reg_cache;
 108         jit_state_t *_jit = block->_jit;
 109
 110         lightrec_clean_regs(reg_cache, _jit);
 111
 112         lightrec_load_imm(reg_cache, _jit, JIT_V0, block->pc,
 113                           block->pc + (offset << 2));
 114         jit_subi(LIGHTREC_REG_CYCLE, LIGHTREC_REG_CYCLE, state->cycles);
 115
 116         lightrec_jump_to_eob(state, _jit);
 117 }
 118
 119 static void rec_special_JR(struct lightrec_cstate *state, const struct block *block, u16 offset)
 120 {
 121         union code c = block->opcode_list[offset].c;
 122
 123         _jit_name(block->_jit, __func__);
 124         lightrec_emit_end_of_block(state, block, offset, c.r.rs, 0, 31, 0, true);
 125 }
 126
 127 static void rec_special_JALR(struct lightrec_cstate *state, const struct block *block, u16 offset)
 128 {
 129         union code c = block->opcode_list[offset].c;
 130
 131         _jit_name(block->_jit, __func__);
 132         lightrec_emit_end_of_block(state, block, offset, c.r.rs, 0, c.r.rd,
 133                                    get_branch_pc(block, offset, 2), true);
 134 }
 135
 136 static void rec_J(struct lightrec_cstate *state, const struct block *block, u16 offset)
 137 {
 138         union code c = block->opcode_list[offset].c;
 139
 140         _jit_name(block->_jit, __func__);
 141         lightrec_emit_end_of_block(state, block, offset, -1,
 142                                    (block->pc & 0xf0000000) | (c.j.imm << 2),
 143                                    31, 0, true);
 144 }
 145
 146 static void rec_JAL(struct lightrec_cstate *state, const struct block *block, u16 offset)
 147 {
 148         union code c = block->opcode_list[offset].c;
 149
 150         _jit_name(block->_jit, __func__);
 151         lightrec_emit_end_of_block(state, block, offset, -1,
 152                                    (block->pc & 0xf0000000) | (c.j.imm << 2),
 153                                    31, get_branch_pc(block, offset, 2), true);
 154 }
 155
 156 static void lightrec_do_early_unload(struct lightrec_cstate *state,
 157                                      const struct block *block, u16 offset)
 158 {
 159         struct regcache *reg_cache = state->reg_cache;
 160         const struct opcode *op = &block->opcode_list[offset];
 161         jit_state_t *_jit = block->_jit;
 162         unsigned int i;
 163         u8 reg;
 164         struct {
 165                 u8 reg, op;
 166         } reg_ops[3] = {
 167                 { op->r.rd, LIGHTREC_FLAGS_GET_RD(op->flags), },
 168                 { op->i.rt, LIGHTREC_FLAGS_GET_RT(op->flags), },
 169                 { op->i.rs, LIGHTREC_FLAGS_GET_RS(op->flags), },
 170         };
 171
 172         for (i = 0; i < ARRAY_SIZE(reg_ops); i++) {
 173                 reg = reg_ops[i].reg;
 174
 175                 switch (reg_ops[i].op) {
 176                 case LIGHTREC_REG_UNLOAD:
 177                         lightrec_clean_reg_if_loaded(reg_cache, _jit, reg, true);
 178                         break;
 179
 180                 case LIGHTREC_REG_DISCARD:
 181                         lightrec_discard_reg_if_loaded(reg_cache, reg);
 182                         break;
 183
 184                 case LIGHTREC_REG_CLEAN:
 185                         lightrec_clean_reg_if_loaded(reg_cache, _jit, reg, false);
 186                         break;
 187                 default:
 188                         break;
 189                 };
 190         }
 191 }
 192
 193 static void rec_b(struct lightrec_cstate *state, const struct block *block, u16 offset,
 194                   jit_code_t code, jit_code_t code2, u32 link, bool unconditional, bool bz)
 195 {
 196         struct regcache *reg_cache = state->reg_cache;
 197         struct native_register *regs_backup;
 198         jit_state_t *_jit = block->_jit;
 199         struct lightrec_branch *branch;
 200         const struct opcode *op = &block->opcode_list[offset],
 201                             *next = &block->opcode_list[offset + 1];
 202         jit_node_t *addr;
 203         bool is_forward = (s16)op->i.imm >= -1;
 204         int op_cycles = lightrec_cycles_of_opcode(op->c);
 205         u32 target_offset, cycles = state->cycles + op_cycles;
 206         bool no_indirection = false;
 207         u32 next_pc;
 208         u8 rs, rt;
 209
 210         jit_note(__FILE__, __LINE__);
 211
 212         if (!op_flag_no_ds(op->flags))
 213                 cycles += lightrec_cycles_of_opcode(next->c);
 214
 215         state->cycles = -op_cycles;
 216
 217         if (!unconditional) {
 218                 rs = lightrec_alloc_reg_in(reg_cache, _jit, op->i.rs, REG_EXT);
 219                 rt = bz ? 0 : lightrec_alloc_reg_in(reg_cache,
 220                                                     _jit, op->i.rt, REG_EXT);
 221
 222                 /* Unload dead registers before evaluating the branch */
 223                 if (OPT_EARLY_UNLOAD)
 224                         lightrec_do_early_unload(state, block, offset);
 225
 226                 if (op_flag_local_branch(op->flags) &&
 227                     (op_flag_no_ds(op->flags) || !next->opcode) &&
 228                     is_forward && !lightrec_has_dirty_regs(reg_cache))
 229                         no_indirection = true;
 230
 231                 if (no_indirection)
 232                         pr_debug("Using no indirection for branch at offset 0x%hx\n", offset << 2);
 233         }
 234
 235         if (cycles)
 236                 jit_subi(LIGHTREC_REG_CYCLE, LIGHTREC_REG_CYCLE, cycles);
 237
 238         if (!unconditional) {
 239                 /* Generate the branch opcode */
 240                 if (!no_indirection)
 241                         addr = jit_new_node_pww(code, NULL, rs, rt);
 242
 243                 lightrec_free_regs(reg_cache);
 244                 regs_backup = lightrec_regcache_enter_branch(reg_cache);
 245         }
 246
 247         if (op_flag_local_branch(op->flags)) {
 248                 /* Recompile the delay slot */
 249                 if (!op_flag_no_ds(op->flags) && next->opcode)
 250                         lightrec_rec_opcode(state, block, offset + 1);
 251
 252                 if (link)
 253                         update_ra_register(reg_cache, _jit, 31, block->pc, link);
 254
 255                 /* Clean remaining registers */
 256                 lightrec_clean_regs(reg_cache, _jit);
 257
 258                 target_offset = offset + 1 + (s16)op->i.imm
 259                         - !!op_flag_no_ds(op->flags);
 260                 pr_debug("Adding local branch to offset 0x%x\n",
 261                          target_offset << 2);
 262                 branch = &state->local_branches[
 263                         state->nb_local_branches++];
 264
 265                 branch->target = target_offset;
 266
 267                 if (no_indirection)
 268                         branch->branch = jit_new_node_pww(code2, NULL, rs, rt);
 269                 else if (is_forward)
 270                         branch->branch = jit_b();
 271                 else
 272                         branch->branch = jit_bgti(LIGHTREC_REG_CYCLE, 0);
 273         }
 274
 275         if (!op_flag_local_branch(op->flags) || !is_forward) {
 276                 next_pc = get_branch_pc(block, offset, 1 + (s16)op->i.imm);
 277                 lightrec_emit_end_of_block(state, block, offset, -1, next_pc,
 278                                            31, link, false);
 279         }
 280
 281         if (!unconditional) {
 282                 if (!no_indirection)
 283                         jit_patch(addr);
 284
 285                 lightrec_regcache_leave_branch(reg_cache, regs_backup);
 286
 287                 if (bz && link)
 288                         update_ra_register(reg_cache, _jit, 31, block->pc, link);
 289
 290                 if (!op_flag_no_ds(op->flags) && next->opcode)
 291                         lightrec_rec_opcode(state, block, offset + 1);
 292         }
 293 }
 294
 295 static void rec_BNE(struct lightrec_cstate *state,
 296                     const struct block *block, u16 offset)
 297 {
 298         union code c = block->opcode_list[offset].c;
 299
 300         _jit_name(block->_jit, __func__);
 301
 302         if (c.i.rt == 0)
 303                 rec_b(state, block, offset, jit_code_beqi, jit_code_bnei, 0, false, true);
 304         else
 305                 rec_b(state, block, offset, jit_code_beqr, jit_code_bner, 0, false, false);
 306 }
 307
 308 static void rec_BEQ(struct lightrec_cstate *state,
 309                     const struct block *block, u16 offset)
 310 {
 311         union code c = block->opcode_list[offset].c;
 312
 313         _jit_name(block->_jit, __func__);
 314
 315         if (c.i.rt == 0)
 316                 rec_b(state, block, offset, jit_code_bnei, jit_code_beqi, 0, c.i.rs == 0, true);
 317         else
 318                 rec_b(state, block, offset, jit_code_bner, jit_code_beqr, 0, c.i.rs == c.i.rt, false);
 319 }
 320
 321 static void rec_BLEZ(struct lightrec_cstate *state,
 322                      const struct block *block, u16 offset)
 323 {
 324         union code c = block->opcode_list[offset].c;
 325
 326         _jit_name(block->_jit, __func__);
 327         rec_b(state, block, offset, jit_code_bgti, jit_code_blei, 0, c.i.rs == 0, true);
 328 }
 329
 330 static void rec_BGTZ(struct lightrec_cstate *state,
 331                      const struct block *block, u16 offset)
 332 {
 333         _jit_name(block->_jit, __func__);
 334         rec_b(state, block, offset, jit_code_blei, jit_code_bgti, 0, false, true);
 335 }
 336
 337 static void rec_regimm_BLTZ(struct lightrec_cstate *state,
 338                             const struct block *block, u16 offset)
 339 {
 340         _jit_name(block->_jit, __func__);
 341         rec_b(state, block, offset, jit_code_bgei, jit_code_blti, 0, false, true);
 342 }
 343
 344 static void rec_regimm_BLTZAL(struct lightrec_cstate *state,
 345                               const struct block *block, u16 offset)
 346 {
 347         _jit_name(block->_jit, __func__);
 348         rec_b(state, block, offset, jit_code_bgei, jit_code_blti,
 349               get_branch_pc(block, offset, 2), false, true);
 350 }
 351
 352 static void rec_regimm_BGEZ(struct lightrec_cstate *state,
 353                             const struct block *block, u16 offset)
 354 {
 355         union code c = block->opcode_list[offset].c;
 356
 357         _jit_name(block->_jit, __func__);
 358         rec_b(state, block, offset, jit_code_blti, jit_code_bgei, 0, !c.i.rs, true);
 359 }
 360
 361 static void rec_regimm_BGEZAL(struct lightrec_cstate *state,
 362                               const struct block *block, u16 offset)
 363 {
 364         const struct opcode *op = &block->opcode_list[offset];
 365         _jit_name(block->_jit, __func__);
 366         rec_b(state, block, offset, jit_code_blti, jit_code_bgei,
 367               get_branch_pc(block, offset, 2),
 368               !op->i.rs, true);
 369 }
 370
 371 static void rec_alu_imm(struct lightrec_cstate *state, const struct block *block,
 372                         u16 offset, jit_code_t code, bool slti)
 373 {
 374         struct regcache *reg_cache = state->reg_cache;
 375         union code c = block->opcode_list[offset].c;
 376         jit_state_t *_jit = block->_jit;
 377         u8 rs, rt, out_flags = REG_EXT;
 378
 379         if (slti)
 380                 out_flags |= REG_ZEXT;
 381
 382         jit_note(__FILE__, __LINE__);
 383         rs = lightrec_alloc_reg_in(reg_cache, _jit, c.i.rs, REG_EXT);
 384         rt = lightrec_alloc_reg_out(reg_cache, _jit, c.i.rt, out_flags);
 385
 386         jit_new_node_www(code, rt, rs, (s32)(s16) c.i.imm);
 387
 388         lightrec_free_reg(reg_cache, rs);
 389         lightrec_free_reg(reg_cache, rt);
 390 }
 391
 392 static void rec_alu_special(struct lightrec_cstate *state, const struct block *block,
 393                             u16 offset, jit_code_t code, bool out_ext)
 394 {
 395         struct regcache *reg_cache = state->reg_cache;
 396         union code c = block->opcode_list[offset].c;
 397         jit_state_t *_jit = block->_jit;
 398         u8 rd, rt, rs;
 399
 400         jit_note(__FILE__, __LINE__);
 401         rs = lightrec_alloc_reg_in(reg_cache, _jit, c.r.rs, REG_EXT);
 402         rt = lightrec_alloc_reg_in(reg_cache, _jit, c.r.rt, REG_EXT);
 403         rd = lightrec_alloc_reg_out(reg_cache, _jit, c.r.rd,
 404                                     out_ext ? REG_EXT | REG_ZEXT : 0);
 405
 406         jit_new_node_www(code, rd, rs, rt);
 407
 408         lightrec_free_reg(reg_cache, rs);
 409         lightrec_free_reg(reg_cache, rt);
 410         lightrec_free_reg(reg_cache, rd);
 411 }
 412
 413 static void rec_alu_shiftv(struct lightrec_cstate *state, const struct block *block,
 414                            u16 offset, jit_code_t code)
 415 {
 416         struct regcache *reg_cache = state->reg_cache;
 417         union code c = block->opcode_list[offset].c;
 418         jit_state_t *_jit = block->_jit;
 419         u8 rd, rt, rs, temp, flags = 0;
 420
 421         jit_note(__FILE__, __LINE__);
 422         rs = lightrec_alloc_reg_in(reg_cache, _jit, c.r.rs, 0);
 423
 424         if (code == jit_code_rshr)
 425                 flags = REG_EXT;
 426         else if (code == jit_code_rshr_u)
 427                 flags = REG_ZEXT;
 428
 429         rt = lightrec_alloc_reg_in(reg_cache, _jit, c.r.rt, flags);
 430         rd = lightrec_alloc_reg_out(reg_cache, _jit, c.r.rd, flags);
 431
 432         if (rs != rd && rt != rd) {
 433                 jit_andi(rd, rs, 0x1f);
 434                 jit_new_node_www(code, rd, rt, rd);
 435         } else {
 436                 temp = lightrec_alloc_reg_temp(reg_cache, _jit);
 437                 jit_andi(temp, rs, 0x1f);
 438                 jit_new_node_www(code, rd, rt, temp);
 439                 lightrec_free_reg(reg_cache, temp);
 440         }
 441
 442         lightrec_free_reg(reg_cache, rs);
 443         lightrec_free_reg(reg_cache, rt);
 444         lightrec_free_reg(reg_cache, rd);
 445 }
 446
 447 static void rec_movi(struct lightrec_cstate *state,
 448                      const struct block *block, u16 offset)
 449 {
 450         struct regcache *reg_cache = state->reg_cache;
 451         union code c = block->opcode_list[offset].c;
 452         jit_state_t *_jit = block->_jit;
 453         u16 flags = REG_EXT;
 454         u8 rt;
 455
 456         if (!(c.i.imm & 0x8000))
 457                 flags |= REG_ZEXT;
 458
 459         rt = lightrec_alloc_reg_out(reg_cache, _jit, c.i.rt, flags);
 460
 461         jit_movi(rt, (s32)(s16) c.i.imm);
 462
 463         lightrec_free_reg(reg_cache, rt);
 464 }
 465
 466 static void rec_ADDIU(struct lightrec_cstate *state,
 467                       const struct block *block, u16 offset)
 468 {
 469         _jit_name(block->_jit, __func__);
 470
 471         if (block->opcode_list[offset].c.i.rs)
 472                 rec_alu_imm(state, block, offset, jit_code_addi, false);
 473         else
 474                 rec_movi(state, block, offset);
 475 }
 476
 477 static void rec_ADDI(struct lightrec_cstate *state,
 478                      const struct block *block, u16 offset)
 479 {
 480         /* TODO: Handle the exception? */
 481         _jit_name(block->_jit, __func__);
 482         rec_ADDIU(state, block, offset);
 483 }
 484
 485 static void rec_SLTIU(struct lightrec_cstate *state,
 486                       const struct block *block, u16 offset)
 487 {
 488         _jit_name(block->_jit, __func__);
 489         rec_alu_imm(state, block, offset, jit_code_lti_u, true);
 490 }
 491
 492 static void rec_SLTI(struct lightrec_cstate *state,
 493                      const struct block *block, u16 offset)
 494 {
 495         _jit_name(block->_jit, __func__);
 496         rec_alu_imm(state, block, offset, jit_code_lti, true);
 497 }
 498
 499 static void rec_ANDI(struct lightrec_cstate *state,
 500                      const struct block *block, u16 offset)
 501 {
 502         struct regcache *reg_cache = state->reg_cache;
 503         union code c = block->opcode_list[offset].c;
 504         jit_state_t *_jit = block->_jit;
 505         u8 rs, rt;
 506
 507         _jit_name(block->_jit, __func__);
 508         jit_note(__FILE__, __LINE__);
 509         rs = lightrec_alloc_reg_in(reg_cache, _jit, c.i.rs, 0);
 510         rt = lightrec_alloc_reg_out(reg_cache, _jit, c.i.rt,
 511                                     REG_EXT | REG_ZEXT);
 512
 513         /* PSX code uses ANDI 0xff / ANDI 0xffff a lot, which are basically
 514          * casts to uint8_t / uint16_t. */
 515         if (c.i.imm == 0xff)
 516                 jit_extr_uc(rt, rs);
 517         else if (c.i.imm == 0xffff)
 518                 jit_extr_us(rt, rs);
 519         else
 520                 jit_andi(rt, rs, (u32)(u16) c.i.imm);
 521
 522         lightrec_free_reg(reg_cache, rs);
 523         lightrec_free_reg(reg_cache, rt);
 524 }
 525
 526 static void rec_alu_or_xor(struct lightrec_cstate *state, const struct block *block,
 527                            u16 offset, jit_code_t code)
 528 {
 529         struct regcache *reg_cache = state->reg_cache;
 530         union code c = block->opcode_list[offset].c;
 531         jit_state_t *_jit = block->_jit;
 532         u8 rs, rt, flags;
 533
 534         jit_note(__FILE__, __LINE__);
 535         rs = lightrec_alloc_reg_in(reg_cache, _jit, c.i.rs, 0);
 536         rt = lightrec_alloc_reg_out(reg_cache, _jit, c.i.rt, 0);
 537
 538         flags = lightrec_get_reg_in_flags(reg_cache, rs);
 539         lightrec_set_reg_out_flags(reg_cache, rt, flags);
 540
 541         jit_new_node_www(code, rt, rs, (u32)(u16) c.i.imm);
 542
 543         lightrec_free_reg(reg_cache, rs);
 544         lightrec_free_reg(reg_cache, rt);
 545 }
 546
 547
 548 static void rec_ORI(struct lightrec_cstate *state,
 549                     const struct block *block, u16 offset)
 550 {
 551         _jit_name(block->_jit, __func__);
 552         rec_alu_or_xor(state, block, offset, jit_code_ori);
 553 }
 554
 555 static void rec_XORI(struct lightrec_cstate *state,
 556                      const struct block *block, u16 offset)
 557 {
 558         _jit_name(block->_jit, __func__);
 559         rec_alu_or_xor(state, block, offset, jit_code_xori);
 560 }
 561
 562 static void rec_LUI(struct lightrec_cstate *state,
 563                     const struct block *block, u16 offset)
 564 {
 565         struct regcache *reg_cache = state->reg_cache;
 566         union code c = block->opcode_list[offset].c;
 567         jit_state_t *_jit = block->_jit;
 568         u8 rt, flags = REG_EXT;
 569
 570         jit_name(__func__);
 571         jit_note(__FILE__, __LINE__);
 572
 573         if (!(c.i.imm & BIT(15)))
 574                 flags |= REG_ZEXT;
 575
 576         rt = lightrec_alloc_reg_out(reg_cache, _jit, c.i.rt, flags);
 577
 578         jit_movi(rt, (s32)(c.i.imm << 16));
 579
 580         lightrec_free_reg(reg_cache, rt);
 581 }
 582
 583 static void rec_special_ADDU(struct lightrec_cstate *state,
 584                              const struct block *block, u16 offset)
 585 {
 586         _jit_name(block->_jit, __func__);
 587         rec_alu_special(state, block, offset, jit_code_addr, false);
 588 }
 589
 590 static void rec_special_ADD(struct lightrec_cstate *state,
 591                             const struct block *block, u16 offset)
 592 {
 593         /* TODO: Handle the exception? */
 594         _jit_name(block->_jit, __func__);
 595         rec_alu_special(state, block, offset, jit_code_addr, false);
 596 }
 597
 598 static void rec_special_SUBU(struct lightrec_cstate *state,
 599                              const struct block *block, u16 offset)
 600 {
 601         _jit_name(block->_jit, __func__);
 602         rec_alu_special(state, block, offset, jit_code_subr, false);
 603 }
 604
 605 static void rec_special_SUB(struct lightrec_cstate *state,
 606                             const struct block *block, u16 offset)
 607 {
 608         /* TODO: Handle the exception? */
 609         _jit_name(block->_jit, __func__);
 610         rec_alu_special(state, block, offset, jit_code_subr, false);
 611 }
 612
 613 static void rec_special_AND(struct lightrec_cstate *state,
 614                             const struct block *block, u16 offset)
 615 {
 616         struct regcache *reg_cache = state->reg_cache;
 617         union code c = block->opcode_list[offset].c;
 618         jit_state_t *_jit = block->_jit;
 619         u8 rd, rt, rs, flags_rs, flags_rt, flags_rd;
 620
 621         _jit_name(block->_jit, __func__);
 622         jit_note(__FILE__, __LINE__);
 623         rs = lightrec_alloc_reg_in(reg_cache, _jit, c.r.rs, 0);
 624         rt = lightrec_alloc_reg_in(reg_cache, _jit, c.r.rt, 0);
 625         rd = lightrec_alloc_reg_out(reg_cache, _jit, c.r.rd, 0);
 626
 627         flags_rs = lightrec_get_reg_in_flags(reg_cache, rs);
 628         flags_rt = lightrec_get_reg_in_flags(reg_cache, rt);
 629
 630         /* Z(rd) = Z(rs) | Z(rt) */
 631         flags_rd = REG_ZEXT & (flags_rs | flags_rt);
 632
 633         /* E(rd) = (E(rt) & Z(rt)) | (E(rs) & Z(rs)) | (E(rs) & E(rt)) */
 634         if (((flags_rs & REG_EXT) && (flags_rt & REG_ZEXT)) ||
 635             ((flags_rt & REG_EXT) && (flags_rs & REG_ZEXT)) ||
 636             (REG_EXT & flags_rs & flags_rt))
 637                 flags_rd |= REG_EXT;
 638
 639         lightrec_set_reg_out_flags(reg_cache, rd, flags_rd);
 640
 641         jit_andr(rd, rs, rt);
 642
 643         lightrec_free_reg(reg_cache, rs);
 644         lightrec_free_reg(reg_cache, rt);
 645         lightrec_free_reg(reg_cache, rd);
 646 }
 647
 648 static void rec_special_or_nor(struct lightrec_cstate *state,
 649                                const struct block *block, u16 offset, bool nor)
 650 {
 651         struct regcache *reg_cache = state->reg_cache;
 652         union code c = block->opcode_list[offset].c;
 653         jit_state_t *_jit = block->_jit;
 654         u8 rd, rt, rs, flags_rs, flags_rt, flags_rd = 0;
 655
 656         jit_note(__FILE__, __LINE__);
 657         rs = lightrec_alloc_reg_in(reg_cache, _jit, c.r.rs, 0);
 658         rt = lightrec_alloc_reg_in(reg_cache, _jit, c.r.rt, 0);
 659         rd = lightrec_alloc_reg_out(reg_cache, _jit, c.r.rd, 0);
 660
 661         flags_rs = lightrec_get_reg_in_flags(reg_cache, rs);
 662         flags_rt = lightrec_get_reg_in_flags(reg_cache, rt);
 663
 664         /* or: Z(rd) = Z(rs) & Z(rt)
 665          * nor: Z(rd) = 0 */
 666         if (!nor)
 667                 flags_rd = REG_ZEXT & flags_rs & flags_rt;
 668
 669         /* E(rd) = E(rs) & E(rt) */
 670         if (REG_EXT & flags_rs & flags_rt)
 671                 flags_rd |= REG_EXT;
 672
 673         lightrec_set_reg_out_flags(reg_cache, rd, flags_rd);
 674
 675         jit_orr(rd, rs, rt);
 676
 677         if (nor)
 678                 jit_comr(rd, rd);
 679
 680         lightrec_free_reg(reg_cache, rs);
 681         lightrec_free_reg(reg_cache, rt);
 682         lightrec_free_reg(reg_cache, rd);
 683 }
 684
 685 static void rec_special_OR(struct lightrec_cstate *state,
 686                            const struct block *block, u16 offset)
 687 {
 688         _jit_name(block->_jit, __func__);
 689         rec_special_or_nor(state, block, offset, false);
 690 }
 691
 692 static void rec_special_NOR(struct lightrec_cstate *state,
 693                             const struct block *block, u16 offset)
 694 {
 695         _jit_name(block->_jit, __func__);
 696         rec_special_or_nor(state, block, offset, true);
 697 }
 698
 699 static void rec_special_XOR(struct lightrec_cstate *state,
 700                             const struct block *block, u16 offset)
 701 {
 702         struct regcache *reg_cache = state->reg_cache;
 703         union code c = block->opcode_list[offset].c;
 704         jit_state_t *_jit = block->_jit;
 705         u8 rd, rt, rs, flags_rs, flags_rt, flags_rd;
 706
 707         _jit_name(block->_jit, __func__);
 708
 709         jit_note(__FILE__, __LINE__);
 710         rs = lightrec_alloc_reg_in(reg_cache, _jit, c.r.rs, 0);
 711         rt = lightrec_alloc_reg_in(reg_cache, _jit, c.r.rt, 0);
 712         rd = lightrec_alloc_reg_out(reg_cache, _jit, c.r.rd, 0);
 713
 714         flags_rs = lightrec_get_reg_in_flags(reg_cache, rs);
 715         flags_rt = lightrec_get_reg_in_flags(reg_cache, rt);
 716
 717         /* Z(rd) = Z(rs) & Z(rt) */
 718         flags_rd = REG_ZEXT & flags_rs & flags_rt;
 719
 720         /* E(rd) = E(rs) & E(rt) */
 721         flags_rd |= REG_EXT & flags_rs & flags_rt;
 722
 723         lightrec_set_reg_out_flags(reg_cache, rd, flags_rd);
 724
 725         jit_xorr(rd, rs, rt);
 726
 727         lightrec_free_reg(reg_cache, rs);
 728         lightrec_free_reg(reg_cache, rt);
 729         lightrec_free_reg(reg_cache, rd);
 730 }
 731
 732 static void rec_special_SLTU(struct lightrec_cstate *state,
 733                              const struct block *block, u16 offset)
 734 {
 735         _jit_name(block->_jit, __func__);
 736         rec_alu_special(state, block, offset, jit_code_ltr_u, true);
 737 }
 738
 739 static void rec_special_SLT(struct lightrec_cstate *state,
 740                             const struct block *block, u16 offset)
 741 {
 742         _jit_name(block->_jit, __func__);
 743         rec_alu_special(state, block, offset, jit_code_ltr, true);
 744 }
 745
 746 static void rec_special_SLLV(struct lightrec_cstate *state,
 747                              const struct block *block, u16 offset)
 748 {
 749         _jit_name(block->_jit, __func__);
 750         rec_alu_shiftv(state, block, offset, jit_code_lshr);
 751 }
 752
 753 static void rec_special_SRLV(struct lightrec_cstate *state,
 754                              const struct block *block, u16 offset)
 755 {
 756         _jit_name(block->_jit, __func__);
 757         rec_alu_shiftv(state, block, offset, jit_code_rshr_u);
 758 }
 759
 760 static void rec_special_SRAV(struct lightrec_cstate *state,
 761                              const struct block *block, u16 offset)
 762 {
 763         _jit_name(block->_jit, __func__);
 764         rec_alu_shiftv(state, block, offset, jit_code_rshr);
 765 }
 766
 767 static void rec_alu_shift(struct lightrec_cstate *state, const struct block *block,
 768                           u16 offset, jit_code_t code)
 769 {
 770         struct regcache *reg_cache = state->reg_cache;
 771         union code c = block->opcode_list[offset].c;
 772         jit_state_t *_jit = block->_jit;
 773         u8 rd, rt, flags = 0;
 774
 775         jit_note(__FILE__, __LINE__);
 776
 777         if (code == jit_code_rshi)
 778                 flags = REG_EXT;
 779         else if (code == jit_code_rshi_u)
 780                 flags = REG_ZEXT;
 781
 782         rt = lightrec_alloc_reg_in(reg_cache, _jit, c.r.rt, flags);
 783
 784         /* Input reg is zero-extended, if we SRL at least by one bit, we know
 785          * the output reg will be both zero-extended and sign-extended. */
 786         if (code == jit_code_rshi_u && c.r.imm)
 787                 flags |= REG_EXT;
 788         rd = lightrec_alloc_reg_out(reg_cache, _jit, c.r.rd, flags);
 789
 790         jit_new_node_www(code, rd, rt, c.r.imm);
 791
 792         lightrec_free_reg(reg_cache, rt);
 793         lightrec_free_reg(reg_cache, rd);
 794 }
 795
 796 static void rec_special_SLL(struct lightrec_cstate *state,
 797                             const struct block *block, u16 offset)
 798 {
 799         _jit_name(block->_jit, __func__);
 800         rec_alu_shift(state, block, offset, jit_code_lshi);
 801 }
 802
 803 static void rec_special_SRL(struct lightrec_cstate *state,
 804                             const struct block *block, u16 offset)
 805 {
 806         _jit_name(block->_jit, __func__);
 807         rec_alu_shift(state, block, offset, jit_code_rshi_u);
 808 }
 809
 810 static void rec_special_SRA(struct lightrec_cstate *state,
 811                             const struct block *block, u16 offset)
 812 {
 813         _jit_name(block->_jit, __func__);
 814         rec_alu_shift(state, block, offset, jit_code_rshi);
 815 }
 816
 817 static void rec_alu_mult(struct lightrec_cstate *state,
 818                          const struct block *block, u16 offset, bool is_signed)
 819 {
 820         struct regcache *reg_cache = state->reg_cache;
 821         union code c = block->opcode_list[offset].c;
 822         u32 flags = block->opcode_list[offset].flags;
 823         u8 reg_lo = get_mult_div_lo(c);
 824         u8 reg_hi = get_mult_div_hi(c);
 825         jit_state_t *_jit = block->_jit;
 826         u8 lo, hi, rs, rt, rflags = 0;
 827
 828         jit_note(__FILE__, __LINE__);
 829
 830         if (is_signed)
 831                 rflags = REG_EXT;
 832         else
 833                 rflags = REG_ZEXT;
 834
 835         rs = lightrec_alloc_reg_in(reg_cache, _jit, c.r.rs, rflags);
 836         rt = lightrec_alloc_reg_in(reg_cache, _jit, c.r.rt, rflags);
 837
 838         if (!op_flag_no_lo(flags))
 839                 lo = lightrec_alloc_reg_out(reg_cache, _jit, reg_lo, 0);
 840         else if (__WORDSIZE == 32)
 841                 lo = lightrec_alloc_reg_temp(reg_cache, _jit);
 842
 843         if (!op_flag_no_hi(flags))
 844                 hi = lightrec_alloc_reg_out(reg_cache, _jit, reg_hi, REG_EXT);
 845
 846         if (__WORDSIZE == 32) {
 847                 /* On 32-bit systems, do a 32*32->64 bit operation, or a 32*32->32 bit
 848                  * operation if the MULT was detected a 32-bit only. */
 849                 if (!op_flag_no_hi(flags)) {
 850                         if (is_signed)
 851                                 jit_qmulr(lo, hi, rs, rt);
 852                         else
 853                                 jit_qmulr_u(lo, hi, rs, rt);
 854                 } else {
 855                         jit_mulr(lo, rs, rt);
 856                 }
 857         } else {
 858                 /* On 64-bit systems, do a 64*64->64 bit operation. */
 859                 if (op_flag_no_lo(flags)) {
 860                         jit_mulr(hi, rs, rt);
 861                         jit_rshi(hi, hi, 32);
 862                 } else {
 863                         jit_mulr(lo, rs, rt);
 864
 865                         /* The 64-bit output value is in $lo, store the upper 32 bits in $hi */
 866                         if (!op_flag_no_hi(flags))
 867                                 jit_rshi(hi, lo, 32);
 868                 }
 869         }
 870
 871         lightrec_free_reg(reg_cache, rs);
 872         lightrec_free_reg(reg_cache, rt);
 873         if (!op_flag_no_lo(flags) || __WORDSIZE == 32)
 874                 lightrec_free_reg(reg_cache, lo);
 875         if (!op_flag_no_hi(flags))
 876                 lightrec_free_reg(reg_cache, hi);
 877 }
 878
 879 static void rec_alu_div(struct lightrec_cstate *state,
 880                         const struct block *block, u16 offset, bool is_signed)
 881 {
 882         struct regcache *reg_cache = state->reg_cache;
 883         union code c = block->opcode_list[offset].c;
 884         u32 flags = block->opcode_list[offset].flags;
 885         bool no_check = op_flag_no_div_check(flags);
 886         u8 reg_lo = get_mult_div_lo(c);
 887         u8 reg_hi = get_mult_div_hi(c);
 888         jit_state_t *_jit = block->_jit;
 889         jit_node_t *branch, *to_end;
 890         u8 lo = 0, hi = 0, rs, rt, rflags = 0;
 891
 892         jit_note(__FILE__, __LINE__);
 893
 894         if (is_signed)
 895                 rflags = REG_EXT;
 896         else
 897                 rflags = REG_ZEXT;
 898
 899         rs = lightrec_alloc_reg_in(reg_cache, _jit, c.r.rs, rflags);
 900         rt = lightrec_alloc_reg_in(reg_cache, _jit, c.r.rt, rflags);
 901
 902         if (!op_flag_no_lo(flags))
 903                 lo = lightrec_alloc_reg_out(reg_cache, _jit, reg_lo, 0);
 904
 905         if (!op_flag_no_hi(flags))
 906                 hi = lightrec_alloc_reg_out(reg_cache, _jit, reg_hi, 0);
 907
 908         /* Jump to special handler if dividing by zero  */
 909         if (!no_check)
 910                 branch = jit_beqi(rt, 0);
 911
 912         if (op_flag_no_lo(flags)) {
 913                 if (is_signed)
 914                         jit_remr(hi, rs, rt);
 915                 else
 916                         jit_remr_u(hi, rs, rt);
 917         } else if (op_flag_no_hi(flags)) {
 918                 if (is_signed)
 919                         jit_divr(lo, rs, rt);
 920                 else
 921                         jit_divr_u(lo, rs, rt);
 922         } else {
 923                 if (is_signed)
 924                         jit_qdivr(lo, hi, rs, rt);
 925                 else
 926                         jit_qdivr_u(lo, hi, rs, rt);
 927         }
 928
 929         if (!no_check) {
 930                 /* Jump above the div-by-zero handler */
 931                 to_end = jit_b();
 932
 933                 jit_patch(branch);
 934
 935                 if (!op_flag_no_lo(flags)) {
 936                         if (is_signed) {
 937                                 jit_ltr(lo, rs, rt);
 938                                 jit_lshi(lo, lo, 1);
 939                                 jit_subi(lo, lo, 1);
 940                         } else {
 941                                 jit_subi(lo, rt, 1);
 942                         }
 943                 }
 944
 945                 if (!op_flag_no_hi(flags))
 946                         jit_movr(hi, rs);
 947
 948                 jit_patch(to_end);
 949         }
 950
 951         lightrec_free_reg(reg_cache, rs);
 952         lightrec_free_reg(reg_cache, rt);
 953
 954         if (!op_flag_no_lo(flags))
 955                 lightrec_free_reg(reg_cache, lo);
 956
 957         if (!op_flag_no_hi(flags))
 958                 lightrec_free_reg(reg_cache, hi);
 959 }
 960
 961 static void rec_special_MULT(struct lightrec_cstate *state,
 962                              const struct block *block, u16 offset)
 963 {
 964         _jit_name(block->_jit, __func__);
 965         rec_alu_mult(state, block, offset, true);
 966 }
 967
 968 static void rec_special_MULTU(struct lightrec_cstate *state,
 969                               const struct block *block, u16 offset)
 970 {
 971         _jit_name(block->_jit, __func__);
 972         rec_alu_mult(state, block, offset, false);
 973 }
 974
 975 static void rec_special_DIV(struct lightrec_cstate *state,
 976                             const struct block *block, u16 offset)
 977 {
 978         _jit_name(block->_jit, __func__);
 979         rec_alu_div(state, block, offset, true);
 980 }
 981
 982 static void rec_special_DIVU(struct lightrec_cstate *state,
 983                              const struct block *block, u16 offset)
 984 {
 985         _jit_name(block->_jit, __func__);
 986         rec_alu_div(state, block, offset, false);
 987 }
 988
 989 static void rec_alu_mv_lo_hi(struct lightrec_cstate *state,
 990                              const struct block *block, u8 dst, u8 src)
 991 {
 992         struct regcache *reg_cache = state->reg_cache;
 993         jit_state_t *_jit = block->_jit;
 994
 995         jit_note(__FILE__, __LINE__);
 996         src = lightrec_alloc_reg_in(reg_cache, _jit, src, 0);
 997         dst = lightrec_alloc_reg_out(reg_cache, _jit, dst, REG_EXT);
 998
 999         jit_extr_i(dst, src);
1000
1001         lightrec_free_reg(reg_cache, src);
1002         lightrec_free_reg(reg_cache, dst);
1003 }
1004
1005 static void rec_special_MFHI(struct lightrec_cstate *state,
1006                              const struct block *block, u16 offset)
1007 {
1008         union code c = block->opcode_list[offset].c;
1009
1010         _jit_name(block->_jit, __func__);
1011         rec_alu_mv_lo_hi(state, block, c.r.rd, REG_HI);
1012 }
1013
1014 static void rec_special_MTHI(struct lightrec_cstate *state,
1015                              const struct block *block, u16 offset)
1016 {
1017         union code c = block->opcode_list[offset].c;
1018
1019         _jit_name(block->_jit, __func__);
1020         rec_alu_mv_lo_hi(state, block, REG_HI, c.r.rs);
1021 }
1022
1023 static void rec_special_MFLO(struct lightrec_cstate *state,
1024                              const struct block *block, u16 offset)
1025 {
1026         union code c = block->opcode_list[offset].c;
1027
1028         _jit_name(block->_jit, __func__);
1029         rec_alu_mv_lo_hi(state, block, c.r.rd, REG_LO);
1030 }
1031
1032 static void rec_special_MTLO(struct lightrec_cstate *state,
1033                              const struct block *block, u16 offset)
1034 {
1035         union code c = block->opcode_list[offset].c;
1036
1037         _jit_name(block->_jit, __func__);
1038         rec_alu_mv_lo_hi(state, block, REG_LO, c.r.rs);
1039 }
1040
1041 static void call_to_c_wrapper(struct lightrec_cstate *state,
1042                               const struct block *block, u32 arg,
1043                               enum c_wrappers wrapper)
1044 {
1045         struct regcache *reg_cache = state->reg_cache;
1046         jit_state_t *_jit = block->_jit;
1047         s8 tmp, tmp2;
1048
1049         /* Make sure JIT_R1 is not mapped; it will be used in the C wrapper. */
1050         tmp2 = lightrec_alloc_reg(reg_cache, _jit, JIT_R1);
1051
1052         tmp = lightrec_get_reg_with_value(reg_cache,
1053                                           (intptr_t) state->state->wrappers_eps[wrapper]);
1054         if (tmp < 0) {
1055                 tmp = lightrec_alloc_reg_temp(reg_cache, _jit);
1056                 jit_ldxi(tmp, LIGHTREC_REG_STATE,
1057                          offsetof(struct lightrec_state, wrappers_eps[wrapper]));
1058
1059                 lightrec_temp_set_value(reg_cache, tmp,
1060                                         (intptr_t) state->state->wrappers_eps[wrapper]);
1061         }
1062
1063         lightrec_free_reg(reg_cache, tmp2);
1064
1065 #ifdef __mips__
1066         /* On MIPS, register t9 is always used as the target register for JALR.
1067          * Therefore if it does not contain the target address we must
1068          * invalidate it. */
1069         if (tmp != _T9)
1070                 lightrec_unload_reg(reg_cache, _jit, _T9);
1071 #endif
1072
1073         jit_prepare();
1074         jit_pushargi(arg);
1075
1076         lightrec_regcache_mark_live(reg_cache, _jit);
1077         jit_callr(tmp);
1078
1079         lightrec_free_reg(reg_cache, tmp);
1080         lightrec_regcache_mark_live(reg_cache, _jit);
1081 }
1082
1083 static void rec_io(struct lightrec_cstate *state,
1084                    const struct block *block, u16 offset,
1085                    bool load_rt, bool read_rt)
1086 {
1087         struct regcache *reg_cache = state->reg_cache;
1088         jit_state_t *_jit = block->_jit;
1089         union code c = block->opcode_list[offset].c;
1090         u32 flags = block->opcode_list[offset].flags;
1091         bool is_tagged = LIGHTREC_FLAGS_GET_IO_MODE(flags);
1092         u32 lut_entry;
1093
1094         jit_note(__FILE__, __LINE__);
1095
1096         lightrec_clean_reg_if_loaded(reg_cache, _jit, c.i.rs, false);
1097
1098         if (read_rt && likely(c.i.rt))
1099                 lightrec_clean_reg_if_loaded(reg_cache, _jit, c.i.rt, true);
1100         else if (load_rt)
1101                 lightrec_clean_reg_if_loaded(reg_cache, _jit, c.i.rt, false);
1102
1103         if (is_tagged) {
1104                 call_to_c_wrapper(state, block, c.opcode, C_WRAPPER_RW);
1105         } else {
1106                 lut_entry = lightrec_get_lut_entry(block);
1107                 call_to_c_wrapper(state, block, (lut_entry << 16) | offset,
1108                                   C_WRAPPER_RW_GENERIC);
1109         }
1110 }
1111
1112 static u32 rec_ram_mask(struct lightrec_state *state)
1113 {
1114         return (RAM_SIZE << (state->mirrors_mapped * 2)) - 1;
1115 }
1116
1117 static u32 rec_io_mask(const struct lightrec_state *state)
1118 {
1119         u32 length = state->maps[PSX_MAP_HW_REGISTERS].length;
1120
1121         return GENMASK(31 - clz32(length - 1), 0);
1122 }
1123
1124 static void rec_store_memory(struct lightrec_cstate *cstate,
1125                              const struct block *block,
1126                              u16 offset, jit_code_t code,
1127                              jit_code_t swap_code,
1128                              uintptr_t addr_offset, u32 addr_mask,
1129                              bool invalidate)
1130 {
1131         const struct lightrec_state *state = cstate->state;
1132         struct regcache *reg_cache = cstate->reg_cache;
1133         struct opcode *op = &block->opcode_list[offset];
1134         jit_state_t *_jit = block->_jit;
1135         union code c = op->c;
1136         u8 rs, rt, tmp, tmp2, tmp3, addr_reg, addr_reg2;
1137         s16 imm = (s16)c.i.imm;
1138         s32 simm = (s32)imm << (1 - lut_is_32bit(state));
1139         s32 lut_offt = offsetof(struct lightrec_state, code_lut);
1140         bool no_mask = op_flag_no_mask(op->flags);
1141         bool add_imm = c.i.imm &&
1142                 ((!state->mirrors_mapped && !no_mask) || (invalidate &&
1143                 ((imm & 0x3) || simm + lut_offt != (s16)(simm + lut_offt))));
1144         bool need_tmp = !no_mask || addr_offset || add_imm || invalidate;
1145         bool swc2 = c.i.op == OP_SWC2;
1146         u8 in_reg = swc2 ? REG_CP2_TEMP : c.i.rt;
1147
1148         rt = lightrec_alloc_reg_in(reg_cache, _jit, in_reg, 0);
1149         rs = lightrec_alloc_reg_in(reg_cache, _jit, c.i.rs, 0);
1150         if (need_tmp)
1151                 tmp = lightrec_alloc_reg_temp(reg_cache, _jit);
1152
1153         addr_reg = rs;
1154
1155         if (add_imm) {
1156                 jit_addi(tmp, addr_reg, (s16)c.i.imm);
1157                 addr_reg = tmp;
1158                 imm = 0;
1159         } else if (simm) {
1160                 lut_offt += simm;
1161         }
1162
1163         if (!no_mask) {
1164                 jit_andi(tmp, addr_reg, addr_mask);
1165                 addr_reg = tmp;
1166         }
1167
1168         if (addr_offset) {
1169                 tmp2 = lightrec_alloc_reg_temp(reg_cache, _jit);
1170                 jit_addi(tmp2, addr_reg, addr_offset);
1171                 addr_reg2 = tmp2;
1172         } else {
1173                 addr_reg2 = addr_reg;
1174         }
1175
1176         if (is_big_endian() && swap_code && in_reg) {
1177                 tmp3 = lightrec_alloc_reg_temp(reg_cache, _jit);
1178
1179                 jit_new_node_ww(swap_code, tmp3, rt);
1180                 jit_new_node_www(code, imm, addr_reg2, tmp3);
1181
1182                 lightrec_free_reg(reg_cache, tmp3);
1183         } else {
1184                 jit_new_node_www(code, imm, addr_reg2, rt);
1185         }
1186
1187         lightrec_free_reg(reg_cache, rt);
1188
1189         if (invalidate) {
1190                 tmp3 = lightrec_alloc_reg_in(reg_cache, _jit, 0, 0);
1191
1192                 if (c.i.op != OP_SW) {
1193                         jit_andi(tmp, addr_reg, ~3);
1194                         addr_reg = tmp;
1195                 }
1196
1197                 if (!lut_is_32bit(state)) {
1198                         jit_lshi(tmp, addr_reg, 1);
1199                         addr_reg = tmp;
1200                 }
1201
1202                 if (addr_reg == rs && c.i.rs == 0) {
1203                         addr_reg = LIGHTREC_REG_STATE;
1204                 } else {
1205                         jit_addr(tmp, addr_reg, LIGHTREC_REG_STATE);
1206                         addr_reg = tmp;
1207                 }
1208
1209                 if (lut_is_32bit(state))
1210                         jit_stxi_i(lut_offt, addr_reg, tmp3);
1211                 else
1212                         jit_stxi(lut_offt, addr_reg, tmp3);
1213
1214                 lightrec_free_reg(reg_cache, tmp3);
1215         }
1216
1217         if (addr_offset)
1218                 lightrec_free_reg(reg_cache, tmp2);
1219         if (need_tmp)
1220                 lightrec_free_reg(reg_cache, tmp);
1221         lightrec_free_reg(reg_cache, rs);
1222 }
1223
1224 static void rec_store_ram(struct lightrec_cstate *cstate,
1225                           const struct block *block,
1226                           u16 offset, jit_code_t code,
1227                           jit_code_t swap_code, bool invalidate)
1228 {
1229         struct lightrec_state *state = cstate->state;
1230
1231         _jit_note(block->_jit, __FILE__, __LINE__);
1232
1233         return rec_store_memory(cstate, block, offset, code, swap_code,
1234                                 state->offset_ram, rec_ram_mask(state),
1235                                 invalidate);
1236 }
1237
1238 static void rec_store_scratch(struct lightrec_cstate *cstate,
1239                               const struct block *block, u16 offset,
1240                               jit_code_t code, jit_code_t swap_code)
1241 {
1242         _jit_note(block->_jit, __FILE__, __LINE__);
1243
1244         return rec_store_memory(cstate, block, offset, code, swap_code,
1245                                 cstate->state->offset_scratch,
1246                                 0x1fffffff, false);
1247 }
1248
1249 static void rec_store_io(struct lightrec_cstate *cstate,
1250                          const struct block *block, u16 offset,
1251                          jit_code_t code, jit_code_t swap_code)
1252 {
1253         _jit_note(block->_jit, __FILE__, __LINE__);
1254
1255         return rec_store_memory(cstate, block, offset, code, swap_code,
1256                                 cstate->state->offset_io,
1257                                 rec_io_mask(cstate->state), false);
1258 }
1259
1260 static void rec_store_direct_no_invalidate(struct lightrec_cstate *cstate,
1261                                            const struct block *block,
1262                                            u16 offset, jit_code_t code,
1263                                            jit_code_t swap_code)
1264 {
1265         struct lightrec_state *state = cstate->state;
1266         struct regcache *reg_cache = cstate->reg_cache;
1267         union code c = block->opcode_list[offset].c;
1268         jit_state_t *_jit = block->_jit;
1269         jit_node_t *to_not_ram, *to_end;
1270         bool swc2 = c.i.op == OP_SWC2;
1271         u8 tmp, tmp2, rs, rt, in_reg = swc2 ? REG_CP2_TEMP : c.i.rt;
1272         s16 imm;
1273
1274         jit_note(__FILE__, __LINE__);
1275         rs = lightrec_alloc_reg_in(reg_cache, _jit, c.i.rs, 0);
1276         tmp = lightrec_alloc_reg_temp(reg_cache, _jit);
1277
1278         if (state->offset_ram || state->offset_scratch)
1279                 tmp2 = lightrec_alloc_reg_temp(reg_cache, _jit);
1280
1281         /* Convert to KUNSEG and avoid RAM mirrors */
1282         if (state->mirrors_mapped) {
1283                 imm = (s16)c.i.imm;
1284                 jit_andi(tmp, rs, 0x1f800000 | (4 * RAM_SIZE - 1));
1285         } else if (c.i.imm) {
1286                 imm = 0;
1287                 jit_addi(tmp, rs, (s16)c.i.imm);
1288                 jit_andi(tmp, tmp, 0x1f800000 | (RAM_SIZE - 1));
1289         } else {
1290                 imm = 0;
1291                 jit_andi(tmp, rs, 0x1f800000 | (RAM_SIZE - 1));
1292         }
1293
1294         lightrec_free_reg(reg_cache, rs);
1295
1296         if (state->offset_ram != state->offset_scratch) {
1297                 to_not_ram = jit_bmsi(tmp, BIT(28));
1298
1299                 jit_movi(tmp2, state->offset_ram);
1300
1301                 to_end = jit_b();
1302                 jit_patch(to_not_ram);
1303
1304                 jit_movi(tmp2, state->offset_scratch);
1305                 jit_patch(to_end);
1306         } else if (state->offset_ram) {
1307                 jit_movi(tmp2, state->offset_ram);
1308         }
1309
1310         if (state->offset_ram || state->offset_scratch) {
1311                 jit_addr(tmp, tmp, tmp2);
1312                 lightrec_free_reg(reg_cache, tmp2);
1313         }
1314
1315         rt = lightrec_alloc_reg_in(reg_cache, _jit, in_reg, 0);
1316
1317         if (is_big_endian() && swap_code && in_reg) {
1318                 tmp2 = lightrec_alloc_reg_temp(reg_cache, _jit);
1319
1320                 jit_new_node_ww(swap_code, tmp2, rt);
1321                 jit_new_node_www(code, imm, tmp, tmp2);
1322
1323                 lightrec_free_reg(reg_cache, tmp2);
1324         } else {
1325                 jit_new_node_www(code, imm, tmp, rt);
1326         }
1327
1328         lightrec_free_reg(reg_cache, rt);
1329         lightrec_free_reg(reg_cache, tmp);
1330 }
1331
1332 static void rec_store_direct(struct lightrec_cstate *cstate, const struct block *block,
1333                              u16 offset, jit_code_t code, jit_code_t swap_code)
1334 {
1335         struct lightrec_state *state = cstate->state;
1336         u32 ram_size = state->mirrors_mapped ? RAM_SIZE * 4 : RAM_SIZE;
1337         struct regcache *reg_cache = cstate->reg_cache;
1338         union code c = block->opcode_list[offset].c;
1339         jit_state_t *_jit = block->_jit;
1340         jit_node_t *to_not_ram, *to_end;
1341         bool swc2 = c.i.op == OP_SWC2;
1342         u8 tmp, tmp2, tmp3, masked_reg, rs, rt;
1343         u8 in_reg = swc2 ? REG_CP2_TEMP : c.i.rt;
1344
1345         jit_note(__FILE__, __LINE__);
1346
1347         rs = lightrec_alloc_reg_in(reg_cache, _jit, c.i.rs, 0);
1348         tmp2 = lightrec_alloc_reg_temp(reg_cache, _jit);
1349         tmp3 = lightrec_alloc_reg_in(reg_cache, _jit, 0, 0);
1350
1351         /* Convert to KUNSEG and avoid RAM mirrors */
1352         if (c.i.imm) {
1353                 jit_addi(tmp2, rs, (s16)c.i.imm);
1354                 jit_andi(tmp2, tmp2, 0x1f800000 | (ram_size - 1));
1355         } else {
1356                 jit_andi(tmp2, rs, 0x1f800000 | (ram_size - 1));
1357         }
1358
1359         lightrec_free_reg(reg_cache, rs);
1360         tmp = lightrec_alloc_reg_temp(reg_cache, _jit);
1361
1362         if (state->offset_ram != state->offset_scratch) {
1363                 to_not_ram = jit_bgti(tmp2, ram_size);
1364                 masked_reg = tmp2;
1365         } else {
1366                 jit_lti_u(tmp, tmp2, ram_size);
1367                 jit_movnr(tmp, tmp2, tmp);
1368                 masked_reg = tmp;
1369         }
1370
1371         /* Compute the offset to the code LUT */
1372         if (c.i.op == OP_SW)
1373                 jit_andi(tmp, masked_reg, RAM_SIZE - 1);
1374         else
1375                 jit_andi(tmp, masked_reg, (RAM_SIZE - 1) & ~3);
1376
1377         if (!lut_is_32bit(state))
1378                 jit_lshi(tmp, tmp, 1);
1379         jit_addr(tmp, LIGHTREC_REG_STATE, tmp);
1380
1381         /* Write NULL to the code LUT to invalidate any block that's there */
1382         if (lut_is_32bit(state))
1383                 jit_stxi_i(offsetof(struct lightrec_state, code_lut), tmp, tmp3);
1384         else
1385                 jit_stxi(offsetof(struct lightrec_state, code_lut), tmp, tmp3);
1386
1387         if (state->offset_ram != state->offset_scratch) {
1388                 jit_movi(tmp, state->offset_ram);
1389
1390                 to_end = jit_b();
1391                 jit_patch(to_not_ram);
1392         }
1393
1394         if (state->offset_ram || state->offset_scratch)
1395                 jit_movi(tmp, state->offset_scratch);
1396
1397         if (state->offset_ram != state->offset_scratch)
1398                 jit_patch(to_end);
1399
1400         if (state->offset_ram || state->offset_scratch)
1401                 jit_addr(tmp2, tmp2, tmp);
1402
1403         lightrec_free_reg(reg_cache, tmp);
1404         lightrec_free_reg(reg_cache, tmp3);
1405
1406         rt = lightrec_alloc_reg_in(reg_cache, _jit, in_reg, 0);
1407
1408         if (is_big_endian() && swap_code && in_reg) {
1409                 tmp = lightrec_alloc_reg_temp(reg_cache, _jit);
1410
1411                 jit_new_node_ww(swap_code, tmp, rt);
1412                 jit_new_node_www(code, 0, tmp2, tmp);
1413
1414                 lightrec_free_reg(reg_cache, tmp);
1415         } else {
1416                 jit_new_node_www(code, 0, tmp2, rt);
1417         }
1418
1419         lightrec_free_reg(reg_cache, rt);
1420         lightrec_free_reg(reg_cache, tmp2);
1421 }
1422
1423 static void rec_store(struct lightrec_cstate *state,
1424                       const struct block *block, u16 offset,
1425                       jit_code_t code, jit_code_t swap_code)
1426 {
1427         u32 flags = block->opcode_list[offset].flags;
1428         u32 mode = LIGHTREC_FLAGS_GET_IO_MODE(flags);
1429         bool no_invalidate = op_flag_no_invalidate(flags) ||
1430                 state->state->invalidate_from_dma_only;
1431         union code c = block->opcode_list[offset].c;
1432         bool is_swc2 = c.i.op == OP_SWC2;
1433
1434         if (is_swc2) {
1435                 switch (mode) {
1436                 case LIGHTREC_IO_RAM:
1437                 case LIGHTREC_IO_SCRATCH:
1438                 case LIGHTREC_IO_DIRECT:
1439                 case LIGHTREC_IO_DIRECT_HW:
1440                         rec_cp2_do_mfc2(state, block, offset, c.i.rt, REG_CP2_TEMP);
1441                         break;
1442                 default:
1443                         break;
1444                 }
1445         }
1446
1447         switch (mode) {
1448         case LIGHTREC_IO_RAM:
1449                 rec_store_ram(state, block, offset, code,
1450                               swap_code, !no_invalidate);
1451                 break;
1452         case LIGHTREC_IO_SCRATCH:
1453                 rec_store_scratch(state, block, offset, code, swap_code);
1454                 break;
1455         case LIGHTREC_IO_DIRECT:
1456                 if (no_invalidate) {
1457                         rec_store_direct_no_invalidate(state, block, offset,
1458                                                        code, swap_code);
1459                 } else {
1460                         rec_store_direct(state, block, offset, code, swap_code);
1461                 }
1462                 break;
1463         case LIGHTREC_IO_DIRECT_HW:
1464                 rec_store_io(state, block, offset, code, swap_code);
1465                 break;
1466         default:
1467                 rec_io(state, block, offset, true, false);
1468                 return;
1469         }
1470
1471         if (is_swc2)
1472                 lightrec_discard_reg_if_loaded(state->reg_cache, REG_CP2_TEMP);
1473 }
1474
1475 static void rec_SB(struct lightrec_cstate *state,
1476                    const struct block *block, u16 offset)
1477 {
1478         _jit_name(block->_jit, __func__);
1479         rec_store(state, block, offset, jit_code_stxi_c, 0);
1480 }
1481
1482 static void rec_SH(struct lightrec_cstate *state,
1483                    const struct block *block, u16 offset)
1484 {
1485         _jit_name(block->_jit, __func__);
1486         rec_store(state, block, offset,
1487                   jit_code_stxi_s, jit_code_bswapr_us);
1488 }
1489
1490 static void rec_SW(struct lightrec_cstate *state,
1491                    const struct block *block, u16 offset)
1492
1493 {
1494         union code c = block->opcode_list[offset].c;
1495
1496         _jit_name(block->_jit, c.i.op == OP_SWC2 ? "rec_SWC2" : "rec_SW");
1497         rec_store(state, block, offset,
1498                   jit_code_stxi_i, jit_code_bswapr_ui);
1499 }
1500
1501 static void rec_SWL(struct lightrec_cstate *state,
1502                     const struct block *block, u16 offset)
1503 {
1504         _jit_name(block->_jit, __func__);
1505         rec_io(state, block, offset, true, false);
1506 }
1507
1508 static void rec_SWR(struct lightrec_cstate *state,
1509                     const struct block *block, u16 offset)
1510 {
1511         _jit_name(block->_jit, __func__);
1512         rec_io(state, block, offset, true, false);
1513 }
1514
1515 static void rec_load_memory(struct lightrec_cstate *cstate,
1516                             const struct block *block, u16 offset,
1517                             jit_code_t code, jit_code_t swap_code, bool is_unsigned,
1518                             uintptr_t addr_offset, u32 addr_mask)
1519 {
1520         struct regcache *reg_cache = cstate->reg_cache;
1521         struct opcode *op = &block->opcode_list[offset];
1522         jit_state_t *_jit = block->_jit;
1523         u8 rs, rt, out_reg, addr_reg, flags = REG_EXT;
1524         bool no_mask = op_flag_no_mask(op->flags);
1525         union code c = op->c;
1526         s16 imm;
1527
1528         if (c.i.op == OP_LWC2)
1529                 out_reg = REG_CP2_TEMP;
1530         else if (c.i.rt)
1531                 out_reg = c.i.rt;
1532         else
1533                 return;
1534
1535         if (is_unsigned)
1536                 flags |= REG_ZEXT;
1537
1538         rs = lightrec_alloc_reg_in(reg_cache, _jit, c.i.rs, 0);
1539         rt = lightrec_alloc_reg_out(reg_cache, _jit, out_reg, flags);
1540
1541         if (!cstate->state->mirrors_mapped && c.i.imm && !no_mask) {
1542                 jit_addi(rt, rs, (s16)c.i.imm);
1543                 addr_reg = rt;
1544                 imm = 0;
1545         } else {
1546                 addr_reg = rs;
1547                 imm = (s16)c.i.imm;
1548         }
1549
1550         if (!no_mask) {
1551                 jit_andi(rt, addr_reg, addr_mask);
1552                 addr_reg = rt;
1553         }
1554
1555         if (addr_offset) {
1556                 jit_addi(rt, addr_reg, addr_offset);
1557                 addr_reg = rt;
1558         }
1559
1560         jit_new_node_www(code, rt, addr_reg, imm);
1561
1562         if (is_big_endian() && swap_code) {
1563                 jit_new_node_ww(swap_code, rt, rt);
1564
1565                 if (c.i.op == OP_LH)
1566                         jit_extr_s(rt, rt);
1567                 else if (c.i.op == OP_LW && __WORDSIZE == 64)
1568                         jit_extr_i(rt, rt);
1569         }
1570
1571         lightrec_free_reg(reg_cache, rs);
1572         lightrec_free_reg(reg_cache, rt);
1573 }
1574
1575 static void rec_load_ram(struct lightrec_cstate *cstate,
1576                          const struct block *block, u16 offset,
1577                          jit_code_t code, jit_code_t swap_code, bool is_unsigned)
1578 {
1579         _jit_note(block->_jit, __FILE__, __LINE__);
1580
1581         rec_load_memory(cstate, block, offset, code, swap_code, is_unsigned,
1582                         cstate->state->offset_ram, rec_ram_mask(cstate->state));
1583 }
1584
1585 static void rec_load_bios(struct lightrec_cstate *cstate,
1586                           const struct block *block, u16 offset,
1587                           jit_code_t code, jit_code_t swap_code, bool is_unsigned)
1588 {
1589         _jit_note(block->_jit, __FILE__, __LINE__);
1590
1591         rec_load_memory(cstate, block, offset, code, swap_code, is_unsigned,
1592                         cstate->state->offset_bios, 0x1fffffff);
1593 }
1594
1595 static void rec_load_scratch(struct lightrec_cstate *cstate,
1596                              const struct block *block, u16 offset,
1597                              jit_code_t code, jit_code_t swap_code, bool is_unsigned)
1598 {
1599         _jit_note(block->_jit, __FILE__, __LINE__);
1600
1601         rec_load_memory(cstate, block, offset, code, swap_code, is_unsigned,
1602                         cstate->state->offset_scratch, 0x1fffffff);
1603 }
1604
1605 static void rec_load_io(struct lightrec_cstate *cstate,
1606                         const struct block *block, u16 offset,
1607                         jit_code_t code, jit_code_t swap_code, bool is_unsigned)
1608 {
1609         _jit_note(block->_jit, __FILE__, __LINE__);
1610
1611         rec_load_memory(cstate, block, offset, code, swap_code, is_unsigned,
1612                         cstate->state->offset_io, rec_io_mask(cstate->state));
1613 }
1614
1615 static void rec_load_direct(struct lightrec_cstate *cstate,
1616                             const struct block *block, u16 offset,
1617                             jit_code_t code, jit_code_t swap_code,
1618                             bool is_unsigned)
1619 {
1620         struct lightrec_state *state = cstate->state;
1621         struct regcache *reg_cache = cstate->reg_cache;
1622         union code c = block->opcode_list[offset].c;
1623         jit_state_t *_jit = block->_jit;
1624         jit_node_t *to_not_ram, *to_not_bios, *to_end, *to_end2;
1625         u8 tmp, rs, rt, out_reg, addr_reg, flags = REG_EXT;
1626         s16 imm;
1627
1628         if (c.i.op == OP_LWC2)
1629                 out_reg = REG_CP2_TEMP;
1630         else if (c.i.rt)
1631                 out_reg = c.i.rt;
1632         else
1633                 return;
1634
1635         if (is_unsigned)
1636                 flags |= REG_ZEXT;
1637
1638         jit_note(__FILE__, __LINE__);
1639         rs = lightrec_alloc_reg_in(reg_cache, _jit, c.i.rs, 0);
1640         rt = lightrec_alloc_reg_out(reg_cache, _jit, out_reg, flags);
1641
1642         if ((state->offset_ram == state->offset_bios &&
1643             state->offset_ram == state->offset_scratch &&
1644             state->mirrors_mapped) || !c.i.imm) {
1645                 addr_reg = rs;
1646                 imm = (s16)c.i.imm;
1647         } else {
1648                 jit_addi(rt, rs, (s16)c.i.imm);
1649                 addr_reg = rt;
1650                 imm = 0;
1651
1652                 if (c.i.rs != c.i.rt)
1653                         lightrec_free_reg(reg_cache, rs);
1654         }
1655
1656         tmp = lightrec_alloc_reg_temp(reg_cache, _jit);
1657
1658         if (state->offset_ram == state->offset_bios &&
1659             state->offset_ram == state->offset_scratch) {
1660                 if (!state->mirrors_mapped) {
1661                         jit_andi(tmp, addr_reg, BIT(28));
1662                         jit_rshi_u(tmp, tmp, 28 - 22);
1663                         jit_ori(tmp, tmp, 0x1f800000 | (RAM_SIZE - 1));
1664                         jit_andr(rt, addr_reg, tmp);
1665                 } else {
1666                         jit_andi(rt, addr_reg, 0x1fffffff);
1667                 }
1668
1669                 if (state->offset_ram)
1670                         jit_movi(tmp, state->offset_ram);
1671         } else {
1672                 to_not_ram = jit_bmsi(addr_reg, BIT(28));
1673
1674                 /* Convert to KUNSEG and avoid RAM mirrors */
1675                 jit_andi(rt, addr_reg, RAM_SIZE - 1);
1676
1677                 if (state->offset_ram)
1678                         jit_movi(tmp, state->offset_ram);
1679
1680                 to_end = jit_b();
1681
1682                 jit_patch(to_not_ram);
1683
1684                 if (state->offset_bios != state->offset_scratch)
1685                         to_not_bios = jit_bmci(addr_reg, BIT(22));
1686
1687                 /* Convert to KUNSEG */
1688                 jit_andi(rt, addr_reg, 0x1fc00000 | (BIOS_SIZE - 1));
1689
1690                 jit_movi(tmp, state->offset_bios);
1691
1692                 if (state->offset_bios != state->offset_scratch) {
1693                         to_end2 = jit_b();
1694
1695                         jit_patch(to_not_bios);
1696
1697                         /* Convert to KUNSEG */
1698                         jit_andi(rt, addr_reg, 0x1f800fff);
1699
1700                         if (state->offset_scratch)
1701                                 jit_movi(tmp, state->offset_scratch);
1702
1703                         jit_patch(to_end2);
1704                 }
1705
1706                 jit_patch(to_end);
1707         }
1708
1709         if (state->offset_ram || state->offset_bios || state->offset_scratch)
1710                 jit_addr(rt, rt, tmp);
1711
1712         jit_new_node_www(code, rt, rt, imm);
1713
1714         if (is_big_endian() && swap_code) {
1715                 jit_new_node_ww(swap_code, rt, rt);
1716
1717                 if (c.i.op == OP_LH)
1718                         jit_extr_s(rt, rt);
1719                 else if (c.i.op == OP_LW && __WORDSIZE == 64)
1720                         jit_extr_i(rt, rt);
1721         }
1722
1723         lightrec_free_reg(reg_cache, addr_reg);
1724         lightrec_free_reg(reg_cache, rt);
1725         lightrec_free_reg(reg_cache, tmp);
1726 }
1727
1728 static void rec_load(struct lightrec_cstate *state, const struct block *block,
1729                      u16 offset, jit_code_t code, jit_code_t swap_code,
1730                      bool is_unsigned)
1731 {
1732         const struct opcode *op = &block->opcode_list[offset];
1733         u32 flags = op->flags;
1734
1735         switch (LIGHTREC_FLAGS_GET_IO_MODE(flags)) {
1736         case LIGHTREC_IO_RAM:
1737                 rec_load_ram(state, block, offset, code, swap_code, is_unsigned);
1738                 break;
1739         case LIGHTREC_IO_BIOS:
1740                 rec_load_bios(state, block, offset, code, swap_code, is_unsigned);
1741                 break;
1742         case LIGHTREC_IO_SCRATCH:
1743                 rec_load_scratch(state, block, offset, code, swap_code, is_unsigned);
1744                 break;
1745         case LIGHTREC_IO_DIRECT_HW:
1746                 rec_load_io(state, block, offset, code, swap_code, is_unsigned);
1747                 break;
1748         case LIGHTREC_IO_DIRECT:
1749                 rec_load_direct(state, block, offset, code, swap_code, is_unsigned);
1750                 break;
1751         default:
1752                 rec_io(state, block, offset, false, true);
1753                 return;
1754         }
1755
1756         if (op->i.op == OP_LWC2) {
1757                 rec_cp2_do_mtc2(state, block, offset, op->i.rt, REG_CP2_TEMP);
1758                 lightrec_discard_reg_if_loaded(state->reg_cache, REG_CP2_TEMP);
1759         }
1760 }
1761
1762 static void rec_LB(struct lightrec_cstate *state, const struct block *block, u16 offset)
1763 {
1764         _jit_name(block->_jit, __func__);
1765         rec_load(state, block, offset, jit_code_ldxi_c, 0, false);
1766 }
1767
1768 static void rec_LBU(struct lightrec_cstate *state, const struct block *block, u16 offset)
1769 {
1770         _jit_name(block->_jit, __func__);
1771         rec_load(state, block, offset, jit_code_ldxi_uc, 0, true);
1772 }
1773
1774 static void rec_LH(struct lightrec_cstate *state, const struct block *block, u16 offset)
1775 {
1776         jit_code_t code = is_big_endian() ? jit_code_ldxi_us : jit_code_ldxi_s;
1777
1778         _jit_name(block->_jit, __func__);
1779         rec_load(state, block, offset, code, jit_code_bswapr_us, false);
1780 }
1781
1782 static void rec_LHU(struct lightrec_cstate *state, const struct block *block, u16 offset)
1783 {
1784         _jit_name(block->_jit, __func__);
1785         rec_load(state, block, offset, jit_code_ldxi_us, jit_code_bswapr_us, true);
1786 }
1787
1788 static void rec_LWL(struct lightrec_cstate *state, const struct block *block, u16 offset)
1789 {
1790         _jit_name(block->_jit, __func__);
1791         rec_io(state, block, offset, true, true);
1792 }
1793
1794 static void rec_LWR(struct lightrec_cstate *state, const struct block *block, u16 offset)
1795 {
1796         _jit_name(block->_jit, __func__);
1797         rec_io(state, block, offset, true, true);
1798 }
1799
1800 static void rec_LW(struct lightrec_cstate *state, const struct block *block, u16 offset)
1801 {
1802         union code c = block->opcode_list[offset].c;
1803         jit_code_t code;
1804
1805         if (is_big_endian() && __WORDSIZE == 64)
1806                 code = jit_code_ldxi_ui;
1807         else
1808                 code = jit_code_ldxi_i;
1809
1810         _jit_name(block->_jit, c.i.op == OP_LWC2 ? "rec_LWC2" : "rec_LW");
1811         rec_load(state, block, offset, code, jit_code_bswapr_ui, false);
1812 }
1813
1814 static void rec_break_syscall(struct lightrec_cstate *state,
1815                               const struct block *block, u16 offset,
1816                               u32 exit_code)
1817 {
1818         struct regcache *reg_cache = state->reg_cache;
1819         jit_state_t *_jit = block->_jit;
1820         u8 tmp;
1821
1822         _jit_note(block->_jit, __FILE__, __LINE__);
1823
1824         tmp = lightrec_alloc_reg_temp(reg_cache, _jit);
1825
1826         jit_movi(tmp, exit_code);
1827         jit_stxi_i(offsetof(struct lightrec_state, exit_flags),
1828                    LIGHTREC_REG_STATE, tmp);
1829
1830         lightrec_free_reg(reg_cache, tmp);
1831
1832         /* TODO: the return address should be "pc - 4" if we're a delay slot */
1833         lightrec_emit_end_of_block(state, block, offset, -1,
1834                                    get_ds_pc(block, offset, 0),
1835                                    31, 0, true);
1836 }
1837
1838 static void rec_special_SYSCALL(struct lightrec_cstate *state,
1839                                 const struct block *block, u16 offset)
1840 {
1841         _jit_name(block->_jit, __func__);
1842         rec_break_syscall(state, block, offset, LIGHTREC_EXIT_SYSCALL);
1843 }
1844
1845 static void rec_special_BREAK(struct lightrec_cstate *state,
1846                               const struct block *block, u16 offset)
1847 {
1848         _jit_name(block->_jit, __func__);
1849         rec_break_syscall(state, block, offset, LIGHTREC_EXIT_BREAK);
1850 }
1851
1852 static void rec_mfc(struct lightrec_cstate *state, const struct block *block, u16 offset)
1853 {
1854         struct regcache *reg_cache = state->reg_cache;
1855         union code c = block->opcode_list[offset].c;
1856         jit_state_t *_jit = block->_jit;
1857
1858         jit_note(__FILE__, __LINE__);
1859
1860         if (c.i.op != OP_SWC2)
1861                 lightrec_clean_reg_if_loaded(reg_cache, _jit, c.i.rt, true);
1862
1863         call_to_c_wrapper(state, block, c.opcode, C_WRAPPER_MFC);
1864 }
1865
1866 static void rec_mtc(struct lightrec_cstate *state, const struct block *block, u16 offset)
1867 {
1868         struct regcache *reg_cache = state->reg_cache;
1869         union code c = block->opcode_list[offset].c;
1870         jit_state_t *_jit = block->_jit;
1871
1872         jit_note(__FILE__, __LINE__);
1873         lightrec_clean_reg_if_loaded(reg_cache, _jit, c.i.rs, false);
1874         lightrec_clean_reg_if_loaded(reg_cache, _jit, c.i.rt, false);
1875
1876         call_to_c_wrapper(state, block, c.opcode, C_WRAPPER_MTC);
1877
1878         if (c.i.op == OP_CP0 &&
1879             !op_flag_no_ds(block->opcode_list[offset].flags) &&
1880             (c.r.rd == 12 || c.r.rd == 13))
1881                 lightrec_emit_end_of_block(state, block, offset, -1,
1882                                            get_ds_pc(block, offset, 1),
1883                                            0, 0, true);
1884 }
1885
1886 static void
1887 rec_mfc0(struct lightrec_cstate *state, const struct block *block, u16 offset)
1888 {
1889         struct regcache *reg_cache = state->reg_cache;
1890         union code c = block->opcode_list[offset].c;
1891         jit_state_t *_jit = block->_jit;
1892         u8 rt;
1893
1894         jit_note(__FILE__, __LINE__);
1895
1896         rt = lightrec_alloc_reg_out(reg_cache, _jit, c.i.rt, REG_EXT);
1897
1898         jit_ldxi_i(rt, LIGHTREC_REG_STATE,
1899                    offsetof(struct lightrec_state, regs.cp0[c.r.rd]));
1900
1901         lightrec_free_reg(reg_cache, rt);
1902 }
1903
1904 static bool block_in_bios(const struct lightrec_cstate *state,
1905                           const struct block *block)
1906 {
1907         const struct lightrec_mem_map *bios = &state->state->maps[PSX_MAP_BIOS];
1908         u32 pc = kunseg(block->pc);
1909
1910         return pc >= bios->pc && pc < bios->pc + bios->length;
1911 }
1912
1913 static void
1914 rec_mtc0(struct lightrec_cstate *state, const struct block *block, u16 offset)
1915 {
1916         struct regcache *reg_cache = state->reg_cache;
1917         const union code c = block->opcode_list[offset].c;
1918         jit_state_t *_jit = block->_jit;
1919         u8 rt, tmp = 0, tmp2, status;
1920         jit_node_t *to_end;
1921
1922         jit_note(__FILE__, __LINE__);
1923
1924         switch(c.r.rd) {
1925         case 1:
1926         case 4:
1927         case 8:
1928         case 14:
1929         case 15:
1930                 /* Those registers are read-only */
1931                 return;
1932         default:
1933                 break;
1934         }
1935
1936         if (/*block_in_bios(state, block) &&*/ c.r.rd == 12) {
1937                 /* If we are running code from the BIOS, handle writes to the
1938                  * Status register in C. BIOS code may toggle bit 16 which will
1939                  * map/unmap the RAM, while game code cannot do that. */
1940                 /*  ^ wrong, it can execute from 0xa0000000 with isolated cache */
1941                 rec_mtc(state, block, offset);
1942                 return;
1943         }
1944
1945         rt = lightrec_alloc_reg_in(reg_cache, _jit, c.i.rt, 0);
1946
1947         if (c.r.rd != 13) {
1948                 jit_stxi_i(offsetof(struct lightrec_state, regs.cp0[c.r.rd]),
1949                            LIGHTREC_REG_STATE, rt);
1950         }
1951
1952         if (c.r.rd == 12 || c.r.rd == 13) {
1953                 tmp = lightrec_alloc_reg_temp(reg_cache, _jit);
1954                 jit_ldxi_i(tmp, LIGHTREC_REG_STATE,
1955                            offsetof(struct lightrec_state, regs.cp0[13]));
1956
1957                 tmp2 = lightrec_alloc_reg_temp(reg_cache, _jit);
1958         }
1959
1960         if (c.r.rd == 12) {
1961                 status = rt;
1962         } else if (c.r.rd == 13) {
1963                 /* Cause = (Cause & ~0x0300) | (value & 0x0300) */
1964                 jit_andi(tmp2, rt, 0x0300);
1965                 jit_ori(tmp, tmp, 0x0300);
1966                 jit_xori(tmp, tmp, 0x0300);
1967                 jit_orr(tmp, tmp, tmp2);
1968                 jit_ldxi_i(tmp2, LIGHTREC_REG_STATE,
1969                            offsetof(struct lightrec_state, regs.cp0[12]));
1970                 jit_stxi_i(offsetof(struct lightrec_state, regs.cp0[13]),
1971                            LIGHTREC_REG_STATE, tmp);
1972                 status = tmp2;
1973         }
1974
1975         if (c.r.rd == 12 || c.r.rd == 13) {
1976                 /* Exit dynarec in case there's a software interrupt.
1977                  * exit_flags = !!(status & tmp & 0x0300) & status; */
1978                 jit_andr(tmp, tmp, status);
1979                 jit_andi(tmp, tmp, 0x0300);
1980                 jit_nei(tmp, tmp, 0);
1981                 jit_andr(tmp, tmp, status);
1982         }
1983
1984         if (c.r.rd == 12) {
1985                 /* Exit dynarec in case we unmask a hardware interrupt.
1986                  * exit_flags = !(~status & 0x401) */
1987
1988                 jit_comr(tmp2, status);
1989                 jit_andi(tmp2, tmp2, 0x401);
1990                 jit_eqi(tmp2, tmp2, 0);
1991                 jit_orr(tmp, tmp, tmp2);
1992         }
1993
1994         lightrec_free_reg(reg_cache, rt);
1995
1996         if (c.r.rd == 12 || c.r.rd == 13) {
1997                 to_end = jit_beqi(tmp, 0);
1998
1999                 jit_ldxi_i(tmp2, LIGHTREC_REG_STATE,
2000                            offsetof(struct lightrec_state, target_cycle));
2001                 jit_subr(tmp2, tmp2, LIGHTREC_REG_CYCLE);
2002                 jit_movi(LIGHTREC_REG_CYCLE, 0);
2003                 jit_stxi_i(offsetof(struct lightrec_state, target_cycle),
2004                            LIGHTREC_REG_STATE, tmp2);
2005                 jit_stxi_i(offsetof(struct lightrec_state, current_cycle),
2006                            LIGHTREC_REG_STATE, tmp2);
2007
2008
2009                 jit_patch(to_end);
2010         }
2011
2012         if (!op_flag_no_ds(block->opcode_list[offset].flags) &&
2013             (c.r.rd == 12 || c.r.rd == 13)) {
2014                 state->cycles += lightrec_cycles_of_opcode(c);
2015                 lightrec_emit_eob(state, block, offset + 1);
2016         }
2017 }
2018
2019 static void rec_cp0_MFC0(struct lightrec_cstate *state,
2020                          const struct block *block, u16 offset)
2021 {
2022         _jit_name(block->_jit, __func__);
2023         rec_mfc0(state, block, offset);
2024 }
2025
2026 static void rec_cp0_CFC0(struct lightrec_cstate *state,
2027                          const struct block *block, u16 offset)
2028 {
2029         _jit_name(block->_jit, __func__);
2030         rec_mfc0(state, block, offset);
2031 }
2032
2033 static void rec_cp0_MTC0(struct lightrec_cstate *state,
2034                          const struct block *block, u16 offset)
2035 {
2036         _jit_name(block->_jit, __func__);
2037         rec_mtc0(state, block, offset);
2038 }
2039
2040 static void rec_cp0_CTC0(struct lightrec_cstate *state,
2041                          const struct block *block, u16 offset)
2042 {
2043         _jit_name(block->_jit, __func__);
2044         rec_mtc0(state, block, offset);
2045 }
2046
2047 static unsigned int cp2d_i_offset(u8 reg)
2048 {
2049         return offsetof(struct lightrec_state, regs.cp2d[reg]);
2050 }
2051
2052 static unsigned int cp2d_s_offset(u8 reg)
2053 {
2054         return cp2d_i_offset(reg) + is_big_endian() * 2;
2055 }
2056
2057 static unsigned int cp2c_i_offset(u8 reg)
2058 {
2059         return offsetof(struct lightrec_state, regs.cp2c[reg]);
2060 }
2061
2062 static unsigned int cp2c_s_offset(u8 reg)
2063 {
2064         return cp2c_i_offset(reg) + is_big_endian() * 2;
2065 }
2066
2067 static void rec_cp2_do_mfc2(struct lightrec_cstate *state,
2068                             const struct block *block, u16 offset,
2069                             u8 reg, u8 out_reg)
2070 {
2071         struct regcache *reg_cache = state->reg_cache;
2072         jit_state_t *_jit = block->_jit;
2073         const u32 zext_regs = 0x300f0080;
2074         u8 rt, tmp, tmp2, tmp3, out, flags;
2075         unsigned int i;
2076
2077         _jit_name(block->_jit, __func__);
2078
2079         if (state->state->ops.cop2_notify) {
2080                 /* We must call cop2_notify, handle that in C. */
2081                 rec_mfc(state, block, offset);
2082                 return;
2083         }
2084
2085         flags = (zext_regs & BIT(reg)) ? REG_ZEXT : REG_EXT;
2086         rt = lightrec_alloc_reg_out(reg_cache, _jit, out_reg, flags);
2087
2088         if (reg == 15)
2089                 reg = 14;
2090
2091         switch (reg) {
2092         case 1:
2093         case 3:
2094         case 5:
2095         case 8:
2096         case 9:
2097         case 10:
2098         case 11:
2099                 jit_ldxi_s(rt, LIGHTREC_REG_STATE, cp2d_s_offset(reg));
2100                 break;
2101         case 7:
2102         case 16:
2103         case 17:
2104         case 18:
2105         case 19:
2106                 jit_ldxi_us(rt, LIGHTREC_REG_STATE, cp2d_s_offset(reg));
2107                 break;
2108         case 28:
2109         case 29:
2110                 tmp = lightrec_alloc_reg_temp(reg_cache, _jit);
2111                 tmp2 = lightrec_alloc_reg_temp(reg_cache, _jit);
2112                 tmp3 = lightrec_alloc_reg_temp(reg_cache, _jit);
2113
2114                 for (i = 0; i < 3; i++) {
2115                         out = i == 0 ? rt : tmp;
2116
2117                         jit_ldxi_s(tmp, LIGHTREC_REG_STATE, cp2d_s_offset(9 + i));
2118                         jit_movi(tmp2, 0x1f);
2119                         jit_rshi(out, tmp, 7);
2120
2121                         jit_ltr(tmp3, tmp2, out);
2122                         jit_movnr(out, tmp2, tmp3);
2123
2124                         jit_gei(tmp2, out, 0);
2125                         jit_movzr(out, tmp2, tmp2);
2126
2127                         if (i > 0) {
2128                                 jit_lshi(tmp, tmp, 5 * i);
2129                                 jit_orr(rt, rt, tmp);
2130                         }
2131                 }
2132
2133
2134                 lightrec_free_reg(reg_cache, tmp);
2135                 lightrec_free_reg(reg_cache, tmp2);
2136                 lightrec_free_reg(reg_cache, tmp3);
2137                 break;
2138         default:
2139                 jit_ldxi_i(rt, LIGHTREC_REG_STATE, cp2d_i_offset(reg));
2140                 break;
2141         }
2142
2143         lightrec_free_reg(reg_cache, rt);
2144 }
2145
2146 static void rec_cp2_basic_MFC2(struct lightrec_cstate *state,
2147                                const struct block *block, u16 offset)
2148 {
2149         const union code c = block->opcode_list[offset].c;
2150
2151         rec_cp2_do_mfc2(state, block, offset, c.r.rd, c.r.rt);
2152 }
2153
2154 static void rec_cp2_basic_CFC2(struct lightrec_cstate *state,
2155                                const struct block *block, u16 offset)
2156 {
2157         struct regcache *reg_cache = state->reg_cache;
2158         const union code c = block->opcode_list[offset].c;
2159         jit_state_t *_jit = block->_jit;
2160         u8 rt;
2161
2162         _jit_name(block->_jit, __func__);
2163
2164         if (state->state->ops.cop2_notify) {
2165                 /* We must call cop2_notify, handle that in C. */
2166                 rec_mfc(state, block, offset);
2167                 return;
2168         }
2169
2170         switch (c.r.rd) {
2171         case 4:
2172         case 12:
2173         case 20:
2174         case 26:
2175         case 27:
2176         case 29:
2177         case 30:
2178                 rt = lightrec_alloc_reg_out(reg_cache, _jit, c.r.rt, REG_EXT);
2179                 jit_ldxi_s(rt, LIGHTREC_REG_STATE, cp2c_s_offset(c.r.rd));
2180                 break;
2181         default:
2182                 rt = lightrec_alloc_reg_out(reg_cache, _jit, c.r.rt, REG_ZEXT);
2183                 jit_ldxi_ui(rt, LIGHTREC_REG_STATE, cp2c_i_offset(c.r.rd));
2184                 break;
2185         }
2186
2187         lightrec_free_reg(reg_cache, rt);
2188 }
2189
2190 static void rec_cp2_do_mtc2(struct lightrec_cstate *state,
2191                             const struct block *block, u16 offset,
2192                             u8 reg, u8 in_reg)
2193 {
2194         struct regcache *reg_cache = state->reg_cache;
2195         jit_state_t *_jit = block->_jit;
2196         jit_node_t *loop, *to_loop;
2197         u8 rt, tmp, tmp2, flags = 0;
2198
2199         _jit_name(block->_jit, __func__);
2200
2201         if (state->state->ops.cop2_notify) {
2202                 /* We must call cop2_notify, handle that in C. */
2203                 rec_mtc(state, block, offset);
2204                 return;
2205         }
2206
2207         if (reg == 31)
2208                 return;
2209
2210         if (reg == 30)
2211                 flags |= REG_EXT;
2212
2213         rt = lightrec_alloc_reg_in(reg_cache, _jit, in_reg, flags);
2214
2215         switch (reg) {
2216         case 15:
2217                 tmp = lightrec_alloc_reg_temp(reg_cache, _jit);
2218                 jit_ldxi_i(tmp, LIGHTREC_REG_STATE, cp2d_i_offset(13));
2219
2220                 tmp2 = lightrec_alloc_reg_temp(reg_cache, _jit);
2221                 jit_ldxi_i(tmp2, LIGHTREC_REG_STATE, cp2d_i_offset(14));
2222
2223                 jit_stxi_i(cp2d_i_offset(12), LIGHTREC_REG_STATE, tmp);
2224                 jit_stxi_i(cp2d_i_offset(13), LIGHTREC_REG_STATE, tmp2);
2225                 jit_stxi_i(cp2d_i_offset(14), LIGHTREC_REG_STATE, rt);
2226
2227                 lightrec_free_reg(reg_cache, tmp);
2228                 lightrec_free_reg(reg_cache, tmp2);
2229                 break;
2230         case 28:
2231                 tmp = lightrec_alloc_reg_temp(reg_cache, _jit);
2232
2233                 jit_lshi(tmp, rt, 7);
2234                 jit_andi(tmp, tmp, 0xf80);
2235                 jit_stxi_s(cp2d_s_offset(9), LIGHTREC_REG_STATE, tmp);
2236
2237                 jit_lshi(tmp, rt, 2);
2238                 jit_andi(tmp, tmp, 0xf80);
2239                 jit_stxi_s(cp2d_s_offset(10), LIGHTREC_REG_STATE, tmp);
2240
2241                 jit_rshi(tmp, rt, 3);
2242                 jit_andi(tmp, tmp, 0xf80);
2243                 jit_stxi_s(cp2d_s_offset(11), LIGHTREC_REG_STATE, tmp);
2244
2245                 lightrec_free_reg(reg_cache, tmp);
2246                 break;
2247         case 30:
2248                 tmp = lightrec_alloc_reg_temp(reg_cache, _jit);
2249                 tmp2 = lightrec_alloc_reg_temp(reg_cache, _jit);
2250
2251                 /* if (rt < 0) rt = ~rt; */
2252                 jit_rshi(tmp, rt, 31);
2253                 jit_xorr(tmp, rt, tmp);
2254
2255                 /* We know the sign bit is 0. Left-shift by 1 to start the algorithm */
2256                 jit_lshi(tmp, tmp, 1);
2257                 jit_movi(tmp2, 33);
2258
2259                 /* Decrement tmp2 and right-shift the value by 1 until it equals zero */
2260                 loop = jit_label();
2261                 jit_subi(tmp2, tmp2, 1);
2262                 jit_rshi_u(tmp, tmp, 1);
2263                 to_loop = jit_bnei(tmp, 0);
2264
2265                 jit_patch_at(to_loop, loop);
2266
2267                 jit_stxi_i(cp2d_i_offset(31), LIGHTREC_REG_STATE, tmp2);
2268                 jit_stxi_i(cp2d_i_offset(30), LIGHTREC_REG_STATE, rt);
2269
2270                 lightrec_free_reg(reg_cache, tmp);
2271                 lightrec_free_reg(reg_cache, tmp2);
2272                 break;
2273         default:
2274                 jit_stxi_i(cp2d_i_offset(reg), LIGHTREC_REG_STATE, rt);
2275                 break;
2276         }
2277
2278         lightrec_free_reg(reg_cache, rt);
2279 }
2280
2281 static void rec_cp2_basic_MTC2(struct lightrec_cstate *state,
2282                                const struct block *block, u16 offset)
2283 {
2284         const union code c = block->opcode_list[offset].c;
2285
2286         rec_cp2_do_mtc2(state, block, offset, c.r.rd, c.r.rt);
2287 }
2288
2289 static void rec_cp2_basic_CTC2(struct lightrec_cstate *state,
2290                                const struct block *block, u16 offset)
2291 {
2292         struct regcache *reg_cache = state->reg_cache;
2293         const union code c = block->opcode_list[offset].c;
2294         jit_state_t *_jit = block->_jit;
2295         u8 rt, tmp, tmp2;
2296
2297         _jit_name(block->_jit, __func__);
2298
2299         if (state->state->ops.cop2_notify) {
2300                 /* We must call cop2_notify, handle that in C. */
2301                 rec_mtc(state, block, offset);
2302                 return;
2303         }
2304
2305         rt = lightrec_alloc_reg_in(reg_cache, _jit, c.r.rt, 0);
2306
2307         switch (c.r.rd) {
2308         case 4:
2309         case 12:
2310         case 20:
2311         case 26:
2312         case 27:
2313         case 29:
2314         case 30:
2315                 jit_stxi_s(cp2c_s_offset(c.r.rd), LIGHTREC_REG_STATE, rt);
2316                 break;
2317         case 31:
2318                 tmp = lightrec_alloc_reg_temp(reg_cache, _jit);
2319                 tmp2 = lightrec_alloc_reg_temp(reg_cache, _jit);
2320
2321                 jit_andi(tmp, rt, 0x7f87e000);
2322                 jit_nei(tmp, tmp, 0);
2323                 jit_lshi(tmp, tmp, 31);
2324
2325                 jit_andi(tmp2, rt, 0x7ffff000);
2326                 jit_orr(tmp, tmp2, tmp);
2327
2328                 jit_stxi_i(cp2c_i_offset(31), LIGHTREC_REG_STATE, tmp);
2329
2330                 lightrec_free_reg(reg_cache, tmp);
2331                 lightrec_free_reg(reg_cache, tmp2);
2332                 break;
2333
2334         default:
2335                 jit_stxi_i(cp2c_i_offset(c.r.rd), LIGHTREC_REG_STATE, rt);
2336         }
2337
2338         lightrec_free_reg(reg_cache, rt);
2339 }
2340
2341 static void rec_cp0_RFE(struct lightrec_cstate *state,
2342                         const struct block *block, u16 offset)
2343 {
2344         struct regcache *reg_cache = state->reg_cache;
2345         jit_state_t *_jit = block->_jit;
2346         u8 status, tmp;
2347
2348         jit_name(__func__);
2349         jit_note(__FILE__, __LINE__);
2350
2351         status = lightrec_alloc_reg_temp(reg_cache, _jit);
2352         jit_ldxi_i(status, LIGHTREC_REG_STATE,
2353                    offsetof(struct lightrec_state, regs.cp0[12]));
2354
2355         tmp = lightrec_alloc_reg_temp(reg_cache, _jit);
2356
2357         /* status = ((status >> 2) & 0xf) | status & ~0xf; */
2358         jit_rshi(tmp, status, 2);
2359         jit_andi(tmp, tmp, 0xf);
2360         jit_andi(status, status, ~0xful);
2361         jit_orr(status, status, tmp);
2362
2363         jit_ldxi_i(tmp, LIGHTREC_REG_STATE,
2364                    offsetof(struct lightrec_state, regs.cp0[13]));
2365         jit_stxi_i(offsetof(struct lightrec_state, regs.cp0[12]),
2366                    LIGHTREC_REG_STATE, status);
2367
2368         /* Exit dynarec in case there's a software interrupt.
2369          * exit_flags = !!(status & cause & 0x0300) & status; */
2370         jit_andr(tmp, tmp, status);
2371         jit_andi(tmp, tmp, 0x0300);
2372         jit_nei(tmp, tmp, 0);
2373         jit_andr(tmp, tmp, status);
2374         jit_stxi_i(offsetof(struct lightrec_state, exit_flags),
2375                    LIGHTREC_REG_STATE, tmp);
2376
2377         lightrec_free_reg(reg_cache, status);
2378         lightrec_free_reg(reg_cache, tmp);
2379 }
2380
2381 static void rec_CP(struct lightrec_cstate *state,
2382                    const struct block *block, u16 offset)
2383 {
2384         union code c = block->opcode_list[offset].c;
2385         jit_state_t *_jit = block->_jit;
2386
2387         jit_name(__func__);
2388         jit_note(__FILE__, __LINE__);
2389
2390         call_to_c_wrapper(state, block, c.opcode, C_WRAPPER_CP);
2391 }
2392
2393 static void rec_meta_MOV(struct lightrec_cstate *state,
2394                          const struct block *block, u16 offset)
2395 {
2396         struct regcache *reg_cache = state->reg_cache;
2397         const struct opcode *op = &block->opcode_list[offset];
2398         union code c = op->c;
2399         jit_state_t *_jit = block->_jit;
2400         bool unload_rd;
2401         u8 rs, rd;
2402
2403         _jit_name(block->_jit, __func__);
2404         jit_note(__FILE__, __LINE__);
2405
2406         unload_rd = OPT_EARLY_UNLOAD
2407                 && LIGHTREC_FLAGS_GET_RD(op->flags) == LIGHTREC_REG_UNLOAD;
2408
2409         if (c.r.rs || unload_rd)
2410                 rs = lightrec_alloc_reg_in(reg_cache, _jit, c.r.rs, 0);
2411
2412         if (unload_rd) {
2413                 /* If the destination register will be unloaded right after the
2414                  * MOV meta-opcode, we don't actually need to write any host
2415                  * register - we can just store the source register directly to
2416                  * the register cache, at the offset corresponding to the
2417                  * destination register. */
2418                 lightrec_discard_reg_if_loaded(reg_cache, c.r.rd);
2419
2420                 jit_stxi_i(offsetof(struct lightrec_state, regs.gpr)
2421                            + c.r.rd << 2, LIGHTREC_REG_STATE, rs);
2422
2423                 lightrec_free_reg(reg_cache, rs);
2424         } else {
2425                 rd = lightrec_alloc_reg_out(reg_cache, _jit, c.r.rd, REG_EXT);
2426
2427                 if (c.r.rs == 0)
2428                         jit_movi(rd, 0);
2429                 else
2430                         jit_extr_i(rd, rs);
2431
2432                 lightrec_free_reg(reg_cache, rd);
2433         }
2434
2435         if (c.r.rs || unload_rd)
2436                 lightrec_free_reg(reg_cache, rs);
2437 }
2438
2439 static void rec_meta_EXTC_EXTS(struct lightrec_cstate *state,
2440                                const struct block *block,
2441                                u16 offset)
2442 {
2443         struct regcache *reg_cache = state->reg_cache;
2444         union code c = block->opcode_list[offset].c;
2445         jit_state_t *_jit = block->_jit;
2446         u8 rs, rt;
2447
2448         _jit_name(block->_jit, __func__);
2449         jit_note(__FILE__, __LINE__);
2450
2451         rs = lightrec_alloc_reg_in(reg_cache, _jit, c.i.rs, 0);
2452         rt = lightrec_alloc_reg_out(reg_cache, _jit, c.i.rt, REG_EXT);
2453
2454         if (c.i.op == OP_META_EXTC)
2455                 jit_extr_c(rt, rs);
2456         else
2457                 jit_extr_s(rt, rs);
2458
2459         lightrec_free_reg(reg_cache, rs);
2460         lightrec_free_reg(reg_cache, rt);
2461 }
2462
2463 static void rec_meta_MULT2(struct lightrec_cstate *state,
2464                            const struct block *block,
2465                            u16 offset)
2466 {
2467         struct regcache *reg_cache = state->reg_cache;
2468         union code c = block->opcode_list[offset].c;
2469         jit_state_t *_jit = block->_jit;
2470         u8 reg_lo = get_mult_div_lo(c);
2471         u8 reg_hi = get_mult_div_hi(c);
2472         u32 flags = block->opcode_list[offset].flags;
2473         bool is_signed = c.i.op == OP_META_MULT2;
2474         u8 rs, lo, hi, rflags = 0, hiflags = 0;
2475         unsigned int i;
2476
2477         if (!op_flag_no_hi(flags) && c.r.op < 32) {
2478                 rflags = is_signed ? REG_EXT : REG_ZEXT;
2479                 hiflags = is_signed ? REG_EXT : (REG_EXT | REG_ZEXT);
2480         }
2481
2482         _jit_name(block->_jit, __func__);
2483         jit_note(__FILE__, __LINE__);
2484
2485         rs = lightrec_alloc_reg_in(reg_cache, _jit, c.i.rs, rflags);
2486
2487         /*
2488          * We must handle the case where one of the output registers is our rs
2489          * input register. Thanksfully, computing LO/HI can be done in any
2490          * order. Here, we make sure that the computation that overwrites the
2491          * input register is always performed last.
2492          */
2493         for (i = 0; i < 2; i++) {
2494                 if ((!i ^ (reg_lo == c.i.rs)) && !op_flag_no_lo(flags)) {
2495                         lo = lightrec_alloc_reg_out(reg_cache, _jit, reg_lo, 0);
2496
2497                         if (c.r.op < 32)
2498                                 jit_lshi(lo, rs, c.r.op);
2499                         else
2500                                 jit_movi(lo, 0);
2501
2502                         lightrec_free_reg(reg_cache, lo);
2503                         continue;
2504                 }
2505
2506                 if ((!!i ^ (reg_lo == c.i.rs)) && !op_flag_no_hi(flags)) {
2507                         hi = lightrec_alloc_reg_out(reg_cache, _jit,
2508                                                     reg_hi, hiflags);
2509
2510                         if (c.r.op >= 32)
2511                                 jit_lshi(hi, rs, c.r.op - 32);
2512                         else if (is_signed)
2513                                 jit_rshi(hi, rs, 32 - c.r.op);
2514                         else
2515                                 jit_rshi_u(hi, rs, 32 - c.r.op);
2516
2517                         lightrec_free_reg(reg_cache, hi);
2518                 }
2519         }
2520
2521         lightrec_free_reg(reg_cache, rs);
2522
2523         _jit_name(block->_jit, __func__);
2524         jit_note(__FILE__, __LINE__);
2525 }
2526
2527 static const lightrec_rec_func_t rec_standard[64] = {
2528         SET_DEFAULT_ELM(rec_standard, unknown_opcode),
2529         [OP_SPECIAL]            = rec_SPECIAL,
2530         [OP_REGIMM]             = rec_REGIMM,
2531         [OP_J]                  = rec_J,
2532         [OP_JAL]                = rec_JAL,
2533         [OP_BEQ]                = rec_BEQ,
2534         [OP_BNE]                = rec_BNE,
2535         [OP_BLEZ]               = rec_BLEZ,
2536         [OP_BGTZ]               = rec_BGTZ,
2537         [OP_ADDI]               = rec_ADDI,
2538         [OP_ADDIU]              = rec_ADDIU,
2539         [OP_SLTI]               = rec_SLTI,
2540         [OP_SLTIU]              = rec_SLTIU,
2541         [OP_ANDI]               = rec_ANDI,
2542         [OP_ORI]                = rec_ORI,
2543         [OP_XORI]               = rec_XORI,
2544         [OP_LUI]                = rec_LUI,
2545         [OP_CP0]                = rec_CP0,
2546         [OP_CP2]                = rec_CP2,
2547         [OP_LB]                 = rec_LB,
2548         [OP_LH]                 = rec_LH,
2549         [OP_LWL]                = rec_LWL,
2550         [OP_LW]                 = rec_LW,
2551         [OP_LBU]                = rec_LBU,
2552         [OP_LHU]                = rec_LHU,
2553         [OP_LWR]                = rec_LWR,
2554         [OP_SB]                 = rec_SB,
2555         [OP_SH]                 = rec_SH,
2556         [OP_SWL]                = rec_SWL,
2557         [OP_SW]                 = rec_SW,
2558         [OP_SWR]                = rec_SWR,
2559         [OP_LWC2]               = rec_LW,
2560         [OP_SWC2]               = rec_SW,
2561
2562         [OP_META_MOV]           = rec_meta_MOV,
2563         [OP_META_EXTC]          = rec_meta_EXTC_EXTS,
2564         [OP_META_EXTS]          = rec_meta_EXTC_EXTS,
2565         [OP_META_MULT2]         = rec_meta_MULT2,
2566         [OP_META_MULTU2]        = rec_meta_MULT2,
2567 };
2568
2569 static const lightrec_rec_func_t rec_special[64] = {
2570         SET_DEFAULT_ELM(rec_special, unknown_opcode),
2571         [OP_SPECIAL_SLL]        = rec_special_SLL,
2572         [OP_SPECIAL_SRL]        = rec_special_SRL,
2573         [OP_SPECIAL_SRA]        = rec_special_SRA,
2574         [OP_SPECIAL_SLLV]       = rec_special_SLLV,
2575         [OP_SPECIAL_SRLV]       = rec_special_SRLV,
2576         [OP_SPECIAL_SRAV]       = rec_special_SRAV,
2577         [OP_SPECIAL_JR]         = rec_special_JR,
2578         [OP_SPECIAL_JALR]       = rec_special_JALR,
2579         [OP_SPECIAL_SYSCALL]    = rec_special_SYSCALL,
2580         [OP_SPECIAL_BREAK]      = rec_special_BREAK,
2581         [OP_SPECIAL_MFHI]       = rec_special_MFHI,
2582         [OP_SPECIAL_MTHI]       = rec_special_MTHI,
2583         [OP_SPECIAL_MFLO]       = rec_special_MFLO,
2584         [OP_SPECIAL_MTLO]       = rec_special_MTLO,
2585         [OP_SPECIAL_MULT]       = rec_special_MULT,
2586         [OP_SPECIAL_MULTU]      = rec_special_MULTU,
2587         [OP_SPECIAL_DIV]        = rec_special_DIV,
2588         [OP_SPECIAL_DIVU]       = rec_special_DIVU,
2589         [OP_SPECIAL_ADD]        = rec_special_ADD,
2590         [OP_SPECIAL_ADDU]       = rec_special_ADDU,
2591         [OP_SPECIAL_SUB]        = rec_special_SUB,
2592         [OP_SPECIAL_SUBU]       = rec_special_SUBU,
2593         [OP_SPECIAL_AND]        = rec_special_AND,
2594         [OP_SPECIAL_OR]         = rec_special_OR,
2595         [OP_SPECIAL_XOR]        = rec_special_XOR,
2596         [OP_SPECIAL_NOR]        = rec_special_NOR,
2597         [OP_SPECIAL_SLT]        = rec_special_SLT,
2598         [OP_SPECIAL_SLTU]       = rec_special_SLTU,
2599 };
2600
2601 static const lightrec_rec_func_t rec_regimm[64] = {
2602         SET_DEFAULT_ELM(rec_regimm, unknown_opcode),
2603         [OP_REGIMM_BLTZ]        = rec_regimm_BLTZ,
2604         [OP_REGIMM_BGEZ]        = rec_regimm_BGEZ,
2605         [OP_REGIMM_BLTZAL]      = rec_regimm_BLTZAL,
2606         [OP_REGIMM_BGEZAL]      = rec_regimm_BGEZAL,
2607 };
2608
2609 static const lightrec_rec_func_t rec_cp0[64] = {
2610         SET_DEFAULT_ELM(rec_cp0, rec_CP),
2611         [OP_CP0_MFC0]           = rec_cp0_MFC0,
2612         [OP_CP0_CFC0]           = rec_cp0_CFC0,
2613         [OP_CP0_MTC0]           = rec_cp0_MTC0,
2614         [OP_CP0_CTC0]           = rec_cp0_CTC0,
2615         [OP_CP0_RFE]            = rec_cp0_RFE,
2616 };
2617
2618 static const lightrec_rec_func_t rec_cp2_basic[64] = {
2619         SET_DEFAULT_ELM(rec_cp2_basic, rec_CP),
2620         [OP_CP2_BASIC_MFC2]     = rec_cp2_basic_MFC2,
2621         [OP_CP2_BASIC_CFC2]     = rec_cp2_basic_CFC2,
2622         [OP_CP2_BASIC_MTC2]     = rec_cp2_basic_MTC2,
2623         [OP_CP2_BASIC_CTC2]     = rec_cp2_basic_CTC2,
2624 };
2625
2626 static void rec_SPECIAL(struct lightrec_cstate *state,
2627                         const struct block *block, u16 offset)
2628 {
2629         union code c = block->opcode_list[offset].c;
2630         lightrec_rec_func_t f = rec_special[c.r.op];
2631
2632         if (!HAS_DEFAULT_ELM && unlikely(!f))
2633                 unknown_opcode(state, block, offset);
2634         else
2635                 (*f)(state, block, offset);
2636 }
2637
2638 static void rec_REGIMM(struct lightrec_cstate *state,
2639                        const struct block *block, u16 offset)
2640 {
2641         union code c = block->opcode_list[offset].c;
2642         lightrec_rec_func_t f = rec_regimm[c.r.rt];
2643
2644         if (!HAS_DEFAULT_ELM && unlikely(!f))
2645                 unknown_opcode(state, block, offset);
2646         else
2647                 (*f)(state, block, offset);
2648 }
2649
2650 static void rec_CP0(struct lightrec_cstate *state,
2651                     const struct block *block, u16 offset)
2652 {
2653         union code c = block->opcode_list[offset].c;
2654         lightrec_rec_func_t f = rec_cp0[c.r.rs];
2655
2656         if (!HAS_DEFAULT_ELM && unlikely(!f))
2657                 rec_CP(state, block, offset);
2658         else
2659                 (*f)(state, block, offset);
2660 }
2661
2662 static void rec_CP2(struct lightrec_cstate *state,
2663                     const struct block *block, u16 offset)
2664 {
2665         union code c = block->opcode_list[offset].c;
2666
2667         if (c.r.op == OP_CP2_BASIC) {
2668                 lightrec_rec_func_t f = rec_cp2_basic[c.r.rs];
2669
2670                 if (HAS_DEFAULT_ELM || likely(f)) {
2671                         (*f)(state, block, offset);
2672                         return;
2673                 }
2674         }
2675
2676         rec_CP(state, block, offset);
2677 }
2678
2679 void lightrec_rec_opcode(struct lightrec_cstate *state,
2680                          const struct block *block, u16 offset)
2681 {
2682         struct regcache *reg_cache = state->reg_cache;
2683         struct lightrec_branch_target *target;
2684         const struct opcode *op = &block->opcode_list[offset];
2685         jit_state_t *_jit = block->_jit;
2686         lightrec_rec_func_t f;
2687         u16 unload_offset;
2688
2689         if (op_flag_sync(op->flags)) {
2690                 if (state->cycles)
2691                         jit_subi(LIGHTREC_REG_CYCLE, LIGHTREC_REG_CYCLE, state->cycles);
2692                 state->cycles = 0;
2693
2694                 lightrec_storeback_regs(reg_cache, _jit);
2695                 lightrec_regcache_reset(reg_cache);
2696
2697                 pr_debug("Adding branch target at offset 0x%x\n", offset << 2);
2698                 target = &state->targets[state->nb_targets++];
2699                 target->offset = offset;
2700                 target->label = jit_indirect();
2701         }
2702
2703         if (likely(op->opcode)) {
2704                 f = rec_standard[op->i.op];
2705
2706                 if (!HAS_DEFAULT_ELM && unlikely(!f))
2707                         unknown_opcode(state, block, offset);
2708                 else
2709                         (*f)(state, block, offset);
2710         }
2711
2712         if (OPT_EARLY_UNLOAD) {
2713                 unload_offset = offset +
2714                         (has_delay_slot(op->c) && !op_flag_no_ds(op->flags));
2715
2716                 lightrec_do_early_unload(state, block, unload_offset);
2717         }
2718 }