deps/lightrec/emitter.c

   1 // SPDX-License-Identifier: LGPL-2.1-or-later
   2 /*
   3  * Copyright (C) 2014-2021 Paul Cercueil <paul@crapouillou.net>
   4  */
   5
   6 #include "blockcache.h"
   7 #include "debug.h"
   8 #include "disassembler.h"
   9 #include "emitter.h"
  10 #include "lightning-wrapper.h"
  11 #include "optimizer.h"
  12 #include "regcache.h"
  13
  14 #include <stdbool.h>
  15 #include <stddef.h>
  16
  17 typedef void (*lightrec_rec_func_t)(struct lightrec_cstate *, const struct block *, u16);
  18
  19 /* Forward declarations */
  20 static void rec_SPECIAL(struct lightrec_cstate *state, const struct block *block, u16 offset);
  21 static void rec_REGIMM(struct lightrec_cstate *state, const struct block *block, u16 offset);
  22 static void rec_CP0(struct lightrec_cstate *state, const struct block *block, u16 offset);
  23 static void rec_CP2(struct lightrec_cstate *state, const struct block *block, u16 offset);
  24
  25 static void unknown_opcode(struct lightrec_cstate *state, const struct block *block, u16 offset)
  26 {
  27         pr_warn("Unknown opcode: 0x%08x at PC 0x%08x\n",
  28                 block->opcode_list[offset].c.opcode,
  29                 block->pc + (offset << 2));
  30 }
  31
  32 static void
  33 lightrec_jump_to_eob(struct lightrec_cstate *state, jit_state_t *_jit)
  34 {
  35         /* Prevent jit_jmpi() from using our cycles register as a temporary */
  36         jit_live(LIGHTREC_REG_CYCLE);
  37
  38         jit_patch_abs(jit_jmpi(), state->state->eob_wrapper_func);
  39 }
  40
  41 static void lightrec_emit_end_of_block(struct lightrec_cstate *state,
  42                                        const struct block *block, u16 offset,
  43                                        s8 reg_new_pc, u32 imm, u8 ra_reg,
  44                                        u32 link, bool update_cycles)
  45 {
  46         struct regcache *reg_cache = state->reg_cache;
  47         jit_state_t *_jit = block->_jit;
  48         const struct opcode *op = &block->opcode_list[offset],
  49                             *next = &block->opcode_list[offset + 1];
  50         u32 cycles = state->cycles + lightrec_cycles_of_opcode(op->c);
  51
  52         jit_note(__FILE__, __LINE__);
  53
  54         if (link) {
  55                 /* Update the $ra register */
  56                 u8 link_reg = lightrec_alloc_reg_out(reg_cache, _jit, ra_reg, 0);
  57                 jit_movi(link_reg, link);
  58                 lightrec_free_reg(reg_cache, link_reg);
  59         }
  60
  61         if (reg_new_pc < 0) {
  62                 reg_new_pc = lightrec_alloc_reg(reg_cache, _jit, JIT_V0);
  63                 lightrec_lock_reg(reg_cache, _jit, reg_new_pc);
  64
  65                 jit_movi(reg_new_pc, imm);
  66         }
  67
  68         if (has_delay_slot(op->c) &&
  69             !op_flag_no_ds(op->flags) && !op_flag_local_branch(op->flags)) {
  70                 cycles += lightrec_cycles_of_opcode(next->c);
  71
  72                 /* Recompile the delay slot */
  73                 if (next->c.opcode)
  74                         lightrec_rec_opcode(state, block, offset + 1);
  75         }
  76
  77         /* Clean the remaining registers */
  78         lightrec_clean_regs(reg_cache, _jit);
  79
  80         jit_movr(JIT_V0, reg_new_pc);
  81
  82         if (cycles && update_cycles) {
  83                 jit_subi(LIGHTREC_REG_CYCLE, LIGHTREC_REG_CYCLE, cycles);
  84                 pr_debug("EOB: %u cycles\n", cycles);
  85         }
  86
  87         lightrec_jump_to_eob(state, _jit);
  88 }
  89
  90 void lightrec_emit_eob(struct lightrec_cstate *state,
  91                        const struct block *block, u16 offset)
  92 {
  93         struct regcache *reg_cache = state->reg_cache;
  94         jit_state_t *_jit = block->_jit;
  95
  96         lightrec_clean_regs(reg_cache, _jit);
  97
  98         jit_movi(JIT_V0, block->pc + (offset << 2));
  99         jit_subi(LIGHTREC_REG_CYCLE, LIGHTREC_REG_CYCLE, state->cycles);
 100
 101         lightrec_jump_to_eob(state, _jit);
 102 }
 103
 104 static u8 get_jr_jalr_reg(struct lightrec_cstate *state, const struct block *block, u16 offset)
 105 {
 106         struct regcache *reg_cache = state->reg_cache;
 107         jit_state_t *_jit = block->_jit;
 108         const struct opcode *op = &block->opcode_list[offset];
 109         u8 rs;
 110
 111         rs = lightrec_request_reg_in(reg_cache, _jit, op->r.rs, JIT_V0);
 112         lightrec_lock_reg(reg_cache, _jit, rs);
 113
 114         return rs;
 115 }
 116
 117 static void rec_special_JR(struct lightrec_cstate *state, const struct block *block, u16 offset)
 118 {
 119         u8 rs = get_jr_jalr_reg(state, block, offset);
 120
 121         _jit_name(block->_jit, __func__);
 122         lightrec_emit_end_of_block(state, block, offset, rs, 0, 31, 0, true);
 123 }
 124
 125 static void rec_special_JALR(struct lightrec_cstate *state, const struct block *block, u16 offset)
 126 {
 127         u8 rs = get_jr_jalr_reg(state, block, offset);
 128         union code c = block->opcode_list[offset].c;
 129
 130         _jit_name(block->_jit, __func__);
 131         lightrec_emit_end_of_block(state, block, offset, rs, 0, c.r.rd,
 132                                    get_branch_pc(block, offset, 2), true);
 133 }
 134
 135 static void rec_J(struct lightrec_cstate *state, const struct block *block, u16 offset)
 136 {
 137         union code c = block->opcode_list[offset].c;
 138
 139         _jit_name(block->_jit, __func__);
 140         lightrec_emit_end_of_block(state, block, offset, -1,
 141                                    (block->pc & 0xf0000000) | (c.j.imm << 2),
 142                                    31, 0, true);
 143 }
 144
 145 static void rec_JAL(struct lightrec_cstate *state, const struct block *block, u16 offset)
 146 {
 147         union code c = block->opcode_list[offset].c;
 148
 149         _jit_name(block->_jit, __func__);
 150         lightrec_emit_end_of_block(state, block, offset, -1,
 151                                    (block->pc & 0xf0000000) | (c.j.imm << 2),
 152                                    31, get_branch_pc(block, offset, 2), true);
 153 }
 154
 155 static void lightrec_do_early_unload(struct lightrec_cstate *state,
 156                                      const struct block *block, u16 offset)
 157 {
 158         struct regcache *reg_cache = state->reg_cache;
 159         const struct opcode *op = &block->opcode_list[offset];
 160         jit_state_t *_jit = block->_jit;
 161         unsigned int i;
 162         u8 reg;
 163         struct {
 164                 u8 reg, op;
 165         } reg_ops[3] = {
 166                 { op->r.rd, LIGHTREC_FLAGS_GET_RD(op->flags), },
 167                 { op->i.rt, LIGHTREC_FLAGS_GET_RT(op->flags), },
 168                 { op->i.rs, LIGHTREC_FLAGS_GET_RS(op->flags), },
 169         };
 170
 171         for (i = 0; i < ARRAY_SIZE(reg_ops); i++) {
 172                 reg = reg_ops[i].reg;
 173
 174                 switch (reg_ops[i].op) {
 175                 case LIGHTREC_REG_UNLOAD:
 176                         lightrec_clean_reg_if_loaded(reg_cache, _jit, reg, true);
 177                         break;
 178
 179                 case LIGHTREC_REG_DISCARD:
 180                         lightrec_discard_reg_if_loaded(reg_cache, reg);
 181                         break;
 182
 183                 case LIGHTREC_REG_CLEAN:
 184                         lightrec_clean_reg_if_loaded(reg_cache, _jit, reg, false);
 185                         break;
 186                 default:
 187                         break;
 188                 };
 189         }
 190 }
 191
 192 static void rec_b(struct lightrec_cstate *state, const struct block *block, u16 offset,
 193                   jit_code_t code, jit_code_t code2, u32 link, bool unconditional, bool bz)
 194 {
 195         struct regcache *reg_cache = state->reg_cache;
 196         struct native_register *regs_backup;
 197         jit_state_t *_jit = block->_jit;
 198         struct lightrec_branch *branch;
 199         const struct opcode *op = &block->opcode_list[offset],
 200                             *next = &block->opcode_list[offset + 1];
 201         jit_node_t *addr;
 202         u8 link_reg, rs, rt;
 203         bool is_forward = (s16)op->i.imm >= -1;
 204         int op_cycles = lightrec_cycles_of_opcode(op->c);
 205         u32 target_offset, cycles = state->cycles + op_cycles;
 206         bool no_indirection = false;
 207         u32 next_pc;
 208
 209         jit_note(__FILE__, __LINE__);
 210
 211         if (!op_flag_no_ds(op->flags))
 212                 cycles += lightrec_cycles_of_opcode(next->c);
 213
 214         state->cycles = -op_cycles;
 215
 216         if (!unconditional) {
 217                 rs = lightrec_alloc_reg_in(reg_cache, _jit, op->i.rs, REG_EXT);
 218                 rt = bz ? 0 : lightrec_alloc_reg_in(reg_cache,
 219                                                     _jit, op->i.rt, REG_EXT);
 220
 221                 /* Unload dead registers before evaluating the branch */
 222                 if (OPT_EARLY_UNLOAD)
 223                         lightrec_do_early_unload(state, block, offset);
 224
 225                 if (op_flag_local_branch(op->flags) &&
 226                     (op_flag_no_ds(op->flags) || !next->opcode) &&
 227                     is_forward && !lightrec_has_dirty_regs(reg_cache))
 228                         no_indirection = true;
 229
 230                 if (no_indirection)
 231                         pr_debug("Using no indirection for branch at offset 0x%hx\n", offset << 2);
 232         }
 233
 234         if (cycles)
 235                 jit_subi(LIGHTREC_REG_CYCLE, LIGHTREC_REG_CYCLE, cycles);
 236
 237         if (!unconditional) {
 238                 /* Generate the branch opcode */
 239                 if (!no_indirection)
 240                         addr = jit_new_node_pww(code, NULL, rs, rt);
 241
 242                 lightrec_free_regs(reg_cache);
 243                 regs_backup = lightrec_regcache_enter_branch(reg_cache);
 244         }
 245
 246         if (op_flag_local_branch(op->flags)) {
 247                 /* Recompile the delay slot */
 248                 if (!op_flag_no_ds(op->flags) && next->opcode)
 249                         lightrec_rec_opcode(state, block, offset + 1);
 250
 251                 if (link) {
 252                         /* Update the $ra register */
 253                         link_reg = lightrec_alloc_reg_out(reg_cache, _jit, 31, 0);
 254                         jit_movi(link_reg, link);
 255                         lightrec_free_reg(reg_cache, link_reg);
 256                 }
 257
 258                 /* Clean remaining registers */
 259                 lightrec_clean_regs(reg_cache, _jit);
 260
 261                 target_offset = offset + 1 + (s16)op->i.imm
 262                         - !!op_flag_no_ds(op->flags);
 263                 pr_debug("Adding local branch to offset 0x%x\n",
 264                          target_offset << 2);
 265                 branch = &state->local_branches[
 266                         state->nb_local_branches++];
 267
 268                 branch->target = target_offset;
 269
 270                 if (no_indirection)
 271                         branch->branch = jit_new_node_pww(code2, NULL, rs, rt);
 272                 else if (is_forward)
 273                         branch->branch = jit_b();
 274                 else
 275                         branch->branch = jit_bgti(LIGHTREC_REG_CYCLE, 0);
 276         }
 277
 278         if (!op_flag_local_branch(op->flags) || !is_forward) {
 279                 next_pc = get_branch_pc(block, offset, 1 + (s16)op->i.imm);
 280                 lightrec_emit_end_of_block(state, block, offset, -1, next_pc,
 281                                            31, link, false);
 282         }
 283
 284         if (!unconditional) {
 285                 if (!no_indirection)
 286                         jit_patch(addr);
 287
 288                 lightrec_regcache_leave_branch(reg_cache, regs_backup);
 289
 290                 if (bz && link) {
 291                         /* Update the $ra register */
 292                         link_reg = lightrec_alloc_reg_out(reg_cache, _jit,
 293                                                           31, REG_EXT);
 294                         jit_movi(link_reg, (s32)link);
 295                         lightrec_free_reg(reg_cache, link_reg);
 296                 }
 297
 298                 if (!op_flag_no_ds(op->flags) && next->opcode)
 299                         lightrec_rec_opcode(state, block, offset + 1);
 300         }
 301 }
 302
 303 static void rec_BNE(struct lightrec_cstate *state,
 304                     const struct block *block, u16 offset)
 305 {
 306         union code c = block->opcode_list[offset].c;
 307
 308         _jit_name(block->_jit, __func__);
 309
 310         if (c.i.rt == 0)
 311                 rec_b(state, block, offset, jit_code_beqi, jit_code_bnei, 0, false, true);
 312         else
 313                 rec_b(state, block, offset, jit_code_beqr, jit_code_bner, 0, false, false);
 314 }
 315
 316 static void rec_BEQ(struct lightrec_cstate *state,
 317                     const struct block *block, u16 offset)
 318 {
 319         union code c = block->opcode_list[offset].c;
 320
 321         _jit_name(block->_jit, __func__);
 322
 323         if (c.i.rt == 0)
 324                 rec_b(state, block, offset, jit_code_bnei, jit_code_beqi, 0, c.i.rs == 0, true);
 325         else
 326                 rec_b(state, block, offset, jit_code_bner, jit_code_beqr, 0, c.i.rs == c.i.rt, false);
 327 }
 328
 329 static void rec_BLEZ(struct lightrec_cstate *state,
 330                      const struct block *block, u16 offset)
 331 {
 332         union code c = block->opcode_list[offset].c;
 333
 334         _jit_name(block->_jit, __func__);
 335         rec_b(state, block, offset, jit_code_bgti, jit_code_blei, 0, c.i.rs == 0, true);
 336 }
 337
 338 static void rec_BGTZ(struct lightrec_cstate *state,
 339                      const struct block *block, u16 offset)
 340 {
 341         _jit_name(block->_jit, __func__);
 342         rec_b(state, block, offset, jit_code_blei, jit_code_bgti, 0, false, true);
 343 }
 344
 345 static void rec_regimm_BLTZ(struct lightrec_cstate *state,
 346                             const struct block *block, u16 offset)
 347 {
 348         _jit_name(block->_jit, __func__);
 349         rec_b(state, block, offset, jit_code_bgei, jit_code_blti, 0, false, true);
 350 }
 351
 352 static void rec_regimm_BLTZAL(struct lightrec_cstate *state,
 353                               const struct block *block, u16 offset)
 354 {
 355         _jit_name(block->_jit, __func__);
 356         rec_b(state, block, offset, jit_code_bgei, jit_code_blti,
 357               get_branch_pc(block, offset, 2), false, true);
 358 }
 359
 360 static void rec_regimm_BGEZ(struct lightrec_cstate *state,
 361                             const struct block *block, u16 offset)
 362 {
 363         union code c = block->opcode_list[offset].c;
 364
 365         _jit_name(block->_jit, __func__);
 366         rec_b(state, block, offset, jit_code_blti, jit_code_bgei, 0, !c.i.rs, true);
 367 }
 368
 369 static void rec_regimm_BGEZAL(struct lightrec_cstate *state,
 370                               const struct block *block, u16 offset)
 371 {
 372         const struct opcode *op = &block->opcode_list[offset];
 373         _jit_name(block->_jit, __func__);
 374         rec_b(state, block, offset, jit_code_blti, jit_code_bgei,
 375               get_branch_pc(block, offset, 2),
 376               !op->i.rs, true);
 377 }
 378
 379 static void rec_alu_imm(struct lightrec_cstate *state, const struct block *block,
 380                         u16 offset, jit_code_t code, bool slti)
 381 {
 382         struct regcache *reg_cache = state->reg_cache;
 383         union code c = block->opcode_list[offset].c;
 384         jit_state_t *_jit = block->_jit;
 385         u8 rs, rt, out_flags = REG_EXT;
 386
 387         if (slti)
 388                 out_flags |= REG_ZEXT;
 389
 390         jit_note(__FILE__, __LINE__);
 391         rs = lightrec_alloc_reg_in(reg_cache, _jit, c.i.rs, REG_EXT);
 392         rt = lightrec_alloc_reg_out(reg_cache, _jit, c.i.rt, out_flags);
 393
 394         jit_new_node_www(code, rt, rs, (s32)(s16) c.i.imm);
 395
 396         lightrec_free_reg(reg_cache, rs);
 397         lightrec_free_reg(reg_cache, rt);
 398 }
 399
 400 static void rec_alu_special(struct lightrec_cstate *state, const struct block *block,
 401                             u16 offset, jit_code_t code, bool out_ext)
 402 {
 403         struct regcache *reg_cache = state->reg_cache;
 404         union code c = block->opcode_list[offset].c;
 405         jit_state_t *_jit = block->_jit;
 406         u8 rd, rt, rs;
 407
 408         jit_note(__FILE__, __LINE__);
 409         rs = lightrec_alloc_reg_in(reg_cache, _jit, c.r.rs, REG_EXT);
 410         rt = lightrec_alloc_reg_in(reg_cache, _jit, c.r.rt, REG_EXT);
 411         rd = lightrec_alloc_reg_out(reg_cache, _jit, c.r.rd,
 412                                     out_ext ? REG_EXT | REG_ZEXT : 0);
 413
 414         jit_new_node_www(code, rd, rs, rt);
 415
 416         lightrec_free_reg(reg_cache, rs);
 417         lightrec_free_reg(reg_cache, rt);
 418         lightrec_free_reg(reg_cache, rd);
 419 }
 420
 421 static void rec_alu_shiftv(struct lightrec_cstate *state, const struct block *block,
 422                            u16 offset, jit_code_t code)
 423 {
 424         struct regcache *reg_cache = state->reg_cache;
 425         union code c = block->opcode_list[offset].c;
 426         jit_state_t *_jit = block->_jit;
 427         u8 rd, rt, rs, temp, flags = 0;
 428
 429         jit_note(__FILE__, __LINE__);
 430         rs = lightrec_alloc_reg_in(reg_cache, _jit, c.r.rs, 0);
 431
 432         if (code == jit_code_rshr)
 433                 flags = REG_EXT;
 434         else if (code == jit_code_rshr_u)
 435                 flags = REG_ZEXT;
 436
 437         rt = lightrec_alloc_reg_in(reg_cache, _jit, c.r.rt, flags);
 438         rd = lightrec_alloc_reg_out(reg_cache, _jit, c.r.rd, flags);
 439
 440         if (rs != rd && rt != rd) {
 441                 jit_andi(rd, rs, 0x1f);
 442                 jit_new_node_www(code, rd, rt, rd);
 443         } else {
 444                 temp = lightrec_alloc_reg_temp(reg_cache, _jit);
 445                 jit_andi(temp, rs, 0x1f);
 446                 jit_new_node_www(code, rd, rt, temp);
 447                 lightrec_free_reg(reg_cache, temp);
 448         }
 449
 450         lightrec_free_reg(reg_cache, rs);
 451         lightrec_free_reg(reg_cache, rt);
 452         lightrec_free_reg(reg_cache, rd);
 453 }
 454
 455 static void rec_movi(struct lightrec_cstate *state,
 456                      const struct block *block, u16 offset)
 457 {
 458         struct regcache *reg_cache = state->reg_cache;
 459         union code c = block->opcode_list[offset].c;
 460         jit_state_t *_jit = block->_jit;
 461         u16 flags = REG_EXT;
 462         u8 rt;
 463
 464         if (!(c.i.imm & 0x8000))
 465                 flags |= REG_ZEXT;
 466
 467         rt = lightrec_alloc_reg_out(reg_cache, _jit, c.i.rt, flags);
 468
 469         jit_movi(rt, (s32)(s16) c.i.imm);
 470
 471         lightrec_free_reg(reg_cache, rt);
 472 }
 473
 474 static void rec_ADDIU(struct lightrec_cstate *state,
 475                       const struct block *block, u16 offset)
 476 {
 477         _jit_name(block->_jit, __func__);
 478
 479         if (block->opcode_list[offset].c.i.rs)
 480                 rec_alu_imm(state, block, offset, jit_code_addi, false);
 481         else
 482                 rec_movi(state, block, offset);
 483 }
 484
 485 static void rec_ADDI(struct lightrec_cstate *state,
 486                      const struct block *block, u16 offset)
 487 {
 488         /* TODO: Handle the exception? */
 489         _jit_name(block->_jit, __func__);
 490         rec_ADDIU(state, block, offset);
 491 }
 492
 493 static void rec_SLTIU(struct lightrec_cstate *state,
 494                       const struct block *block, u16 offset)
 495 {
 496         _jit_name(block->_jit, __func__);
 497         rec_alu_imm(state, block, offset, jit_code_lti_u, true);
 498 }
 499
 500 static void rec_SLTI(struct lightrec_cstate *state,
 501                      const struct block *block, u16 offset)
 502 {
 503         _jit_name(block->_jit, __func__);
 504         rec_alu_imm(state, block, offset, jit_code_lti, true);
 505 }
 506
 507 static void rec_ANDI(struct lightrec_cstate *state,
 508                      const struct block *block, u16 offset)
 509 {
 510         struct regcache *reg_cache = state->reg_cache;
 511         union code c = block->opcode_list[offset].c;
 512         jit_state_t *_jit = block->_jit;
 513         u8 rs, rt;
 514
 515         _jit_name(block->_jit, __func__);
 516         jit_note(__FILE__, __LINE__);
 517         rs = lightrec_alloc_reg_in(reg_cache, _jit, c.i.rs, 0);
 518         rt = lightrec_alloc_reg_out(reg_cache, _jit, c.i.rt,
 519                                     REG_EXT | REG_ZEXT);
 520
 521         /* PSX code uses ANDI 0xff / ANDI 0xffff a lot, which are basically
 522          * casts to uint8_t / uint16_t. */
 523         if (c.i.imm == 0xff)
 524                 jit_extr_uc(rt, rs);
 525         else if (c.i.imm == 0xffff)
 526                 jit_extr_us(rt, rs);
 527         else
 528                 jit_andi(rt, rs, (u32)(u16) c.i.imm);
 529
 530         lightrec_free_reg(reg_cache, rs);
 531         lightrec_free_reg(reg_cache, rt);
 532 }
 533
 534 static void rec_alu_or_xor(struct lightrec_cstate *state, const struct block *block,
 535                            u16 offset, jit_code_t code)
 536 {
 537         struct regcache *reg_cache = state->reg_cache;
 538         union code c = block->opcode_list[offset].c;
 539         jit_state_t *_jit = block->_jit;
 540         u8 rs, rt, flags;
 541
 542         jit_note(__FILE__, __LINE__);
 543         rs = lightrec_alloc_reg_in(reg_cache, _jit, c.i.rs, 0);
 544         rt = lightrec_alloc_reg_out(reg_cache, _jit, c.i.rt, 0);
 545
 546         flags = lightrec_get_reg_in_flags(reg_cache, rs);
 547         lightrec_set_reg_out_flags(reg_cache, rt, flags);
 548
 549         jit_new_node_www(code, rt, rs, (u32)(u16) c.i.imm);
 550
 551         lightrec_free_reg(reg_cache, rs);
 552         lightrec_free_reg(reg_cache, rt);
 553 }
 554
 555
 556 static void rec_ORI(struct lightrec_cstate *state,
 557                     const struct block *block, u16 offset)
 558 {
 559         _jit_name(block->_jit, __func__);
 560         rec_alu_or_xor(state, block, offset, jit_code_ori);
 561 }
 562
 563 static void rec_XORI(struct lightrec_cstate *state,
 564                      const struct block *block, u16 offset)
 565 {
 566         _jit_name(block->_jit, __func__);
 567         rec_alu_or_xor(state, block, offset, jit_code_xori);
 568 }
 569
 570 static void rec_LUI(struct lightrec_cstate *state,
 571                     const struct block *block, u16 offset)
 572 {
 573         struct regcache *reg_cache = state->reg_cache;
 574         union code c = block->opcode_list[offset].c;
 575         jit_state_t *_jit = block->_jit;
 576         u8 rt, flags = REG_EXT;
 577
 578         jit_name(__func__);
 579         jit_note(__FILE__, __LINE__);
 580
 581         if (!(c.i.imm & BIT(15)))
 582                 flags |= REG_ZEXT;
 583
 584         rt = lightrec_alloc_reg_out(reg_cache, _jit, c.i.rt, flags);
 585
 586         jit_movi(rt, (s32)(c.i.imm << 16));
 587
 588         lightrec_free_reg(reg_cache, rt);
 589 }
 590
 591 static void rec_special_ADDU(struct lightrec_cstate *state,
 592                              const struct block *block, u16 offset)
 593 {
 594         _jit_name(block->_jit, __func__);
 595         rec_alu_special(state, block, offset, jit_code_addr, false);
 596 }
 597
 598 static void rec_special_ADD(struct lightrec_cstate *state,
 599                             const struct block *block, u16 offset)
 600 {
 601         /* TODO: Handle the exception? */
 602         _jit_name(block->_jit, __func__);
 603         rec_alu_special(state, block, offset, jit_code_addr, false);
 604 }
 605
 606 static void rec_special_SUBU(struct lightrec_cstate *state,
 607                              const struct block *block, u16 offset)
 608 {
 609         _jit_name(block->_jit, __func__);
 610         rec_alu_special(state, block, offset, jit_code_subr, false);
 611 }
 612
 613 static void rec_special_SUB(struct lightrec_cstate *state,
 614                             const struct block *block, u16 offset)
 615 {
 616         /* TODO: Handle the exception? */
 617         _jit_name(block->_jit, __func__);
 618         rec_alu_special(state, block, offset, jit_code_subr, false);
 619 }
 620
 621 static void rec_special_AND(struct lightrec_cstate *state,
 622                             const struct block *block, u16 offset)
 623 {
 624         struct regcache *reg_cache = state->reg_cache;
 625         union code c = block->opcode_list[offset].c;
 626         jit_state_t *_jit = block->_jit;
 627         u8 rd, rt, rs, flags_rs, flags_rt, flags_rd;
 628
 629         _jit_name(block->_jit, __func__);
 630         jit_note(__FILE__, __LINE__);
 631         rs = lightrec_alloc_reg_in(reg_cache, _jit, c.r.rs, 0);
 632         rt = lightrec_alloc_reg_in(reg_cache, _jit, c.r.rt, 0);
 633         rd = lightrec_alloc_reg_out(reg_cache, _jit, c.r.rd, 0);
 634
 635         flags_rs = lightrec_get_reg_in_flags(reg_cache, rs);
 636         flags_rt = lightrec_get_reg_in_flags(reg_cache, rt);
 637
 638         /* Z(rd) = Z(rs) | Z(rt) */
 639         flags_rd = REG_ZEXT & (flags_rs | flags_rt);
 640
 641         /* E(rd) = (E(rt) & Z(rt)) | (E(rs) & Z(rs)) | (E(rs) & E(rt)) */
 642         if (((flags_rs & REG_EXT) && (flags_rt & REG_ZEXT)) ||
 643             ((flags_rt & REG_EXT) && (flags_rs & REG_ZEXT)) ||
 644             (REG_EXT & flags_rs & flags_rt))
 645                 flags_rd |= REG_EXT;
 646
 647         lightrec_set_reg_out_flags(reg_cache, rd, flags_rd);
 648
 649         jit_andr(rd, rs, rt);
 650
 651         lightrec_free_reg(reg_cache, rs);
 652         lightrec_free_reg(reg_cache, rt);
 653         lightrec_free_reg(reg_cache, rd);
 654 }
 655
 656 static void rec_special_or_nor(struct lightrec_cstate *state,
 657                                const struct block *block, u16 offset, bool nor)
 658 {
 659         struct regcache *reg_cache = state->reg_cache;
 660         union code c = block->opcode_list[offset].c;
 661         jit_state_t *_jit = block->_jit;
 662         u8 rd, rt, rs, flags_rs, flags_rt, flags_rd = 0;
 663
 664         jit_note(__FILE__, __LINE__);
 665         rs = lightrec_alloc_reg_in(reg_cache, _jit, c.r.rs, 0);
 666         rt = lightrec_alloc_reg_in(reg_cache, _jit, c.r.rt, 0);
 667         rd = lightrec_alloc_reg_out(reg_cache, _jit, c.r.rd, 0);
 668
 669         flags_rs = lightrec_get_reg_in_flags(reg_cache, rs);
 670         flags_rt = lightrec_get_reg_in_flags(reg_cache, rt);
 671
 672         /* or: Z(rd) = Z(rs) & Z(rt)
 673          * nor: Z(rd) = 0 */
 674         if (!nor)
 675                 flags_rd = REG_ZEXT & flags_rs & flags_rt;
 676
 677         /* E(rd) = E(rs) & E(rt) */
 678         if (REG_EXT & flags_rs & flags_rt)
 679                 flags_rd |= REG_EXT;
 680
 681         lightrec_set_reg_out_flags(reg_cache, rd, flags_rd);
 682
 683         jit_orr(rd, rs, rt);
 684
 685         if (nor)
 686                 jit_comr(rd, rd);
 687
 688         lightrec_free_reg(reg_cache, rs);
 689         lightrec_free_reg(reg_cache, rt);
 690         lightrec_free_reg(reg_cache, rd);
 691 }
 692
 693 static void rec_special_OR(struct lightrec_cstate *state,
 694                            const struct block *block, u16 offset)
 695 {
 696         _jit_name(block->_jit, __func__);
 697         rec_special_or_nor(state, block, offset, false);
 698 }
 699
 700 static void rec_special_NOR(struct lightrec_cstate *state,
 701                             const struct block *block, u16 offset)
 702 {
 703         _jit_name(block->_jit, __func__);
 704         rec_special_or_nor(state, block, offset, true);
 705 }
 706
 707 static void rec_special_XOR(struct lightrec_cstate *state,
 708                             const struct block *block, u16 offset)
 709 {
 710         struct regcache *reg_cache = state->reg_cache;
 711         union code c = block->opcode_list[offset].c;
 712         jit_state_t *_jit = block->_jit;
 713         u8 rd, rt, rs, flags_rs, flags_rt, flags_rd;
 714
 715         _jit_name(block->_jit, __func__);
 716
 717         jit_note(__FILE__, __LINE__);
 718         rs = lightrec_alloc_reg_in(reg_cache, _jit, c.r.rs, 0);
 719         rt = lightrec_alloc_reg_in(reg_cache, _jit, c.r.rt, 0);
 720         rd = lightrec_alloc_reg_out(reg_cache, _jit, c.r.rd, 0);
 721
 722         flags_rs = lightrec_get_reg_in_flags(reg_cache, rs);
 723         flags_rt = lightrec_get_reg_in_flags(reg_cache, rt);
 724
 725         /* Z(rd) = Z(rs) & Z(rt) */
 726         flags_rd = REG_ZEXT & flags_rs & flags_rt;
 727
 728         /* E(rd) = E(rs) & E(rt) */
 729         flags_rd |= REG_EXT & flags_rs & flags_rt;
 730
 731         lightrec_set_reg_out_flags(reg_cache, rd, flags_rd);
 732
 733         jit_xorr(rd, rs, rt);
 734
 735         lightrec_free_reg(reg_cache, rs);
 736         lightrec_free_reg(reg_cache, rt);
 737         lightrec_free_reg(reg_cache, rd);
 738 }
 739
 740 static void rec_special_SLTU(struct lightrec_cstate *state,
 741                              const struct block *block, u16 offset)
 742 {
 743         _jit_name(block->_jit, __func__);
 744         rec_alu_special(state, block, offset, jit_code_ltr_u, true);
 745 }
 746
 747 static void rec_special_SLT(struct lightrec_cstate *state,
 748                             const struct block *block, u16 offset)
 749 {
 750         _jit_name(block->_jit, __func__);
 751         rec_alu_special(state, block, offset, jit_code_ltr, true);
 752 }
 753
 754 static void rec_special_SLLV(struct lightrec_cstate *state,
 755                              const struct block *block, u16 offset)
 756 {
 757         _jit_name(block->_jit, __func__);
 758         rec_alu_shiftv(state, block, offset, jit_code_lshr);
 759 }
 760
 761 static void rec_special_SRLV(struct lightrec_cstate *state,
 762                              const struct block *block, u16 offset)
 763 {
 764         _jit_name(block->_jit, __func__);
 765         rec_alu_shiftv(state, block, offset, jit_code_rshr_u);
 766 }
 767
 768 static void rec_special_SRAV(struct lightrec_cstate *state,
 769                              const struct block *block, u16 offset)
 770 {
 771         _jit_name(block->_jit, __func__);
 772         rec_alu_shiftv(state, block, offset, jit_code_rshr);
 773 }
 774
 775 static void rec_alu_shift(struct lightrec_cstate *state, const struct block *block,
 776                           u16 offset, jit_code_t code)
 777 {
 778         struct regcache *reg_cache = state->reg_cache;
 779         union code c = block->opcode_list[offset].c;
 780         jit_state_t *_jit = block->_jit;
 781         u8 rd, rt, flags = 0;
 782
 783         jit_note(__FILE__, __LINE__);
 784
 785         if (code == jit_code_rshi)
 786                 flags = REG_EXT;
 787         else if (code == jit_code_rshi_u)
 788                 flags = REG_ZEXT;
 789
 790         rt = lightrec_alloc_reg_in(reg_cache, _jit, c.r.rt, flags);
 791
 792         /* Input reg is zero-extended, if we SRL at least by one bit, we know
 793          * the output reg will be both zero-extended and sign-extended. */
 794         if (code == jit_code_rshi_u && c.r.imm)
 795                 flags |= REG_EXT;
 796         rd = lightrec_alloc_reg_out(reg_cache, _jit, c.r.rd, flags);
 797
 798         jit_new_node_www(code, rd, rt, c.r.imm);
 799
 800         lightrec_free_reg(reg_cache, rt);
 801         lightrec_free_reg(reg_cache, rd);
 802 }
 803
 804 static void rec_special_SLL(struct lightrec_cstate *state,
 805                             const struct block *block, u16 offset)
 806 {
 807         _jit_name(block->_jit, __func__);
 808         rec_alu_shift(state, block, offset, jit_code_lshi);
 809 }
 810
 811 static void rec_special_SRL(struct lightrec_cstate *state,
 812                             const struct block *block, u16 offset)
 813 {
 814         _jit_name(block->_jit, __func__);
 815         rec_alu_shift(state, block, offset, jit_code_rshi_u);
 816 }
 817
 818 static void rec_special_SRA(struct lightrec_cstate *state,
 819                             const struct block *block, u16 offset)
 820 {
 821         _jit_name(block->_jit, __func__);
 822         rec_alu_shift(state, block, offset, jit_code_rshi);
 823 }
 824
 825 static void rec_alu_mult(struct lightrec_cstate *state,
 826                          const struct block *block, u16 offset, bool is_signed)
 827 {
 828         struct regcache *reg_cache = state->reg_cache;
 829         union code c = block->opcode_list[offset].c;
 830         u32 flags = block->opcode_list[offset].flags;
 831         u8 reg_lo = get_mult_div_lo(c);
 832         u8 reg_hi = get_mult_div_hi(c);
 833         jit_state_t *_jit = block->_jit;
 834         u8 lo, hi, rs, rt, rflags = 0;
 835
 836         jit_note(__FILE__, __LINE__);
 837
 838         if (is_signed)
 839                 rflags = REG_EXT;
 840         else
 841                 rflags = REG_ZEXT;
 842
 843         rs = lightrec_alloc_reg_in(reg_cache, _jit, c.r.rs, rflags);
 844         rt = lightrec_alloc_reg_in(reg_cache, _jit, c.r.rt, rflags);
 845
 846         if (!op_flag_no_lo(flags))
 847                 lo = lightrec_alloc_reg_out(reg_cache, _jit, reg_lo, 0);
 848         else if (__WORDSIZE == 32)
 849                 lo = lightrec_alloc_reg_temp(reg_cache, _jit);
 850
 851         if (!op_flag_no_hi(flags))
 852                 hi = lightrec_alloc_reg_out(reg_cache, _jit, reg_hi, REG_EXT);
 853
 854         if (__WORDSIZE == 32) {
 855                 /* On 32-bit systems, do a 32*32->64 bit operation, or a 32*32->32 bit
 856                  * operation if the MULT was detected a 32-bit only. */
 857                 if (!op_flag_no_hi(flags)) {
 858                         if (is_signed)
 859                                 jit_qmulr(lo, hi, rs, rt);
 860                         else
 861                                 jit_qmulr_u(lo, hi, rs, rt);
 862                 } else {
 863                         jit_mulr(lo, rs, rt);
 864                 }
 865         } else {
 866                 /* On 64-bit systems, do a 64*64->64 bit operation. */
 867                 if (op_flag_no_lo(flags)) {
 868                         jit_mulr(hi, rs, rt);
 869                         jit_rshi(hi, hi, 32);
 870                 } else {
 871                         jit_mulr(lo, rs, rt);
 872
 873                         /* The 64-bit output value is in $lo, store the upper 32 bits in $hi */
 874                         if (!op_flag_no_hi(flags))
 875                                 jit_rshi(hi, lo, 32);
 876                 }
 877         }
 878
 879         lightrec_free_reg(reg_cache, rs);
 880         lightrec_free_reg(reg_cache, rt);
 881         if (!op_flag_no_lo(flags) || __WORDSIZE == 32)
 882                 lightrec_free_reg(reg_cache, lo);
 883         if (!op_flag_no_hi(flags))
 884                 lightrec_free_reg(reg_cache, hi);
 885 }
 886
 887 static void rec_alu_div(struct lightrec_cstate *state,
 888                         const struct block *block, u16 offset, bool is_signed)
 889 {
 890         struct regcache *reg_cache = state->reg_cache;
 891         union code c = block->opcode_list[offset].c;
 892         u32 flags = block->opcode_list[offset].flags;
 893         bool no_check = op_flag_no_div_check(flags);
 894         u8 reg_lo = get_mult_div_lo(c);
 895         u8 reg_hi = get_mult_div_hi(c);
 896         jit_state_t *_jit = block->_jit;
 897         jit_node_t *branch, *to_end;
 898         u8 lo = 0, hi = 0, rs, rt, rflags = 0;
 899
 900         jit_note(__FILE__, __LINE__);
 901
 902         if (is_signed)
 903                 rflags = REG_EXT;
 904         else
 905                 rflags = REG_ZEXT;
 906
 907         rs = lightrec_alloc_reg_in(reg_cache, _jit, c.r.rs, rflags);
 908         rt = lightrec_alloc_reg_in(reg_cache, _jit, c.r.rt, rflags);
 909
 910         if (!op_flag_no_lo(flags))
 911                 lo = lightrec_alloc_reg_out(reg_cache, _jit, reg_lo, 0);
 912
 913         if (!op_flag_no_hi(flags))
 914                 hi = lightrec_alloc_reg_out(reg_cache, _jit, reg_hi, 0);
 915
 916         /* Jump to special handler if dividing by zero  */
 917         if (!no_check)
 918                 branch = jit_beqi(rt, 0);
 919
 920         if (op_flag_no_lo(flags)) {
 921                 if (is_signed)
 922                         jit_remr(hi, rs, rt);
 923                 else
 924                         jit_remr_u(hi, rs, rt);
 925         } else if (op_flag_no_hi(flags)) {
 926                 if (is_signed)
 927                         jit_divr(lo, rs, rt);
 928                 else
 929                         jit_divr_u(lo, rs, rt);
 930         } else {
 931                 if (is_signed)
 932                         jit_qdivr(lo, hi, rs, rt);
 933                 else
 934                         jit_qdivr_u(lo, hi, rs, rt);
 935         }
 936
 937         if (!no_check) {
 938                 /* Jump above the div-by-zero handler */
 939                 to_end = jit_b();
 940
 941                 jit_patch(branch);
 942
 943                 if (!op_flag_no_lo(flags)) {
 944                         if (is_signed) {
 945                                 jit_lti(lo, rs, 0);
 946                                 jit_lshi(lo, lo, 1);
 947                                 jit_subi(lo, lo, 1);
 948                         } else {
 949                                 jit_movi(lo, 0xffffffff);
 950                         }
 951                 }
 952
 953                 if (!op_flag_no_hi(flags))
 954                         jit_movr(hi, rs);
 955
 956                 jit_patch(to_end);
 957         }
 958
 959         lightrec_free_reg(reg_cache, rs);
 960         lightrec_free_reg(reg_cache, rt);
 961
 962         if (!op_flag_no_lo(flags))
 963                 lightrec_free_reg(reg_cache, lo);
 964
 965         if (!op_flag_no_hi(flags))
 966                 lightrec_free_reg(reg_cache, hi);
 967 }
 968
 969 static void rec_special_MULT(struct lightrec_cstate *state,
 970                              const struct block *block, u16 offset)
 971 {
 972         _jit_name(block->_jit, __func__);
 973         rec_alu_mult(state, block, offset, true);
 974 }
 975
 976 static void rec_special_MULTU(struct lightrec_cstate *state,
 977                               const struct block *block, u16 offset)
 978 {
 979         _jit_name(block->_jit, __func__);
 980         rec_alu_mult(state, block, offset, false);
 981 }
 982
 983 static void rec_special_DIV(struct lightrec_cstate *state,
 984                             const struct block *block, u16 offset)
 985 {
 986         _jit_name(block->_jit, __func__);
 987         rec_alu_div(state, block, offset, true);
 988 }
 989
 990 static void rec_special_DIVU(struct lightrec_cstate *state,
 991                              const struct block *block, u16 offset)
 992 {
 993         _jit_name(block->_jit, __func__);
 994         rec_alu_div(state, block, offset, false);
 995 }
 996
 997 static void rec_alu_mv_lo_hi(struct lightrec_cstate *state,
 998                              const struct block *block, u8 dst, u8 src)
 999 {
1000         struct regcache *reg_cache = state->reg_cache;
1001         jit_state_t *_jit = block->_jit;
1002
1003         jit_note(__FILE__, __LINE__);
1004         src = lightrec_alloc_reg_in(reg_cache, _jit, src, 0);
1005         dst = lightrec_alloc_reg_out(reg_cache, _jit, dst, REG_EXT);
1006
1007         jit_extr_i(dst, src);
1008
1009         lightrec_free_reg(reg_cache, src);
1010         lightrec_free_reg(reg_cache, dst);
1011 }
1012
1013 static void rec_special_MFHI(struct lightrec_cstate *state,
1014                              const struct block *block, u16 offset)
1015 {
1016         union code c = block->opcode_list[offset].c;
1017
1018         _jit_name(block->_jit, __func__);
1019         rec_alu_mv_lo_hi(state, block, c.r.rd, REG_HI);
1020 }
1021
1022 static void rec_special_MTHI(struct lightrec_cstate *state,
1023                              const struct block *block, u16 offset)
1024 {
1025         union code c = block->opcode_list[offset].c;
1026
1027         _jit_name(block->_jit, __func__);
1028         rec_alu_mv_lo_hi(state, block, REG_HI, c.r.rs);
1029 }
1030
1031 static void rec_special_MFLO(struct lightrec_cstate *state,
1032                              const struct block *block, u16 offset)
1033 {
1034         union code c = block->opcode_list[offset].c;
1035
1036         _jit_name(block->_jit, __func__);
1037         rec_alu_mv_lo_hi(state, block, c.r.rd, REG_LO);
1038 }
1039
1040 static void rec_special_MTLO(struct lightrec_cstate *state,
1041                              const struct block *block, u16 offset)
1042 {
1043         union code c = block->opcode_list[offset].c;
1044
1045         _jit_name(block->_jit, __func__);
1046         rec_alu_mv_lo_hi(state, block, REG_LO, c.r.rs);
1047 }
1048
1049 static void call_to_c_wrapper(struct lightrec_cstate *state,
1050                               const struct block *block, u32 arg,
1051                               enum c_wrappers wrapper)
1052 {
1053         struct regcache *reg_cache = state->reg_cache;
1054         jit_state_t *_jit = block->_jit;
1055         s8 tmp, tmp2;
1056
1057         /* Make sure JIT_R1 is not mapped; it will be used in the C wrapper. */
1058         tmp2 = lightrec_alloc_reg(reg_cache, _jit, JIT_R1);
1059
1060         tmp = lightrec_get_reg_with_value(reg_cache,
1061                                           (intptr_t) state->state->wrappers_eps[wrapper]);
1062         if (tmp < 0) {
1063                 tmp = lightrec_alloc_reg_temp(reg_cache, _jit);
1064                 jit_ldxi(tmp, LIGHTREC_REG_STATE,
1065                          offsetof(struct lightrec_state, wrappers_eps[wrapper]));
1066
1067                 lightrec_temp_set_value(reg_cache, tmp,
1068                                         (intptr_t) state->state->wrappers_eps[wrapper]);
1069         }
1070
1071         lightrec_free_reg(reg_cache, tmp2);
1072
1073 #ifdef __mips__
1074         /* On MIPS, register t9 is always used as the target register for JALR.
1075          * Therefore if it does not contain the target address we must
1076          * invalidate it. */
1077         if (tmp != _T9)
1078                 lightrec_unload_reg(reg_cache, _jit, _T9);
1079 #endif
1080
1081         jit_prepare();
1082         jit_pushargi(arg);
1083
1084         lightrec_regcache_mark_live(reg_cache, _jit);
1085         jit_callr(tmp);
1086
1087         lightrec_free_reg(reg_cache, tmp);
1088         lightrec_regcache_mark_live(reg_cache, _jit);
1089 }
1090
1091 static void rec_io(struct lightrec_cstate *state,
1092                    const struct block *block, u16 offset,
1093                    bool load_rt, bool read_rt)
1094 {
1095         struct regcache *reg_cache = state->reg_cache;
1096         jit_state_t *_jit = block->_jit;
1097         union code c = block->opcode_list[offset].c;
1098         u32 flags = block->opcode_list[offset].flags;
1099         bool is_tagged = LIGHTREC_FLAGS_GET_IO_MODE(flags);
1100         u32 lut_entry;
1101
1102         jit_note(__FILE__, __LINE__);
1103
1104         lightrec_clean_reg_if_loaded(reg_cache, _jit, c.i.rs, false);
1105
1106         if (read_rt && likely(c.i.rt))
1107                 lightrec_clean_reg_if_loaded(reg_cache, _jit, c.i.rt, true);
1108         else if (load_rt)
1109                 lightrec_clean_reg_if_loaded(reg_cache, _jit, c.i.rt, false);
1110
1111         if (is_tagged) {
1112                 call_to_c_wrapper(state, block, c.opcode, C_WRAPPER_RW);
1113         } else {
1114                 lut_entry = lightrec_get_lut_entry(block);
1115                 call_to_c_wrapper(state, block, (lut_entry << 16) | offset,
1116                                   C_WRAPPER_RW_GENERIC);
1117         }
1118 }
1119
1120 static u32 rec_ram_mask(struct lightrec_state *state)
1121 {
1122         return (RAM_SIZE << (state->mirrors_mapped * 2)) - 1;
1123 }
1124
1125 static u32 rec_io_mask(const struct lightrec_state *state)
1126 {
1127         u32 length = state->maps[PSX_MAP_HW_REGISTERS].length;
1128
1129         return GENMASK(31 - clz32(length - 1), 0);
1130 }
1131
1132 static void rec_store_memory(struct lightrec_cstate *cstate,
1133                              const struct block *block,
1134                              u16 offset, jit_code_t code,
1135                              jit_code_t swap_code,
1136                              uintptr_t addr_offset, u32 addr_mask,
1137                              bool invalidate)
1138 {
1139         const struct lightrec_state *state = cstate->state;
1140         struct regcache *reg_cache = cstate->reg_cache;
1141         struct opcode *op = &block->opcode_list[offset];
1142         jit_state_t *_jit = block->_jit;
1143         union code c = op->c;
1144         u8 rs, rt, tmp, tmp2, tmp3, addr_reg, addr_reg2;
1145         s16 imm = (s16)c.i.imm;
1146         s32 simm = (s32)imm << (1 - lut_is_32bit(state));
1147         s32 lut_offt = offsetof(struct lightrec_state, code_lut);
1148         bool no_mask = op_flag_no_mask(op->flags);
1149         bool add_imm = c.i.imm &&
1150                 ((!state->mirrors_mapped && !no_mask) || (invalidate &&
1151                 ((imm & 0x3) || simm + lut_offt != (s16)(simm + lut_offt))));
1152         bool need_tmp = !no_mask || addr_offset || add_imm || invalidate;
1153
1154         rt = lightrec_alloc_reg_in(reg_cache, _jit, c.i.rt, 0);
1155         rs = lightrec_alloc_reg_in(reg_cache, _jit, c.i.rs, 0);
1156         if (need_tmp)
1157                 tmp = lightrec_alloc_reg_temp(reg_cache, _jit);
1158
1159         addr_reg = rs;
1160
1161         if (add_imm) {
1162                 jit_addi(tmp, addr_reg, (s16)c.i.imm);
1163                 addr_reg = tmp;
1164                 imm = 0;
1165         } else if (simm) {
1166                 lut_offt += simm;
1167         }
1168
1169         if (!no_mask) {
1170                 jit_andi(tmp, addr_reg, addr_mask);
1171                 addr_reg = tmp;
1172         }
1173
1174         if (addr_offset) {
1175                 tmp2 = lightrec_alloc_reg_temp(reg_cache, _jit);
1176                 jit_addi(tmp2, addr_reg, addr_offset);
1177                 addr_reg2 = tmp2;
1178         } else {
1179                 addr_reg2 = addr_reg;
1180         }
1181
1182         if (is_big_endian() && swap_code && c.i.rt) {
1183                 tmp3 = lightrec_alloc_reg_temp(reg_cache, _jit);
1184
1185                 jit_new_node_ww(swap_code, tmp3, rt);
1186                 jit_new_node_www(code, imm, addr_reg2, tmp3);
1187
1188                 lightrec_free_reg(reg_cache, tmp3);
1189         } else {
1190                 jit_new_node_www(code, imm, addr_reg2, rt);
1191         }
1192
1193         lightrec_free_reg(reg_cache, rt);
1194
1195         if (invalidate) {
1196                 tmp3 = lightrec_alloc_reg_in(reg_cache, _jit, 0, 0);
1197
1198                 if (c.i.op != OP_SW) {
1199                         jit_andi(tmp, addr_reg, ~3);
1200                         addr_reg = tmp;
1201                 }
1202
1203                 if (!lut_is_32bit(state)) {
1204                         jit_lshi(tmp, addr_reg, 1);
1205                         addr_reg = tmp;
1206                 }
1207
1208                 if (addr_reg == rs && c.i.rs == 0) {
1209                         addr_reg = LIGHTREC_REG_STATE;
1210                 } else {
1211                         jit_addr(tmp, addr_reg, LIGHTREC_REG_STATE);
1212                         addr_reg = tmp;
1213                 }
1214
1215                 if (lut_is_32bit(state))
1216                         jit_stxi_i(lut_offt, addr_reg, tmp3);
1217                 else
1218                         jit_stxi(lut_offt, addr_reg, tmp3);
1219
1220                 lightrec_free_reg(reg_cache, tmp3);
1221         }
1222
1223         if (addr_offset)
1224                 lightrec_free_reg(reg_cache, tmp2);
1225         if (need_tmp)
1226                 lightrec_free_reg(reg_cache, tmp);
1227         lightrec_free_reg(reg_cache, rs);
1228 }
1229
1230 static void rec_store_ram(struct lightrec_cstate *cstate,
1231                           const struct block *block,
1232                           u16 offset, jit_code_t code,
1233                           jit_code_t swap_code, bool invalidate)
1234 {
1235         struct lightrec_state *state = cstate->state;
1236
1237         _jit_note(block->_jit, __FILE__, __LINE__);
1238
1239         return rec_store_memory(cstate, block, offset, code, swap_code,
1240                                 state->offset_ram, rec_ram_mask(state),
1241                                 invalidate);
1242 }
1243
1244 static void rec_store_scratch(struct lightrec_cstate *cstate,
1245                               const struct block *block, u16 offset,
1246                               jit_code_t code, jit_code_t swap_code)
1247 {
1248         _jit_note(block->_jit, __FILE__, __LINE__);
1249
1250         return rec_store_memory(cstate, block, offset, code, swap_code,
1251                                 cstate->state->offset_scratch,
1252                                 0x1fffffff, false);
1253 }
1254
1255 static void rec_store_io(struct lightrec_cstate *cstate,
1256                          const struct block *block, u16 offset,
1257                          jit_code_t code, jit_code_t swap_code)
1258 {
1259         _jit_note(block->_jit, __FILE__, __LINE__);
1260
1261         return rec_store_memory(cstate, block, offset, code, swap_code,
1262                                 cstate->state->offset_io,
1263                                 rec_io_mask(cstate->state), false);
1264 }
1265
1266 static void rec_store_direct_no_invalidate(struct lightrec_cstate *cstate,
1267                                            const struct block *block,
1268                                            u16 offset, jit_code_t code,
1269                                            jit_code_t swap_code)
1270 {
1271         struct lightrec_state *state = cstate->state;
1272         struct regcache *reg_cache = cstate->reg_cache;
1273         union code c = block->opcode_list[offset].c;
1274         jit_state_t *_jit = block->_jit;
1275         jit_node_t *to_not_ram, *to_end;
1276         u8 tmp, tmp2, rs, rt;
1277         s16 imm;
1278
1279         jit_note(__FILE__, __LINE__);
1280         rs = lightrec_alloc_reg_in(reg_cache, _jit, c.i.rs, 0);
1281         tmp = lightrec_alloc_reg_temp(reg_cache, _jit);
1282
1283         if (state->offset_ram || state->offset_scratch)
1284                 tmp2 = lightrec_alloc_reg_temp(reg_cache, _jit);
1285
1286         /* Convert to KUNSEG and avoid RAM mirrors */
1287         if (state->mirrors_mapped) {
1288                 imm = (s16)c.i.imm;
1289                 jit_andi(tmp, rs, 0x1f800000 | (4 * RAM_SIZE - 1));
1290         } else if (c.i.imm) {
1291                 imm = 0;
1292                 jit_addi(tmp, rs, (s16)c.i.imm);
1293                 jit_andi(tmp, tmp, 0x1f800000 | (RAM_SIZE - 1));
1294         } else {
1295                 imm = 0;
1296                 jit_andi(tmp, rs, 0x1f800000 | (RAM_SIZE - 1));
1297         }
1298
1299         lightrec_free_reg(reg_cache, rs);
1300
1301         if (state->offset_ram != state->offset_scratch) {
1302                 to_not_ram = jit_bmsi(tmp, BIT(28));
1303
1304                 jit_movi(tmp2, state->offset_ram);
1305
1306                 to_end = jit_b();
1307                 jit_patch(to_not_ram);
1308
1309                 jit_movi(tmp2, state->offset_scratch);
1310                 jit_patch(to_end);
1311         } else if (state->offset_ram) {
1312                 jit_movi(tmp2, state->offset_ram);
1313         }
1314
1315         if (state->offset_ram || state->offset_scratch) {
1316                 jit_addr(tmp, tmp, tmp2);
1317                 lightrec_free_reg(reg_cache, tmp2);
1318         }
1319
1320         rt = lightrec_alloc_reg_in(reg_cache, _jit, c.i.rt, 0);
1321
1322         if (is_big_endian() && swap_code && c.i.rt) {
1323                 tmp2 = lightrec_alloc_reg_temp(reg_cache, _jit);
1324
1325                 jit_new_node_ww(swap_code, tmp2, rt);
1326                 jit_new_node_www(code, imm, tmp, tmp2);
1327
1328                 lightrec_free_reg(reg_cache, tmp2);
1329         } else {
1330                 jit_new_node_www(code, imm, tmp, rt);
1331         }
1332
1333         lightrec_free_reg(reg_cache, rt);
1334         lightrec_free_reg(reg_cache, tmp);
1335 }
1336
1337 static void rec_store_direct(struct lightrec_cstate *cstate, const struct block *block,
1338                              u16 offset, jit_code_t code, jit_code_t swap_code)
1339 {
1340         struct lightrec_state *state = cstate->state;
1341         u32 ram_size = state->mirrors_mapped ? RAM_SIZE * 4 : RAM_SIZE;
1342         struct regcache *reg_cache = cstate->reg_cache;
1343         union code c = block->opcode_list[offset].c;
1344         jit_state_t *_jit = block->_jit;
1345         jit_node_t *to_not_ram, *to_end;
1346         u8 tmp, tmp2, tmp3, rs, rt;
1347
1348         jit_note(__FILE__, __LINE__);
1349
1350         rs = lightrec_alloc_reg_in(reg_cache, _jit, c.i.rs, 0);
1351         tmp2 = lightrec_alloc_reg_temp(reg_cache, _jit);
1352         tmp3 = lightrec_alloc_reg_in(reg_cache, _jit, 0, 0);
1353
1354         /* Convert to KUNSEG and avoid RAM mirrors */
1355         if (c.i.imm) {
1356                 jit_addi(tmp2, rs, (s16)c.i.imm);
1357                 jit_andi(tmp2, tmp2, 0x1f800000 | (ram_size - 1));
1358         } else {
1359                 jit_andi(tmp2, rs, 0x1f800000 | (ram_size - 1));
1360         }
1361
1362         lightrec_free_reg(reg_cache, rs);
1363         tmp = lightrec_alloc_reg_temp(reg_cache, _jit);
1364
1365         to_not_ram = jit_bgti(tmp2, ram_size);
1366
1367         /* Compute the offset to the code LUT */
1368         jit_andi(tmp, tmp2, (RAM_SIZE - 1) & ~3);
1369         if (!lut_is_32bit(state))
1370                 jit_lshi(tmp, tmp, 1);
1371         jit_addr(tmp, LIGHTREC_REG_STATE, tmp);
1372
1373         /* Write NULL to the code LUT to invalidate any block that's there */
1374         if (lut_is_32bit(state))
1375                 jit_stxi_i(offsetof(struct lightrec_state, code_lut), tmp, tmp3);
1376         else
1377                 jit_stxi(offsetof(struct lightrec_state, code_lut), tmp, tmp3);
1378
1379         if (state->offset_ram != state->offset_scratch) {
1380                 jit_movi(tmp, state->offset_ram);
1381
1382                 to_end = jit_b();
1383         }
1384
1385         jit_patch(to_not_ram);
1386
1387         if (state->offset_ram || state->offset_scratch)
1388                 jit_movi(tmp, state->offset_scratch);
1389
1390         if (state->offset_ram != state->offset_scratch)
1391                 jit_patch(to_end);
1392
1393         if (state->offset_ram || state->offset_scratch)
1394                 jit_addr(tmp2, tmp2, tmp);
1395
1396         lightrec_free_reg(reg_cache, tmp);
1397         lightrec_free_reg(reg_cache, tmp3);
1398
1399         rt = lightrec_alloc_reg_in(reg_cache, _jit, c.i.rt, 0);
1400
1401         if (is_big_endian() && swap_code && c.i.rt) {
1402                 tmp = lightrec_alloc_reg_temp(reg_cache, _jit);
1403
1404                 jit_new_node_ww(swap_code, tmp, rt);
1405                 jit_new_node_www(code, 0, tmp2, tmp);
1406
1407                 lightrec_free_reg(reg_cache, tmp);
1408         } else {
1409                 jit_new_node_www(code, 0, tmp2, rt);
1410         }
1411
1412         lightrec_free_reg(reg_cache, rt);
1413         lightrec_free_reg(reg_cache, tmp2);
1414 }
1415
1416 static void rec_store(struct lightrec_cstate *state,
1417                       const struct block *block, u16 offset,
1418                       jit_code_t code, jit_code_t swap_code)
1419 {
1420         u32 flags = block->opcode_list[offset].flags;
1421         bool no_invalidate = op_flag_no_invalidate(flags) ||
1422                 state->state->invalidate_from_dma_only;
1423
1424         switch (LIGHTREC_FLAGS_GET_IO_MODE(flags)) {
1425         case LIGHTREC_IO_RAM:
1426                 rec_store_ram(state, block, offset, code,
1427                               swap_code, !no_invalidate);
1428                 break;
1429         case LIGHTREC_IO_SCRATCH:
1430                 rec_store_scratch(state, block, offset, code, swap_code);
1431                 break;
1432         case LIGHTREC_IO_DIRECT:
1433                 if (no_invalidate) {
1434                         rec_store_direct_no_invalidate(state, block, offset,
1435                                                        code, swap_code);
1436                 } else {
1437                         rec_store_direct(state, block, offset, code, swap_code);
1438                 }
1439                 break;
1440         case LIGHTREC_IO_DIRECT_HW:
1441                 rec_store_io(state, block, offset, code, swap_code);
1442                 break;
1443         default:
1444                 rec_io(state, block, offset, true, false);
1445                 break;
1446         }
1447 }
1448
1449 static void rec_SB(struct lightrec_cstate *state,
1450                    const struct block *block, u16 offset)
1451 {
1452         _jit_name(block->_jit, __func__);
1453         rec_store(state, block, offset, jit_code_stxi_c, 0);
1454 }
1455
1456 static void rec_SH(struct lightrec_cstate *state,
1457                    const struct block *block, u16 offset)
1458 {
1459         _jit_name(block->_jit, __func__);
1460         rec_store(state, block, offset,
1461                   jit_code_stxi_s, jit_code_bswapr_us);
1462 }
1463
1464 static void rec_SW(struct lightrec_cstate *state,
1465                    const struct block *block, u16 offset)
1466
1467 {
1468         _jit_name(block->_jit, __func__);
1469         rec_store(state, block, offset,
1470                   jit_code_stxi_i, jit_code_bswapr_ui);
1471 }
1472
1473 static void rec_SWL(struct lightrec_cstate *state,
1474                     const struct block *block, u16 offset)
1475 {
1476         _jit_name(block->_jit, __func__);
1477         rec_io(state, block, offset, true, false);
1478 }
1479
1480 static void rec_SWR(struct lightrec_cstate *state,
1481                     const struct block *block, u16 offset)
1482 {
1483         _jit_name(block->_jit, __func__);
1484         rec_io(state, block, offset, true, false);
1485 }
1486
1487 static void rec_SWC2(struct lightrec_cstate *state,
1488                      const struct block *block, u16 offset)
1489 {
1490         _jit_name(block->_jit, __func__);
1491         rec_io(state, block, offset, false, false);
1492 }
1493
1494 static void rec_load_memory(struct lightrec_cstate *cstate,
1495                             const struct block *block, u16 offset,
1496                             jit_code_t code, jit_code_t swap_code, bool is_unsigned,
1497                             uintptr_t addr_offset, u32 addr_mask)
1498 {
1499         struct regcache *reg_cache = cstate->reg_cache;
1500         struct opcode *op = &block->opcode_list[offset];
1501         jit_state_t *_jit = block->_jit;
1502         u8 rs, rt, addr_reg, flags = REG_EXT;
1503         bool no_mask = op_flag_no_mask(op->flags);
1504         union code c = op->c;
1505         s16 imm;
1506
1507         if (!c.i.rt)
1508                 return;
1509
1510         if (is_unsigned)
1511                 flags |= REG_ZEXT;
1512
1513         rs = lightrec_alloc_reg_in(reg_cache, _jit, c.i.rs, 0);
1514         rt = lightrec_alloc_reg_out(reg_cache, _jit, c.i.rt, flags);
1515
1516         if (!cstate->state->mirrors_mapped && c.i.imm && !no_mask) {
1517                 jit_addi(rt, rs, (s16)c.i.imm);
1518                 addr_reg = rt;
1519                 imm = 0;
1520         } else {
1521                 addr_reg = rs;
1522                 imm = (s16)c.i.imm;
1523         }
1524
1525         if (!no_mask) {
1526                 jit_andi(rt, addr_reg, addr_mask);
1527                 addr_reg = rt;
1528         }
1529
1530         if (addr_offset) {
1531                 jit_addi(rt, addr_reg, addr_offset);
1532                 addr_reg = rt;
1533         }
1534
1535         jit_new_node_www(code, rt, addr_reg, imm);
1536
1537         if (is_big_endian() && swap_code) {
1538                 jit_new_node_ww(swap_code, rt, rt);
1539
1540                 if (c.i.op == OP_LH)
1541                         jit_extr_s(rt, rt);
1542                 else if (c.i.op == OP_LW && __WORDSIZE == 64)
1543                         jit_extr_i(rt, rt);
1544         }
1545
1546         lightrec_free_reg(reg_cache, rs);
1547         lightrec_free_reg(reg_cache, rt);
1548 }
1549
1550 static void rec_load_ram(struct lightrec_cstate *cstate,
1551                          const struct block *block, u16 offset,
1552                          jit_code_t code, jit_code_t swap_code, bool is_unsigned)
1553 {
1554         _jit_note(block->_jit, __FILE__, __LINE__);
1555
1556         rec_load_memory(cstate, block, offset, code, swap_code, is_unsigned,
1557                         cstate->state->offset_ram, rec_ram_mask(cstate->state));
1558 }
1559
1560 static void rec_load_bios(struct lightrec_cstate *cstate,
1561                           const struct block *block, u16 offset,
1562                           jit_code_t code, jit_code_t swap_code, bool is_unsigned)
1563 {
1564         _jit_note(block->_jit, __FILE__, __LINE__);
1565
1566         rec_load_memory(cstate, block, offset, code, swap_code, is_unsigned,
1567                         cstate->state->offset_bios, 0x1fffffff);
1568 }
1569
1570 static void rec_load_scratch(struct lightrec_cstate *cstate,
1571                              const struct block *block, u16 offset,
1572                              jit_code_t code, jit_code_t swap_code, bool is_unsigned)
1573 {
1574         _jit_note(block->_jit, __FILE__, __LINE__);
1575
1576         rec_load_memory(cstate, block, offset, code, swap_code, is_unsigned,
1577                         cstate->state->offset_scratch, 0x1fffffff);
1578 }
1579
1580 static void rec_load_io(struct lightrec_cstate *cstate,
1581                         const struct block *block, u16 offset,
1582                         jit_code_t code, jit_code_t swap_code, bool is_unsigned)
1583 {
1584         _jit_note(block->_jit, __FILE__, __LINE__);
1585
1586         rec_load_memory(cstate, block, offset, code, swap_code, is_unsigned,
1587                         cstate->state->offset_io, rec_io_mask(cstate->state));
1588 }
1589
1590 static void rec_load_direct(struct lightrec_cstate *cstate,
1591                             const struct block *block, u16 offset,
1592                             jit_code_t code, jit_code_t swap_code,
1593                             bool is_unsigned)
1594 {
1595         struct lightrec_state *state = cstate->state;
1596         struct regcache *reg_cache = cstate->reg_cache;
1597         union code c = block->opcode_list[offset].c;
1598         jit_state_t *_jit = block->_jit;
1599         jit_node_t *to_not_ram, *to_not_bios, *to_end, *to_end2;
1600         u8 tmp, rs, rt, addr_reg, flags = REG_EXT;
1601         s16 imm;
1602
1603         if (!c.i.rt)
1604                 return;
1605
1606         if (is_unsigned)
1607                 flags |= REG_ZEXT;
1608
1609         jit_note(__FILE__, __LINE__);
1610         rs = lightrec_alloc_reg_in(reg_cache, _jit, c.i.rs, 0);
1611         rt = lightrec_alloc_reg_out(reg_cache, _jit, c.i.rt, flags);
1612
1613         if ((state->offset_ram == state->offset_bios &&
1614             state->offset_ram == state->offset_scratch &&
1615             state->mirrors_mapped) || !c.i.imm) {
1616                 addr_reg = rs;
1617                 imm = (s16)c.i.imm;
1618         } else {
1619                 jit_addi(rt, rs, (s16)c.i.imm);
1620                 addr_reg = rt;
1621                 imm = 0;
1622
1623                 if (c.i.rs != c.i.rt)
1624                         lightrec_free_reg(reg_cache, rs);
1625         }
1626
1627         tmp = lightrec_alloc_reg_temp(reg_cache, _jit);
1628
1629         if (state->offset_ram == state->offset_bios &&
1630             state->offset_ram == state->offset_scratch) {
1631                 if (!state->mirrors_mapped) {
1632                         jit_andi(tmp, addr_reg, BIT(28));
1633                         jit_rshi_u(tmp, tmp, 28 - 22);
1634                         jit_ori(tmp, tmp, 0x1f800000 | (RAM_SIZE - 1));
1635                         jit_andr(rt, addr_reg, tmp);
1636                 } else {
1637                         jit_andi(rt, addr_reg, 0x1fffffff);
1638                 }
1639
1640                 if (state->offset_ram)
1641                         jit_movi(tmp, state->offset_ram);
1642         } else {
1643                 to_not_ram = jit_bmsi(addr_reg, BIT(28));
1644
1645                 /* Convert to KUNSEG and avoid RAM mirrors */
1646                 jit_andi(rt, addr_reg, RAM_SIZE - 1);
1647
1648                 if (state->offset_ram)
1649                         jit_movi(tmp, state->offset_ram);
1650
1651                 to_end = jit_b();
1652
1653                 jit_patch(to_not_ram);
1654
1655                 if (state->offset_bios != state->offset_scratch)
1656                         to_not_bios = jit_bmci(addr_reg, BIT(22));
1657
1658                 /* Convert to KUNSEG */
1659                 jit_andi(rt, addr_reg, 0x1fc00000 | (BIOS_SIZE - 1));
1660
1661                 jit_movi(tmp, state->offset_bios);
1662
1663                 if (state->offset_bios != state->offset_scratch) {
1664                         to_end2 = jit_b();
1665
1666                         jit_patch(to_not_bios);
1667
1668                         /* Convert to KUNSEG */
1669                         jit_andi(rt, addr_reg, 0x1f800fff);
1670
1671                         if (state->offset_scratch)
1672                                 jit_movi(tmp, state->offset_scratch);
1673
1674                         jit_patch(to_end2);
1675                 }
1676
1677                 jit_patch(to_end);
1678         }
1679
1680         if (state->offset_ram || state->offset_bios || state->offset_scratch)
1681                 jit_addr(rt, rt, tmp);
1682
1683         jit_new_node_www(code, rt, rt, imm);
1684
1685         if (is_big_endian() && swap_code) {
1686                 jit_new_node_ww(swap_code, rt, rt);
1687
1688                 if (c.i.op == OP_LH)
1689                         jit_extr_s(rt, rt);
1690                 else if (c.i.op == OP_LW && __WORDSIZE == 64)
1691                         jit_extr_i(rt, rt);
1692         }
1693
1694         lightrec_free_reg(reg_cache, addr_reg);
1695         lightrec_free_reg(reg_cache, rt);
1696         lightrec_free_reg(reg_cache, tmp);
1697 }
1698
1699 static void rec_load(struct lightrec_cstate *state, const struct block *block,
1700                      u16 offset, jit_code_t code, jit_code_t swap_code,
1701                      bool is_unsigned)
1702 {
1703         u32 flags = block->opcode_list[offset].flags;
1704
1705         switch (LIGHTREC_FLAGS_GET_IO_MODE(flags)) {
1706         case LIGHTREC_IO_RAM:
1707                 rec_load_ram(state, block, offset, code, swap_code, is_unsigned);
1708                 break;
1709         case LIGHTREC_IO_BIOS:
1710                 rec_load_bios(state, block, offset, code, swap_code, is_unsigned);
1711                 break;
1712         case LIGHTREC_IO_SCRATCH:
1713                 rec_load_scratch(state, block, offset, code, swap_code, is_unsigned);
1714                 break;
1715         case LIGHTREC_IO_DIRECT_HW:
1716                 rec_load_io(state, block, offset, code, swap_code, is_unsigned);
1717                 break;
1718         case LIGHTREC_IO_DIRECT:
1719                 rec_load_direct(state, block, offset, code, swap_code, is_unsigned);
1720                 break;
1721         default:
1722                 rec_io(state, block, offset, false, true);
1723                 break;
1724         }
1725 }
1726
1727 static void rec_LB(struct lightrec_cstate *state, const struct block *block, u16 offset)
1728 {
1729         _jit_name(block->_jit, __func__);
1730         rec_load(state, block, offset, jit_code_ldxi_c, 0, false);
1731 }
1732
1733 static void rec_LBU(struct lightrec_cstate *state, const struct block *block, u16 offset)
1734 {
1735         _jit_name(block->_jit, __func__);
1736         rec_load(state, block, offset, jit_code_ldxi_uc, 0, true);
1737 }
1738
1739 static void rec_LH(struct lightrec_cstate *state, const struct block *block, u16 offset)
1740 {
1741         jit_code_t code = is_big_endian() ? jit_code_ldxi_us : jit_code_ldxi_s;
1742
1743         _jit_name(block->_jit, __func__);
1744         rec_load(state, block, offset, code, jit_code_bswapr_us, false);
1745 }
1746
1747 static void rec_LHU(struct lightrec_cstate *state, const struct block *block, u16 offset)
1748 {
1749         _jit_name(block->_jit, __func__);
1750         rec_load(state, block, offset, jit_code_ldxi_us, jit_code_bswapr_us, true);
1751 }
1752
1753 static void rec_LWL(struct lightrec_cstate *state, const struct block *block, u16 offset)
1754 {
1755         _jit_name(block->_jit, __func__);
1756         rec_io(state, block, offset, true, true);
1757 }
1758
1759 static void rec_LWR(struct lightrec_cstate *state, const struct block *block, u16 offset)
1760 {
1761         _jit_name(block->_jit, __func__);
1762         rec_io(state, block, offset, true, true);
1763 }
1764
1765 static void rec_LW(struct lightrec_cstate *state, const struct block *block, u16 offset)
1766 {
1767         jit_code_t code;
1768
1769         if (is_big_endian() && __WORDSIZE == 64)
1770                 code = jit_code_ldxi_ui;
1771         else
1772                 code = jit_code_ldxi_i;
1773
1774         _jit_name(block->_jit, __func__);
1775         rec_load(state, block, offset, code, jit_code_bswapr_ui, false);
1776 }
1777
1778 static void rec_LWC2(struct lightrec_cstate *state, const struct block *block, u16 offset)
1779 {
1780         _jit_name(block->_jit, __func__);
1781         rec_io(state, block, offset, false, false);
1782 }
1783
1784 static void rec_break_syscall(struct lightrec_cstate *state,
1785                               const struct block *block, u16 offset,
1786                               u32 exit_code)
1787 {
1788         struct regcache *reg_cache = state->reg_cache;
1789         jit_state_t *_jit = block->_jit;
1790         u8 tmp;
1791
1792         _jit_note(block->_jit, __FILE__, __LINE__);
1793
1794         tmp = lightrec_alloc_reg_temp(reg_cache, _jit);
1795
1796         jit_movi(tmp, exit_code);
1797         jit_stxi_i(offsetof(struct lightrec_state, exit_flags),
1798                    LIGHTREC_REG_STATE, tmp);
1799
1800         lightrec_free_reg(reg_cache, tmp);
1801
1802         /* TODO: the return address should be "pc - 4" if we're a delay slot */
1803         lightrec_emit_end_of_block(state, block, offset, -1,
1804                                    get_ds_pc(block, offset, 0),
1805                                    31, 0, true);
1806 }
1807
1808 static void rec_special_SYSCALL(struct lightrec_cstate *state,
1809                                 const struct block *block, u16 offset)
1810 {
1811         _jit_name(block->_jit, __func__);
1812         rec_break_syscall(state, block, offset, LIGHTREC_EXIT_SYSCALL);
1813 }
1814
1815 static void rec_special_BREAK(struct lightrec_cstate *state,
1816                               const struct block *block, u16 offset)
1817 {
1818         _jit_name(block->_jit, __func__);
1819         rec_break_syscall(state, block, offset, LIGHTREC_EXIT_BREAK);
1820 }
1821
1822 static void rec_mfc(struct lightrec_cstate *state, const struct block *block, u16 offset)
1823 {
1824         struct regcache *reg_cache = state->reg_cache;
1825         union code c = block->opcode_list[offset].c;
1826         jit_state_t *_jit = block->_jit;
1827
1828         jit_note(__FILE__, __LINE__);
1829         lightrec_clean_reg_if_loaded(reg_cache, _jit, c.i.rt, true);
1830
1831         call_to_c_wrapper(state, block, c.opcode, C_WRAPPER_MFC);
1832 }
1833
1834 static void rec_mtc(struct lightrec_cstate *state, const struct block *block, u16 offset)
1835 {
1836         struct regcache *reg_cache = state->reg_cache;
1837         union code c = block->opcode_list[offset].c;
1838         jit_state_t *_jit = block->_jit;
1839
1840         jit_note(__FILE__, __LINE__);
1841         lightrec_clean_reg_if_loaded(reg_cache, _jit, c.i.rs, false);
1842         lightrec_clean_reg_if_loaded(reg_cache, _jit, c.i.rt, false);
1843
1844         call_to_c_wrapper(state, block, c.opcode, C_WRAPPER_MTC);
1845
1846         if (c.i.op == OP_CP0 &&
1847             !op_flag_no_ds(block->opcode_list[offset].flags) &&
1848             (c.r.rd == 12 || c.r.rd == 13))
1849                 lightrec_emit_end_of_block(state, block, offset, -1,
1850                                            get_ds_pc(block, offset, 1),
1851                                            0, 0, true);
1852 }
1853
1854 static void
1855 rec_mfc0(struct lightrec_cstate *state, const struct block *block, u16 offset)
1856 {
1857         struct regcache *reg_cache = state->reg_cache;
1858         union code c = block->opcode_list[offset].c;
1859         jit_state_t *_jit = block->_jit;
1860         u8 rt;
1861
1862         jit_note(__FILE__, __LINE__);
1863
1864         rt = lightrec_alloc_reg_out(reg_cache, _jit, c.i.rt, REG_EXT);
1865
1866         jit_ldxi_i(rt, LIGHTREC_REG_STATE,
1867                    offsetof(struct lightrec_state, regs.cp0[c.r.rd]));
1868
1869         lightrec_free_reg(reg_cache, rt);
1870 }
1871
1872 static bool block_in_bios(const struct lightrec_cstate *state,
1873                           const struct block *block)
1874 {
1875         const struct lightrec_mem_map *bios = &state->state->maps[PSX_MAP_BIOS];
1876         u32 pc = kunseg(block->pc);
1877
1878         return pc >= bios->pc && pc < bios->pc + bios->length;
1879 }
1880
1881 static void
1882 rec_mtc0(struct lightrec_cstate *state, const struct block *block, u16 offset)
1883 {
1884         struct regcache *reg_cache = state->reg_cache;
1885         const union code c = block->opcode_list[offset].c;
1886         jit_state_t *_jit = block->_jit;
1887         u8 rt, tmp = 0, tmp2, status;
1888
1889         jit_note(__FILE__, __LINE__);
1890
1891         switch(c.r.rd) {
1892         case 1:
1893         case 4:
1894         case 8:
1895         case 14:
1896         case 15:
1897                 /* Those registers are read-only */
1898                 return;
1899         default:
1900                 break;
1901         }
1902
1903         if (block_in_bios(state, block) && c.r.rd == 12) {
1904                 /* If we are running code from the BIOS, handle writes to the
1905                  * Status register in C. BIOS code may toggle bit 16 which will
1906                  * map/unmap the RAM, while game code cannot do that. */
1907                 rec_mtc(state, block, offset);
1908                 return;
1909         }
1910
1911         rt = lightrec_alloc_reg_in(reg_cache, _jit, c.i.rt, 0);
1912
1913         if (c.r.rd != 13) {
1914                 jit_stxi_i(offsetof(struct lightrec_state, regs.cp0[c.r.rd]),
1915                            LIGHTREC_REG_STATE, rt);
1916         }
1917
1918         if (c.r.rd == 12 || c.r.rd == 13) {
1919                 tmp = lightrec_alloc_reg_temp(reg_cache, _jit);
1920                 jit_ldxi_i(tmp, LIGHTREC_REG_STATE,
1921                            offsetof(struct lightrec_state, regs.cp0[13]));
1922
1923                 tmp2 = lightrec_alloc_reg_temp(reg_cache, _jit);
1924         }
1925
1926         if (c.r.rd == 12) {
1927                 status = rt;
1928         } else if (c.r.rd == 13) {
1929                 /* Cause = (Cause & ~0x0300) | (value & 0x0300) */
1930                 jit_andi(tmp2, rt, 0x0300);
1931                 jit_ori(tmp, tmp, 0x0300);
1932                 jit_xori(tmp, tmp, 0x0300);
1933                 jit_orr(tmp, tmp, tmp2);
1934                 jit_ldxi_i(tmp2, LIGHTREC_REG_STATE,
1935                            offsetof(struct lightrec_state, regs.cp0[12]));
1936                 jit_stxi_i(offsetof(struct lightrec_state, regs.cp0[13]),
1937                            LIGHTREC_REG_STATE, tmp);
1938                 status = tmp2;
1939         }
1940
1941         if (c.r.rd == 12 || c.r.rd == 13) {
1942                 /* Exit dynarec in case there's a software interrupt.
1943                  * exit_flags = !!(status & tmp & 0x0300) & status; */
1944                 jit_andr(tmp, tmp, status);
1945                 jit_andi(tmp, tmp, 0x0300);
1946                 jit_nei(tmp, tmp, 0);
1947                 jit_andr(tmp, tmp, status);
1948         }
1949
1950         if (c.r.rd == 12) {
1951                 /* Exit dynarec in case we unmask a hardware interrupt.
1952                  * exit_flags = !(~status & 0x401) */
1953
1954                 jit_comr(tmp2, status);
1955                 jit_andi(tmp2, tmp2, 0x401);
1956                 jit_eqi(tmp2, tmp2, 0);
1957                 jit_orr(tmp, tmp, tmp2);
1958         }
1959
1960         if (c.r.rd == 12 || c.r.rd == 13) {
1961                 jit_stxi_i(offsetof(struct lightrec_state, exit_flags),
1962                            LIGHTREC_REG_STATE, tmp);
1963
1964                 lightrec_free_reg(reg_cache, tmp);
1965                 lightrec_free_reg(reg_cache, tmp2);
1966         }
1967
1968         lightrec_free_reg(reg_cache, rt);
1969
1970         if (!op_flag_no_ds(block->opcode_list[offset].flags) &&
1971             (c.r.rd == 12 || c.r.rd == 13)) {
1972                 state->cycles += lightrec_cycles_of_opcode(c);
1973                 lightrec_emit_eob(state, block, offset + 1);
1974         }
1975 }
1976
1977 static void rec_cp0_MFC0(struct lightrec_cstate *state,
1978                          const struct block *block, u16 offset)
1979 {
1980         _jit_name(block->_jit, __func__);
1981         rec_mfc0(state, block, offset);
1982 }
1983
1984 static void rec_cp0_CFC0(struct lightrec_cstate *state,
1985                          const struct block *block, u16 offset)
1986 {
1987         _jit_name(block->_jit, __func__);
1988         rec_mfc0(state, block, offset);
1989 }
1990
1991 static void rec_cp0_MTC0(struct lightrec_cstate *state,
1992                          const struct block *block, u16 offset)
1993 {
1994         _jit_name(block->_jit, __func__);
1995         rec_mtc0(state, block, offset);
1996 }
1997
1998 static void rec_cp0_CTC0(struct lightrec_cstate *state,
1999                          const struct block *block, u16 offset)
2000 {
2001         _jit_name(block->_jit, __func__);
2002         rec_mtc0(state, block, offset);
2003 }
2004
2005 static unsigned int cp2d_i_offset(u8 reg)
2006 {
2007         return offsetof(struct lightrec_state, regs.cp2d[reg]);
2008 }
2009
2010 static unsigned int cp2d_s_offset(u8 reg)
2011 {
2012         return cp2d_i_offset(reg) + is_big_endian() * 2;
2013 }
2014
2015 static unsigned int cp2c_i_offset(u8 reg)
2016 {
2017         return offsetof(struct lightrec_state, regs.cp2c[reg]);
2018 }
2019
2020 static unsigned int cp2c_s_offset(u8 reg)
2021 {
2022         return cp2c_i_offset(reg) + is_big_endian() * 2;
2023 }
2024
2025 static void rec_cp2_basic_MFC2(struct lightrec_cstate *state,
2026                                const struct block *block, u16 offset)
2027 {
2028         struct regcache *reg_cache = state->reg_cache;
2029         const union code c = block->opcode_list[offset].c;
2030         jit_state_t *_jit = block->_jit;
2031         const u32 zext_regs = 0x300f0080;
2032         u8 rt, tmp, tmp2, tmp3, out, flags;
2033         u8 reg = c.r.rd == 15 ? 14 : c.r.rd;
2034         unsigned int i;
2035
2036         _jit_name(block->_jit, __func__);
2037
2038         if (state->state->ops.cop2_notify) {
2039                 /* We must call cop2_notify, handle that in C. */
2040                 rec_mfc(state, block, offset);
2041                 return;
2042         }
2043
2044         flags = (zext_regs & BIT(reg)) ? REG_ZEXT : REG_EXT;
2045         rt = lightrec_alloc_reg_out(reg_cache, _jit, c.r.rt, flags);
2046
2047         switch (reg) {
2048         case 1:
2049         case 3:
2050         case 5:
2051         case 8:
2052         case 9:
2053         case 10:
2054         case 11:
2055                 jit_ldxi_s(rt, LIGHTREC_REG_STATE, cp2d_s_offset(reg));
2056                 break;
2057         case 7:
2058         case 16:
2059         case 17:
2060         case 18:
2061         case 19:
2062                 jit_ldxi_us(rt, LIGHTREC_REG_STATE, cp2d_s_offset(reg));
2063                 break;
2064         case 28:
2065         case 29:
2066                 tmp = lightrec_alloc_reg_temp(reg_cache, _jit);
2067                 tmp2 = lightrec_alloc_reg_temp(reg_cache, _jit);
2068                 tmp3 = lightrec_alloc_reg_temp(reg_cache, _jit);
2069
2070                 for (i = 0; i < 3; i++) {
2071                         out = i == 0 ? rt : tmp;
2072
2073                         jit_ldxi_s(tmp, LIGHTREC_REG_STATE, cp2d_s_offset(9 + i));
2074                         jit_movi(tmp2, 0x1f);
2075                         jit_rshi(out, tmp, 7);
2076
2077                         jit_ltr(tmp3, tmp2, out);
2078                         jit_movnr(out, tmp2, tmp3);
2079
2080                         jit_gei(tmp2, out, 0);
2081                         jit_movzr(out, tmp2, tmp2);
2082
2083                         if (i > 0) {
2084                                 jit_lshi(tmp, tmp, 5 * i);
2085                                 jit_orr(rt, rt, tmp);
2086                         }
2087                 }
2088
2089
2090                 lightrec_free_reg(reg_cache, tmp);
2091                 lightrec_free_reg(reg_cache, tmp2);
2092                 lightrec_free_reg(reg_cache, tmp3);
2093                 break;
2094         default:
2095                 jit_ldxi_i(rt, LIGHTREC_REG_STATE, cp2d_i_offset(reg));
2096                 break;
2097         }
2098
2099         lightrec_free_reg(reg_cache, rt);
2100 }
2101
2102 static void rec_cp2_basic_CFC2(struct lightrec_cstate *state,
2103                                const struct block *block, u16 offset)
2104 {
2105         struct regcache *reg_cache = state->reg_cache;
2106         const union code c = block->opcode_list[offset].c;
2107         jit_state_t *_jit = block->_jit;
2108         u8 rt;
2109
2110         _jit_name(block->_jit, __func__);
2111
2112         if (state->state->ops.cop2_notify) {
2113                 /* We must call cop2_notify, handle that in C. */
2114                 rec_mfc(state, block, offset);
2115                 return;
2116         }
2117
2118         switch (c.r.rd) {
2119         case 4:
2120         case 12:
2121         case 20:
2122         case 26:
2123         case 27:
2124         case 29:
2125         case 30:
2126                 rt = lightrec_alloc_reg_out(reg_cache, _jit, c.r.rt, REG_EXT);
2127                 jit_ldxi_s(rt, LIGHTREC_REG_STATE, cp2c_s_offset(c.r.rd));
2128                 break;
2129         default:
2130                 rt = lightrec_alloc_reg_out(reg_cache, _jit, c.r.rt, REG_ZEXT);
2131                 jit_ldxi_i(rt, LIGHTREC_REG_STATE, cp2c_i_offset(c.r.rd));
2132                 break;
2133         }
2134
2135         lightrec_free_reg(reg_cache, rt);
2136 }
2137
2138 static void rec_cp2_basic_MTC2(struct lightrec_cstate *state,
2139                                const struct block *block, u16 offset)
2140 {
2141         struct regcache *reg_cache = state->reg_cache;
2142         const union code c = block->opcode_list[offset].c;
2143         jit_state_t *_jit = block->_jit;
2144         jit_node_t *loop, *to_loop;
2145         u8 rt, tmp, tmp2, flags = 0;
2146
2147         _jit_name(block->_jit, __func__);
2148
2149         if (state->state->ops.cop2_notify) {
2150                 /* We must call cop2_notify, handle that in C. */
2151                 rec_mtc(state, block, offset);
2152                 return;
2153         }
2154
2155         if (c.r.rd == 31)
2156                 return;
2157
2158         if (c.r.rd == 30)
2159                 flags |= REG_EXT;
2160
2161         rt = lightrec_alloc_reg_in(reg_cache, _jit, c.r.rt, flags);
2162
2163         switch (c.r.rd) {
2164         case 15:
2165                 tmp = lightrec_alloc_reg_temp(reg_cache, _jit);
2166                 jit_ldxi_i(tmp, LIGHTREC_REG_STATE, cp2d_i_offset(13));
2167
2168                 tmp2 = lightrec_alloc_reg_temp(reg_cache, _jit);
2169                 jit_ldxi_i(tmp2, LIGHTREC_REG_STATE, cp2d_i_offset(14));
2170
2171                 jit_stxi_i(cp2d_i_offset(12), LIGHTREC_REG_STATE, tmp);
2172                 jit_stxi_i(cp2d_i_offset(13), LIGHTREC_REG_STATE, tmp2);
2173                 jit_stxi_i(cp2d_i_offset(14), LIGHTREC_REG_STATE, rt);
2174
2175                 lightrec_free_reg(reg_cache, tmp);
2176                 lightrec_free_reg(reg_cache, tmp2);
2177                 break;
2178         case 28:
2179                 tmp = lightrec_alloc_reg_temp(reg_cache, _jit);
2180
2181                 jit_lshi(tmp, rt, 7);
2182                 jit_andi(tmp, tmp, 0xf80);
2183                 jit_stxi_s(cp2d_s_offset(9), LIGHTREC_REG_STATE, tmp);
2184
2185                 jit_lshi(tmp, rt, 2);
2186                 jit_andi(tmp, tmp, 0xf80);
2187                 jit_stxi_s(cp2d_s_offset(10), LIGHTREC_REG_STATE, tmp);
2188
2189                 jit_rshi(tmp, rt, 3);
2190                 jit_andi(tmp, tmp, 0xf80);
2191                 jit_stxi_s(cp2d_s_offset(11), LIGHTREC_REG_STATE, tmp);
2192
2193                 lightrec_free_reg(reg_cache, tmp);
2194                 break;
2195         case 30:
2196                 tmp = lightrec_alloc_reg_temp(reg_cache, _jit);
2197                 tmp2 = lightrec_alloc_reg_temp(reg_cache, _jit);
2198
2199                 /* if (rt < 0) rt = ~rt; */
2200                 jit_rshi(tmp, rt, 31);
2201                 jit_xorr(tmp, rt, tmp);
2202
2203                 /* We know the sign bit is 0. Left-shift by 1 to start the algorithm */
2204                 jit_lshi(tmp, tmp, 1);
2205                 jit_movi(tmp2, 33);
2206
2207                 /* Decrement tmp2 and right-shift the value by 1 until it equals zero */
2208                 loop = jit_label();
2209                 jit_subi(tmp2, tmp2, 1);
2210                 jit_rshi_u(tmp, tmp, 1);
2211                 to_loop = jit_bnei(tmp, 0);
2212
2213                 jit_patch_at(to_loop, loop);
2214
2215                 jit_stxi_i(cp2d_i_offset(31), LIGHTREC_REG_STATE, tmp2);
2216                 jit_stxi_i(cp2d_i_offset(30), LIGHTREC_REG_STATE, rt);
2217
2218                 lightrec_free_reg(reg_cache, tmp);
2219                 lightrec_free_reg(reg_cache, tmp2);
2220                 break;
2221         default:
2222                 jit_stxi_i(cp2d_i_offset(c.r.rd), LIGHTREC_REG_STATE, rt);
2223                 break;
2224         }
2225
2226         lightrec_free_reg(reg_cache, rt);
2227 }
2228
2229 static void rec_cp2_basic_CTC2(struct lightrec_cstate *state,
2230                                const struct block *block, u16 offset)
2231 {
2232         struct regcache *reg_cache = state->reg_cache;
2233         const union code c = block->opcode_list[offset].c;
2234         jit_state_t *_jit = block->_jit;
2235         u8 rt, tmp, tmp2;
2236
2237         _jit_name(block->_jit, __func__);
2238
2239         if (state->state->ops.cop2_notify) {
2240                 /* We must call cop2_notify, handle that in C. */
2241                 rec_mtc(state, block, offset);
2242                 return;
2243         }
2244
2245         rt = lightrec_alloc_reg_in(reg_cache, _jit, c.r.rt, 0);
2246
2247         switch (c.r.rd) {
2248         case 4:
2249         case 12:
2250         case 20:
2251         case 26:
2252         case 27:
2253         case 29:
2254         case 30:
2255                 jit_stxi_s(cp2c_s_offset(c.r.rd), LIGHTREC_REG_STATE, rt);
2256                 break;
2257         case 31:
2258                 tmp = lightrec_alloc_reg_temp(reg_cache, _jit);
2259                 tmp2 = lightrec_alloc_reg_temp(reg_cache, _jit);
2260
2261                 jit_andi(tmp, rt, 0x7f87e000);
2262                 jit_nei(tmp, tmp, 0);
2263                 jit_lshi(tmp, tmp, 31);
2264
2265                 jit_andi(tmp2, rt, 0x7ffff000);
2266                 jit_orr(tmp, tmp2, tmp);
2267
2268                 jit_stxi_i(cp2c_i_offset(31), LIGHTREC_REG_STATE, tmp);
2269
2270                 lightrec_free_reg(reg_cache, tmp);
2271                 lightrec_free_reg(reg_cache, tmp2);
2272                 break;
2273
2274         default:
2275                 jit_stxi_i(cp2c_i_offset(c.r.rd), LIGHTREC_REG_STATE, rt);
2276         }
2277
2278         lightrec_free_reg(reg_cache, rt);
2279 }
2280
2281 static void rec_cp0_RFE(struct lightrec_cstate *state,
2282                         const struct block *block, u16 offset)
2283 {
2284         struct regcache *reg_cache = state->reg_cache;
2285         jit_state_t *_jit = block->_jit;
2286         u8 status, tmp;
2287
2288         jit_name(__func__);
2289         jit_note(__FILE__, __LINE__);
2290
2291         status = lightrec_alloc_reg_temp(reg_cache, _jit);
2292         jit_ldxi_i(status, LIGHTREC_REG_STATE,
2293                    offsetof(struct lightrec_state, regs.cp0[12]));
2294
2295         tmp = lightrec_alloc_reg_temp(reg_cache, _jit);
2296
2297         /* status = ((status >> 2) & 0xf) | status & ~0xf; */
2298         jit_rshi(tmp, status, 2);
2299         jit_andi(tmp, tmp, 0xf);
2300         jit_andi(status, status, ~0xful);
2301         jit_orr(status, status, tmp);
2302
2303         jit_ldxi_i(tmp, LIGHTREC_REG_STATE,
2304                    offsetof(struct lightrec_state, regs.cp0[13]));
2305         jit_stxi_i(offsetof(struct lightrec_state, regs.cp0[12]),
2306                    LIGHTREC_REG_STATE, status);
2307
2308         /* Exit dynarec in case there's a software interrupt.
2309          * exit_flags = !!(status & cause & 0x0300) & status; */
2310         jit_andr(tmp, tmp, status);
2311         jit_andi(tmp, tmp, 0x0300);
2312         jit_nei(tmp, tmp, 0);
2313         jit_andr(tmp, tmp, status);
2314         jit_stxi_i(offsetof(struct lightrec_state, exit_flags),
2315                    LIGHTREC_REG_STATE, tmp);
2316
2317         lightrec_free_reg(reg_cache, status);
2318         lightrec_free_reg(reg_cache, tmp);
2319 }
2320
2321 static void rec_CP(struct lightrec_cstate *state,
2322                    const struct block *block, u16 offset)
2323 {
2324         union code c = block->opcode_list[offset].c;
2325         jit_state_t *_jit = block->_jit;
2326
2327         jit_name(__func__);
2328         jit_note(__FILE__, __LINE__);
2329
2330         call_to_c_wrapper(state, block, c.opcode, C_WRAPPER_CP);
2331 }
2332
2333 static void rec_meta_MOV(struct lightrec_cstate *state,
2334                          const struct block *block, u16 offset)
2335 {
2336         struct regcache *reg_cache = state->reg_cache;
2337         union code c = block->opcode_list[offset].c;
2338         jit_state_t *_jit = block->_jit;
2339         u8 rs, rd;
2340
2341         _jit_name(block->_jit, __func__);
2342         jit_note(__FILE__, __LINE__);
2343         if (c.r.rs)
2344                 rs = lightrec_alloc_reg_in(reg_cache, _jit, c.r.rs, 0);
2345         rd = lightrec_alloc_reg_out(reg_cache, _jit, c.r.rd, REG_EXT);
2346
2347         if (c.r.rs == 0)
2348                 jit_movi(rd, 0);
2349         else
2350                 jit_extr_i(rd, rs);
2351
2352         if (c.r.rs)
2353                 lightrec_free_reg(reg_cache, rs);
2354         lightrec_free_reg(reg_cache, rd);
2355 }
2356
2357 static void rec_meta_EXTC_EXTS(struct lightrec_cstate *state,
2358                                const struct block *block,
2359                                u16 offset)
2360 {
2361         struct regcache *reg_cache = state->reg_cache;
2362         union code c = block->opcode_list[offset].c;
2363         jit_state_t *_jit = block->_jit;
2364         u8 rs, rt;
2365
2366         _jit_name(block->_jit, __func__);
2367         jit_note(__FILE__, __LINE__);
2368
2369         rs = lightrec_alloc_reg_in(reg_cache, _jit, c.i.rs, 0);
2370         rt = lightrec_alloc_reg_out(reg_cache, _jit, c.i.rt, REG_EXT);
2371
2372         if (c.i.op == OP_META_EXTC)
2373                 jit_extr_c(rt, rs);
2374         else
2375                 jit_extr_s(rt, rs);
2376
2377         lightrec_free_reg(reg_cache, rs);
2378         lightrec_free_reg(reg_cache, rt);
2379 }
2380
2381 static void rec_meta_MULT2(struct lightrec_cstate *state,
2382                            const struct block *block,
2383                            u16 offset)
2384 {
2385         struct regcache *reg_cache = state->reg_cache;
2386         union code c = block->opcode_list[offset].c;
2387         jit_state_t *_jit = block->_jit;
2388         u8 reg_lo = get_mult_div_lo(c);
2389         u8 reg_hi = get_mult_div_hi(c);
2390         u32 flags = block->opcode_list[offset].flags;
2391         bool is_signed = c.i.op == OP_META_MULT2;
2392         u8 rs, lo, hi, rflags = 0, hiflags = 0;
2393
2394         if (!op_flag_no_hi(flags) && c.r.op < 32) {
2395                 rflags = is_signed ? REG_EXT : REG_ZEXT;
2396                 hiflags = is_signed ? REG_EXT : (REG_EXT | REG_ZEXT);
2397         }
2398
2399         _jit_name(block->_jit, __func__);
2400         jit_note(__FILE__, __LINE__);
2401
2402         rs = lightrec_alloc_reg_in(reg_cache, _jit, c.i.rs, rflags);
2403
2404         if (!op_flag_no_lo(flags)) {
2405                 lo = lightrec_alloc_reg_out(reg_cache, _jit, reg_lo, 0);
2406
2407                 if (c.r.op < 32)
2408                         jit_lshi(lo, rs, c.r.op);
2409                 else
2410                         jit_movi(lo, 0);
2411
2412                 lightrec_free_reg(reg_cache, lo);
2413         }
2414
2415         if (!op_flag_no_hi(flags)) {
2416                 hi = lightrec_alloc_reg_out(reg_cache, _jit, reg_hi, hiflags);
2417
2418                 if (c.r.op >= 32)
2419                         jit_lshi(hi, rs, c.r.op - 32);
2420                 else if (is_signed)
2421                         jit_rshi(hi, rs, 32 - c.r.op);
2422                 else
2423                         jit_rshi_u(hi, rs, 32 - c.r.op);
2424
2425                 lightrec_free_reg(reg_cache, hi);
2426         }
2427
2428         lightrec_free_reg(reg_cache, rs);
2429
2430         _jit_name(block->_jit, __func__);
2431         jit_note(__FILE__, __LINE__);
2432 }
2433
2434 static const lightrec_rec_func_t rec_standard[64] = {
2435         SET_DEFAULT_ELM(rec_standard, unknown_opcode),
2436         [OP_SPECIAL]            = rec_SPECIAL,
2437         [OP_REGIMM]             = rec_REGIMM,
2438         [OP_J]                  = rec_J,
2439         [OP_JAL]                = rec_JAL,
2440         [OP_BEQ]                = rec_BEQ,
2441         [OP_BNE]                = rec_BNE,
2442         [OP_BLEZ]               = rec_BLEZ,
2443         [OP_BGTZ]               = rec_BGTZ,
2444         [OP_ADDI]               = rec_ADDI,
2445         [OP_ADDIU]              = rec_ADDIU,
2446         [OP_SLTI]               = rec_SLTI,
2447         [OP_SLTIU]              = rec_SLTIU,
2448         [OP_ANDI]               = rec_ANDI,
2449         [OP_ORI]                = rec_ORI,
2450         [OP_XORI]               = rec_XORI,
2451         [OP_LUI]                = rec_LUI,
2452         [OP_CP0]                = rec_CP0,
2453         [OP_CP2]                = rec_CP2,
2454         [OP_LB]                 = rec_LB,
2455         [OP_LH]                 = rec_LH,
2456         [OP_LWL]                = rec_LWL,
2457         [OP_LW]                 = rec_LW,
2458         [OP_LBU]                = rec_LBU,
2459         [OP_LHU]                = rec_LHU,
2460         [OP_LWR]                = rec_LWR,
2461         [OP_SB]                 = rec_SB,
2462         [OP_SH]                 = rec_SH,
2463         [OP_SWL]                = rec_SWL,
2464         [OP_SW]                 = rec_SW,
2465         [OP_SWR]                = rec_SWR,
2466         [OP_LWC2]               = rec_LWC2,
2467         [OP_SWC2]               = rec_SWC2,
2468
2469         [OP_META_MOV]           = rec_meta_MOV,
2470         [OP_META_EXTC]          = rec_meta_EXTC_EXTS,
2471         [OP_META_EXTS]          = rec_meta_EXTC_EXTS,
2472         [OP_META_MULT2]         = rec_meta_MULT2,
2473         [OP_META_MULTU2]        = rec_meta_MULT2,
2474 };
2475
2476 static const lightrec_rec_func_t rec_special[64] = {
2477         SET_DEFAULT_ELM(rec_special, unknown_opcode),
2478         [OP_SPECIAL_SLL]        = rec_special_SLL,
2479         [OP_SPECIAL_SRL]        = rec_special_SRL,
2480         [OP_SPECIAL_SRA]        = rec_special_SRA,
2481         [OP_SPECIAL_SLLV]       = rec_special_SLLV,
2482         [OP_SPECIAL_SRLV]       = rec_special_SRLV,
2483         [OP_SPECIAL_SRAV]       = rec_special_SRAV,
2484         [OP_SPECIAL_JR]         = rec_special_JR,
2485         [OP_SPECIAL_JALR]       = rec_special_JALR,
2486         [OP_SPECIAL_SYSCALL]    = rec_special_SYSCALL,
2487         [OP_SPECIAL_BREAK]      = rec_special_BREAK,
2488         [OP_SPECIAL_MFHI]       = rec_special_MFHI,
2489         [OP_SPECIAL_MTHI]       = rec_special_MTHI,
2490         [OP_SPECIAL_MFLO]       = rec_special_MFLO,
2491         [OP_SPECIAL_MTLO]       = rec_special_MTLO,
2492         [OP_SPECIAL_MULT]       = rec_special_MULT,
2493         [OP_SPECIAL_MULTU]      = rec_special_MULTU,
2494         [OP_SPECIAL_DIV]        = rec_special_DIV,
2495         [OP_SPECIAL_DIVU]       = rec_special_DIVU,
2496         [OP_SPECIAL_ADD]        = rec_special_ADD,
2497         [OP_SPECIAL_ADDU]       = rec_special_ADDU,
2498         [OP_SPECIAL_SUB]        = rec_special_SUB,
2499         [OP_SPECIAL_SUBU]       = rec_special_SUBU,
2500         [OP_SPECIAL_AND]        = rec_special_AND,
2501         [OP_SPECIAL_OR]         = rec_special_OR,
2502         [OP_SPECIAL_XOR]        = rec_special_XOR,
2503         [OP_SPECIAL_NOR]        = rec_special_NOR,
2504         [OP_SPECIAL_SLT]        = rec_special_SLT,
2505         [OP_SPECIAL_SLTU]       = rec_special_SLTU,
2506 };
2507
2508 static const lightrec_rec_func_t rec_regimm[64] = {
2509         SET_DEFAULT_ELM(rec_regimm, unknown_opcode),
2510         [OP_REGIMM_BLTZ]        = rec_regimm_BLTZ,
2511         [OP_REGIMM_BGEZ]        = rec_regimm_BGEZ,
2512         [OP_REGIMM_BLTZAL]      = rec_regimm_BLTZAL,
2513         [OP_REGIMM_BGEZAL]      = rec_regimm_BGEZAL,
2514 };
2515
2516 static const lightrec_rec_func_t rec_cp0[64] = {
2517         SET_DEFAULT_ELM(rec_cp0, rec_CP),
2518         [OP_CP0_MFC0]           = rec_cp0_MFC0,
2519         [OP_CP0_CFC0]           = rec_cp0_CFC0,
2520         [OP_CP0_MTC0]           = rec_cp0_MTC0,
2521         [OP_CP0_CTC0]           = rec_cp0_CTC0,
2522         [OP_CP0_RFE]            = rec_cp0_RFE,
2523 };
2524
2525 static const lightrec_rec_func_t rec_cp2_basic[64] = {
2526         SET_DEFAULT_ELM(rec_cp2_basic, rec_CP),
2527         [OP_CP2_BASIC_MFC2]     = rec_cp2_basic_MFC2,
2528         [OP_CP2_BASIC_CFC2]     = rec_cp2_basic_CFC2,
2529         [OP_CP2_BASIC_MTC2]     = rec_cp2_basic_MTC2,
2530         [OP_CP2_BASIC_CTC2]     = rec_cp2_basic_CTC2,
2531 };
2532
2533 static void rec_SPECIAL(struct lightrec_cstate *state,
2534                         const struct block *block, u16 offset)
2535 {
2536         union code c = block->opcode_list[offset].c;
2537         lightrec_rec_func_t f = rec_special[c.r.op];
2538
2539         if (!HAS_DEFAULT_ELM && unlikely(!f))
2540                 unknown_opcode(state, block, offset);
2541         else
2542                 (*f)(state, block, offset);
2543 }
2544
2545 static void rec_REGIMM(struct lightrec_cstate *state,
2546                        const struct block *block, u16 offset)
2547 {
2548         union code c = block->opcode_list[offset].c;
2549         lightrec_rec_func_t f = rec_regimm[c.r.rt];
2550
2551         if (!HAS_DEFAULT_ELM && unlikely(!f))
2552                 unknown_opcode(state, block, offset);
2553         else
2554                 (*f)(state, block, offset);
2555 }
2556
2557 static void rec_CP0(struct lightrec_cstate *state,
2558                     const struct block *block, u16 offset)
2559 {
2560         union code c = block->opcode_list[offset].c;
2561         lightrec_rec_func_t f = rec_cp0[c.r.rs];
2562
2563         if (!HAS_DEFAULT_ELM && unlikely(!f))
2564                 rec_CP(state, block, offset);
2565         else
2566                 (*f)(state, block, offset);
2567 }
2568
2569 static void rec_CP2(struct lightrec_cstate *state,
2570                     const struct block *block, u16 offset)
2571 {
2572         union code c = block->opcode_list[offset].c;
2573
2574         if (c.r.op == OP_CP2_BASIC) {
2575                 lightrec_rec_func_t f = rec_cp2_basic[c.r.rs];
2576
2577                 if (HAS_DEFAULT_ELM || likely(f)) {
2578                         (*f)(state, block, offset);
2579                         return;
2580                 }
2581         }
2582
2583         rec_CP(state, block, offset);
2584 }
2585
2586 void lightrec_rec_opcode(struct lightrec_cstate *state,
2587                          const struct block *block, u16 offset)
2588 {
2589         struct regcache *reg_cache = state->reg_cache;
2590         struct lightrec_branch_target *target;
2591         const struct opcode *op = &block->opcode_list[offset];
2592         jit_state_t *_jit = block->_jit;
2593         lightrec_rec_func_t f;
2594         u16 unload_offset;
2595
2596         if (op_flag_sync(op->flags)) {
2597                 if (state->cycles)
2598                         jit_subi(LIGHTREC_REG_CYCLE, LIGHTREC_REG_CYCLE, state->cycles);
2599                 state->cycles = 0;
2600
2601                 lightrec_storeback_regs(reg_cache, _jit);
2602                 lightrec_regcache_reset(reg_cache);
2603
2604                 pr_debug("Adding branch target at offset 0x%x\n", offset << 2);
2605                 target = &state->targets[state->nb_targets++];
2606                 target->offset = offset;
2607                 target->label = jit_indirect();
2608         }
2609
2610         if (likely(op->opcode)) {
2611                 f = rec_standard[op->i.op];
2612
2613                 if (!HAS_DEFAULT_ELM && unlikely(!f))
2614                         unknown_opcode(state, block, offset);
2615                 else
2616                         (*f)(state, block, offset);
2617         }
2618
2619         if (OPT_EARLY_UNLOAD) {
2620                 unload_offset = offset +
2621                         (has_delay_slot(op->c) && !op_flag_no_ds(op->flags));
2622
2623                 lightrec_do_early_unload(state, block, unload_offset);
2624         }
2625 }