deps/lightrec/emitter.c

   1 // SPDX-License-Identifier: LGPL-2.1-or-later
   2 /*
   3  * Copyright (C) 2014-2021 Paul Cercueil <paul@crapouillou.net>
   4  */
   5
   6 #include "blockcache.h"
   7 #include "debug.h"
   8 #include "disassembler.h"
   9 #include "emitter.h"
  10 #include "lightning-wrapper.h"
  11 #include "optimizer.h"
  12 #include "regcache.h"
  13
  14 #include <stdbool.h>
  15 #include <stddef.h>
  16
  17 typedef void (*lightrec_rec_func_t)(struct lightrec_cstate *, const struct block *, u16);
  18
  19 /* Forward declarations */
  20 static void rec_SPECIAL(struct lightrec_cstate *state, const struct block *block, u16 offset);
  21 static void rec_REGIMM(struct lightrec_cstate *state, const struct block *block, u16 offset);
  22 static void rec_CP0(struct lightrec_cstate *state, const struct block *block, u16 offset);
  23 static void rec_CP2(struct lightrec_cstate *state, const struct block *block, u16 offset);
  24
  25 static void unknown_opcode(struct lightrec_cstate *state, const struct block *block, u16 offset)
  26 {
  27         pr_warn("Unknown opcode: 0x%08x at PC 0x%08x\n",
  28                 block->opcode_list[offset].c.opcode,
  29                 block->pc + (offset << 2));
  30 }
  31
  32 static void
  33 lightrec_jump_to_eob(struct lightrec_cstate *state, jit_state_t *_jit)
  34 {
  35         /* Prevent jit_jmpi() from using our cycles register as a temporary */
  36         jit_live(LIGHTREC_REG_CYCLE);
  37
  38         jit_patch_abs(jit_jmpi(), state->state->eob_wrapper_func);
  39 }
  40
  41 static void lightrec_emit_end_of_block(struct lightrec_cstate *state,
  42                                        const struct block *block, u16 offset,
  43                                        s8 reg_new_pc, u32 imm, u8 ra_reg,
  44                                        u32 link, bool update_cycles)
  45 {
  46         struct regcache *reg_cache = state->reg_cache;
  47         jit_state_t *_jit = block->_jit;
  48         const struct opcode *op = &block->opcode_list[offset],
  49                             *next = &block->opcode_list[offset + 1];
  50         u32 cycles = state->cycles + lightrec_cycles_of_opcode(op->c);
  51
  52         jit_note(__FILE__, __LINE__);
  53
  54         if (link) {
  55                 /* Update the $ra register */
  56                 u8 link_reg = lightrec_alloc_reg_out(reg_cache, _jit, ra_reg, 0);
  57                 jit_movi(link_reg, link);
  58                 lightrec_free_reg(reg_cache, link_reg);
  59         }
  60
  61         if (reg_new_pc < 0) {
  62                 reg_new_pc = lightrec_alloc_reg(reg_cache, _jit, JIT_V0);
  63                 lightrec_lock_reg(reg_cache, _jit, reg_new_pc);
  64
  65                 jit_movi(reg_new_pc, imm);
  66         }
  67
  68         if (has_delay_slot(op->c) &&
  69             !op_flag_no_ds(op->flags) && !op_flag_local_branch(op->flags)) {
  70                 cycles += lightrec_cycles_of_opcode(next->c);
  71
  72                 /* Recompile the delay slot */
  73                 if (next->c.opcode)
  74                         lightrec_rec_opcode(state, block, offset + 1);
  75         }
  76
  77         /* Clean the remaining registers */
  78         lightrec_clean_regs(reg_cache, _jit);
  79
  80         jit_movr(JIT_V0, reg_new_pc);
  81
  82         if (cycles && update_cycles) {
  83                 jit_subi(LIGHTREC_REG_CYCLE, LIGHTREC_REG_CYCLE, cycles);
  84                 pr_debug("EOB: %u cycles\n", cycles);
  85         }
  86
  87         lightrec_jump_to_eob(state, _jit);
  88 }
  89
  90 void lightrec_emit_eob(struct lightrec_cstate *state,
  91                        const struct block *block, u16 offset)
  92 {
  93         struct regcache *reg_cache = state->reg_cache;
  94         jit_state_t *_jit = block->_jit;
  95
  96         lightrec_clean_regs(reg_cache, _jit);
  97
  98         jit_movi(JIT_V0, block->pc + (offset << 2));
  99         jit_subi(LIGHTREC_REG_CYCLE, LIGHTREC_REG_CYCLE, state->cycles);
 100
 101         lightrec_jump_to_eob(state, _jit);
 102 }
 103
 104 static u8 get_jr_jalr_reg(struct lightrec_cstate *state, const struct block *block, u16 offset)
 105 {
 106         struct regcache *reg_cache = state->reg_cache;
 107         jit_state_t *_jit = block->_jit;
 108         const struct opcode *op = &block->opcode_list[offset];
 109         u8 rs;
 110
 111         rs = lightrec_request_reg_in(reg_cache, _jit, op->r.rs, JIT_V0);
 112         lightrec_lock_reg(reg_cache, _jit, rs);
 113
 114         return rs;
 115 }
 116
 117 static void rec_special_JR(struct lightrec_cstate *state, const struct block *block, u16 offset)
 118 {
 119         u8 rs = get_jr_jalr_reg(state, block, offset);
 120
 121         _jit_name(block->_jit, __func__);
 122         lightrec_emit_end_of_block(state, block, offset, rs, 0, 31, 0, true);
 123 }
 124
 125 static void rec_special_JALR(struct lightrec_cstate *state, const struct block *block, u16 offset)
 126 {
 127         u8 rs = get_jr_jalr_reg(state, block, offset);
 128         union code c = block->opcode_list[offset].c;
 129
 130         _jit_name(block->_jit, __func__);
 131         lightrec_emit_end_of_block(state, block, offset, rs, 0, c.r.rd,
 132                                    get_branch_pc(block, offset, 2), true);
 133 }
 134
 135 static void rec_J(struct lightrec_cstate *state, const struct block *block, u16 offset)
 136 {
 137         union code c = block->opcode_list[offset].c;
 138
 139         _jit_name(block->_jit, __func__);
 140         lightrec_emit_end_of_block(state, block, offset, -1,
 141                                    (block->pc & 0xf0000000) | (c.j.imm << 2),
 142                                    31, 0, true);
 143 }
 144
 145 static void rec_JAL(struct lightrec_cstate *state, const struct block *block, u16 offset)
 146 {
 147         union code c = block->opcode_list[offset].c;
 148
 149         _jit_name(block->_jit, __func__);
 150         lightrec_emit_end_of_block(state, block, offset, -1,
 151                                    (block->pc & 0xf0000000) | (c.j.imm << 2),
 152                                    31, get_branch_pc(block, offset, 2), true);
 153 }
 154
 155 static void lightrec_do_early_unload(struct lightrec_cstate *state,
 156                                      const struct block *block, u16 offset)
 157 {
 158         struct regcache *reg_cache = state->reg_cache;
 159         const struct opcode *op = &block->opcode_list[offset];
 160         jit_state_t *_jit = block->_jit;
 161         unsigned int i;
 162         u8 reg;
 163         struct {
 164                 u8 reg, op;
 165         } reg_ops[3] = {
 166                 { op->r.rd, LIGHTREC_FLAGS_GET_RD(op->flags), },
 167                 { op->i.rt, LIGHTREC_FLAGS_GET_RT(op->flags), },
 168                 { op->i.rs, LIGHTREC_FLAGS_GET_RS(op->flags), },
 169         };
 170
 171         for (i = 0; i < ARRAY_SIZE(reg_ops); i++) {
 172                 reg = reg_ops[i].reg;
 173
 174                 switch (reg_ops[i].op) {
 175                 case LIGHTREC_REG_UNLOAD:
 176                         lightrec_clean_reg_if_loaded(reg_cache, _jit, reg, true);
 177                         break;
 178
 179                 case LIGHTREC_REG_DISCARD:
 180                         lightrec_discard_reg_if_loaded(reg_cache, reg);
 181                         break;
 182
 183                 case LIGHTREC_REG_CLEAN:
 184                         lightrec_clean_reg_if_loaded(reg_cache, _jit, reg, false);
 185                         break;
 186                 default:
 187                         break;
 188                 };
 189         }
 190 }
 191
 192 static void rec_b(struct lightrec_cstate *state, const struct block *block, u16 offset,
 193                   jit_code_t code, jit_code_t code2, u32 link, bool unconditional, bool bz)
 194 {
 195         struct regcache *reg_cache = state->reg_cache;
 196         struct native_register *regs_backup;
 197         jit_state_t *_jit = block->_jit;
 198         struct lightrec_branch *branch;
 199         const struct opcode *op = &block->opcode_list[offset],
 200                             *next = &block->opcode_list[offset + 1];
 201         jit_node_t *addr;
 202         u8 link_reg, rs, rt;
 203         bool is_forward = (s16)op->i.imm >= -1;
 204         int op_cycles = lightrec_cycles_of_opcode(op->c);
 205         u32 target_offset, cycles = state->cycles + op_cycles;
 206         bool no_indirection = false;
 207         u32 next_pc;
 208
 209         jit_note(__FILE__, __LINE__);
 210
 211         if (!op_flag_no_ds(op->flags))
 212                 cycles += lightrec_cycles_of_opcode(next->c);
 213
 214         state->cycles = -op_cycles;
 215
 216         if (!unconditional) {
 217                 rs = lightrec_alloc_reg_in(reg_cache, _jit, op->i.rs, REG_EXT);
 218                 rt = bz ? 0 : lightrec_alloc_reg_in(reg_cache,
 219                                                     _jit, op->i.rt, REG_EXT);
 220
 221                 /* Unload dead registers before evaluating the branch */
 222                 if (OPT_EARLY_UNLOAD)
 223                         lightrec_do_early_unload(state, block, offset);
 224
 225                 if (op_flag_local_branch(op->flags) &&
 226                     (op_flag_no_ds(op->flags) || !next->opcode) &&
 227                     is_forward && !lightrec_has_dirty_regs(reg_cache))
 228                         no_indirection = true;
 229
 230                 if (no_indirection)
 231                         pr_debug("Using no indirection for branch at offset 0x%hx\n", offset << 2);
 232         }
 233
 234         if (cycles)
 235                 jit_subi(LIGHTREC_REG_CYCLE, LIGHTREC_REG_CYCLE, cycles);
 236
 237         if (!unconditional) {
 238                 /* Generate the branch opcode */
 239                 if (!no_indirection)
 240                         addr = jit_new_node_pww(code, NULL, rs, rt);
 241
 242                 lightrec_free_regs(reg_cache);
 243                 regs_backup = lightrec_regcache_enter_branch(reg_cache);
 244         }
 245
 246         if (op_flag_local_branch(op->flags)) {
 247                 /* Recompile the delay slot */
 248                 if (!op_flag_no_ds(op->flags) && next->opcode)
 249                         lightrec_rec_opcode(state, block, offset + 1);
 250
 251                 if (link) {
 252                         /* Update the $ra register */
 253                         link_reg = lightrec_alloc_reg_out(reg_cache, _jit, 31, 0);
 254                         jit_movi(link_reg, link);
 255                         lightrec_free_reg(reg_cache, link_reg);
 256                 }
 257
 258                 /* Clean remaining registers */
 259                 lightrec_clean_regs(reg_cache, _jit);
 260
 261                 target_offset = offset + 1 + (s16)op->i.imm
 262                         - !!op_flag_no_ds(op->flags);
 263                 pr_debug("Adding local branch to offset 0x%x\n",
 264                          target_offset << 2);
 265                 branch = &state->local_branches[
 266                         state->nb_local_branches++];
 267
 268                 branch->target = target_offset;
 269
 270                 if (no_indirection)
 271                         branch->branch = jit_new_node_pww(code2, NULL, rs, rt);
 272                 else if (is_forward)
 273                         branch->branch = jit_b();
 274                 else
 275                         branch->branch = jit_bgti(LIGHTREC_REG_CYCLE, 0);
 276         }
 277
 278         if (!op_flag_local_branch(op->flags) || !is_forward) {
 279                 next_pc = get_branch_pc(block, offset, 1 + (s16)op->i.imm);
 280                 lightrec_emit_end_of_block(state, block, offset, -1, next_pc,
 281                                            31, link, false);
 282         }
 283
 284         if (!unconditional) {
 285                 if (!no_indirection)
 286                         jit_patch(addr);
 287
 288                 lightrec_regcache_leave_branch(reg_cache, regs_backup);
 289
 290                 if (bz && link) {
 291                         /* Update the $ra register */
 292                         link_reg = lightrec_alloc_reg_out(reg_cache, _jit,
 293                                                           31, REG_EXT);
 294                         jit_movi(link_reg, (s32)link);
 295                         lightrec_free_reg(reg_cache, link_reg);
 296                 }
 297
 298                 if (!op_flag_no_ds(op->flags) && next->opcode)
 299                         lightrec_rec_opcode(state, block, offset + 1);
 300         }
 301 }
 302
 303 static void rec_BNE(struct lightrec_cstate *state,
 304                     const struct block *block, u16 offset)
 305 {
 306         union code c = block->opcode_list[offset].c;
 307
 308         _jit_name(block->_jit, __func__);
 309
 310         if (c.i.rt == 0)
 311                 rec_b(state, block, offset, jit_code_beqi, jit_code_bnei, 0, false, true);
 312         else
 313                 rec_b(state, block, offset, jit_code_beqr, jit_code_bner, 0, false, false);
 314 }
 315
 316 static void rec_BEQ(struct lightrec_cstate *state,
 317                     const struct block *block, u16 offset)
 318 {
 319         union code c = block->opcode_list[offset].c;
 320
 321         _jit_name(block->_jit, __func__);
 322
 323         if (c.i.rt == 0)
 324                 rec_b(state, block, offset, jit_code_bnei, jit_code_beqi, 0, c.i.rs == 0, true);
 325         else
 326                 rec_b(state, block, offset, jit_code_bner, jit_code_beqr, 0, c.i.rs == c.i.rt, false);
 327 }
 328
 329 static void rec_BLEZ(struct lightrec_cstate *state,
 330                      const struct block *block, u16 offset)
 331 {
 332         union code c = block->opcode_list[offset].c;
 333
 334         _jit_name(block->_jit, __func__);
 335         rec_b(state, block, offset, jit_code_bgti, jit_code_blei, 0, c.i.rs == 0, true);
 336 }
 337
 338 static void rec_BGTZ(struct lightrec_cstate *state,
 339                      const struct block *block, u16 offset)
 340 {
 341         _jit_name(block->_jit, __func__);
 342         rec_b(state, block, offset, jit_code_blei, jit_code_bgti, 0, false, true);
 343 }
 344
 345 static void rec_regimm_BLTZ(struct lightrec_cstate *state,
 346                             const struct block *block, u16 offset)
 347 {
 348         _jit_name(block->_jit, __func__);
 349         rec_b(state, block, offset, jit_code_bgei, jit_code_blti, 0, false, true);
 350 }
 351
 352 static void rec_regimm_BLTZAL(struct lightrec_cstate *state,
 353                               const struct block *block, u16 offset)
 354 {
 355         _jit_name(block->_jit, __func__);
 356         rec_b(state, block, offset, jit_code_bgei, jit_code_blti,
 357               get_branch_pc(block, offset, 2), false, true);
 358 }
 359
 360 static void rec_regimm_BGEZ(struct lightrec_cstate *state,
 361                             const struct block *block, u16 offset)
 362 {
 363         union code c = block->opcode_list[offset].c;
 364
 365         _jit_name(block->_jit, __func__);
 366         rec_b(state, block, offset, jit_code_blti, jit_code_bgei, 0, !c.i.rs, true);
 367 }
 368
 369 static void rec_regimm_BGEZAL(struct lightrec_cstate *state,
 370                               const struct block *block, u16 offset)
 371 {
 372         const struct opcode *op = &block->opcode_list[offset];
 373         _jit_name(block->_jit, __func__);
 374         rec_b(state, block, offset, jit_code_blti, jit_code_bgei,
 375               get_branch_pc(block, offset, 2),
 376               !op->i.rs, true);
 377 }
 378
 379 static void rec_alu_imm(struct lightrec_cstate *state, const struct block *block,
 380                         u16 offset, jit_code_t code, bool slti)
 381 {
 382         struct regcache *reg_cache = state->reg_cache;
 383         union code c = block->opcode_list[offset].c;
 384         jit_state_t *_jit = block->_jit;
 385         u8 rs, rt, out_flags = REG_EXT;
 386
 387         if (slti)
 388                 out_flags |= REG_ZEXT;
 389
 390         jit_note(__FILE__, __LINE__);
 391         rs = lightrec_alloc_reg_in(reg_cache, _jit, c.i.rs, REG_EXT);
 392         rt = lightrec_alloc_reg_out(reg_cache, _jit, c.i.rt, out_flags);
 393
 394         jit_new_node_www(code, rt, rs, (s32)(s16) c.i.imm);
 395
 396         lightrec_free_reg(reg_cache, rs);
 397         lightrec_free_reg(reg_cache, rt);
 398 }
 399
 400 static void rec_alu_special(struct lightrec_cstate *state, const struct block *block,
 401                             u16 offset, jit_code_t code, bool out_ext)
 402 {
 403         struct regcache *reg_cache = state->reg_cache;
 404         union code c = block->opcode_list[offset].c;
 405         jit_state_t *_jit = block->_jit;
 406         u8 rd, rt, rs;
 407
 408         jit_note(__FILE__, __LINE__);
 409         rs = lightrec_alloc_reg_in(reg_cache, _jit, c.r.rs, REG_EXT);
 410         rt = lightrec_alloc_reg_in(reg_cache, _jit, c.r.rt, REG_EXT);
 411         rd = lightrec_alloc_reg_out(reg_cache, _jit, c.r.rd,
 412                                     out_ext ? REG_EXT | REG_ZEXT : 0);
 413
 414         jit_new_node_www(code, rd, rs, rt);
 415
 416         lightrec_free_reg(reg_cache, rs);
 417         lightrec_free_reg(reg_cache, rt);
 418         lightrec_free_reg(reg_cache, rd);
 419 }
 420
 421 static void rec_alu_shiftv(struct lightrec_cstate *state, const struct block *block,
 422                            u16 offset, jit_code_t code)
 423 {
 424         struct regcache *reg_cache = state->reg_cache;
 425         union code c = block->opcode_list[offset].c;
 426         jit_state_t *_jit = block->_jit;
 427         u8 rd, rt, rs, temp, flags = 0;
 428
 429         jit_note(__FILE__, __LINE__);
 430         rs = lightrec_alloc_reg_in(reg_cache, _jit, c.r.rs, 0);
 431
 432         if (code == jit_code_rshr)
 433                 flags = REG_EXT;
 434         else if (code == jit_code_rshr_u)
 435                 flags = REG_ZEXT;
 436
 437         rt = lightrec_alloc_reg_in(reg_cache, _jit, c.r.rt, flags);
 438         rd = lightrec_alloc_reg_out(reg_cache, _jit, c.r.rd, flags);
 439
 440         if (rs != rd && rt != rd) {
 441                 jit_andi(rd, rs, 0x1f);
 442                 jit_new_node_www(code, rd, rt, rd);
 443         } else {
 444                 temp = lightrec_alloc_reg_temp(reg_cache, _jit);
 445                 jit_andi(temp, rs, 0x1f);
 446                 jit_new_node_www(code, rd, rt, temp);
 447                 lightrec_free_reg(reg_cache, temp);
 448         }
 449
 450         lightrec_free_reg(reg_cache, rs);
 451         lightrec_free_reg(reg_cache, rt);
 452         lightrec_free_reg(reg_cache, rd);
 453 }
 454
 455 static void rec_movi(struct lightrec_cstate *state,
 456                      const struct block *block, u16 offset)
 457 {
 458         struct regcache *reg_cache = state->reg_cache;
 459         union code c = block->opcode_list[offset].c;
 460         jit_state_t *_jit = block->_jit;
 461         u16 flags = REG_EXT;
 462         u8 rt;
 463
 464         if (!(c.i.imm & 0x8000))
 465                 flags |= REG_ZEXT;
 466
 467         rt = lightrec_alloc_reg_out(reg_cache, _jit, c.i.rt, flags);
 468
 469         jit_movi(rt, (s32)(s16) c.i.imm);
 470
 471         lightrec_free_reg(reg_cache, rt);
 472 }
 473
 474 static void rec_ADDIU(struct lightrec_cstate *state,
 475                       const struct block *block, u16 offset)
 476 {
 477         _jit_name(block->_jit, __func__);
 478
 479         if (block->opcode_list[offset].c.i.rs)
 480                 rec_alu_imm(state, block, offset, jit_code_addi, false);
 481         else
 482                 rec_movi(state, block, offset);
 483 }
 484
 485 static void rec_ADDI(struct lightrec_cstate *state,
 486                      const struct block *block, u16 offset)
 487 {
 488         /* TODO: Handle the exception? */
 489         _jit_name(block->_jit, __func__);
 490         rec_ADDIU(state, block, offset);
 491 }
 492
 493 static void rec_SLTIU(struct lightrec_cstate *state,
 494                       const struct block *block, u16 offset)
 495 {
 496         _jit_name(block->_jit, __func__);
 497         rec_alu_imm(state, block, offset, jit_code_lti_u, true);
 498 }
 499
 500 static void rec_SLTI(struct lightrec_cstate *state,
 501                      const struct block *block, u16 offset)
 502 {
 503         _jit_name(block->_jit, __func__);
 504         rec_alu_imm(state, block, offset, jit_code_lti, true);
 505 }
 506
 507 static void rec_ANDI(struct lightrec_cstate *state,
 508                      const struct block *block, u16 offset)
 509 {
 510         struct regcache *reg_cache = state->reg_cache;
 511         union code c = block->opcode_list[offset].c;
 512         jit_state_t *_jit = block->_jit;
 513         u8 rs, rt;
 514
 515         _jit_name(block->_jit, __func__);
 516         jit_note(__FILE__, __LINE__);
 517         rs = lightrec_alloc_reg_in(reg_cache, _jit, c.i.rs, 0);
 518         rt = lightrec_alloc_reg_out(reg_cache, _jit, c.i.rt,
 519                                     REG_EXT | REG_ZEXT);
 520
 521         /* PSX code uses ANDI 0xff / ANDI 0xffff a lot, which are basically
 522          * casts to uint8_t / uint16_t. */
 523         if (c.i.imm == 0xff)
 524                 jit_extr_uc(rt, rs);
 525         else if (c.i.imm == 0xffff)
 526                 jit_extr_us(rt, rs);
 527         else
 528                 jit_andi(rt, rs, (u32)(u16) c.i.imm);
 529
 530         lightrec_free_reg(reg_cache, rs);
 531         lightrec_free_reg(reg_cache, rt);
 532 }
 533
 534 static void rec_alu_or_xor(struct lightrec_cstate *state, const struct block *block,
 535                            u16 offset, jit_code_t code)
 536 {
 537         struct regcache *reg_cache = state->reg_cache;
 538         union code c = block->opcode_list[offset].c;
 539         jit_state_t *_jit = block->_jit;
 540         u8 rs, rt, flags;
 541
 542         jit_note(__FILE__, __LINE__);
 543         rs = lightrec_alloc_reg_in(reg_cache, _jit, c.i.rs, 0);
 544         rt = lightrec_alloc_reg_out(reg_cache, _jit, c.i.rt, 0);
 545
 546         flags = lightrec_get_reg_in_flags(reg_cache, rs);
 547         lightrec_set_reg_out_flags(reg_cache, rt, flags);
 548
 549         jit_new_node_www(code, rt, rs, (u32)(u16) c.i.imm);
 550
 551         lightrec_free_reg(reg_cache, rs);
 552         lightrec_free_reg(reg_cache, rt);
 553 }
 554
 555
 556 static void rec_ORI(struct lightrec_cstate *state,
 557                     const struct block *block, u16 offset)
 558 {
 559         _jit_name(block->_jit, __func__);
 560         rec_alu_or_xor(state, block, offset, jit_code_ori);
 561 }
 562
 563 static void rec_XORI(struct lightrec_cstate *state,
 564                      const struct block *block, u16 offset)
 565 {
 566         _jit_name(block->_jit, __func__);
 567         rec_alu_or_xor(state, block, offset, jit_code_xori);
 568 }
 569
 570 static void rec_LUI(struct lightrec_cstate *state,
 571                     const struct block *block, u16 offset)
 572 {
 573         struct regcache *reg_cache = state->reg_cache;
 574         union code c = block->opcode_list[offset].c;
 575         jit_state_t *_jit = block->_jit;
 576         u8 rt, flags = REG_EXT;
 577
 578         jit_name(__func__);
 579         jit_note(__FILE__, __LINE__);
 580
 581         if (!(c.i.imm & BIT(15)))
 582                 flags |= REG_ZEXT;
 583
 584         rt = lightrec_alloc_reg_out(reg_cache, _jit, c.i.rt, flags);
 585
 586         jit_movi(rt, (s32)(c.i.imm << 16));
 587
 588         lightrec_free_reg(reg_cache, rt);
 589 }
 590
 591 static void rec_special_ADDU(struct lightrec_cstate *state,
 592                              const struct block *block, u16 offset)
 593 {
 594         _jit_name(block->_jit, __func__);
 595         rec_alu_special(state, block, offset, jit_code_addr, false);
 596 }
 597
 598 static void rec_special_ADD(struct lightrec_cstate *state,
 599                             const struct block *block, u16 offset)
 600 {
 601         /* TODO: Handle the exception? */
 602         _jit_name(block->_jit, __func__);
 603         rec_alu_special(state, block, offset, jit_code_addr, false);
 604 }
 605
 606 static void rec_special_SUBU(struct lightrec_cstate *state,
 607                              const struct block *block, u16 offset)
 608 {
 609         _jit_name(block->_jit, __func__);
 610         rec_alu_special(state, block, offset, jit_code_subr, false);
 611 }
 612
 613 static void rec_special_SUB(struct lightrec_cstate *state,
 614                             const struct block *block, u16 offset)
 615 {
 616         /* TODO: Handle the exception? */
 617         _jit_name(block->_jit, __func__);
 618         rec_alu_special(state, block, offset, jit_code_subr, false);
 619 }
 620
 621 static void rec_special_AND(struct lightrec_cstate *state,
 622                             const struct block *block, u16 offset)
 623 {
 624         struct regcache *reg_cache = state->reg_cache;
 625         union code c = block->opcode_list[offset].c;
 626         jit_state_t *_jit = block->_jit;
 627         u8 rd, rt, rs, flags_rs, flags_rt, flags_rd;
 628
 629         _jit_name(block->_jit, __func__);
 630         jit_note(__FILE__, __LINE__);
 631         rs = lightrec_alloc_reg_in(reg_cache, _jit, c.r.rs, 0);
 632         rt = lightrec_alloc_reg_in(reg_cache, _jit, c.r.rt, 0);
 633         rd = lightrec_alloc_reg_out(reg_cache, _jit, c.r.rd, 0);
 634
 635         flags_rs = lightrec_get_reg_in_flags(reg_cache, rs);
 636         flags_rt = lightrec_get_reg_in_flags(reg_cache, rt);
 637
 638         /* Z(rd) = Z(rs) | Z(rt) */
 639         flags_rd = REG_ZEXT & (flags_rs | flags_rt);
 640
 641         /* E(rd) = (E(rt) & Z(rt)) | (E(rs) & Z(rs)) | (E(rs) & E(rt)) */
 642         if (((flags_rs & REG_EXT) && (flags_rt & REG_ZEXT)) ||
 643             ((flags_rt & REG_EXT) && (flags_rs & REG_ZEXT)) ||
 644             (REG_EXT & flags_rs & flags_rt))
 645                 flags_rd |= REG_EXT;
 646
 647         lightrec_set_reg_out_flags(reg_cache, rd, flags_rd);
 648
 649         jit_andr(rd, rs, rt);
 650
 651         lightrec_free_reg(reg_cache, rs);
 652         lightrec_free_reg(reg_cache, rt);
 653         lightrec_free_reg(reg_cache, rd);
 654 }
 655
 656 static void rec_special_or_nor(struct lightrec_cstate *state,
 657                                const struct block *block, u16 offset, bool nor)
 658 {
 659         struct regcache *reg_cache = state->reg_cache;
 660         union code c = block->opcode_list[offset].c;
 661         jit_state_t *_jit = block->_jit;
 662         u8 rd, rt, rs, flags_rs, flags_rt, flags_rd = 0;
 663
 664         jit_note(__FILE__, __LINE__);
 665         rs = lightrec_alloc_reg_in(reg_cache, _jit, c.r.rs, 0);
 666         rt = lightrec_alloc_reg_in(reg_cache, _jit, c.r.rt, 0);
 667         rd = lightrec_alloc_reg_out(reg_cache, _jit, c.r.rd, 0);
 668
 669         flags_rs = lightrec_get_reg_in_flags(reg_cache, rs);
 670         flags_rt = lightrec_get_reg_in_flags(reg_cache, rt);
 671
 672         /* or: Z(rd) = Z(rs) & Z(rt)
 673          * nor: Z(rd) = 0 */
 674         if (!nor)
 675                 flags_rd = REG_ZEXT & flags_rs & flags_rt;
 676
 677         /* E(rd) = E(rs) & E(rt) */
 678         if (REG_EXT & flags_rs & flags_rt)
 679                 flags_rd |= REG_EXT;
 680
 681         lightrec_set_reg_out_flags(reg_cache, rd, flags_rd);
 682
 683         jit_orr(rd, rs, rt);
 684
 685         if (nor)
 686                 jit_comr(rd, rd);
 687
 688         lightrec_free_reg(reg_cache, rs);
 689         lightrec_free_reg(reg_cache, rt);
 690         lightrec_free_reg(reg_cache, rd);
 691 }
 692
 693 static void rec_special_OR(struct lightrec_cstate *state,
 694                            const struct block *block, u16 offset)
 695 {
 696         _jit_name(block->_jit, __func__);
 697         rec_special_or_nor(state, block, offset, false);
 698 }
 699
 700 static void rec_special_NOR(struct lightrec_cstate *state,
 701                             const struct block *block, u16 offset)
 702 {
 703         _jit_name(block->_jit, __func__);
 704         rec_special_or_nor(state, block, offset, true);
 705 }
 706
 707 static void rec_special_XOR(struct lightrec_cstate *state,
 708                             const struct block *block, u16 offset)
 709 {
 710         struct regcache *reg_cache = state->reg_cache;
 711         union code c = block->opcode_list[offset].c;
 712         jit_state_t *_jit = block->_jit;
 713         u8 rd, rt, rs, flags_rs, flags_rt, flags_rd;
 714
 715         _jit_name(block->_jit, __func__);
 716
 717         jit_note(__FILE__, __LINE__);
 718         rs = lightrec_alloc_reg_in(reg_cache, _jit, c.r.rs, 0);
 719         rt = lightrec_alloc_reg_in(reg_cache, _jit, c.r.rt, 0);
 720         rd = lightrec_alloc_reg_out(reg_cache, _jit, c.r.rd, 0);
 721
 722         flags_rs = lightrec_get_reg_in_flags(reg_cache, rs);
 723         flags_rt = lightrec_get_reg_in_flags(reg_cache, rt);
 724
 725         /* Z(rd) = Z(rs) & Z(rt) */
 726         flags_rd = REG_ZEXT & flags_rs & flags_rt;
 727
 728         /* E(rd) = E(rs) & E(rt) */
 729         flags_rd |= REG_EXT & flags_rs & flags_rt;
 730
 731         lightrec_set_reg_out_flags(reg_cache, rd, flags_rd);
 732
 733         jit_xorr(rd, rs, rt);
 734
 735         lightrec_free_reg(reg_cache, rs);
 736         lightrec_free_reg(reg_cache, rt);
 737         lightrec_free_reg(reg_cache, rd);
 738 }
 739
 740 static void rec_special_SLTU(struct lightrec_cstate *state,
 741                              const struct block *block, u16 offset)
 742 {
 743         _jit_name(block->_jit, __func__);
 744         rec_alu_special(state, block, offset, jit_code_ltr_u, true);
 745 }
 746
 747 static void rec_special_SLT(struct lightrec_cstate *state,
 748                             const struct block *block, u16 offset)
 749 {
 750         _jit_name(block->_jit, __func__);
 751         rec_alu_special(state, block, offset, jit_code_ltr, true);
 752 }
 753
 754 static void rec_special_SLLV(struct lightrec_cstate *state,
 755                              const struct block *block, u16 offset)
 756 {
 757         _jit_name(block->_jit, __func__);
 758         rec_alu_shiftv(state, block, offset, jit_code_lshr);
 759 }
 760
 761 static void rec_special_SRLV(struct lightrec_cstate *state,
 762                              const struct block *block, u16 offset)
 763 {
 764         _jit_name(block->_jit, __func__);
 765         rec_alu_shiftv(state, block, offset, jit_code_rshr_u);
 766 }
 767
 768 static void rec_special_SRAV(struct lightrec_cstate *state,
 769                              const struct block *block, u16 offset)
 770 {
 771         _jit_name(block->_jit, __func__);
 772         rec_alu_shiftv(state, block, offset, jit_code_rshr);
 773 }
 774
 775 static void rec_alu_shift(struct lightrec_cstate *state, const struct block *block,
 776                           u16 offset, jit_code_t code)
 777 {
 778         struct regcache *reg_cache = state->reg_cache;
 779         union code c = block->opcode_list[offset].c;
 780         jit_state_t *_jit = block->_jit;
 781         u8 rd, rt, flags = 0;
 782
 783         jit_note(__FILE__, __LINE__);
 784
 785         if (code == jit_code_rshi)
 786                 flags = REG_EXT;
 787         else if (code == jit_code_rshi_u)
 788                 flags = REG_ZEXT;
 789
 790         rt = lightrec_alloc_reg_in(reg_cache, _jit, c.r.rt, flags);
 791
 792         /* Input reg is zero-extended, if we SRL at least by one bit, we know
 793          * the output reg will be both zero-extended and sign-extended. */
 794         if (code == jit_code_rshi_u && c.r.imm)
 795                 flags |= REG_EXT;
 796         rd = lightrec_alloc_reg_out(reg_cache, _jit, c.r.rd, flags);
 797
 798         jit_new_node_www(code, rd, rt, c.r.imm);
 799
 800         lightrec_free_reg(reg_cache, rt);
 801         lightrec_free_reg(reg_cache, rd);
 802 }
 803
 804 static void rec_special_SLL(struct lightrec_cstate *state,
 805                             const struct block *block, u16 offset)
 806 {
 807         _jit_name(block->_jit, __func__);
 808         rec_alu_shift(state, block, offset, jit_code_lshi);
 809 }
 810
 811 static void rec_special_SRL(struct lightrec_cstate *state,
 812                             const struct block *block, u16 offset)
 813 {
 814         _jit_name(block->_jit, __func__);
 815         rec_alu_shift(state, block, offset, jit_code_rshi_u);
 816 }
 817
 818 static void rec_special_SRA(struct lightrec_cstate *state,
 819                             const struct block *block, u16 offset)
 820 {
 821         _jit_name(block->_jit, __func__);
 822         rec_alu_shift(state, block, offset, jit_code_rshi);
 823 }
 824
 825 static void rec_alu_mult(struct lightrec_cstate *state,
 826                          const struct block *block, u16 offset, bool is_signed)
 827 {
 828         struct regcache *reg_cache = state->reg_cache;
 829         union code c = block->opcode_list[offset].c;
 830         u32 flags = block->opcode_list[offset].flags;
 831         u8 reg_lo = get_mult_div_lo(c);
 832         u8 reg_hi = get_mult_div_hi(c);
 833         jit_state_t *_jit = block->_jit;
 834         u8 lo, hi, rs, rt, rflags = 0;
 835
 836         jit_note(__FILE__, __LINE__);
 837
 838         if (is_signed)
 839                 rflags = REG_EXT;
 840         else
 841                 rflags = REG_ZEXT;
 842
 843         rs = lightrec_alloc_reg_in(reg_cache, _jit, c.r.rs, rflags);
 844         rt = lightrec_alloc_reg_in(reg_cache, _jit, c.r.rt, rflags);
 845
 846         if (!op_flag_no_lo(flags))
 847                 lo = lightrec_alloc_reg_out(reg_cache, _jit, reg_lo, 0);
 848         else if (__WORDSIZE == 32)
 849                 lo = lightrec_alloc_reg_temp(reg_cache, _jit);
 850
 851         if (!op_flag_no_hi(flags))
 852                 hi = lightrec_alloc_reg_out(reg_cache, _jit, reg_hi, REG_EXT);
 853
 854         if (__WORDSIZE == 32) {
 855                 /* On 32-bit systems, do a 32*32->64 bit operation, or a 32*32->32 bit
 856                  * operation if the MULT was detected a 32-bit only. */
 857                 if (!op_flag_no_hi(flags)) {
 858                         if (is_signed)
 859                                 jit_qmulr(lo, hi, rs, rt);
 860                         else
 861                                 jit_qmulr_u(lo, hi, rs, rt);
 862                 } else {
 863                         jit_mulr(lo, rs, rt);
 864                 }
 865         } else {
 866                 /* On 64-bit systems, do a 64*64->64 bit operation. */
 867                 if (op_flag_no_lo(flags)) {
 868                         jit_mulr(hi, rs, rt);
 869                         jit_rshi(hi, hi, 32);
 870                 } else {
 871                         jit_mulr(lo, rs, rt);
 872
 873                         /* The 64-bit output value is in $lo, store the upper 32 bits in $hi */
 874                         if (!op_flag_no_hi(flags))
 875                                 jit_rshi(hi, lo, 32);
 876                 }
 877         }
 878
 879         lightrec_free_reg(reg_cache, rs);
 880         lightrec_free_reg(reg_cache, rt);
 881         if (!op_flag_no_lo(flags) || __WORDSIZE == 32)
 882                 lightrec_free_reg(reg_cache, lo);
 883         if (!op_flag_no_hi(flags))
 884                 lightrec_free_reg(reg_cache, hi);
 885 }
 886
 887 static void rec_alu_div(struct lightrec_cstate *state,
 888                         const struct block *block, u16 offset, bool is_signed)
 889 {
 890         struct regcache *reg_cache = state->reg_cache;
 891         union code c = block->opcode_list[offset].c;
 892         u32 flags = block->opcode_list[offset].flags;
 893         bool no_check = op_flag_no_div_check(flags);
 894         u8 reg_lo = get_mult_div_lo(c);
 895         u8 reg_hi = get_mult_div_hi(c);
 896         jit_state_t *_jit = block->_jit;
 897         jit_node_t *branch, *to_end;
 898         u8 lo = 0, hi = 0, rs, rt, rflags = 0;
 899
 900         jit_note(__FILE__, __LINE__);
 901
 902         if (is_signed)
 903                 rflags = REG_EXT;
 904         else
 905                 rflags = REG_ZEXT;
 906
 907         rs = lightrec_alloc_reg_in(reg_cache, _jit, c.r.rs, rflags);
 908         rt = lightrec_alloc_reg_in(reg_cache, _jit, c.r.rt, rflags);
 909
 910         if (!op_flag_no_lo(flags))
 911                 lo = lightrec_alloc_reg_out(reg_cache, _jit, reg_lo, 0);
 912
 913         if (!op_flag_no_hi(flags))
 914                 hi = lightrec_alloc_reg_out(reg_cache, _jit, reg_hi, 0);
 915
 916         /* Jump to special handler if dividing by zero  */
 917         if (!no_check)
 918                 branch = jit_beqi(rt, 0);
 919
 920         if (op_flag_no_lo(flags)) {
 921                 if (is_signed)
 922                         jit_remr(hi, rs, rt);
 923                 else
 924                         jit_remr_u(hi, rs, rt);
 925         } else if (op_flag_no_hi(flags)) {
 926                 if (is_signed)
 927                         jit_divr(lo, rs, rt);
 928                 else
 929                         jit_divr_u(lo, rs, rt);
 930         } else {
 931                 if (is_signed)
 932                         jit_qdivr(lo, hi, rs, rt);
 933                 else
 934                         jit_qdivr_u(lo, hi, rs, rt);
 935         }
 936
 937         if (!no_check) {
 938                 /* Jump above the div-by-zero handler */
 939                 to_end = jit_b();
 940
 941                 jit_patch(branch);
 942
 943                 if (!op_flag_no_lo(flags)) {
 944                         if (is_signed) {
 945                                 jit_lti(lo, rs, 0);
 946                                 jit_lshi(lo, lo, 1);
 947                                 jit_subi(lo, lo, 1);
 948                         } else {
 949                                 jit_movi(lo, 0xffffffff);
 950                         }
 951                 }
 952
 953                 if (!op_flag_no_hi(flags))
 954                         jit_movr(hi, rs);
 955
 956                 jit_patch(to_end);
 957         }
 958
 959         lightrec_free_reg(reg_cache, rs);
 960         lightrec_free_reg(reg_cache, rt);
 961
 962         if (!op_flag_no_lo(flags))
 963                 lightrec_free_reg(reg_cache, lo);
 964
 965         if (!op_flag_no_hi(flags))
 966                 lightrec_free_reg(reg_cache, hi);
 967 }
 968
 969 static void rec_special_MULT(struct lightrec_cstate *state,
 970                              const struct block *block, u16 offset)
 971 {
 972         _jit_name(block->_jit, __func__);
 973         rec_alu_mult(state, block, offset, true);
 974 }
 975
 976 static void rec_special_MULTU(struct lightrec_cstate *state,
 977                               const struct block *block, u16 offset)
 978 {
 979         _jit_name(block->_jit, __func__);
 980         rec_alu_mult(state, block, offset, false);
 981 }
 982
 983 static void rec_special_DIV(struct lightrec_cstate *state,
 984                             const struct block *block, u16 offset)
 985 {
 986         _jit_name(block->_jit, __func__);
 987         rec_alu_div(state, block, offset, true);
 988 }
 989
 990 static void rec_special_DIVU(struct lightrec_cstate *state,
 991                              const struct block *block, u16 offset)
 992 {
 993         _jit_name(block->_jit, __func__);
 994         rec_alu_div(state, block, offset, false);
 995 }
 996
 997 static void rec_alu_mv_lo_hi(struct lightrec_cstate *state,
 998                              const struct block *block, u8 dst, u8 src)
 999 {
1000         struct regcache *reg_cache = state->reg_cache;
1001         jit_state_t *_jit = block->_jit;
1002
1003         jit_note(__FILE__, __LINE__);
1004         src = lightrec_alloc_reg_in(reg_cache, _jit, src, 0);
1005         dst = lightrec_alloc_reg_out(reg_cache, _jit, dst, REG_EXT);
1006
1007         jit_extr_i(dst, src);
1008
1009         lightrec_free_reg(reg_cache, src);
1010         lightrec_free_reg(reg_cache, dst);
1011 }
1012
1013 static void rec_special_MFHI(struct lightrec_cstate *state,
1014                              const struct block *block, u16 offset)
1015 {
1016         union code c = block->opcode_list[offset].c;
1017
1018         _jit_name(block->_jit, __func__);
1019         rec_alu_mv_lo_hi(state, block, c.r.rd, REG_HI);
1020 }
1021
1022 static void rec_special_MTHI(struct lightrec_cstate *state,
1023                              const struct block *block, u16 offset)
1024 {
1025         union code c = block->opcode_list[offset].c;
1026
1027         _jit_name(block->_jit, __func__);
1028         rec_alu_mv_lo_hi(state, block, REG_HI, c.r.rs);
1029 }
1030
1031 static void rec_special_MFLO(struct lightrec_cstate *state,
1032                              const struct block *block, u16 offset)
1033 {
1034         union code c = block->opcode_list[offset].c;
1035
1036         _jit_name(block->_jit, __func__);
1037         rec_alu_mv_lo_hi(state, block, c.r.rd, REG_LO);
1038 }
1039
1040 static void rec_special_MTLO(struct lightrec_cstate *state,
1041                              const struct block *block, u16 offset)
1042 {
1043         union code c = block->opcode_list[offset].c;
1044
1045         _jit_name(block->_jit, __func__);
1046         rec_alu_mv_lo_hi(state, block, REG_LO, c.r.rs);
1047 }
1048
1049 static void call_to_c_wrapper(struct lightrec_cstate *state,
1050                               const struct block *block, u32 arg,
1051                               enum c_wrappers wrapper)
1052 {
1053         struct regcache *reg_cache = state->reg_cache;
1054         jit_state_t *_jit = block->_jit;
1055         s8 tmp, tmp2;
1056
1057         /* Make sure JIT_R1 is not mapped; it will be used in the C wrapper. */
1058         tmp2 = lightrec_alloc_reg(reg_cache, _jit, JIT_R1);
1059
1060         tmp = lightrec_get_reg_with_value(reg_cache,
1061                                           (intptr_t) state->state->wrappers_eps[wrapper]);
1062         if (tmp < 0) {
1063                 tmp = lightrec_alloc_reg_temp(reg_cache, _jit);
1064                 jit_ldxi(tmp, LIGHTREC_REG_STATE,
1065                          offsetof(struct lightrec_state, wrappers_eps[wrapper]));
1066
1067                 lightrec_temp_set_value(reg_cache, tmp,
1068                                         (intptr_t) state->state->wrappers_eps[wrapper]);
1069         }
1070
1071         lightrec_free_reg(reg_cache, tmp2);
1072
1073 #ifdef __mips__
1074         /* On MIPS, register t9 is always used as the target register for JALR.
1075          * Therefore if it does not contain the target address we must
1076          * invalidate it. */
1077         if (tmp != _T9)
1078                 lightrec_unload_reg(reg_cache, _jit, _T9);
1079 #endif
1080
1081         jit_prepare();
1082         jit_pushargi(arg);
1083
1084         lightrec_regcache_mark_live(reg_cache, _jit);
1085         jit_callr(tmp);
1086
1087         lightrec_free_reg(reg_cache, tmp);
1088         lightrec_regcache_mark_live(reg_cache, _jit);
1089 }
1090
1091 static void rec_io(struct lightrec_cstate *state,
1092                    const struct block *block, u16 offset,
1093                    bool load_rt, bool read_rt)
1094 {
1095         struct regcache *reg_cache = state->reg_cache;
1096         jit_state_t *_jit = block->_jit;
1097         union code c = block->opcode_list[offset].c;
1098         u32 flags = block->opcode_list[offset].flags;
1099         bool is_tagged = LIGHTREC_FLAGS_GET_IO_MODE(flags);
1100         u32 lut_entry;
1101
1102         jit_note(__FILE__, __LINE__);
1103
1104         lightrec_clean_reg_if_loaded(reg_cache, _jit, c.i.rs, false);
1105
1106         if (read_rt && likely(c.i.rt))
1107                 lightrec_clean_reg_if_loaded(reg_cache, _jit, c.i.rt, true);
1108         else if (load_rt)
1109                 lightrec_clean_reg_if_loaded(reg_cache, _jit, c.i.rt, false);
1110
1111         if (is_tagged) {
1112                 call_to_c_wrapper(state, block, c.opcode, C_WRAPPER_RW);
1113         } else {
1114                 lut_entry = lightrec_get_lut_entry(block);
1115                 call_to_c_wrapper(state, block, (lut_entry << 16) | offset,
1116                                   C_WRAPPER_RW_GENERIC);
1117         }
1118 }
1119
1120 static u32 rec_ram_mask(struct lightrec_state *state)
1121 {
1122         return (RAM_SIZE << (state->mirrors_mapped * 2)) - 1;
1123 }
1124
1125 static u32 rec_io_mask(const struct lightrec_state *state)
1126 {
1127         u32 length = state->maps[PSX_MAP_HW_REGISTERS].length;
1128
1129         return GENMASK(31 - clz32(length - 1), 0);
1130 }
1131
1132 static void rec_store_memory(struct lightrec_cstate *cstate,
1133                              const struct block *block,
1134                              u16 offset, jit_code_t code,
1135                              jit_code_t swap_code,
1136                              uintptr_t addr_offset, u32 addr_mask,
1137                              bool invalidate)
1138 {
1139         const struct lightrec_state *state = cstate->state;
1140         struct regcache *reg_cache = cstate->reg_cache;
1141         struct opcode *op = &block->opcode_list[offset];
1142         jit_state_t *_jit = block->_jit;
1143         union code c = op->c;
1144         u8 rs, rt, tmp, tmp2, tmp3, addr_reg, addr_reg2;
1145         s16 imm = (s16)c.i.imm;
1146         s32 simm = (s32)imm << (1 - lut_is_32bit(state));
1147         s32 lut_offt = offsetof(struct lightrec_state, code_lut);
1148         bool no_mask = op_flag_no_mask(op->flags);
1149         bool add_imm = c.i.imm &&
1150                 ((!state->mirrors_mapped && !no_mask) || (invalidate &&
1151                 ((imm & 0x3) || simm + lut_offt != (s16)(simm + lut_offt))));
1152         bool need_tmp = !no_mask || addr_offset || add_imm || invalidate;
1153
1154         rt = lightrec_alloc_reg_in(reg_cache, _jit, c.i.rt, 0);
1155         rs = lightrec_alloc_reg_in(reg_cache, _jit, c.i.rs, 0);
1156         if (need_tmp)
1157                 tmp = lightrec_alloc_reg_temp(reg_cache, _jit);
1158
1159         addr_reg = rs;
1160
1161         if (add_imm) {
1162                 jit_addi(tmp, addr_reg, (s16)c.i.imm);
1163                 addr_reg = tmp;
1164                 imm = 0;
1165         } else if (simm) {
1166                 lut_offt += simm;
1167         }
1168
1169         if (!no_mask) {
1170                 jit_andi(tmp, addr_reg, addr_mask);
1171                 addr_reg = tmp;
1172         }
1173
1174         if (addr_offset) {
1175                 tmp2 = lightrec_alloc_reg_temp(reg_cache, _jit);
1176                 jit_addi(tmp2, addr_reg, addr_offset);
1177                 addr_reg2 = tmp2;
1178         } else {
1179                 addr_reg2 = addr_reg;
1180         }
1181
1182         if (is_big_endian() && swap_code && c.i.rt) {
1183                 tmp3 = lightrec_alloc_reg_temp(reg_cache, _jit);
1184
1185                 jit_new_node_ww(swap_code, tmp3, rt);
1186                 jit_new_node_www(code, imm, addr_reg2, tmp3);
1187
1188                 lightrec_free_reg(reg_cache, tmp3);
1189         } else {
1190                 jit_new_node_www(code, imm, addr_reg2, rt);
1191         }
1192
1193         lightrec_free_reg(reg_cache, rt);
1194
1195         if (invalidate) {
1196                 tmp3 = lightrec_alloc_reg_in(reg_cache, _jit, 0, 0);
1197
1198                 if (c.i.op != OP_SW) {
1199                         jit_andi(tmp, addr_reg, ~3);
1200                         addr_reg = tmp;
1201                 }
1202
1203                 if (!lut_is_32bit(state)) {
1204                         jit_lshi(tmp, addr_reg, 1);
1205                         addr_reg = tmp;
1206                 }
1207
1208                 if (addr_reg == rs && c.i.rs == 0) {
1209                         addr_reg = LIGHTREC_REG_STATE;
1210                 } else {
1211                         jit_addr(tmp, addr_reg, LIGHTREC_REG_STATE);
1212                         addr_reg = tmp;
1213                 }
1214
1215                 if (lut_is_32bit(state))
1216                         jit_stxi_i(lut_offt, addr_reg, tmp3);
1217                 else
1218                         jit_stxi(lut_offt, addr_reg, tmp3);
1219
1220                 lightrec_free_reg(reg_cache, tmp3);
1221         }
1222
1223         if (addr_offset)
1224                 lightrec_free_reg(reg_cache, tmp2);
1225         if (need_tmp)
1226                 lightrec_free_reg(reg_cache, tmp);
1227         lightrec_free_reg(reg_cache, rs);
1228 }
1229
1230 static void rec_store_ram(struct lightrec_cstate *cstate,
1231                           const struct block *block,
1232                           u16 offset, jit_code_t code,
1233                           jit_code_t swap_code, bool invalidate)
1234 {
1235         struct lightrec_state *state = cstate->state;
1236
1237         _jit_note(block->_jit, __FILE__, __LINE__);
1238
1239         return rec_store_memory(cstate, block, offset, code, swap_code,
1240                                 state->offset_ram, rec_ram_mask(state),
1241                                 invalidate);
1242 }
1243
1244 static void rec_store_scratch(struct lightrec_cstate *cstate,
1245                               const struct block *block, u16 offset,
1246                               jit_code_t code, jit_code_t swap_code)
1247 {
1248         _jit_note(block->_jit, __FILE__, __LINE__);
1249
1250         return rec_store_memory(cstate, block, offset, code, swap_code,
1251                                 cstate->state->offset_scratch,
1252                                 0x1fffffff, false);
1253 }
1254
1255 static void rec_store_io(struct lightrec_cstate *cstate,
1256                          const struct block *block, u16 offset,
1257                          jit_code_t code, jit_code_t swap_code)
1258 {
1259         _jit_note(block->_jit, __FILE__, __LINE__);
1260
1261         return rec_store_memory(cstate, block, offset, code, swap_code,
1262                                 cstate->state->offset_io,
1263                                 rec_io_mask(cstate->state), false);
1264 }
1265
1266 static void rec_store_direct_no_invalidate(struct lightrec_cstate *cstate,
1267                                            const struct block *block,
1268                                            u16 offset, jit_code_t code,
1269                                            jit_code_t swap_code)
1270 {
1271         struct lightrec_state *state = cstate->state;
1272         struct regcache *reg_cache = cstate->reg_cache;
1273         union code c = block->opcode_list[offset].c;
1274         jit_state_t *_jit = block->_jit;
1275         jit_node_t *to_not_ram, *to_end;
1276         u8 tmp, tmp2, rs, rt;
1277         s16 imm;
1278
1279         jit_note(__FILE__, __LINE__);
1280         rs = lightrec_alloc_reg_in(reg_cache, _jit, c.i.rs, 0);
1281         tmp = lightrec_alloc_reg_temp(reg_cache, _jit);
1282
1283         if (state->offset_ram || state->offset_scratch)
1284                 tmp2 = lightrec_alloc_reg_temp(reg_cache, _jit);
1285
1286         /* Convert to KUNSEG and avoid RAM mirrors */
1287         if (state->mirrors_mapped) {
1288                 imm = (s16)c.i.imm;
1289                 jit_andi(tmp, rs, 0x1f800000 | (4 * RAM_SIZE - 1));
1290         } else if (c.i.imm) {
1291                 imm = 0;
1292                 jit_addi(tmp, rs, (s16)c.i.imm);
1293                 jit_andi(tmp, tmp, 0x1f800000 | (RAM_SIZE - 1));
1294         } else {
1295                 imm = 0;
1296                 jit_andi(tmp, rs, 0x1f800000 | (RAM_SIZE - 1));
1297         }
1298
1299         lightrec_free_reg(reg_cache, rs);
1300
1301         if (state->offset_ram != state->offset_scratch) {
1302                 to_not_ram = jit_bmsi(tmp, BIT(28));
1303
1304                 jit_movi(tmp2, state->offset_ram);
1305
1306                 to_end = jit_b();
1307                 jit_patch(to_not_ram);
1308
1309                 jit_movi(tmp2, state->offset_scratch);
1310                 jit_patch(to_end);
1311         } else if (state->offset_ram) {
1312                 jit_movi(tmp2, state->offset_ram);
1313         }
1314
1315         if (state->offset_ram || state->offset_scratch) {
1316                 jit_addr(tmp, tmp, tmp2);
1317                 lightrec_free_reg(reg_cache, tmp2);
1318         }
1319
1320         rt = lightrec_alloc_reg_in(reg_cache, _jit, c.i.rt, 0);
1321
1322         if (is_big_endian() && swap_code && c.i.rt) {
1323                 tmp2 = lightrec_alloc_reg_temp(reg_cache, _jit);
1324
1325                 jit_new_node_ww(swap_code, tmp2, rt);
1326                 jit_new_node_www(code, imm, tmp, tmp2);
1327
1328                 lightrec_free_reg(reg_cache, tmp2);
1329         } else {
1330                 jit_new_node_www(code, imm, tmp, rt);
1331         }
1332
1333         lightrec_free_reg(reg_cache, rt);
1334         lightrec_free_reg(reg_cache, tmp);
1335 }
1336
1337 static void rec_store_direct(struct lightrec_cstate *cstate, const struct block *block,
1338                              u16 offset, jit_code_t code, jit_code_t swap_code)
1339 {
1340         struct lightrec_state *state = cstate->state;
1341         u32 ram_size = state->mirrors_mapped ? RAM_SIZE * 4 : RAM_SIZE;
1342         struct regcache *reg_cache = cstate->reg_cache;
1343         union code c = block->opcode_list[offset].c;
1344         jit_state_t *_jit = block->_jit;
1345         jit_node_t *to_not_ram, *to_end;
1346         u8 tmp, tmp2, tmp3, rs, rt;
1347
1348         jit_note(__FILE__, __LINE__);
1349
1350         rs = lightrec_alloc_reg_in(reg_cache, _jit, c.i.rs, 0);
1351         tmp2 = lightrec_alloc_reg_temp(reg_cache, _jit);
1352         tmp3 = lightrec_alloc_reg_in(reg_cache, _jit, 0, 0);
1353
1354         /* Convert to KUNSEG and avoid RAM mirrors */
1355         if (c.i.imm) {
1356                 jit_addi(tmp2, rs, (s16)c.i.imm);
1357                 jit_andi(tmp2, tmp2, 0x1f800000 | (ram_size - 1));
1358         } else {
1359                 jit_andi(tmp2, rs, 0x1f800000 | (ram_size - 1));
1360         }
1361
1362         lightrec_free_reg(reg_cache, rs);
1363         tmp = lightrec_alloc_reg_temp(reg_cache, _jit);
1364
1365         to_not_ram = jit_bgti(tmp2, ram_size);
1366
1367         /* Compute the offset to the code LUT */
1368         jit_andi(tmp, tmp2, (RAM_SIZE - 1) & ~3);
1369         if (!lut_is_32bit(state))
1370                 jit_lshi(tmp, tmp, 1);
1371         jit_addr(tmp, LIGHTREC_REG_STATE, tmp);
1372
1373         /* Write NULL to the code LUT to invalidate any block that's there */
1374         if (lut_is_32bit(state))
1375                 jit_stxi_i(offsetof(struct lightrec_state, code_lut), tmp, tmp3);
1376         else
1377                 jit_stxi(offsetof(struct lightrec_state, code_lut), tmp, tmp3);
1378
1379         if (state->offset_ram != state->offset_scratch) {
1380                 jit_movi(tmp, state->offset_ram);
1381
1382                 to_end = jit_b();
1383         }
1384
1385         jit_patch(to_not_ram);
1386
1387         if (state->offset_ram || state->offset_scratch)
1388                 jit_movi(tmp, state->offset_scratch);
1389
1390         if (state->offset_ram != state->offset_scratch)
1391                 jit_patch(to_end);
1392
1393         if (state->offset_ram || state->offset_scratch)
1394                 jit_addr(tmp2, tmp2, tmp);
1395
1396         lightrec_free_reg(reg_cache, tmp);
1397         lightrec_free_reg(reg_cache, tmp3);
1398
1399         rt = lightrec_alloc_reg_in(reg_cache, _jit, c.i.rt, 0);
1400
1401         if (is_big_endian() && swap_code && c.i.rt) {
1402                 tmp = lightrec_alloc_reg_temp(reg_cache, _jit);
1403
1404                 jit_new_node_ww(swap_code, tmp, rt);
1405                 jit_new_node_www(code, 0, tmp2, tmp);
1406
1407                 lightrec_free_reg(reg_cache, tmp);
1408         } else {
1409                 jit_new_node_www(code, 0, tmp2, rt);
1410         }
1411
1412         lightrec_free_reg(reg_cache, rt);
1413         lightrec_free_reg(reg_cache, tmp2);
1414 }
1415
1416 static void rec_store(struct lightrec_cstate *state,
1417                       const struct block *block, u16 offset,
1418                       jit_code_t code, jit_code_t swap_code)
1419 {
1420         u32 flags = block->opcode_list[offset].flags;
1421         bool no_invalidate = op_flag_no_invalidate(flags) ||
1422                 state->state->invalidate_from_dma_only;
1423
1424         switch (LIGHTREC_FLAGS_GET_IO_MODE(flags)) {
1425         case LIGHTREC_IO_RAM:
1426                 rec_store_ram(state, block, offset, code,
1427                               swap_code, !no_invalidate);
1428                 break;
1429         case LIGHTREC_IO_SCRATCH:
1430                 rec_store_scratch(state, block, offset, code, swap_code);
1431                 break;
1432         case LIGHTREC_IO_DIRECT:
1433                 if (no_invalidate) {
1434                         rec_store_direct_no_invalidate(state, block, offset,
1435                                                        code, swap_code);
1436                 } else {
1437                         rec_store_direct(state, block, offset, code, swap_code);
1438                 }
1439                 break;
1440         case LIGHTREC_IO_DIRECT_HW:
1441                 rec_store_io(state, block, offset, code, swap_code);
1442                 break;
1443         default:
1444                 rec_io(state, block, offset, true, false);
1445                 break;
1446         }
1447 }
1448
1449 static void rec_SB(struct lightrec_cstate *state,
1450                    const struct block *block, u16 offset)
1451 {
1452         _jit_name(block->_jit, __func__);
1453         rec_store(state, block, offset, jit_code_stxi_c, 0);
1454 }
1455
1456 static void rec_SH(struct lightrec_cstate *state,
1457                    const struct block *block, u16 offset)
1458 {
1459         _jit_name(block->_jit, __func__);
1460         rec_store(state, block, offset,
1461                   jit_code_stxi_s, jit_code_bswapr_us);
1462 }
1463
1464 static void rec_SW(struct lightrec_cstate *state,
1465                    const struct block *block, u16 offset)
1466
1467 {
1468         _jit_name(block->_jit, __func__);
1469         rec_store(state, block, offset,
1470                   jit_code_stxi_i, jit_code_bswapr_ui);
1471 }
1472
1473 static void rec_SWL(struct lightrec_cstate *state,
1474                     const struct block *block, u16 offset)
1475 {
1476         _jit_name(block->_jit, __func__);
1477         rec_io(state, block, offset, true, false);
1478 }
1479
1480 static void rec_SWR(struct lightrec_cstate *state,
1481                     const struct block *block, u16 offset)
1482 {
1483         _jit_name(block->_jit, __func__);
1484         rec_io(state, block, offset, true, false);
1485 }
1486
1487 static void rec_SWC2(struct lightrec_cstate *state,
1488                      const struct block *block, u16 offset)
1489 {
1490         _jit_name(block->_jit, __func__);
1491         rec_io(state, block, offset, false, false);
1492 }
1493
1494 static void rec_load_memory(struct lightrec_cstate *cstate,
1495                             const struct block *block, u16 offset,
1496                             jit_code_t code, jit_code_t swap_code, bool is_unsigned,
1497                             uintptr_t addr_offset, u32 addr_mask)
1498 {
1499         struct regcache *reg_cache = cstate->reg_cache;
1500         struct opcode *op = &block->opcode_list[offset];
1501         jit_state_t *_jit = block->_jit;
1502         u8 rs, rt, addr_reg, flags = REG_EXT;
1503         bool no_mask = op_flag_no_mask(op->flags);
1504         union code c = op->c;
1505         s16 imm;
1506
1507         if (!c.i.rt)
1508                 return;
1509
1510         if (is_unsigned)
1511                 flags |= REG_ZEXT;
1512
1513         rs = lightrec_alloc_reg_in(reg_cache, _jit, c.i.rs, 0);
1514         rt = lightrec_alloc_reg_out(reg_cache, _jit, c.i.rt, flags);
1515
1516         if (!cstate->state->mirrors_mapped && c.i.imm && !no_mask) {
1517                 jit_addi(rt, rs, (s16)c.i.imm);
1518                 addr_reg = rt;
1519                 imm = 0;
1520         } else {
1521                 addr_reg = rs;
1522                 imm = (s16)c.i.imm;
1523         }
1524
1525         if (!no_mask) {
1526                 jit_andi(rt, addr_reg, addr_mask);
1527                 addr_reg = rt;
1528         }
1529
1530         if (addr_offset) {
1531                 jit_addi(rt, addr_reg, addr_offset);
1532                 addr_reg = rt;
1533         }
1534
1535         jit_new_node_www(code, rt, addr_reg, imm);
1536
1537         if (is_big_endian() && swap_code) {
1538                 jit_new_node_ww(swap_code, rt, rt);
1539
1540                 if (c.i.op == OP_LH)
1541                         jit_extr_s(rt, rt);
1542                 else if (c.i.op == OP_LW && __WORDSIZE == 64)
1543                         jit_extr_i(rt, rt);
1544         }
1545
1546         lightrec_free_reg(reg_cache, rs);
1547         lightrec_free_reg(reg_cache, rt);
1548 }
1549
1550 static void rec_load_ram(struct lightrec_cstate *cstate,
1551                          const struct block *block, u16 offset,
1552                          jit_code_t code, jit_code_t swap_code, bool is_unsigned)
1553 {
1554         _jit_note(block->_jit, __FILE__, __LINE__);
1555
1556         rec_load_memory(cstate, block, offset, code, swap_code, is_unsigned,
1557                         cstate->state->offset_ram, rec_ram_mask(cstate->state));
1558 }
1559
1560 static void rec_load_bios(struct lightrec_cstate *cstate,
1561                           const struct block *block, u16 offset,
1562                           jit_code_t code, jit_code_t swap_code, bool is_unsigned)
1563 {
1564         _jit_note(block->_jit, __FILE__, __LINE__);
1565
1566         rec_load_memory(cstate, block, offset, code, swap_code, is_unsigned,
1567                         cstate->state->offset_bios, 0x1fffffff);
1568 }
1569
1570 static void rec_load_scratch(struct lightrec_cstate *cstate,
1571                              const struct block *block, u16 offset,
1572                              jit_code_t code, jit_code_t swap_code, bool is_unsigned)
1573 {
1574         _jit_note(block->_jit, __FILE__, __LINE__);
1575
1576         rec_load_memory(cstate, block, offset, code, swap_code, is_unsigned,
1577                         cstate->state->offset_scratch, 0x1fffffff);
1578 }
1579
1580 static void rec_load_io(struct lightrec_cstate *cstate,
1581                         const struct block *block, u16 offset,
1582                         jit_code_t code, jit_code_t swap_code, bool is_unsigned)
1583 {
1584         _jit_note(block->_jit, __FILE__, __LINE__);
1585
1586         rec_load_memory(cstate, block, offset, code, swap_code, is_unsigned,
1587                         cstate->state->offset_io, rec_io_mask(cstate->state));
1588 }
1589
1590 static void rec_load_direct(struct lightrec_cstate *cstate,
1591                             const struct block *block, u16 offset,
1592                             jit_code_t code, jit_code_t swap_code,
1593                             bool is_unsigned)
1594 {
1595         struct lightrec_state *state = cstate->state;
1596         struct regcache *reg_cache = cstate->reg_cache;
1597         union code c = block->opcode_list[offset].c;
1598         jit_state_t *_jit = block->_jit;
1599         jit_node_t *to_not_ram, *to_not_bios, *to_end, *to_end2;
1600         u8 tmp, rs, rt, addr_reg, flags = REG_EXT;
1601         s16 imm;
1602
1603         if (!c.i.rt)
1604                 return;
1605
1606         if (is_unsigned)
1607                 flags |= REG_ZEXT;
1608
1609         jit_note(__FILE__, __LINE__);
1610         rs = lightrec_alloc_reg_in(reg_cache, _jit, c.i.rs, 0);
1611         rt = lightrec_alloc_reg_out(reg_cache, _jit, c.i.rt, flags);
1612
1613         if ((state->offset_ram == state->offset_bios &&
1614             state->offset_ram == state->offset_scratch &&
1615             state->mirrors_mapped) || !c.i.imm) {
1616                 addr_reg = rs;
1617                 imm = (s16)c.i.imm;
1618         } else {
1619                 jit_addi(rt, rs, (s16)c.i.imm);
1620                 addr_reg = rt;
1621                 imm = 0;
1622
1623                 if (c.i.rs != c.i.rt)
1624                         lightrec_free_reg(reg_cache, rs);
1625         }
1626
1627         tmp = lightrec_alloc_reg_temp(reg_cache, _jit);
1628
1629         if (state->offset_ram == state->offset_bios &&
1630             state->offset_ram == state->offset_scratch) {
1631                 if (!state->mirrors_mapped) {
1632                         jit_andi(tmp, addr_reg, BIT(28));
1633                         jit_rshi_u(tmp, tmp, 28 - 22);
1634                         jit_ori(tmp, tmp, 0x1f800000 | (RAM_SIZE - 1));
1635                         jit_andr(rt, addr_reg, tmp);
1636                 } else {
1637                         jit_andi(rt, addr_reg, 0x1fffffff);
1638                 }
1639
1640                 if (state->offset_ram)
1641                         jit_movi(tmp, state->offset_ram);
1642         } else {
1643                 to_not_ram = jit_bmsi(addr_reg, BIT(28));
1644
1645                 /* Convert to KUNSEG and avoid RAM mirrors */
1646                 jit_andi(rt, addr_reg, RAM_SIZE - 1);
1647
1648                 if (state->offset_ram)
1649                         jit_movi(tmp, state->offset_ram);
1650
1651                 to_end = jit_b();
1652
1653                 jit_patch(to_not_ram);
1654
1655                 if (state->offset_bios != state->offset_scratch)
1656                         to_not_bios = jit_bmci(addr_reg, BIT(22));
1657
1658                 /* Convert to KUNSEG */
1659                 jit_andi(rt, addr_reg, 0x1fc00000 | (BIOS_SIZE - 1));
1660
1661                 jit_movi(tmp, state->offset_bios);
1662
1663                 if (state->offset_bios != state->offset_scratch) {
1664                         to_end2 = jit_b();
1665
1666                         jit_patch(to_not_bios);
1667
1668                         /* Convert to KUNSEG */
1669                         jit_andi(rt, addr_reg, 0x1f800fff);
1670
1671                         if (state->offset_scratch)
1672                                 jit_movi(tmp, state->offset_scratch);
1673
1674                         jit_patch(to_end2);
1675                 }
1676
1677                 jit_patch(to_end);
1678         }
1679
1680         if (state->offset_ram || state->offset_bios || state->offset_scratch)
1681                 jit_addr(rt, rt, tmp);
1682
1683         jit_new_node_www(code, rt, rt, imm);
1684
1685         if (is_big_endian() && swap_code) {
1686                 jit_new_node_ww(swap_code, rt, rt);
1687
1688                 if (c.i.op == OP_LH)
1689                         jit_extr_s(rt, rt);
1690                 else if (c.i.op == OP_LW && __WORDSIZE == 64)
1691                         jit_extr_i(rt, rt);
1692         }
1693
1694         lightrec_free_reg(reg_cache, addr_reg);
1695         lightrec_free_reg(reg_cache, rt);
1696         lightrec_free_reg(reg_cache, tmp);
1697 }
1698
1699 static void rec_load(struct lightrec_cstate *state, const struct block *block,
1700                      u16 offset, jit_code_t code, jit_code_t swap_code,
1701                      bool is_unsigned)
1702 {
1703         u32 flags = block->opcode_list[offset].flags;
1704
1705         switch (LIGHTREC_FLAGS_GET_IO_MODE(flags)) {
1706         case LIGHTREC_IO_RAM:
1707                 rec_load_ram(state, block, offset, code, swap_code, is_unsigned);
1708                 break;
1709         case LIGHTREC_IO_BIOS:
1710                 rec_load_bios(state, block, offset, code, swap_code, is_unsigned);
1711                 break;
1712         case LIGHTREC_IO_SCRATCH:
1713                 rec_load_scratch(state, block, offset, code, swap_code, is_unsigned);
1714                 break;
1715         case LIGHTREC_IO_DIRECT_HW:
1716                 rec_load_io(state, block, offset, code, swap_code, is_unsigned);
1717                 break;
1718         case LIGHTREC_IO_DIRECT:
1719                 rec_load_direct(state, block, offset, code, swap_code, is_unsigned);
1720                 break;
1721         default:
1722                 rec_io(state, block, offset, false, true);
1723                 break;
1724         }
1725 }
1726
1727 static void rec_LB(struct lightrec_cstate *state, const struct block *block, u16 offset)
1728 {
1729         _jit_name(block->_jit, __func__);
1730         rec_load(state, block, offset, jit_code_ldxi_c, 0, false);
1731 }
1732
1733 static void rec_LBU(struct lightrec_cstate *state, const struct block *block, u16 offset)
1734 {
1735         _jit_name(block->_jit, __func__);
1736         rec_load(state, block, offset, jit_code_ldxi_uc, 0, true);
1737 }
1738
1739 static void rec_LH(struct lightrec_cstate *state, const struct block *block, u16 offset)
1740 {
1741         jit_code_t code = is_big_endian() ? jit_code_ldxi_us : jit_code_ldxi_s;
1742
1743         _jit_name(block->_jit, __func__);
1744         rec_load(state, block, offset, code, jit_code_bswapr_us, false);
1745 }
1746
1747 static void rec_LHU(struct lightrec_cstate *state, const struct block *block, u16 offset)
1748 {
1749         _jit_name(block->_jit, __func__);
1750         rec_load(state, block, offset, jit_code_ldxi_us, jit_code_bswapr_us, true);
1751 }
1752
1753 static void rec_LWL(struct lightrec_cstate *state, const struct block *block, u16 offset)
1754 {
1755         _jit_name(block->_jit, __func__);
1756         rec_io(state, block, offset, true, true);
1757 }
1758
1759 static void rec_LWR(struct lightrec_cstate *state, const struct block *block, u16 offset)
1760 {
1761         _jit_name(block->_jit, __func__);
1762         rec_io(state, block, offset, true, true);
1763 }
1764
1765 static void rec_LW(struct lightrec_cstate *state, const struct block *block, u16 offset)
1766 {
1767         jit_code_t code;
1768
1769         if (is_big_endian() && __WORDSIZE == 64)
1770                 code = jit_code_ldxi_ui;
1771         else
1772                 code = jit_code_ldxi_i;
1773
1774         _jit_name(block->_jit, __func__);
1775         rec_load(state, block, offset, code, jit_code_bswapr_ui, false);
1776 }
1777
1778 static void rec_LWC2(struct lightrec_cstate *state, const struct block *block, u16 offset)
1779 {
1780         _jit_name(block->_jit, __func__);
1781         rec_io(state, block, offset, false, false);
1782 }
1783
1784 static void rec_break_syscall(struct lightrec_cstate *state,
1785                               const struct block *block, u16 offset,
1786                               u32 exit_code)
1787 {
1788         struct regcache *reg_cache = state->reg_cache;
1789         jit_state_t *_jit = block->_jit;
1790         u8 tmp;
1791
1792         _jit_note(block->_jit, __FILE__, __LINE__);
1793
1794         tmp = lightrec_alloc_reg_temp(reg_cache, _jit);
1795
1796         jit_movi(tmp, exit_code);
1797         jit_stxi_i(offsetof(struct lightrec_state, exit_flags),
1798                    LIGHTREC_REG_STATE, tmp);
1799
1800         lightrec_free_reg(reg_cache, tmp);
1801
1802         /* TODO: the return address should be "pc - 4" if we're a delay slot */
1803         lightrec_emit_end_of_block(state, block, offset, -1,
1804                                    get_ds_pc(block, offset, 0),
1805                                    31, 0, true);
1806 }
1807
1808 static void rec_special_SYSCALL(struct lightrec_cstate *state,
1809                                 const struct block *block, u16 offset)
1810 {
1811         _jit_name(block->_jit, __func__);
1812         rec_break_syscall(state, block, offset, LIGHTREC_EXIT_SYSCALL);
1813 }
1814
1815 static void rec_special_BREAK(struct lightrec_cstate *state,
1816                               const struct block *block, u16 offset)
1817 {
1818         _jit_name(block->_jit, __func__);
1819         rec_break_syscall(state, block, offset, LIGHTREC_EXIT_BREAK);
1820 }
1821
1822 static void rec_mfc(struct lightrec_cstate *state, const struct block *block, u16 offset)
1823 {
1824         struct regcache *reg_cache = state->reg_cache;
1825         union code c = block->opcode_list[offset].c;
1826         jit_state_t *_jit = block->_jit;
1827
1828         jit_note(__FILE__, __LINE__);
1829         lightrec_clean_reg_if_loaded(reg_cache, _jit, c.i.rt, true);
1830
1831         call_to_c_wrapper(state, block, c.opcode, C_WRAPPER_MFC);
1832 }
1833
1834 static void rec_mtc(struct lightrec_cstate *state, const struct block *block, u16 offset)
1835 {
1836         struct regcache *reg_cache = state->reg_cache;
1837         union code c = block->opcode_list[offset].c;
1838         jit_state_t *_jit = block->_jit;
1839
1840         jit_note(__FILE__, __LINE__);
1841         lightrec_clean_reg_if_loaded(reg_cache, _jit, c.i.rs, false);
1842         lightrec_clean_reg_if_loaded(reg_cache, _jit, c.i.rt, false);
1843
1844         call_to_c_wrapper(state, block, c.opcode, C_WRAPPER_MTC);
1845
1846         if (c.i.op == OP_CP0 &&
1847             !op_flag_no_ds(block->opcode_list[offset].flags) &&
1848             (c.r.rd == 12 || c.r.rd == 13))
1849                 lightrec_emit_end_of_block(state, block, offset, -1,
1850                                            get_ds_pc(block, offset, 1),
1851                                            0, 0, true);
1852 }
1853
1854 static void
1855 rec_mfc0(struct lightrec_cstate *state, const struct block *block, u16 offset)
1856 {
1857         struct regcache *reg_cache = state->reg_cache;
1858         union code c = block->opcode_list[offset].c;
1859         jit_state_t *_jit = block->_jit;
1860         u8 rt;
1861
1862         jit_note(__FILE__, __LINE__);
1863
1864         rt = lightrec_alloc_reg_out(reg_cache, _jit, c.i.rt, REG_EXT);
1865
1866         jit_ldxi_i(rt, LIGHTREC_REG_STATE,
1867                    offsetof(struct lightrec_state, regs.cp0[c.r.rd]));
1868
1869         lightrec_free_reg(reg_cache, rt);
1870 }
1871
1872 static bool block_in_bios(const struct lightrec_cstate *state,
1873                           const struct block *block)
1874 {
1875         const struct lightrec_mem_map *bios = &state->state->maps[PSX_MAP_BIOS];
1876         u32 pc = kunseg(block->pc);
1877
1878         return pc >= bios->pc && pc < bios->pc + bios->length;
1879 }
1880
1881 static void
1882 rec_mtc0(struct lightrec_cstate *state, const struct block *block, u16 offset)
1883 {
1884         struct regcache *reg_cache = state->reg_cache;
1885         const union code c = block->opcode_list[offset].c;
1886         jit_state_t *_jit = block->_jit;
1887         u8 rt, tmp = 0, tmp2, status;
1888         jit_node_t *to_end;
1889
1890         jit_note(__FILE__, __LINE__);
1891
1892         switch(c.r.rd) {
1893         case 1:
1894         case 4:
1895         case 8:
1896         case 14:
1897         case 15:
1898                 /* Those registers are read-only */
1899                 return;
1900         default:
1901                 break;
1902         }
1903
1904         if (block_in_bios(state, block) && c.r.rd == 12) {
1905                 /* If we are running code from the BIOS, handle writes to the
1906                  * Status register in C. BIOS code may toggle bit 16 which will
1907                  * map/unmap the RAM, while game code cannot do that. */
1908                 rec_mtc(state, block, offset);
1909                 return;
1910         }
1911
1912         rt = lightrec_alloc_reg_in(reg_cache, _jit, c.i.rt, 0);
1913
1914         if (c.r.rd != 13) {
1915                 jit_stxi_i(offsetof(struct lightrec_state, regs.cp0[c.r.rd]),
1916                            LIGHTREC_REG_STATE, rt);
1917         }
1918
1919         if (c.r.rd == 12 || c.r.rd == 13) {
1920                 tmp = lightrec_alloc_reg_temp(reg_cache, _jit);
1921                 jit_ldxi_i(tmp, LIGHTREC_REG_STATE,
1922                            offsetof(struct lightrec_state, regs.cp0[13]));
1923
1924                 tmp2 = lightrec_alloc_reg_temp(reg_cache, _jit);
1925         }
1926
1927         if (c.r.rd == 12) {
1928                 status = rt;
1929         } else if (c.r.rd == 13) {
1930                 /* Cause = (Cause & ~0x0300) | (value & 0x0300) */
1931                 jit_andi(tmp2, rt, 0x0300);
1932                 jit_ori(tmp, tmp, 0x0300);
1933                 jit_xori(tmp, tmp, 0x0300);
1934                 jit_orr(tmp, tmp, tmp2);
1935                 jit_ldxi_i(tmp2, LIGHTREC_REG_STATE,
1936                            offsetof(struct lightrec_state, regs.cp0[12]));
1937                 jit_stxi_i(offsetof(struct lightrec_state, regs.cp0[13]),
1938                            LIGHTREC_REG_STATE, tmp);
1939                 status = tmp2;
1940         }
1941
1942         if (c.r.rd == 12 || c.r.rd == 13) {
1943                 /* Exit dynarec in case there's a software interrupt.
1944                  * exit_flags = !!(status & tmp & 0x0300) & status; */
1945                 jit_andr(tmp, tmp, status);
1946                 jit_andi(tmp, tmp, 0x0300);
1947                 jit_nei(tmp, tmp, 0);
1948                 jit_andr(tmp, tmp, status);
1949         }
1950
1951         if (c.r.rd == 12) {
1952                 /* Exit dynarec in case we unmask a hardware interrupt.
1953                  * exit_flags = !(~status & 0x401) */
1954
1955                 jit_comr(tmp2, status);
1956                 jit_andi(tmp2, tmp2, 0x401);
1957                 jit_eqi(tmp2, tmp2, 0);
1958                 jit_orr(tmp, tmp, tmp2);
1959         }
1960
1961         lightrec_free_reg(reg_cache, rt);
1962
1963         if (c.r.rd == 12 || c.r.rd == 13) {
1964                 to_end = jit_beqi(tmp, 0);
1965
1966                 jit_ldxi_i(tmp2, LIGHTREC_REG_STATE,
1967                            offsetof(struct lightrec_state, target_cycle));
1968                 jit_subr(tmp2, tmp2, LIGHTREC_REG_CYCLE);
1969                 jit_movi(LIGHTREC_REG_CYCLE, 0);
1970                 jit_stxi_i(offsetof(struct lightrec_state, target_cycle),
1971                            LIGHTREC_REG_STATE, tmp2);
1972                 jit_stxi_i(offsetof(struct lightrec_state, current_cycle),
1973                            LIGHTREC_REG_STATE, tmp2);
1974
1975
1976                 jit_patch(to_end);
1977         }
1978
1979         if (!op_flag_no_ds(block->opcode_list[offset].flags) &&
1980             (c.r.rd == 12 || c.r.rd == 13)) {
1981                 state->cycles += lightrec_cycles_of_opcode(c);
1982                 lightrec_emit_eob(state, block, offset + 1);
1983         }
1984 }
1985
1986 static void rec_cp0_MFC0(struct lightrec_cstate *state,
1987                          const struct block *block, u16 offset)
1988 {
1989         _jit_name(block->_jit, __func__);
1990         rec_mfc0(state, block, offset);
1991 }
1992
1993 static void rec_cp0_CFC0(struct lightrec_cstate *state,
1994                          const struct block *block, u16 offset)
1995 {
1996         _jit_name(block->_jit, __func__);
1997         rec_mfc0(state, block, offset);
1998 }
1999
2000 static void rec_cp0_MTC0(struct lightrec_cstate *state,
2001                          const struct block *block, u16 offset)
2002 {
2003         _jit_name(block->_jit, __func__);
2004         rec_mtc0(state, block, offset);
2005 }
2006
2007 static void rec_cp0_CTC0(struct lightrec_cstate *state,
2008                          const struct block *block, u16 offset)
2009 {
2010         _jit_name(block->_jit, __func__);
2011         rec_mtc0(state, block, offset);
2012 }
2013
2014 static unsigned int cp2d_i_offset(u8 reg)
2015 {
2016         return offsetof(struct lightrec_state, regs.cp2d[reg]);
2017 }
2018
2019 static unsigned int cp2d_s_offset(u8 reg)
2020 {
2021         return cp2d_i_offset(reg) + is_big_endian() * 2;
2022 }
2023
2024 static unsigned int cp2c_i_offset(u8 reg)
2025 {
2026         return offsetof(struct lightrec_state, regs.cp2c[reg]);
2027 }
2028
2029 static unsigned int cp2c_s_offset(u8 reg)
2030 {
2031         return cp2c_i_offset(reg) + is_big_endian() * 2;
2032 }
2033
2034 static void rec_cp2_basic_MFC2(struct lightrec_cstate *state,
2035                                const struct block *block, u16 offset)
2036 {
2037         struct regcache *reg_cache = state->reg_cache;
2038         const union code c = block->opcode_list[offset].c;
2039         jit_state_t *_jit = block->_jit;
2040         const u32 zext_regs = 0x300f0080;
2041         u8 rt, tmp, tmp2, tmp3, out, flags;
2042         u8 reg = c.r.rd == 15 ? 14 : c.r.rd;
2043         unsigned int i;
2044
2045         _jit_name(block->_jit, __func__);
2046
2047         if (state->state->ops.cop2_notify) {
2048                 /* We must call cop2_notify, handle that in C. */
2049                 rec_mfc(state, block, offset);
2050                 return;
2051         }
2052
2053         flags = (zext_regs & BIT(reg)) ? REG_ZEXT : REG_EXT;
2054         rt = lightrec_alloc_reg_out(reg_cache, _jit, c.r.rt, flags);
2055
2056         switch (reg) {
2057         case 1:
2058         case 3:
2059         case 5:
2060         case 8:
2061         case 9:
2062         case 10:
2063         case 11:
2064                 jit_ldxi_s(rt, LIGHTREC_REG_STATE, cp2d_s_offset(reg));
2065                 break;
2066         case 7:
2067         case 16:
2068         case 17:
2069         case 18:
2070         case 19:
2071                 jit_ldxi_us(rt, LIGHTREC_REG_STATE, cp2d_s_offset(reg));
2072                 break;
2073         case 28:
2074         case 29:
2075                 tmp = lightrec_alloc_reg_temp(reg_cache, _jit);
2076                 tmp2 = lightrec_alloc_reg_temp(reg_cache, _jit);
2077                 tmp3 = lightrec_alloc_reg_temp(reg_cache, _jit);
2078
2079                 for (i = 0; i < 3; i++) {
2080                         out = i == 0 ? rt : tmp;
2081
2082                         jit_ldxi_s(tmp, LIGHTREC_REG_STATE, cp2d_s_offset(9 + i));
2083                         jit_movi(tmp2, 0x1f);
2084                         jit_rshi(out, tmp, 7);
2085
2086                         jit_ltr(tmp3, tmp2, out);
2087                         jit_movnr(out, tmp2, tmp3);
2088
2089                         jit_gei(tmp2, out, 0);
2090                         jit_movzr(out, tmp2, tmp2);
2091
2092                         if (i > 0) {
2093                                 jit_lshi(tmp, tmp, 5 * i);
2094                                 jit_orr(rt, rt, tmp);
2095                         }
2096                 }
2097
2098
2099                 lightrec_free_reg(reg_cache, tmp);
2100                 lightrec_free_reg(reg_cache, tmp2);
2101                 lightrec_free_reg(reg_cache, tmp3);
2102                 break;
2103         default:
2104                 jit_ldxi_i(rt, LIGHTREC_REG_STATE, cp2d_i_offset(reg));
2105                 break;
2106         }
2107
2108         lightrec_free_reg(reg_cache, rt);
2109 }
2110
2111 static void rec_cp2_basic_CFC2(struct lightrec_cstate *state,
2112                                const struct block *block, u16 offset)
2113 {
2114         struct regcache *reg_cache = state->reg_cache;
2115         const union code c = block->opcode_list[offset].c;
2116         jit_state_t *_jit = block->_jit;
2117         u8 rt;
2118
2119         _jit_name(block->_jit, __func__);
2120
2121         if (state->state->ops.cop2_notify) {
2122                 /* We must call cop2_notify, handle that in C. */
2123                 rec_mfc(state, block, offset);
2124                 return;
2125         }
2126
2127         switch (c.r.rd) {
2128         case 4:
2129         case 12:
2130         case 20:
2131         case 26:
2132         case 27:
2133         case 29:
2134         case 30:
2135                 rt = lightrec_alloc_reg_out(reg_cache, _jit, c.r.rt, REG_EXT);
2136                 jit_ldxi_s(rt, LIGHTREC_REG_STATE, cp2c_s_offset(c.r.rd));
2137                 break;
2138         default:
2139                 rt = lightrec_alloc_reg_out(reg_cache, _jit, c.r.rt, REG_ZEXT);
2140                 jit_ldxi_ui(rt, LIGHTREC_REG_STATE, cp2c_i_offset(c.r.rd));
2141                 break;
2142         }
2143
2144         lightrec_free_reg(reg_cache, rt);
2145 }
2146
2147 static void rec_cp2_basic_MTC2(struct lightrec_cstate *state,
2148                                const struct block *block, u16 offset)
2149 {
2150         struct regcache *reg_cache = state->reg_cache;
2151         const union code c = block->opcode_list[offset].c;
2152         jit_state_t *_jit = block->_jit;
2153         jit_node_t *loop, *to_loop;
2154         u8 rt, tmp, tmp2, flags = 0;
2155
2156         _jit_name(block->_jit, __func__);
2157
2158         if (state->state->ops.cop2_notify) {
2159                 /* We must call cop2_notify, handle that in C. */
2160                 rec_mtc(state, block, offset);
2161                 return;
2162         }
2163
2164         if (c.r.rd == 31)
2165                 return;
2166
2167         if (c.r.rd == 30)
2168                 flags |= REG_EXT;
2169
2170         rt = lightrec_alloc_reg_in(reg_cache, _jit, c.r.rt, flags);
2171
2172         switch (c.r.rd) {
2173         case 15:
2174                 tmp = lightrec_alloc_reg_temp(reg_cache, _jit);
2175                 jit_ldxi_i(tmp, LIGHTREC_REG_STATE, cp2d_i_offset(13));
2176
2177                 tmp2 = lightrec_alloc_reg_temp(reg_cache, _jit);
2178                 jit_ldxi_i(tmp2, LIGHTREC_REG_STATE, cp2d_i_offset(14));
2179
2180                 jit_stxi_i(cp2d_i_offset(12), LIGHTREC_REG_STATE, tmp);
2181                 jit_stxi_i(cp2d_i_offset(13), LIGHTREC_REG_STATE, tmp2);
2182                 jit_stxi_i(cp2d_i_offset(14), LIGHTREC_REG_STATE, rt);
2183
2184                 lightrec_free_reg(reg_cache, tmp);
2185                 lightrec_free_reg(reg_cache, tmp2);
2186                 break;
2187         case 28:
2188                 tmp = lightrec_alloc_reg_temp(reg_cache, _jit);
2189
2190                 jit_lshi(tmp, rt, 7);
2191                 jit_andi(tmp, tmp, 0xf80);
2192                 jit_stxi_s(cp2d_s_offset(9), LIGHTREC_REG_STATE, tmp);
2193
2194                 jit_lshi(tmp, rt, 2);
2195                 jit_andi(tmp, tmp, 0xf80);
2196                 jit_stxi_s(cp2d_s_offset(10), LIGHTREC_REG_STATE, tmp);
2197
2198                 jit_rshi(tmp, rt, 3);
2199                 jit_andi(tmp, tmp, 0xf80);
2200                 jit_stxi_s(cp2d_s_offset(11), LIGHTREC_REG_STATE, tmp);
2201
2202                 lightrec_free_reg(reg_cache, tmp);
2203                 break;
2204         case 30:
2205                 tmp = lightrec_alloc_reg_temp(reg_cache, _jit);
2206                 tmp2 = lightrec_alloc_reg_temp(reg_cache, _jit);
2207
2208                 /* if (rt < 0) rt = ~rt; */
2209                 jit_rshi(tmp, rt, 31);
2210                 jit_xorr(tmp, rt, tmp);
2211
2212                 /* We know the sign bit is 0. Left-shift by 1 to start the algorithm */
2213                 jit_lshi(tmp, tmp, 1);
2214                 jit_movi(tmp2, 33);
2215
2216                 /* Decrement tmp2 and right-shift the value by 1 until it equals zero */
2217                 loop = jit_label();
2218                 jit_subi(tmp2, tmp2, 1);
2219                 jit_rshi_u(tmp, tmp, 1);
2220                 to_loop = jit_bnei(tmp, 0);
2221
2222                 jit_patch_at(to_loop, loop);
2223
2224                 jit_stxi_i(cp2d_i_offset(31), LIGHTREC_REG_STATE, tmp2);
2225                 jit_stxi_i(cp2d_i_offset(30), LIGHTREC_REG_STATE, rt);
2226
2227                 lightrec_free_reg(reg_cache, tmp);
2228                 lightrec_free_reg(reg_cache, tmp2);
2229                 break;
2230         default:
2231                 jit_stxi_i(cp2d_i_offset(c.r.rd), LIGHTREC_REG_STATE, rt);
2232                 break;
2233         }
2234
2235         lightrec_free_reg(reg_cache, rt);
2236 }
2237
2238 static void rec_cp2_basic_CTC2(struct lightrec_cstate *state,
2239                                const struct block *block, u16 offset)
2240 {
2241         struct regcache *reg_cache = state->reg_cache;
2242         const union code c = block->opcode_list[offset].c;
2243         jit_state_t *_jit = block->_jit;
2244         u8 rt, tmp, tmp2;
2245
2246         _jit_name(block->_jit, __func__);
2247
2248         if (state->state->ops.cop2_notify) {
2249                 /* We must call cop2_notify, handle that in C. */
2250                 rec_mtc(state, block, offset);
2251                 return;
2252         }
2253
2254         rt = lightrec_alloc_reg_in(reg_cache, _jit, c.r.rt, 0);
2255
2256         switch (c.r.rd) {
2257         case 4:
2258         case 12:
2259         case 20:
2260         case 26:
2261         case 27:
2262         case 29:
2263         case 30:
2264                 jit_stxi_s(cp2c_s_offset(c.r.rd), LIGHTREC_REG_STATE, rt);
2265                 break;
2266         case 31:
2267                 tmp = lightrec_alloc_reg_temp(reg_cache, _jit);
2268                 tmp2 = lightrec_alloc_reg_temp(reg_cache, _jit);
2269
2270                 jit_andi(tmp, rt, 0x7f87e000);
2271                 jit_nei(tmp, tmp, 0);
2272                 jit_lshi(tmp, tmp, 31);
2273
2274                 jit_andi(tmp2, rt, 0x7ffff000);
2275                 jit_orr(tmp, tmp2, tmp);
2276
2277                 jit_stxi_i(cp2c_i_offset(31), LIGHTREC_REG_STATE, tmp);
2278
2279                 lightrec_free_reg(reg_cache, tmp);
2280                 lightrec_free_reg(reg_cache, tmp2);
2281                 break;
2282
2283         default:
2284                 jit_stxi_i(cp2c_i_offset(c.r.rd), LIGHTREC_REG_STATE, rt);
2285         }
2286
2287         lightrec_free_reg(reg_cache, rt);
2288 }
2289
2290 static void rec_cp0_RFE(struct lightrec_cstate *state,
2291                         const struct block *block, u16 offset)
2292 {
2293         struct regcache *reg_cache = state->reg_cache;
2294         jit_state_t *_jit = block->_jit;
2295         u8 status, tmp;
2296
2297         jit_name(__func__);
2298         jit_note(__FILE__, __LINE__);
2299
2300         status = lightrec_alloc_reg_temp(reg_cache, _jit);
2301         jit_ldxi_i(status, LIGHTREC_REG_STATE,
2302                    offsetof(struct lightrec_state, regs.cp0[12]));
2303
2304         tmp = lightrec_alloc_reg_temp(reg_cache, _jit);
2305
2306         /* status = ((status >> 2) & 0xf) | status & ~0xf; */
2307         jit_rshi(tmp, status, 2);
2308         jit_andi(tmp, tmp, 0xf);
2309         jit_andi(status, status, ~0xful);
2310         jit_orr(status, status, tmp);
2311
2312         jit_ldxi_i(tmp, LIGHTREC_REG_STATE,
2313                    offsetof(struct lightrec_state, regs.cp0[13]));
2314         jit_stxi_i(offsetof(struct lightrec_state, regs.cp0[12]),
2315                    LIGHTREC_REG_STATE, status);
2316
2317         /* Exit dynarec in case there's a software interrupt.
2318          * exit_flags = !!(status & cause & 0x0300) & status; */
2319         jit_andr(tmp, tmp, status);
2320         jit_andi(tmp, tmp, 0x0300);
2321         jit_nei(tmp, tmp, 0);
2322         jit_andr(tmp, tmp, status);
2323         jit_stxi_i(offsetof(struct lightrec_state, exit_flags),
2324                    LIGHTREC_REG_STATE, tmp);
2325
2326         lightrec_free_reg(reg_cache, status);
2327         lightrec_free_reg(reg_cache, tmp);
2328 }
2329
2330 static void rec_CP(struct lightrec_cstate *state,
2331                    const struct block *block, u16 offset)
2332 {
2333         union code c = block->opcode_list[offset].c;
2334         jit_state_t *_jit = block->_jit;
2335
2336         jit_name(__func__);
2337         jit_note(__FILE__, __LINE__);
2338
2339         call_to_c_wrapper(state, block, c.opcode, C_WRAPPER_CP);
2340 }
2341
2342 static void rec_meta_MOV(struct lightrec_cstate *state,
2343                          const struct block *block, u16 offset)
2344 {
2345         struct regcache *reg_cache = state->reg_cache;
2346         union code c = block->opcode_list[offset].c;
2347         jit_state_t *_jit = block->_jit;
2348         u8 rs, rd;
2349
2350         _jit_name(block->_jit, __func__);
2351         jit_note(__FILE__, __LINE__);
2352         if (c.r.rs)
2353                 rs = lightrec_alloc_reg_in(reg_cache, _jit, c.r.rs, 0);
2354         rd = lightrec_alloc_reg_out(reg_cache, _jit, c.r.rd, REG_EXT);
2355
2356         if (c.r.rs == 0)
2357                 jit_movi(rd, 0);
2358         else
2359                 jit_extr_i(rd, rs);
2360
2361         if (c.r.rs)
2362                 lightrec_free_reg(reg_cache, rs);
2363         lightrec_free_reg(reg_cache, rd);
2364 }
2365
2366 static void rec_meta_EXTC_EXTS(struct lightrec_cstate *state,
2367                                const struct block *block,
2368                                u16 offset)
2369 {
2370         struct regcache *reg_cache = state->reg_cache;
2371         union code c = block->opcode_list[offset].c;
2372         jit_state_t *_jit = block->_jit;
2373         u8 rs, rt;
2374
2375         _jit_name(block->_jit, __func__);
2376         jit_note(__FILE__, __LINE__);
2377
2378         rs = lightrec_alloc_reg_in(reg_cache, _jit, c.i.rs, 0);
2379         rt = lightrec_alloc_reg_out(reg_cache, _jit, c.i.rt, REG_EXT);
2380
2381         if (c.i.op == OP_META_EXTC)
2382                 jit_extr_c(rt, rs);
2383         else
2384                 jit_extr_s(rt, rs);
2385
2386         lightrec_free_reg(reg_cache, rs);
2387         lightrec_free_reg(reg_cache, rt);
2388 }
2389
2390 static void rec_meta_MULT2(struct lightrec_cstate *state,
2391                            const struct block *block,
2392                            u16 offset)
2393 {
2394         struct regcache *reg_cache = state->reg_cache;
2395         union code c = block->opcode_list[offset].c;
2396         jit_state_t *_jit = block->_jit;
2397         u8 reg_lo = get_mult_div_lo(c);
2398         u8 reg_hi = get_mult_div_hi(c);
2399         u32 flags = block->opcode_list[offset].flags;
2400         bool is_signed = c.i.op == OP_META_MULT2;
2401         u8 rs, lo, hi, rflags = 0, hiflags = 0;
2402         unsigned int i;
2403
2404         if (!op_flag_no_hi(flags) && c.r.op < 32) {
2405                 rflags = is_signed ? REG_EXT : REG_ZEXT;
2406                 hiflags = is_signed ? REG_EXT : (REG_EXT | REG_ZEXT);
2407         }
2408
2409         _jit_name(block->_jit, __func__);
2410         jit_note(__FILE__, __LINE__);
2411
2412         rs = lightrec_alloc_reg_in(reg_cache, _jit, c.i.rs, rflags);
2413
2414         /*
2415          * We must handle the case where one of the output registers is our rs
2416          * input register. Thanksfully, computing LO/HI can be done in any
2417          * order. Here, we make sure that the computation that overwrites the
2418          * input register is always performed last.
2419          */
2420         for (i = 0; i < 2; i++) {
2421                 if ((!i ^ (reg_lo == c.i.rs)) && !op_flag_no_lo(flags)) {
2422                         lo = lightrec_alloc_reg_out(reg_cache, _jit, reg_lo, 0);
2423
2424                         if (c.r.op < 32)
2425                                 jit_lshi(lo, rs, c.r.op);
2426                         else
2427                                 jit_movi(lo, 0);
2428
2429                         lightrec_free_reg(reg_cache, lo);
2430                         continue;
2431                 }
2432
2433                 if ((!!i ^ (reg_lo == c.i.rs)) && !op_flag_no_hi(flags)) {
2434                         hi = lightrec_alloc_reg_out(reg_cache, _jit,
2435                                                     reg_hi, hiflags);
2436
2437                         if (c.r.op >= 32)
2438                                 jit_lshi(hi, rs, c.r.op - 32);
2439                         else if (is_signed)
2440                                 jit_rshi(hi, rs, 32 - c.r.op);
2441                         else
2442                                 jit_rshi_u(hi, rs, 32 - c.r.op);
2443
2444                         lightrec_free_reg(reg_cache, hi);
2445                 }
2446         }
2447
2448         lightrec_free_reg(reg_cache, rs);
2449
2450         _jit_name(block->_jit, __func__);
2451         jit_note(__FILE__, __LINE__);
2452 }
2453
2454 static const lightrec_rec_func_t rec_standard[64] = {
2455         SET_DEFAULT_ELM(rec_standard, unknown_opcode),
2456         [OP_SPECIAL]            = rec_SPECIAL,
2457         [OP_REGIMM]             = rec_REGIMM,
2458         [OP_J]                  = rec_J,
2459         [OP_JAL]                = rec_JAL,
2460         [OP_BEQ]                = rec_BEQ,
2461         [OP_BNE]                = rec_BNE,
2462         [OP_BLEZ]               = rec_BLEZ,
2463         [OP_BGTZ]               = rec_BGTZ,
2464         [OP_ADDI]               = rec_ADDI,
2465         [OP_ADDIU]              = rec_ADDIU,
2466         [OP_SLTI]               = rec_SLTI,
2467         [OP_SLTIU]              = rec_SLTIU,
2468         [OP_ANDI]               = rec_ANDI,
2469         [OP_ORI]                = rec_ORI,
2470         [OP_XORI]               = rec_XORI,
2471         [OP_LUI]                = rec_LUI,
2472         [OP_CP0]                = rec_CP0,
2473         [OP_CP2]                = rec_CP2,
2474         [OP_LB]                 = rec_LB,
2475         [OP_LH]                 = rec_LH,
2476         [OP_LWL]                = rec_LWL,
2477         [OP_LW]                 = rec_LW,
2478         [OP_LBU]                = rec_LBU,
2479         [OP_LHU]                = rec_LHU,
2480         [OP_LWR]                = rec_LWR,
2481         [OP_SB]                 = rec_SB,
2482         [OP_SH]                 = rec_SH,
2483         [OP_SWL]                = rec_SWL,
2484         [OP_SW]                 = rec_SW,
2485         [OP_SWR]                = rec_SWR,
2486         [OP_LWC2]               = rec_LWC2,
2487         [OP_SWC2]               = rec_SWC2,
2488
2489         [OP_META_MOV]           = rec_meta_MOV,
2490         [OP_META_EXTC]          = rec_meta_EXTC_EXTS,
2491         [OP_META_EXTS]          = rec_meta_EXTC_EXTS,
2492         [OP_META_MULT2]         = rec_meta_MULT2,
2493         [OP_META_MULTU2]        = rec_meta_MULT2,
2494 };
2495
2496 static const lightrec_rec_func_t rec_special[64] = {
2497         SET_DEFAULT_ELM(rec_special, unknown_opcode),
2498         [OP_SPECIAL_SLL]        = rec_special_SLL,
2499         [OP_SPECIAL_SRL]        = rec_special_SRL,
2500         [OP_SPECIAL_SRA]        = rec_special_SRA,
2501         [OP_SPECIAL_SLLV]       = rec_special_SLLV,
2502         [OP_SPECIAL_SRLV]       = rec_special_SRLV,
2503         [OP_SPECIAL_SRAV]       = rec_special_SRAV,
2504         [OP_SPECIAL_JR]         = rec_special_JR,
2505         [OP_SPECIAL_JALR]       = rec_special_JALR,
2506         [OP_SPECIAL_SYSCALL]    = rec_special_SYSCALL,
2507         [OP_SPECIAL_BREAK]      = rec_special_BREAK,
2508         [OP_SPECIAL_MFHI]       = rec_special_MFHI,
2509         [OP_SPECIAL_MTHI]       = rec_special_MTHI,
2510         [OP_SPECIAL_MFLO]       = rec_special_MFLO,
2511         [OP_SPECIAL_MTLO]       = rec_special_MTLO,
2512         [OP_SPECIAL_MULT]       = rec_special_MULT,
2513         [OP_SPECIAL_MULTU]      = rec_special_MULTU,
2514         [OP_SPECIAL_DIV]        = rec_special_DIV,
2515         [OP_SPECIAL_DIVU]       = rec_special_DIVU,
2516         [OP_SPECIAL_ADD]        = rec_special_ADD,
2517         [OP_SPECIAL_ADDU]       = rec_special_ADDU,
2518         [OP_SPECIAL_SUB]        = rec_special_SUB,
2519         [OP_SPECIAL_SUBU]       = rec_special_SUBU,
2520         [OP_SPECIAL_AND]        = rec_special_AND,
2521         [OP_SPECIAL_OR]         = rec_special_OR,
2522         [OP_SPECIAL_XOR]        = rec_special_XOR,
2523         [OP_SPECIAL_NOR]        = rec_special_NOR,
2524         [OP_SPECIAL_SLT]        = rec_special_SLT,
2525         [OP_SPECIAL_SLTU]       = rec_special_SLTU,
2526 };
2527
2528 static const lightrec_rec_func_t rec_regimm[64] = {
2529         SET_DEFAULT_ELM(rec_regimm, unknown_opcode),
2530         [OP_REGIMM_BLTZ]        = rec_regimm_BLTZ,
2531         [OP_REGIMM_BGEZ]        = rec_regimm_BGEZ,
2532         [OP_REGIMM_BLTZAL]      = rec_regimm_BLTZAL,
2533         [OP_REGIMM_BGEZAL]      = rec_regimm_BGEZAL,
2534 };
2535
2536 static const lightrec_rec_func_t rec_cp0[64] = {
2537         SET_DEFAULT_ELM(rec_cp0, rec_CP),
2538         [OP_CP0_MFC0]           = rec_cp0_MFC0,
2539         [OP_CP0_CFC0]           = rec_cp0_CFC0,
2540         [OP_CP0_MTC0]           = rec_cp0_MTC0,
2541         [OP_CP0_CTC0]           = rec_cp0_CTC0,
2542         [OP_CP0_RFE]            = rec_cp0_RFE,
2543 };
2544
2545 static const lightrec_rec_func_t rec_cp2_basic[64] = {
2546         SET_DEFAULT_ELM(rec_cp2_basic, rec_CP),
2547         [OP_CP2_BASIC_MFC2]     = rec_cp2_basic_MFC2,
2548         [OP_CP2_BASIC_CFC2]     = rec_cp2_basic_CFC2,
2549         [OP_CP2_BASIC_MTC2]     = rec_cp2_basic_MTC2,
2550         [OP_CP2_BASIC_CTC2]     = rec_cp2_basic_CTC2,
2551 };
2552
2553 static void rec_SPECIAL(struct lightrec_cstate *state,
2554                         const struct block *block, u16 offset)
2555 {
2556         union code c = block->opcode_list[offset].c;
2557         lightrec_rec_func_t f = rec_special[c.r.op];
2558
2559         if (!HAS_DEFAULT_ELM && unlikely(!f))
2560                 unknown_opcode(state, block, offset);
2561         else
2562                 (*f)(state, block, offset);
2563 }
2564
2565 static void rec_REGIMM(struct lightrec_cstate *state,
2566                        const struct block *block, u16 offset)
2567 {
2568         union code c = block->opcode_list[offset].c;
2569         lightrec_rec_func_t f = rec_regimm[c.r.rt];
2570
2571         if (!HAS_DEFAULT_ELM && unlikely(!f))
2572                 unknown_opcode(state, block, offset);
2573         else
2574                 (*f)(state, block, offset);
2575 }
2576
2577 static void rec_CP0(struct lightrec_cstate *state,
2578                     const struct block *block, u16 offset)
2579 {
2580         union code c = block->opcode_list[offset].c;
2581         lightrec_rec_func_t f = rec_cp0[c.r.rs];
2582
2583         if (!HAS_DEFAULT_ELM && unlikely(!f))
2584                 rec_CP(state, block, offset);
2585         else
2586                 (*f)(state, block, offset);
2587 }
2588
2589 static void rec_CP2(struct lightrec_cstate *state,
2590                     const struct block *block, u16 offset)
2591 {
2592         union code c = block->opcode_list[offset].c;
2593
2594         if (c.r.op == OP_CP2_BASIC) {
2595                 lightrec_rec_func_t f = rec_cp2_basic[c.r.rs];
2596
2597                 if (HAS_DEFAULT_ELM || likely(f)) {
2598                         (*f)(state, block, offset);
2599                         return;
2600                 }
2601         }
2602
2603         rec_CP(state, block, offset);
2604 }
2605
2606 void lightrec_rec_opcode(struct lightrec_cstate *state,
2607                          const struct block *block, u16 offset)
2608 {
2609         struct regcache *reg_cache = state->reg_cache;
2610         struct lightrec_branch_target *target;
2611         const struct opcode *op = &block->opcode_list[offset];
2612         jit_state_t *_jit = block->_jit;
2613         lightrec_rec_func_t f;
2614         u16 unload_offset;
2615
2616         if (op_flag_sync(op->flags)) {
2617                 if (state->cycles)
2618                         jit_subi(LIGHTREC_REG_CYCLE, LIGHTREC_REG_CYCLE, state->cycles);
2619                 state->cycles = 0;
2620
2621                 lightrec_storeback_regs(reg_cache, _jit);
2622                 lightrec_regcache_reset(reg_cache);
2623
2624                 pr_debug("Adding branch target at offset 0x%x\n", offset << 2);
2625                 target = &state->targets[state->nb_targets++];
2626                 target->offset = offset;
2627                 target->label = jit_indirect();
2628         }
2629
2630         if (likely(op->opcode)) {
2631                 f = rec_standard[op->i.op];
2632
2633                 if (!HAS_DEFAULT_ELM && unlikely(!f))
2634                         unknown_opcode(state, block, offset);
2635                 else
2636                         (*f)(state, block, offset);
2637         }
2638
2639         if (OPT_EARLY_UNLOAD) {
2640                 unload_offset = offset +
2641                         (has_delay_slot(op->c) && !op_flag_no_ds(op->flags));
2642
2643                 lightrec_do_early_unload(state, block, unload_offset);
2644         }
2645 }