deps/lightrec/emitter.c

   1 // SPDX-License-Identifier: LGPL-2.1-or-later
   2 /*
   3  * Copyright (C) 2014-2021 Paul Cercueil <paul@crapouillou.net>
   4  */
   5
   6 #include "blockcache.h"
   7 #include "debug.h"
   8 #include "disassembler.h"
   9 #include "emitter.h"
  10 #include "lightning-wrapper.h"
  11 #include "optimizer.h"
  12 #include "regcache.h"
  13
  14 #include <stdbool.h>
  15 #include <stddef.h>
  16
  17 typedef void (*lightrec_rec_func_t)(struct lightrec_cstate *, const struct block *, u16);
  18
  19 /* Forward declarations */
  20 static void rec_SPECIAL(struct lightrec_cstate *state, const struct block *block, u16 offset);
  21 static void rec_REGIMM(struct lightrec_cstate *state, const struct block *block, u16 offset);
  22 static void rec_CP0(struct lightrec_cstate *state, const struct block *block, u16 offset);
  23 static void rec_CP2(struct lightrec_cstate *state, const struct block *block, u16 offset);
  24
  25 static void unknown_opcode(struct lightrec_cstate *state, const struct block *block, u16 offset)
  26 {
  27         pr_warn("Unknown opcode: 0x%08x at PC 0x%08x\n",
  28                 block->opcode_list[offset].c.opcode,
  29                 block->pc + (offset << 2));
  30 }
  31
  32 static void lightrec_emit_end_of_block(struct lightrec_cstate *state,
  33                                        const struct block *block, u16 offset,
  34                                        s8 reg_new_pc, u32 imm, u8 ra_reg,
  35                                        u32 link, bool update_cycles)
  36 {
  37         struct regcache *reg_cache = state->reg_cache;
  38         u32 cycles = state->cycles;
  39         jit_state_t *_jit = block->_jit;
  40         const struct opcode *op = &block->opcode_list[offset],
  41                             *next = &block->opcode_list[offset + 1];
  42
  43         jit_note(__FILE__, __LINE__);
  44
  45         if (link) {
  46                 /* Update the $ra register */
  47                 u8 link_reg = lightrec_alloc_reg_out(reg_cache, _jit, ra_reg, 0);
  48                 jit_movi(link_reg, link);
  49                 lightrec_free_reg(reg_cache, link_reg);
  50         }
  51
  52         if (reg_new_pc < 0) {
  53                 reg_new_pc = lightrec_alloc_reg(reg_cache, _jit, JIT_V0);
  54                 lightrec_lock_reg(reg_cache, _jit, reg_new_pc);
  55
  56                 jit_movi(reg_new_pc, imm);
  57         }
  58
  59         if (has_delay_slot(op->c) &&
  60             !(op->flags & (LIGHTREC_NO_DS | LIGHTREC_LOCAL_BRANCH))) {
  61                 cycles += lightrec_cycles_of_opcode(next->c);
  62
  63                 /* Recompile the delay slot */
  64                 if (next->c.opcode)
  65                         lightrec_rec_opcode(state, block, offset + 1);
  66         }
  67
  68         /* Store back remaining registers */
  69         lightrec_storeback_regs(reg_cache, _jit);
  70
  71         jit_movr(JIT_V0, reg_new_pc);
  72
  73         if (cycles && update_cycles) {
  74                 jit_subi(LIGHTREC_REG_CYCLE, LIGHTREC_REG_CYCLE, cycles);
  75                 pr_debug("EOB: %u cycles\n", cycles);
  76         }
  77
  78         if (offset + !!(op->flags & LIGHTREC_NO_DS) < block->nb_ops - 1)
  79                 state->branches[state->nb_branches++] = jit_jmpi();
  80 }
  81
  82 void lightrec_emit_eob(struct lightrec_cstate *state, const struct block *block,
  83                        u16 offset, bool after_op)
  84 {
  85         struct regcache *reg_cache = state->reg_cache;
  86         jit_state_t *_jit = block->_jit;
  87         union code c = block->opcode_list[offset].c;
  88         u32 cycles = state->cycles;
  89
  90         if (!after_op)
  91                 cycles -= lightrec_cycles_of_opcode(c);
  92
  93         lightrec_storeback_regs(reg_cache, _jit);
  94
  95         jit_movi(JIT_V0, block->pc + (offset << 2));
  96         jit_subi(LIGHTREC_REG_CYCLE, LIGHTREC_REG_CYCLE, cycles);
  97
  98         state->branches[state->nb_branches++] = jit_jmpi();
  99 }
 100
 101 static u8 get_jr_jalr_reg(struct lightrec_cstate *state, const struct block *block, u16 offset)
 102 {
 103         struct regcache *reg_cache = state->reg_cache;
 104         jit_state_t *_jit = block->_jit;
 105         const struct opcode *op = &block->opcode_list[offset],
 106                             *next = &block->opcode_list[offset + 1];
 107         u8 rs = lightrec_request_reg_in(reg_cache, _jit, op->r.rs, JIT_V0);
 108
 109         /* If the source register is already mapped to JIT_R0 or JIT_R1, and the
 110          * delay slot is a I/O operation, unload the register, since JIT_R0 and
 111          * JIT_R1 are explicitely used by the I/O opcode generators. */
 112         if ((rs == JIT_R0 || rs == JIT_R1) &&
 113             !(op->flags & LIGHTREC_NO_DS) &&
 114             opcode_is_io(next->c) &&
 115             !(next->flags & (LIGHTREC_NO_INVALIDATE | LIGHTREC_DIRECT_IO))) {
 116                 lightrec_unload_reg(reg_cache, _jit, rs);
 117                 lightrec_free_reg(reg_cache, rs);
 118
 119                 rs = lightrec_request_reg_in(reg_cache, _jit, op->r.rs, JIT_V0);
 120         }
 121
 122         lightrec_lock_reg(reg_cache, _jit, rs);
 123
 124         return rs;
 125 }
 126
 127 static void rec_special_JR(struct lightrec_cstate *state, const struct block *block, u16 offset)
 128 {
 129         u8 rs = get_jr_jalr_reg(state, block, offset);
 130
 131         _jit_name(block->_jit, __func__);
 132         lightrec_emit_end_of_block(state, block, offset, rs, 0, 31, 0, true);
 133 }
 134
 135 static void rec_special_JALR(struct lightrec_cstate *state, const struct block *block, u16 offset)
 136 {
 137         u8 rs = get_jr_jalr_reg(state, block, offset);
 138         union code c = block->opcode_list[offset].c;
 139
 140         _jit_name(block->_jit, __func__);
 141         lightrec_emit_end_of_block(state, block, offset, rs, 0, c.r.rd,
 142                                    get_branch_pc(block, offset, 2), true);
 143 }
 144
 145 static void rec_J(struct lightrec_cstate *state, const struct block *block, u16 offset)
 146 {
 147         union code c = block->opcode_list[offset].c;
 148
 149         _jit_name(block->_jit, __func__);
 150         lightrec_emit_end_of_block(state, block, offset, -1,
 151                                    (block->pc & 0xf0000000) | (c.j.imm << 2),
 152                                    31, 0, true);
 153 }
 154
 155 static void rec_JAL(struct lightrec_cstate *state, const struct block *block, u16 offset)
 156 {
 157         union code c = block->opcode_list[offset].c;
 158
 159         _jit_name(block->_jit, __func__);
 160         lightrec_emit_end_of_block(state, block, offset, -1,
 161                                    (block->pc & 0xf0000000) | (c.j.imm << 2),
 162                                    31, get_branch_pc(block, offset, 2), true);
 163 }
 164
 165 static void rec_b(struct lightrec_cstate *state, const struct block *block, u16 offset,
 166                   jit_code_t code, u32 link, bool unconditional, bool bz)
 167 {
 168         struct regcache *reg_cache = state->reg_cache;
 169         struct native_register *regs_backup;
 170         jit_state_t *_jit = block->_jit;
 171         struct lightrec_branch *branch;
 172         const struct opcode *op = &block->opcode_list[offset],
 173                             *next = &block->opcode_list[offset + 1];
 174         jit_node_t *addr;
 175         u8 link_reg;
 176         u32 target_offset, cycles = state->cycles;
 177         bool is_forward = (s16)op->i.imm >= -1;
 178         u32 next_pc;
 179
 180         jit_note(__FILE__, __LINE__);
 181
 182         if (!(op->flags & LIGHTREC_NO_DS))
 183                 cycles += lightrec_cycles_of_opcode(next->c);
 184
 185         state->cycles = 0;
 186
 187         if (cycles)
 188                 jit_subi(LIGHTREC_REG_CYCLE, LIGHTREC_REG_CYCLE, cycles);
 189
 190         if (!unconditional) {
 191                 u8 rs = lightrec_alloc_reg_in(reg_cache, _jit, op->i.rs, REG_EXT),
 192                    rt = bz ? 0 : lightrec_alloc_reg_in(reg_cache,
 193                                                        _jit, op->i.rt, REG_EXT);
 194
 195                 /* Generate the branch opcode */
 196                 addr = jit_new_node_pww(code, NULL, rs, rt);
 197
 198                 lightrec_free_regs(reg_cache);
 199                 regs_backup = lightrec_regcache_enter_branch(reg_cache);
 200         }
 201
 202         if (op->flags & LIGHTREC_LOCAL_BRANCH) {
 203                 if (next && !(op->flags & LIGHTREC_NO_DS)) {
 204                         /* Recompile the delay slot */
 205                         if (next->opcode)
 206                                 lightrec_rec_opcode(state, block, offset + 1);
 207                 }
 208
 209                 if (link) {
 210                         /* Update the $ra register */
 211                         link_reg = lightrec_alloc_reg_out(reg_cache, _jit, 31, 0);
 212                         jit_movi(link_reg, link);
 213                         lightrec_free_reg(reg_cache, link_reg);
 214                 }
 215
 216                 /* Store back remaining registers */
 217                 lightrec_storeback_regs(reg_cache, _jit);
 218
 219                 target_offset = offset + 1 + (s16)op->i.imm
 220                         - !!(OPT_SWITCH_DELAY_SLOTS && (op->flags & LIGHTREC_NO_DS));
 221                 pr_debug("Adding local branch to offset 0x%x\n",
 222                          target_offset << 2);
 223                 branch = &state->local_branches[
 224                         state->nb_local_branches++];
 225
 226                 branch->target = target_offset;
 227                 if (is_forward)
 228                         branch->branch = jit_jmpi();
 229                 else
 230                         branch->branch = jit_bgti(LIGHTREC_REG_CYCLE, 0);
 231         }
 232
 233         if (!(op->flags & LIGHTREC_LOCAL_BRANCH) || !is_forward) {
 234                 next_pc = get_branch_pc(block, offset, 1 + (s16)op->i.imm);
 235                 lightrec_emit_end_of_block(state, block, offset, -1, next_pc,
 236                                            31, link, false);
 237         }
 238
 239         if (!unconditional) {
 240                 jit_patch(addr);
 241                 lightrec_regcache_leave_branch(reg_cache, regs_backup);
 242
 243                 if (bz && link) {
 244                         /* Update the $ra register */
 245                         link_reg = lightrec_alloc_reg_out(reg_cache, _jit,
 246                                                           31, REG_EXT);
 247                         jit_movi(link_reg, (s32)link);
 248                         lightrec_free_reg(reg_cache, link_reg);
 249                 }
 250
 251                 if (!(op->flags & LIGHTREC_NO_DS) && next->opcode)
 252                         lightrec_rec_opcode(state, block, offset + 1);
 253         }
 254 }
 255
 256 static void rec_BNE(struct lightrec_cstate *state,
 257                     const struct block *block, u16 offset)
 258 {
 259         union code c = block->opcode_list[offset].c;
 260
 261         _jit_name(block->_jit, __func__);
 262
 263         if (c.i.rt == 0)
 264                 rec_b(state, block, offset, jit_code_beqi, 0, false, true);
 265         else
 266                 rec_b(state, block, offset, jit_code_beqr, 0, false, false);
 267 }
 268
 269 static void rec_BEQ(struct lightrec_cstate *state,
 270                     const struct block *block, u16 offset)
 271 {
 272         union code c = block->opcode_list[offset].c;
 273
 274         _jit_name(block->_jit, __func__);
 275
 276         if (c.i.rt == 0)
 277                 rec_b(state, block, offset, jit_code_bnei, 0, c.i.rs == 0, true);
 278         else
 279                 rec_b(state, block, offset, jit_code_bner, 0, c.i.rs == c.i.rt, false);
 280 }
 281
 282 static void rec_BLEZ(struct lightrec_cstate *state,
 283                      const struct block *block, u16 offset)
 284 {
 285         union code c = block->opcode_list[offset].c;
 286
 287         _jit_name(block->_jit, __func__);
 288         rec_b(state, block, offset, jit_code_bgti, 0, c.i.rs == 0, true);
 289 }
 290
 291 static void rec_BGTZ(struct lightrec_cstate *state,
 292                      const struct block *block, u16 offset)
 293 {
 294         _jit_name(block->_jit, __func__);
 295         rec_b(state, block, offset, jit_code_blei, 0, false, true);
 296 }
 297
 298 static void rec_regimm_BLTZ(struct lightrec_cstate *state,
 299                             const struct block *block, u16 offset)
 300 {
 301         _jit_name(block->_jit, __func__);
 302         rec_b(state, block, offset, jit_code_bgei, 0, false, true);
 303 }
 304
 305 static void rec_regimm_BLTZAL(struct lightrec_cstate *state,
 306                               const struct block *block, u16 offset)
 307 {
 308         _jit_name(block->_jit, __func__);
 309         rec_b(state, block, offset, jit_code_bgei,
 310               get_branch_pc(block, offset, 2), false, true);
 311 }
 312
 313 static void rec_regimm_BGEZ(struct lightrec_cstate *state,
 314                             const struct block *block, u16 offset)
 315 {
 316         union code c = block->opcode_list[offset].c;
 317
 318         _jit_name(block->_jit, __func__);
 319         rec_b(state, block, offset, jit_code_blti, 0, !c.i.rs, true);
 320 }
 321
 322 static void rec_regimm_BGEZAL(struct lightrec_cstate *state,
 323                               const struct block *block, u16 offset)
 324 {
 325         const struct opcode *op = &block->opcode_list[offset];
 326         _jit_name(block->_jit, __func__);
 327         rec_b(state, block, offset, jit_code_blti,
 328               get_branch_pc(block, offset, 2),
 329               !op->i.rs, true);
 330 }
 331
 332 static void rec_alu_imm(struct lightrec_cstate *state, const struct block *block,
 333                         u16 offset, jit_code_t code, bool slti)
 334 {
 335         struct regcache *reg_cache = state->reg_cache;
 336         union code c = block->opcode_list[offset].c;
 337         jit_state_t *_jit = block->_jit;
 338         u8 rs, rt, out_flags = REG_EXT;
 339
 340         if (slti)
 341                 out_flags |= REG_ZEXT;
 342
 343         jit_note(__FILE__, __LINE__);
 344         rs = lightrec_alloc_reg_in(reg_cache, _jit, c.i.rs, REG_EXT);
 345         rt = lightrec_alloc_reg_out(reg_cache, _jit, c.i.rt, out_flags);
 346
 347         jit_new_node_www(code, rt, rs, (s32)(s16) c.i.imm);
 348
 349         lightrec_free_reg(reg_cache, rs);
 350         lightrec_free_reg(reg_cache, rt);
 351 }
 352
 353 static void rec_alu_special(struct lightrec_cstate *state, const struct block *block,
 354                             u16 offset, jit_code_t code, bool out_ext)
 355 {
 356         struct regcache *reg_cache = state->reg_cache;
 357         union code c = block->opcode_list[offset].c;
 358         jit_state_t *_jit = block->_jit;
 359         u8 rd, rt, rs;
 360
 361         jit_note(__FILE__, __LINE__);
 362         rs = lightrec_alloc_reg_in(reg_cache, _jit, c.r.rs, REG_EXT);
 363         rt = lightrec_alloc_reg_in(reg_cache, _jit, c.r.rt, REG_EXT);
 364         rd = lightrec_alloc_reg_out(reg_cache, _jit, c.r.rd,
 365                                     out_ext ? REG_EXT | REG_ZEXT : 0);
 366
 367         jit_new_node_www(code, rd, rs, rt);
 368
 369         lightrec_free_reg(reg_cache, rs);
 370         lightrec_free_reg(reg_cache, rt);
 371         lightrec_free_reg(reg_cache, rd);
 372 }
 373
 374 static void rec_alu_shiftv(struct lightrec_cstate *state, const struct block *block,
 375                            u16 offset, jit_code_t code)
 376 {
 377         struct regcache *reg_cache = state->reg_cache;
 378         union code c = block->opcode_list[offset].c;
 379         jit_state_t *_jit = block->_jit;
 380         u8 rd, rt, rs, temp, flags = 0;
 381
 382         jit_note(__FILE__, __LINE__);
 383         rs = lightrec_alloc_reg_in(reg_cache, _jit, c.r.rs, 0);
 384
 385         if (code == jit_code_rshr)
 386                 flags = REG_EXT;
 387         else if (code == jit_code_rshr_u)
 388                 flags = REG_ZEXT;
 389
 390         rt = lightrec_alloc_reg_in(reg_cache, _jit, c.r.rt, flags);
 391         rd = lightrec_alloc_reg_out(reg_cache, _jit, c.r.rd, flags);
 392
 393         if (rs != rd && rt != rd) {
 394                 jit_andi(rd, rs, 0x1f);
 395                 jit_new_node_www(code, rd, rt, rd);
 396         } else {
 397                 temp = lightrec_alloc_reg_temp(reg_cache, _jit);
 398                 jit_andi(temp, rs, 0x1f);
 399                 jit_new_node_www(code, rd, rt, temp);
 400                 lightrec_free_reg(reg_cache, temp);
 401         }
 402
 403         lightrec_free_reg(reg_cache, rs);
 404         lightrec_free_reg(reg_cache, rt);
 405         lightrec_free_reg(reg_cache, rd);
 406 }
 407
 408 static void rec_ADDIU(struct lightrec_cstate *state,
 409                       const struct block *block, u16 offset)
 410 {
 411         _jit_name(block->_jit, __func__);
 412         rec_alu_imm(state, block, offset, jit_code_addi, false);
 413 }
 414
 415 static void rec_ADDI(struct lightrec_cstate *state,
 416                      const struct block *block, u16 offset)
 417 {
 418         /* TODO: Handle the exception? */
 419         _jit_name(block->_jit, __func__);
 420         rec_alu_imm(state, block, offset, jit_code_addi, false);
 421 }
 422
 423 static void rec_SLTIU(struct lightrec_cstate *state,
 424                       const struct block *block, u16 offset)
 425 {
 426         _jit_name(block->_jit, __func__);
 427         rec_alu_imm(state, block, offset, jit_code_lti_u, true);
 428 }
 429
 430 static void rec_SLTI(struct lightrec_cstate *state,
 431                      const struct block *block, u16 offset)
 432 {
 433         _jit_name(block->_jit, __func__);
 434         rec_alu_imm(state, block, offset, jit_code_lti, true);
 435 }
 436
 437 static void rec_ANDI(struct lightrec_cstate *state,
 438                      const struct block *block, u16 offset)
 439 {
 440         struct regcache *reg_cache = state->reg_cache;
 441         union code c = block->opcode_list[offset].c;
 442         jit_state_t *_jit = block->_jit;
 443         u8 rs, rt;
 444
 445         _jit_name(block->_jit, __func__);
 446         jit_note(__FILE__, __LINE__);
 447         rs = lightrec_alloc_reg_in(reg_cache, _jit, c.i.rs, 0);
 448         rt = lightrec_alloc_reg_out(reg_cache, _jit, c.i.rt,
 449                                     REG_EXT | REG_ZEXT);
 450
 451         /* PSX code uses ANDI 0xff / ANDI 0xffff a lot, which are basically
 452          * casts to uint8_t / uint16_t. */
 453         if (c.i.imm == 0xff)
 454                 jit_extr_uc(rt, rs);
 455         else if (c.i.imm == 0xffff)
 456                 jit_extr_us(rt, rs);
 457         else
 458                 jit_andi(rt, rs, (u32)(u16) c.i.imm);
 459
 460         lightrec_free_reg(reg_cache, rs);
 461         lightrec_free_reg(reg_cache, rt);
 462 }
 463
 464 static void rec_alu_or_xor(struct lightrec_cstate *state, const struct block *block,
 465                            u16 offset, jit_code_t code)
 466 {
 467         struct regcache *reg_cache = state->reg_cache;
 468         union code c = block->opcode_list[offset].c;
 469         jit_state_t *_jit = block->_jit;
 470         u8 rs, rt, flags;
 471
 472         jit_note(__FILE__, __LINE__);
 473         rs = lightrec_alloc_reg_in(reg_cache, _jit, c.i.rs, 0);
 474         rt = lightrec_alloc_reg_out(reg_cache, _jit, c.i.rt, 0);
 475
 476         flags = lightrec_get_reg_in_flags(reg_cache, rs);
 477         lightrec_set_reg_out_flags(reg_cache, rt, flags);
 478
 479         jit_new_node_www(code, rt, rs, (u32)(u16) c.i.imm);
 480
 481         lightrec_free_reg(reg_cache, rs);
 482         lightrec_free_reg(reg_cache, rt);
 483 }
 484
 485
 486 static void rec_ORI(struct lightrec_cstate *state,
 487                     const struct block *block, u16 offset)
 488 {
 489         _jit_name(block->_jit, __func__);
 490         rec_alu_or_xor(state, block, offset, jit_code_ori);
 491 }
 492
 493 static void rec_XORI(struct lightrec_cstate *state,
 494                      const struct block *block, u16 offset)
 495 {
 496         _jit_name(block->_jit, __func__);
 497         rec_alu_or_xor(state, block, offset, jit_code_xori);
 498 }
 499
 500 static void rec_LUI(struct lightrec_cstate *state,
 501                     const struct block *block, u16 offset)
 502 {
 503         struct regcache *reg_cache = state->reg_cache;
 504         union code c = block->opcode_list[offset].c;
 505         jit_state_t *_jit = block->_jit;
 506         u8 rt, flags = REG_EXT;
 507
 508         jit_name(__func__);
 509         jit_note(__FILE__, __LINE__);
 510
 511         if (!(c.i.imm & BIT(15)))
 512                 flags |= REG_ZEXT;
 513
 514         rt = lightrec_alloc_reg_out(reg_cache, _jit, c.i.rt, flags);
 515
 516         jit_movi(rt, (s32)(c.i.imm << 16));
 517
 518         lightrec_free_reg(reg_cache, rt);
 519 }
 520
 521 static void rec_special_ADDU(struct lightrec_cstate *state,
 522                              const struct block *block, u16 offset)
 523 {
 524         _jit_name(block->_jit, __func__);
 525         rec_alu_special(state, block, offset, jit_code_addr, false);
 526 }
 527
 528 static void rec_special_ADD(struct lightrec_cstate *state,
 529                             const struct block *block, u16 offset)
 530 {
 531         /* TODO: Handle the exception? */
 532         _jit_name(block->_jit, __func__);
 533         rec_alu_special(state, block, offset, jit_code_addr, false);
 534 }
 535
 536 static void rec_special_SUBU(struct lightrec_cstate *state,
 537                              const struct block *block, u16 offset)
 538 {
 539         _jit_name(block->_jit, __func__);
 540         rec_alu_special(state, block, offset, jit_code_subr, false);
 541 }
 542
 543 static void rec_special_SUB(struct lightrec_cstate *state,
 544                             const struct block *block, u16 offset)
 545 {
 546         /* TODO: Handle the exception? */
 547         _jit_name(block->_jit, __func__);
 548         rec_alu_special(state, block, offset, jit_code_subr, false);
 549 }
 550
 551 static void rec_special_AND(struct lightrec_cstate *state,
 552                             const struct block *block, u16 offset)
 553 {
 554         struct regcache *reg_cache = state->reg_cache;
 555         union code c = block->opcode_list[offset].c;
 556         jit_state_t *_jit = block->_jit;
 557         u8 rd, rt, rs, flags_rs, flags_rt, flags_rd;
 558
 559         _jit_name(block->_jit, __func__);
 560         jit_note(__FILE__, __LINE__);
 561         rs = lightrec_alloc_reg_in(reg_cache, _jit, c.r.rs, 0);
 562         rt = lightrec_alloc_reg_in(reg_cache, _jit, c.r.rt, 0);
 563         rd = lightrec_alloc_reg_out(reg_cache, _jit, c.r.rd, 0);
 564
 565         flags_rs = lightrec_get_reg_in_flags(reg_cache, rs);
 566         flags_rt = lightrec_get_reg_in_flags(reg_cache, rt);
 567
 568         /* Z(rd) = Z(rs) | Z(rt) */
 569         flags_rd = REG_ZEXT & (flags_rs | flags_rt);
 570
 571         /* E(rd) = (E(rt) & Z(rt)) | (E(rs) & Z(rs)) | (E(rs) & E(rt)) */
 572         if (((flags_rs & REG_EXT) && (flags_rt & REG_ZEXT)) ||
 573             ((flags_rt & REG_EXT) && (flags_rs & REG_ZEXT)) ||
 574             (REG_EXT & flags_rs & flags_rt))
 575                 flags_rd |= REG_EXT;
 576
 577         lightrec_set_reg_out_flags(reg_cache, rd, flags_rd);
 578
 579         jit_andr(rd, rs, rt);
 580
 581         lightrec_free_reg(reg_cache, rs);
 582         lightrec_free_reg(reg_cache, rt);
 583         lightrec_free_reg(reg_cache, rd);
 584 }
 585
 586 static void rec_special_or_nor(struct lightrec_cstate *state,
 587                                const struct block *block, u16 offset, bool nor)
 588 {
 589         struct regcache *reg_cache = state->reg_cache;
 590         union code c = block->opcode_list[offset].c;
 591         jit_state_t *_jit = block->_jit;
 592         u8 rd, rt, rs, flags_rs, flags_rt, flags_rd = 0;
 593
 594         jit_note(__FILE__, __LINE__);
 595         rs = lightrec_alloc_reg_in(reg_cache, _jit, c.r.rs, 0);
 596         rt = lightrec_alloc_reg_in(reg_cache, _jit, c.r.rt, 0);
 597         rd = lightrec_alloc_reg_out(reg_cache, _jit, c.r.rd, 0);
 598
 599         flags_rs = lightrec_get_reg_in_flags(reg_cache, rs);
 600         flags_rt = lightrec_get_reg_in_flags(reg_cache, rt);
 601
 602         /* or: Z(rd) = Z(rs) & Z(rt)
 603          * nor: Z(rd) = 0 */
 604         if (!nor)
 605                 flags_rd = REG_ZEXT & flags_rs & flags_rt;
 606
 607         /* E(rd) = (E(rs) & E(rt)) | (E(rt) & !Z(rt)) | (E(rs) & !Z(rs)) */
 608         if ((REG_EXT & flags_rs & flags_rt) ||
 609             (flags_rt & (REG_EXT | REG_ZEXT) == REG_EXT) ||
 610             (flags_rs & (REG_EXT | REG_ZEXT) == REG_EXT))
 611                 flags_rd |= REG_EXT;
 612
 613         lightrec_set_reg_out_flags(reg_cache, rd, flags_rd);
 614
 615         jit_orr(rd, rs, rt);
 616
 617         if (nor)
 618                 jit_comr(rd, rd);
 619
 620         lightrec_free_reg(reg_cache, rs);
 621         lightrec_free_reg(reg_cache, rt);
 622         lightrec_free_reg(reg_cache, rd);
 623 }
 624
 625 static void rec_special_OR(struct lightrec_cstate *state,
 626                            const struct block *block, u16 offset)
 627 {
 628         _jit_name(block->_jit, __func__);
 629         rec_special_or_nor(state, block, offset, false);
 630 }
 631
 632 static void rec_special_NOR(struct lightrec_cstate *state,
 633                             const struct block *block, u16 offset)
 634 {
 635         _jit_name(block->_jit, __func__);
 636         rec_special_or_nor(state, block, offset, true);
 637 }
 638
 639 static void rec_special_XOR(struct lightrec_cstate *state,
 640                             const struct block *block, u16 offset)
 641 {
 642         struct regcache *reg_cache = state->reg_cache;
 643         union code c = block->opcode_list[offset].c;
 644         jit_state_t *_jit = block->_jit;
 645         u8 rd, rt, rs, flags_rs, flags_rt, flags_rd;
 646
 647         _jit_name(block->_jit, __func__);
 648
 649         jit_note(__FILE__, __LINE__);
 650         rs = lightrec_alloc_reg_in(reg_cache, _jit, c.r.rs, 0);
 651         rt = lightrec_alloc_reg_in(reg_cache, _jit, c.r.rt, 0);
 652         rd = lightrec_alloc_reg_out(reg_cache, _jit, c.r.rd, 0);
 653
 654         flags_rs = lightrec_get_reg_in_flags(reg_cache, rs);
 655         flags_rt = lightrec_get_reg_in_flags(reg_cache, rt);
 656
 657         /* Z(rd) = Z(rs) & Z(rt) */
 658         flags_rd = REG_ZEXT & flags_rs & flags_rt;
 659
 660         /* E(rd) = E(rs) & E(rt) */
 661         flags_rd |= REG_EXT & flags_rs & flags_rt;
 662
 663         lightrec_set_reg_out_flags(reg_cache, rd, flags_rd);
 664
 665         jit_xorr(rd, rs, rt);
 666
 667         lightrec_free_reg(reg_cache, rs);
 668         lightrec_free_reg(reg_cache, rt);
 669         lightrec_free_reg(reg_cache, rd);
 670 }
 671
 672 static void rec_special_SLTU(struct lightrec_cstate *state,
 673                              const struct block *block, u16 offset)
 674 {
 675         _jit_name(block->_jit, __func__);
 676         rec_alu_special(state, block, offset, jit_code_ltr_u, true);
 677 }
 678
 679 static void rec_special_SLT(struct lightrec_cstate *state,
 680                             const struct block *block, u16 offset)
 681 {
 682         _jit_name(block->_jit, __func__);
 683         rec_alu_special(state, block, offset, jit_code_ltr, true);
 684 }
 685
 686 static void rec_special_SLLV(struct lightrec_cstate *state,
 687                              const struct block *block, u16 offset)
 688 {
 689         _jit_name(block->_jit, __func__);
 690         rec_alu_shiftv(state, block, offset, jit_code_lshr);
 691 }
 692
 693 static void rec_special_SRLV(struct lightrec_cstate *state,
 694                              const struct block *block, u16 offset)
 695 {
 696         _jit_name(block->_jit, __func__);
 697         rec_alu_shiftv(state, block, offset, jit_code_rshr_u);
 698 }
 699
 700 static void rec_special_SRAV(struct lightrec_cstate *state,
 701                              const struct block *block, u16 offset)
 702 {
 703         _jit_name(block->_jit, __func__);
 704         rec_alu_shiftv(state, block, offset, jit_code_rshr);
 705 }
 706
 707 static void rec_alu_shift(struct lightrec_cstate *state, const struct block *block,
 708                           u16 offset, jit_code_t code)
 709 {
 710         struct regcache *reg_cache = state->reg_cache;
 711         union code c = block->opcode_list[offset].c;
 712         jit_state_t *_jit = block->_jit;
 713         u8 rd, rt, flags = 0;
 714
 715         jit_note(__FILE__, __LINE__);
 716
 717         if (code == jit_code_rshi)
 718                 flags = REG_EXT;
 719         else if (code == jit_code_rshi_u)
 720                 flags = REG_ZEXT;
 721
 722         rt = lightrec_alloc_reg_in(reg_cache, _jit, c.r.rt, flags);
 723
 724         /* Input reg is zero-extended, if we SRL at least by one bit, we know
 725          * the output reg will be both zero-extended and sign-extended. */
 726         if (code == jit_code_rshi_u && c.r.imm)
 727                 flags |= REG_EXT;
 728         rd = lightrec_alloc_reg_out(reg_cache, _jit, c.r.rd, flags);
 729
 730         jit_new_node_www(code, rd, rt, c.r.imm);
 731
 732         lightrec_free_reg(reg_cache, rt);
 733         lightrec_free_reg(reg_cache, rd);
 734 }
 735
 736 static void rec_special_SLL(struct lightrec_cstate *state,
 737                             const struct block *block, u16 offset)
 738 {
 739         _jit_name(block->_jit, __func__);
 740         rec_alu_shift(state, block, offset, jit_code_lshi);
 741 }
 742
 743 static void rec_special_SRL(struct lightrec_cstate *state,
 744                             const struct block *block, u16 offset)
 745 {
 746         _jit_name(block->_jit, __func__);
 747         rec_alu_shift(state, block, offset, jit_code_rshi_u);
 748 }
 749
 750 static void rec_special_SRA(struct lightrec_cstate *state,
 751                             const struct block *block, u16 offset)
 752 {
 753         _jit_name(block->_jit, __func__);
 754         rec_alu_shift(state, block, offset, jit_code_rshi);
 755 }
 756
 757 static void rec_alu_mult(struct lightrec_cstate *state,
 758                          const struct block *block, u16 offset, bool is_signed)
 759 {
 760         struct regcache *reg_cache = state->reg_cache;
 761         union code c = block->opcode_list[offset].c;
 762         u16 flags = block->opcode_list[offset].flags;
 763         u8 reg_lo = get_mult_div_lo(c);
 764         u8 reg_hi = get_mult_div_hi(c);
 765         jit_state_t *_jit = block->_jit;
 766         u8 lo, hi, rs, rt, rflags = 0;
 767
 768         jit_note(__FILE__, __LINE__);
 769
 770         if (is_signed)
 771                 rflags = REG_EXT;
 772         else
 773                 rflags = REG_ZEXT;
 774
 775         rs = lightrec_alloc_reg_in(reg_cache, _jit, c.r.rs, rflags);
 776         rt = lightrec_alloc_reg_in(reg_cache, _jit, c.r.rt, rflags);
 777
 778         if (!(flags & LIGHTREC_NO_LO))
 779                 lo = lightrec_alloc_reg_out(reg_cache, _jit, reg_lo, 0);
 780         else if (__WORDSIZE == 32)
 781                 lo = lightrec_alloc_reg_temp(reg_cache, _jit);
 782
 783         if (!(flags & LIGHTREC_NO_HI))
 784                 hi = lightrec_alloc_reg_out(reg_cache, _jit, reg_hi, REG_EXT);
 785
 786         if (__WORDSIZE == 32) {
 787                 /* On 32-bit systems, do a 32*32->64 bit operation, or a 32*32->32 bit
 788                  * operation if the MULT was detected a 32-bit only. */
 789                 if (!(flags & LIGHTREC_NO_HI)) {
 790                         if (is_signed)
 791                                 jit_qmulr(lo, hi, rs, rt);
 792                         else
 793                                 jit_qmulr_u(lo, hi, rs, rt);
 794                 } else {
 795                         jit_mulr(lo, rs, rt);
 796                 }
 797         } else {
 798                 /* On 64-bit systems, do a 64*64->64 bit operation. */
 799                 if (flags & LIGHTREC_NO_LO) {
 800                         jit_mulr(hi, rs, rt);
 801                         jit_rshi(hi, hi, 32);
 802                 } else {
 803                         jit_mulr(lo, rs, rt);
 804
 805                         /* The 64-bit output value is in $lo, store the upper 32 bits in $hi */
 806                         if (!(flags & LIGHTREC_NO_HI))
 807                                 jit_rshi(hi, lo, 32);
 808                 }
 809         }
 810
 811         lightrec_free_reg(reg_cache, rs);
 812         lightrec_free_reg(reg_cache, rt);
 813         if (!(flags & LIGHTREC_NO_LO) || __WORDSIZE == 32)
 814                 lightrec_free_reg(reg_cache, lo);
 815         if (!(flags & LIGHTREC_NO_HI))
 816                 lightrec_free_reg(reg_cache, hi);
 817 }
 818
 819 static void rec_alu_div(struct lightrec_cstate *state,
 820                         const struct block *block, u16 offset, bool is_signed)
 821 {
 822         struct regcache *reg_cache = state->reg_cache;
 823         union code c = block->opcode_list[offset].c;
 824         u16 flags = block->opcode_list[offset].flags;
 825         bool no_check = flags & LIGHTREC_NO_DIV_CHECK;
 826         u8 reg_lo = get_mult_div_lo(c);
 827         u8 reg_hi = get_mult_div_hi(c);
 828         jit_state_t *_jit = block->_jit;
 829         jit_node_t *branch, *to_end;
 830         u8 lo, hi, rs, rt, rflags = 0;
 831
 832         jit_note(__FILE__, __LINE__);
 833
 834         if (is_signed)
 835                 rflags = REG_EXT;
 836         else
 837                 rflags = REG_ZEXT;
 838
 839         rs = lightrec_alloc_reg_in(reg_cache, _jit, c.r.rs, rflags);
 840         rt = lightrec_alloc_reg_in(reg_cache, _jit, c.r.rt, rflags);
 841
 842         if (!(flags & LIGHTREC_NO_LO))
 843                 lo = lightrec_alloc_reg_out(reg_cache, _jit, reg_lo, 0);
 844
 845         if (!(flags & LIGHTREC_NO_HI))
 846                 hi = lightrec_alloc_reg_out(reg_cache, _jit, reg_hi, 0);
 847
 848         /* Jump to special handler if dividing by zero  */
 849         if (!no_check)
 850                 branch = jit_beqi(rt, 0);
 851
 852         if (flags & LIGHTREC_NO_LO) {
 853                 if (is_signed)
 854                         jit_remr(hi, rs, rt);
 855                 else
 856                         jit_remr_u(hi, rs, rt);
 857         } else if (flags & LIGHTREC_NO_HI) {
 858                 if (is_signed)
 859                         jit_divr(lo, rs, rt);
 860                 else
 861                         jit_divr_u(lo, rs, rt);
 862         } else {
 863                 if (is_signed)
 864                         jit_qdivr(lo, hi, rs, rt);
 865                 else
 866                         jit_qdivr_u(lo, hi, rs, rt);
 867         }
 868
 869         if (!no_check) {
 870                 lightrec_regcache_mark_live(reg_cache, _jit);
 871
 872                 /* Jump above the div-by-zero handler */
 873                 to_end = jit_jmpi();
 874
 875                 jit_patch(branch);
 876
 877                 if (!(flags & LIGHTREC_NO_LO)) {
 878                         if (is_signed) {
 879                                 jit_lti(lo, rs, 0);
 880                                 jit_lshi(lo, lo, 1);
 881                                 jit_subi(lo, lo, 1);
 882                         } else {
 883                                 jit_movi(lo, 0xffffffff);
 884                         }
 885                 }
 886
 887                 if (!(flags & LIGHTREC_NO_HI))
 888                         jit_movr(hi, rs);
 889
 890                 jit_patch(to_end);
 891         }
 892
 893         lightrec_free_reg(reg_cache, rs);
 894         lightrec_free_reg(reg_cache, rt);
 895
 896         if (!(flags & LIGHTREC_NO_LO))
 897                 lightrec_free_reg(reg_cache, lo);
 898
 899         if (!(flags & LIGHTREC_NO_HI))
 900                 lightrec_free_reg(reg_cache, hi);
 901 }
 902
 903 static void rec_special_MULT(struct lightrec_cstate *state,
 904                              const struct block *block, u16 offset)
 905 {
 906         _jit_name(block->_jit, __func__);
 907         rec_alu_mult(state, block, offset, true);
 908 }
 909
 910 static void rec_special_MULTU(struct lightrec_cstate *state,
 911                               const struct block *block, u16 offset)
 912 {
 913         _jit_name(block->_jit, __func__);
 914         rec_alu_mult(state, block, offset, false);
 915 }
 916
 917 static void rec_special_DIV(struct lightrec_cstate *state,
 918                             const struct block *block, u16 offset)
 919 {
 920         _jit_name(block->_jit, __func__);
 921         rec_alu_div(state, block, offset, true);
 922 }
 923
 924 static void rec_special_DIVU(struct lightrec_cstate *state,
 925                              const struct block *block, u16 offset)
 926 {
 927         _jit_name(block->_jit, __func__);
 928         rec_alu_div(state, block, offset, false);
 929 }
 930
 931 static void rec_alu_mv_lo_hi(struct lightrec_cstate *state,
 932                              const struct block *block, u8 dst, u8 src)
 933 {
 934         struct regcache *reg_cache = state->reg_cache;
 935         jit_state_t *_jit = block->_jit;
 936
 937         jit_note(__FILE__, __LINE__);
 938         src = lightrec_alloc_reg_in(reg_cache, _jit, src, 0);
 939         dst = lightrec_alloc_reg_out(reg_cache, _jit, dst, REG_EXT);
 940
 941         jit_extr_i(dst, src);
 942
 943         lightrec_free_reg(reg_cache, src);
 944         lightrec_free_reg(reg_cache, dst);
 945 }
 946
 947 static void rec_special_MFHI(struct lightrec_cstate *state,
 948                              const struct block *block, u16 offset)
 949 {
 950         union code c = block->opcode_list[offset].c;
 951
 952         _jit_name(block->_jit, __func__);
 953         rec_alu_mv_lo_hi(state, block, c.r.rd, REG_HI);
 954 }
 955
 956 static void rec_special_MTHI(struct lightrec_cstate *state,
 957                              const struct block *block, u16 offset)
 958 {
 959         union code c = block->opcode_list[offset].c;
 960
 961         _jit_name(block->_jit, __func__);
 962         rec_alu_mv_lo_hi(state, block, REG_HI, c.r.rs);
 963 }
 964
 965 static void rec_special_MFLO(struct lightrec_cstate *state,
 966                              const struct block *block, u16 offset)
 967 {
 968         union code c = block->opcode_list[offset].c;
 969
 970         _jit_name(block->_jit, __func__);
 971         rec_alu_mv_lo_hi(state, block, c.r.rd, REG_LO);
 972 }
 973
 974 static void rec_special_MTLO(struct lightrec_cstate *state,
 975                              const struct block *block, u16 offset)
 976 {
 977         union code c = block->opcode_list[offset].c;
 978
 979         _jit_name(block->_jit, __func__);
 980         rec_alu_mv_lo_hi(state, block, REG_LO, c.r.rs);
 981 }
 982
 983 static void call_to_c_wrapper(struct lightrec_cstate *state, const struct block *block,
 984                               u32 arg, bool with_arg, enum c_wrappers wrapper)
 985 {
 986         struct regcache *reg_cache = state->reg_cache;
 987         jit_state_t *_jit = block->_jit;
 988         u8 tmp, tmp2, tmp3;
 989
 990         if (with_arg)
 991                 tmp3 = lightrec_alloc_reg(reg_cache, _jit, JIT_R1);
 992         tmp2 = lightrec_alloc_reg(reg_cache, _jit, JIT_R0);
 993         tmp = lightrec_alloc_reg_temp(reg_cache, _jit);
 994
 995         jit_ldxi(tmp, LIGHTREC_REG_STATE,
 996                  offsetof(struct lightrec_state, c_wrapper));
 997         jit_ldxi(tmp2, LIGHTREC_REG_STATE,
 998                  offsetof(struct lightrec_state, c_wrappers[wrapper]));
 999         if (with_arg)
1000                 jit_movi(tmp3, arg);
1001
1002         jit_callr(tmp);
1003
1004         lightrec_free_reg(reg_cache, tmp);
1005         lightrec_free_reg(reg_cache, tmp2);
1006         if (with_arg)
1007                 lightrec_free_reg(reg_cache, tmp3);
1008         lightrec_regcache_mark_live(reg_cache, _jit);
1009 }
1010
1011 static void rec_io(struct lightrec_cstate *state,
1012                    const struct block *block, u16 offset,
1013                    bool load_rt, bool read_rt)
1014 {
1015         struct regcache *reg_cache = state->reg_cache;
1016         jit_state_t *_jit = block->_jit;
1017         union code c = block->opcode_list[offset].c;
1018         u16 flags = block->opcode_list[offset].flags;
1019         bool is_tagged = flags & (LIGHTREC_HW_IO | LIGHTREC_DIRECT_IO);
1020         u32 lut_entry;
1021
1022         jit_note(__FILE__, __LINE__);
1023
1024         lightrec_clean_reg_if_loaded(reg_cache, _jit, c.i.rs, false);
1025
1026         if (read_rt && likely(c.i.rt))
1027                 lightrec_clean_reg_if_loaded(reg_cache, _jit, c.i.rt, true);
1028         else if (load_rt)
1029                 lightrec_clean_reg_if_loaded(reg_cache, _jit, c.i.rt, false);
1030
1031         if (is_tagged) {
1032                 call_to_c_wrapper(state, block, c.opcode, true, C_WRAPPER_RW);
1033         } else {
1034                 lut_entry = lightrec_get_lut_entry(block);
1035                 call_to_c_wrapper(state, block, (lut_entry << 16) | offset,
1036                                   true, C_WRAPPER_RW_GENERIC);
1037         }
1038 }
1039
1040 static void rec_store_direct_no_invalidate(struct lightrec_cstate *cstate,
1041                                            const struct block *block,
1042                                            u16 offset, jit_code_t code)
1043 {
1044         struct lightrec_state *state = cstate->state;
1045         struct regcache *reg_cache = cstate->reg_cache;
1046         union code c = block->opcode_list[offset].c;
1047         jit_state_t *_jit = block->_jit;
1048         jit_node_t *to_not_ram, *to_end;
1049         u8 tmp, tmp2, rs, rt;
1050         s16 imm;
1051
1052         jit_note(__FILE__, __LINE__);
1053         rs = lightrec_alloc_reg_in(reg_cache, _jit, c.i.rs, 0);
1054         tmp = lightrec_alloc_reg_temp(reg_cache, _jit);
1055
1056         if (state->offset_ram || state->offset_scratch)
1057                 tmp2 = lightrec_alloc_reg_temp(reg_cache, _jit);
1058
1059         /* Convert to KUNSEG and avoid RAM mirrors */
1060         if (state->mirrors_mapped) {
1061                 imm = (s16)c.i.imm;
1062                 jit_andi(tmp, rs, 0x1f800000 | (4 * RAM_SIZE - 1));
1063         } else if (c.i.imm) {
1064                 imm = 0;
1065                 jit_addi(tmp, rs, (s16)c.i.imm);
1066                 jit_andi(tmp, tmp, 0x1f800000 | (RAM_SIZE - 1));
1067         } else {
1068                 imm = 0;
1069                 jit_andi(tmp, rs, 0x1f800000 | (RAM_SIZE - 1));
1070         }
1071
1072         lightrec_free_reg(reg_cache, rs);
1073
1074         if (state->offset_ram != state->offset_scratch) {
1075                 to_not_ram = jit_bmsi(tmp, BIT(28));
1076
1077                 lightrec_regcache_mark_live(reg_cache, _jit);
1078
1079                 jit_movi(tmp2, state->offset_ram);
1080
1081                 to_end = jit_jmpi();
1082                 jit_patch(to_not_ram);
1083
1084                 jit_movi(tmp2, state->offset_scratch);
1085                 jit_patch(to_end);
1086         } else if (state->offset_ram) {
1087                 jit_movi(tmp2, state->offset_ram);
1088         }
1089
1090         if (state->offset_ram || state->offset_scratch) {
1091                 jit_addr(tmp, tmp, tmp2);
1092                 lightrec_free_reg(reg_cache, tmp2);
1093         }
1094
1095         rt = lightrec_alloc_reg_in(reg_cache, _jit, c.i.rt, 0);
1096         jit_new_node_www(code, imm, tmp, rt);
1097
1098         lightrec_free_reg(reg_cache, rt);
1099         lightrec_free_reg(reg_cache, tmp);
1100 }
1101
1102 static void rec_store_direct(struct lightrec_cstate *cstate, const struct block *block,
1103                              u16 offset, jit_code_t code)
1104 {
1105         struct lightrec_state *state = cstate->state;
1106         u32 ram_size = state->mirrors_mapped ? RAM_SIZE * 4 : RAM_SIZE;
1107         struct regcache *reg_cache = cstate->reg_cache;
1108         union code c = block->opcode_list[offset].c;
1109         jit_state_t *_jit = block->_jit;
1110         jit_node_t *to_not_ram, *to_end;
1111         u8 tmp, tmp2, tmp3, rs, rt;
1112
1113         jit_note(__FILE__, __LINE__);
1114
1115         rs = lightrec_alloc_reg_in(reg_cache, _jit, c.i.rs, 0);
1116         tmp2 = lightrec_alloc_reg_temp(reg_cache, _jit);
1117         tmp3 = lightrec_alloc_reg_in(reg_cache, _jit, 0, 0);
1118
1119         /* Convert to KUNSEG and avoid RAM mirrors */
1120         if (c.i.imm) {
1121                 jit_addi(tmp2, rs, (s16)c.i.imm);
1122                 jit_andi(tmp2, tmp2, 0x1f800000 | (ram_size - 1));
1123         } else {
1124                 jit_andi(tmp2, rs, 0x1f800000 | (ram_size - 1));
1125         }
1126
1127         lightrec_free_reg(reg_cache, rs);
1128         tmp = lightrec_alloc_reg_temp(reg_cache, _jit);
1129
1130         to_not_ram = jit_bgti(tmp2, ram_size);
1131
1132         lightrec_regcache_mark_live(reg_cache, _jit);
1133
1134         /* Compute the offset to the code LUT */
1135         jit_andi(tmp, tmp2, (RAM_SIZE - 1) & ~3);
1136         if (__WORDSIZE == 64)
1137                 jit_lshi(tmp, tmp, 1);
1138         jit_addr(tmp, LIGHTREC_REG_STATE, tmp);
1139
1140         /* Write NULL to the code LUT to invalidate any block that's there */
1141         jit_stxi(offsetof(struct lightrec_state, code_lut), tmp, tmp3);
1142
1143         if (state->offset_ram != state->offset_scratch) {
1144                 jit_movi(tmp, state->offset_ram);
1145
1146                 to_end = jit_jmpi();
1147         }
1148
1149         jit_patch(to_not_ram);
1150
1151         if (state->offset_ram || state->offset_scratch)
1152                 jit_movi(tmp, state->offset_scratch);
1153
1154         if (state->offset_ram != state->offset_scratch)
1155                 jit_patch(to_end);
1156
1157         if (state->offset_ram || state->offset_scratch)
1158                 jit_addr(tmp2, tmp2, tmp);
1159
1160         lightrec_free_reg(reg_cache, tmp);
1161         lightrec_free_reg(reg_cache, tmp3);
1162
1163         rt = lightrec_alloc_reg_in(reg_cache, _jit, c.i.rt, 0);
1164         jit_new_node_www(code, 0, tmp2, rt);
1165
1166         lightrec_free_reg(reg_cache, rt);
1167         lightrec_free_reg(reg_cache, tmp2);
1168 }
1169
1170 static void rec_store(struct lightrec_cstate *state,
1171                       const struct block *block, u16 offset, jit_code_t code)
1172 {
1173         u16 flags = block->opcode_list[offset].flags;
1174
1175         if (flags & LIGHTREC_NO_INVALIDATE) {
1176                 rec_store_direct_no_invalidate(state, block, offset, code);
1177         } else if (flags & LIGHTREC_DIRECT_IO) {
1178                 if (state->state->invalidate_from_dma_only)
1179                         rec_store_direct_no_invalidate(state, block, offset, code);
1180                 else
1181                         rec_store_direct(state, block, offset, code);
1182         } else {
1183                 rec_io(state, block, offset, true, false);
1184         }
1185 }
1186
1187 static void rec_SB(struct lightrec_cstate *state,
1188                    const struct block *block, u16 offset)
1189 {
1190         _jit_name(block->_jit, __func__);
1191         rec_store(state, block, offset, jit_code_stxi_c);
1192 }
1193
1194 static void rec_SH(struct lightrec_cstate *state,
1195                    const struct block *block, u16 offset)
1196 {
1197         _jit_name(block->_jit, __func__);
1198         rec_store(state, block, offset, jit_code_stxi_s);
1199 }
1200
1201 static void rec_SW(struct lightrec_cstate *state,
1202                    const struct block *block, u16 offset)
1203
1204 {
1205         _jit_name(block->_jit, __func__);
1206         rec_store(state, block, offset, jit_code_stxi_i);
1207 }
1208
1209 static void rec_SWL(struct lightrec_cstate *state,
1210                     const struct block *block, u16 offset)
1211 {
1212         _jit_name(block->_jit, __func__);
1213         rec_io(state, block, offset, true, false);
1214 }
1215
1216 static void rec_SWR(struct lightrec_cstate *state,
1217                     const struct block *block, u16 offset)
1218 {
1219         _jit_name(block->_jit, __func__);
1220         rec_io(state, block, offset, true, false);
1221 }
1222
1223 static void rec_SWC2(struct lightrec_cstate *state,
1224                      const struct block *block, u16 offset)
1225 {
1226         _jit_name(block->_jit, __func__);
1227         rec_io(state, block, offset, false, false);
1228 }
1229
1230 static void rec_load_direct(struct lightrec_cstate *cstate, const struct block *block,
1231                             u16 offset, jit_code_t code, bool is_unsigned)
1232 {
1233         struct lightrec_state *state = cstate->state;
1234         struct regcache *reg_cache = cstate->reg_cache;
1235         union code c = block->opcode_list[offset].c;
1236         jit_state_t *_jit = block->_jit;
1237         jit_node_t *to_not_ram, *to_not_bios, *to_end, *to_end2;
1238         u8 tmp, rs, rt, addr_reg, flags = REG_EXT;
1239         s16 imm;
1240
1241         if (!c.i.rt)
1242                 return;
1243
1244         if (is_unsigned)
1245                 flags |= REG_ZEXT;
1246
1247         jit_note(__FILE__, __LINE__);
1248         rs = lightrec_alloc_reg_in(reg_cache, _jit, c.i.rs, 0);
1249         rt = lightrec_alloc_reg_out(reg_cache, _jit, c.i.rt, flags);
1250
1251         if ((state->offset_ram == state->offset_bios &&
1252             state->offset_ram == state->offset_scratch &&
1253             state->mirrors_mapped) || !c.i.imm) {
1254                 addr_reg = rs;
1255                 imm = (s16)c.i.imm;
1256         } else {
1257                 jit_addi(rt, rs, (s16)c.i.imm);
1258                 addr_reg = rt;
1259                 imm = 0;
1260
1261                 if (c.i.rs != c.i.rt)
1262                         lightrec_free_reg(reg_cache, rs);
1263         }
1264
1265         tmp = lightrec_alloc_reg_temp(reg_cache, _jit);
1266
1267         if (state->offset_ram == state->offset_bios &&
1268             state->offset_ram == state->offset_scratch) {
1269                 if (!state->mirrors_mapped) {
1270                         jit_andi(tmp, addr_reg, BIT(28));
1271                         jit_rshi_u(tmp, tmp, 28 - 22);
1272                         jit_ori(tmp, tmp, 0x1f800000 | (RAM_SIZE - 1));
1273                         jit_andr(rt, addr_reg, tmp);
1274                 } else {
1275                         jit_andi(rt, addr_reg, 0x1fffffff);
1276                 }
1277
1278                 if (state->offset_ram)
1279                         jit_movi(tmp, state->offset_ram);
1280         } else {
1281                 to_not_ram = jit_bmsi(addr_reg, BIT(28));
1282
1283                 lightrec_regcache_mark_live(reg_cache, _jit);
1284
1285                 /* Convert to KUNSEG and avoid RAM mirrors */
1286                 jit_andi(rt, addr_reg, RAM_SIZE - 1);
1287
1288                 if (state->offset_ram)
1289                         jit_movi(tmp, state->offset_ram);
1290
1291                 to_end = jit_jmpi();
1292
1293                 jit_patch(to_not_ram);
1294
1295                 if (state->offset_bios != state->offset_scratch)
1296                         to_not_bios = jit_bmci(addr_reg, BIT(22));
1297
1298                 /* Convert to KUNSEG */
1299                 jit_andi(rt, addr_reg, 0x1fc00000 | (BIOS_SIZE - 1));
1300
1301                 jit_movi(tmp, state->offset_bios);
1302
1303                 if (state->offset_bios != state->offset_scratch) {
1304                         to_end2 = jit_jmpi();
1305
1306                         jit_patch(to_not_bios);
1307
1308                         /* Convert to KUNSEG */
1309                         jit_andi(rt, addr_reg, 0x1f800fff);
1310
1311                         if (state->offset_scratch)
1312                                 jit_movi(tmp, state->offset_scratch);
1313
1314                         jit_patch(to_end2);
1315                 }
1316
1317                 jit_patch(to_end);
1318         }
1319
1320         if (state->offset_ram || state->offset_bios || state->offset_scratch)
1321                 jit_addr(rt, rt, tmp);
1322
1323         jit_new_node_www(code, rt, rt, imm);
1324
1325         lightrec_free_reg(reg_cache, addr_reg);
1326         lightrec_free_reg(reg_cache, rt);
1327         lightrec_free_reg(reg_cache, tmp);
1328 }
1329
1330 static void rec_load(struct lightrec_cstate *state, const struct block *block,
1331                      u16 offset, jit_code_t code, bool is_unsigned)
1332 {
1333         u16 flags = block->opcode_list[offset].flags;
1334
1335         if (flags & LIGHTREC_DIRECT_IO)
1336                 rec_load_direct(state, block, offset, code, is_unsigned);
1337         else
1338                 rec_io(state, block, offset, false, true);
1339 }
1340
1341 static void rec_LB(struct lightrec_cstate *state, const struct block *block, u16 offset)
1342 {
1343         _jit_name(block->_jit, __func__);
1344         rec_load(state, block, offset, jit_code_ldxi_c, false);
1345 }
1346
1347 static void rec_LBU(struct lightrec_cstate *state, const struct block *block, u16 offset)
1348 {
1349         _jit_name(block->_jit, __func__);
1350         rec_load(state, block, offset, jit_code_ldxi_uc, true);
1351 }
1352
1353 static void rec_LH(struct lightrec_cstate *state, const struct block *block, u16 offset)
1354 {
1355         _jit_name(block->_jit, __func__);
1356         rec_load(state, block, offset, jit_code_ldxi_s, false);
1357 }
1358
1359 static void rec_LHU(struct lightrec_cstate *state, const struct block *block, u16 offset)
1360 {
1361         _jit_name(block->_jit, __func__);
1362         rec_load(state, block, offset, jit_code_ldxi_us, true);
1363 }
1364
1365 static void rec_LWL(struct lightrec_cstate *state, const struct block *block, u16 offset)
1366 {
1367         _jit_name(block->_jit, __func__);
1368         rec_io(state, block, offset, true, true);
1369 }
1370
1371 static void rec_LWR(struct lightrec_cstate *state, const struct block *block, u16 offset)
1372 {
1373         _jit_name(block->_jit, __func__);
1374         rec_io(state, block, offset, true, true);
1375 }
1376
1377 static void rec_LW(struct lightrec_cstate *state, const struct block *block, u16 offset)
1378 {
1379         _jit_name(block->_jit, __func__);
1380         rec_load(state, block, offset, jit_code_ldxi_i, false);
1381 }
1382
1383 static void rec_LWC2(struct lightrec_cstate *state, const struct block *block, u16 offset)
1384 {
1385         _jit_name(block->_jit, __func__);
1386         rec_io(state, block, offset, false, false);
1387 }
1388
1389 static void rec_break_syscall(struct lightrec_cstate *state,
1390                               const struct block *block, u16 offset, bool is_break)
1391 {
1392         _jit_note(block->_jit, __FILE__, __LINE__);
1393
1394         if (is_break)
1395                 call_to_c_wrapper(state, block, 0, false, C_WRAPPER_BREAK);
1396         else
1397                 call_to_c_wrapper(state, block, 0, false, C_WRAPPER_SYSCALL);
1398
1399         /* TODO: the return address should be "pc - 4" if we're a delay slot */
1400         lightrec_emit_end_of_block(state, block, offset, -1,
1401                                    get_ds_pc(block, offset, 0),
1402                                    31, 0, true);
1403 }
1404
1405 static void rec_special_SYSCALL(struct lightrec_cstate *state,
1406                                 const struct block *block, u16 offset)
1407 {
1408         _jit_name(block->_jit, __func__);
1409         rec_break_syscall(state, block, offset, false);
1410 }
1411
1412 static void rec_special_BREAK(struct lightrec_cstate *state,
1413                               const struct block *block, u16 offset)
1414 {
1415         _jit_name(block->_jit, __func__);
1416         rec_break_syscall(state, block, offset, true);
1417 }
1418
1419 static void rec_mfc(struct lightrec_cstate *state,
1420                     const struct block *block, u16 offset)
1421 {
1422         struct regcache *reg_cache = state->reg_cache;
1423         union code c = block->opcode_list[offset].c;
1424         jit_state_t *_jit = block->_jit;
1425
1426         jit_note(__FILE__, __LINE__);
1427         lightrec_clean_reg_if_loaded(reg_cache, _jit, c.i.rt, true);
1428
1429         call_to_c_wrapper(state, block, c.opcode, true, C_WRAPPER_MFC);
1430 }
1431
1432 static void rec_mtc(struct lightrec_cstate *state,
1433                     const struct block *block, u16 offset)
1434 {
1435         struct regcache *reg_cache = state->reg_cache;
1436         union code c = block->opcode_list[offset].c;
1437         jit_state_t *_jit = block->_jit;
1438
1439         jit_note(__FILE__, __LINE__);
1440         lightrec_clean_reg_if_loaded(reg_cache, _jit, c.i.rs, false);
1441         lightrec_clean_reg_if_loaded(reg_cache, _jit, c.i.rt, false);
1442
1443         call_to_c_wrapper(state, block, c.opcode, true, C_WRAPPER_MTC);
1444
1445         if (c.i.op == OP_CP0 &&
1446             !(block->opcode_list[offset].flags & LIGHTREC_NO_DS) &&
1447             (c.r.rd == 12 || c.r.rd == 13))
1448                 lightrec_emit_end_of_block(state, block, offset, -1,
1449                                            get_ds_pc(block, offset, 1),
1450                                            0, 0, true);
1451 }
1452
1453 static void
1454 rec_mfc0(struct lightrec_cstate *state, const struct block *block, u16 offset)
1455 {
1456         struct regcache *reg_cache = state->reg_cache;
1457         union code c = block->opcode_list[offset].c;
1458         jit_state_t *_jit = block->_jit;
1459         u8 rt;
1460
1461         jit_note(__FILE__, __LINE__);
1462
1463         rt = lightrec_alloc_reg_out(reg_cache, _jit, c.i.rt, REG_EXT);
1464
1465         jit_ldxi_i(rt, LIGHTREC_REG_STATE,
1466                    offsetof(struct lightrec_state, regs.cp0[c.r.rd]));
1467
1468         lightrec_free_reg(reg_cache, rt);
1469 }
1470
1471 static bool block_in_bios(const struct lightrec_cstate *state,
1472                           const struct block *block)
1473 {
1474         const struct lightrec_mem_map *bios = &state->state->maps[PSX_MAP_BIOS];
1475         u32 pc = kunseg(block->pc);
1476
1477         return pc >= bios->pc && pc < bios->pc + bios->length;
1478 }
1479
1480 static void
1481 rec_mtc0(struct lightrec_cstate *state, const struct block *block, u16 offset)
1482 {
1483         struct regcache *reg_cache = state->reg_cache;
1484         const union code c = block->opcode_list[offset].c;
1485         jit_state_t *_jit = block->_jit;
1486         u8 rt, tmp, tmp2, status;
1487
1488         jit_note(__FILE__, __LINE__);
1489
1490         switch(c.r.rd) {
1491         case 1:
1492         case 4:
1493         case 8:
1494         case 14:
1495         case 15:
1496                 /* Those registers are read-only */
1497                 return;
1498         default:
1499                 break;
1500         }
1501
1502         if (block_in_bios(state, block) && c.r.rd == 12) {
1503                 /* If we are running code from the BIOS, handle writes to the
1504                  * Status register in C. BIOS code may toggle bit 16 which will
1505                  * map/unmap the RAM, while game code cannot do that. */
1506                 rec_mtc(state, block, offset);
1507                 return;
1508         }
1509
1510         rt = lightrec_alloc_reg_in(reg_cache, _jit, c.i.rt, 0);
1511
1512         if (c.r.rd != 13) {
1513                 jit_stxi_i(offsetof(struct lightrec_state, regs.cp0[c.r.rd]),
1514                            LIGHTREC_REG_STATE, rt);
1515         }
1516
1517         if (c.r.rd == 12 || c.r.rd == 13) {
1518                 tmp = lightrec_alloc_reg_temp(reg_cache, _jit);
1519                 jit_ldxi_i(tmp, LIGHTREC_REG_STATE,
1520                            offsetof(struct lightrec_state, regs.cp0[13]));
1521         }
1522
1523         if (c.r.rd == 12) {
1524                 status = rt;
1525         } else if (c.r.rd == 13) {
1526                 tmp2 = lightrec_alloc_reg_temp(reg_cache, _jit);
1527
1528                 /* Cause = (Cause & ~0x0300) | (value & 0x0300) */
1529                 jit_andi(tmp2, rt, 0x0300);
1530                 jit_ori(tmp, tmp, 0x0300);
1531                 jit_xori(tmp, tmp, 0x0300);
1532                 jit_orr(tmp, tmp, tmp2);
1533                 jit_ldxi_i(tmp2, LIGHTREC_REG_STATE,
1534                            offsetof(struct lightrec_state, regs.cp0[12]));
1535                 jit_stxi_i(offsetof(struct lightrec_state, regs.cp0[13]),
1536                            LIGHTREC_REG_STATE, tmp);
1537                 status = tmp2;
1538         }
1539
1540         if (c.r.rd == 12 || c.r.rd == 13) {
1541                 /* Exit dynarec in case there's a software interrupt.
1542                  * exit_flags = !!(status & tmp & 0x0300) & status; */
1543                 jit_andr(tmp, tmp, status);
1544                 jit_andi(tmp, tmp, 0x0300);
1545                 jit_nei(tmp, tmp, 0);
1546                 jit_andr(tmp, tmp, status);
1547                 jit_stxi_i(offsetof(struct lightrec_state, exit_flags),
1548                            LIGHTREC_REG_STATE, tmp);
1549
1550                 lightrec_free_reg(reg_cache, tmp);
1551         }
1552
1553         if (c.r.rd == 13)
1554                 lightrec_free_reg(reg_cache, tmp2);
1555
1556         lightrec_free_reg(reg_cache, rt);
1557
1558         if (!(block->opcode_list[offset].flags & LIGHTREC_NO_DS) &&
1559             (c.r.rd == 12 || c.r.rd == 13))
1560                 lightrec_emit_eob(state, block, offset + 1, true);
1561 }
1562
1563 static void rec_cp0_MFC0(struct lightrec_cstate *state,
1564                          const struct block *block, u16 offset)
1565 {
1566         _jit_name(block->_jit, __func__);
1567         rec_mfc0(state, block, offset);
1568 }
1569
1570 static void rec_cp0_CFC0(struct lightrec_cstate *state,
1571                          const struct block *block, u16 offset)
1572 {
1573         _jit_name(block->_jit, __func__);
1574         rec_mfc0(state, block, offset);
1575 }
1576
1577 static void rec_cp0_MTC0(struct lightrec_cstate *state,
1578                          const struct block *block, u16 offset)
1579 {
1580         _jit_name(block->_jit, __func__);
1581         rec_mtc0(state, block, offset);
1582 }
1583
1584 static void rec_cp0_CTC0(struct lightrec_cstate *state,
1585                          const struct block *block, u16 offset)
1586 {
1587         _jit_name(block->_jit, __func__);
1588         rec_mtc0(state, block, offset);
1589 }
1590
1591 static void rec_cp2_basic_MFC2(struct lightrec_cstate *state,
1592                                const struct block *block, u16 offset)
1593 {
1594         _jit_name(block->_jit, __func__);
1595         rec_mfc(state, block, offset);
1596 }
1597
1598 static void rec_cp2_basic_CFC2(struct lightrec_cstate *state,
1599                                const struct block *block, u16 offset)
1600 {
1601         _jit_name(block->_jit, __func__);
1602         rec_mfc(state, block, offset);
1603 }
1604
1605 static void rec_cp2_basic_MTC2(struct lightrec_cstate *state,
1606                                const struct block *block, u16 offset)
1607 {
1608         _jit_name(block->_jit, __func__);
1609         rec_mtc(state, block, offset);
1610 }
1611
1612 static void rec_cp2_basic_CTC2(struct lightrec_cstate *state,
1613                                const struct block *block, u16 offset)
1614 {
1615         _jit_name(block->_jit, __func__);
1616         rec_mtc(state, block, offset);
1617 }
1618
1619 static void rec_cp0_RFE(struct lightrec_cstate *state,
1620                         const struct block *block, u16 offset)
1621 {
1622         struct regcache *reg_cache = state->reg_cache;
1623         jit_state_t *_jit = block->_jit;
1624         u8 status, tmp;
1625
1626         jit_name(__func__);
1627         jit_note(__FILE__, __LINE__);
1628
1629         status = lightrec_alloc_reg_temp(reg_cache, _jit);
1630         jit_ldxi_i(status, LIGHTREC_REG_STATE,
1631                    offsetof(struct lightrec_state, regs.cp0[12]));
1632
1633         tmp = lightrec_alloc_reg_temp(reg_cache, _jit);
1634
1635         /* status = ((status >> 2) & 0xf) | status & ~0xf; */
1636         jit_rshi(tmp, status, 2);
1637         jit_andi(tmp, tmp, 0xf);
1638         jit_andi(status, status, ~0xful);
1639         jit_orr(status, status, tmp);
1640
1641         jit_ldxi_i(tmp, LIGHTREC_REG_STATE,
1642                    offsetof(struct lightrec_state, regs.cp0[13]));
1643         jit_stxi_i(offsetof(struct lightrec_state, regs.cp0[12]),
1644                    LIGHTREC_REG_STATE, status);
1645
1646         /* Exit dynarec in case there's a software interrupt.
1647          * exit_flags = !!(status & cause & 0x0300) & status; */
1648         jit_andr(tmp, tmp, status);
1649         jit_andi(tmp, tmp, 0x0300);
1650         jit_nei(tmp, tmp, 0);
1651         jit_andr(tmp, tmp, status);
1652         jit_stxi_i(offsetof(struct lightrec_state, exit_flags),
1653                    LIGHTREC_REG_STATE, tmp);
1654
1655         lightrec_free_reg(reg_cache, status);
1656         lightrec_free_reg(reg_cache, tmp);
1657 }
1658
1659 static void rec_CP(struct lightrec_cstate *state,
1660                    const struct block *block, u16 offset)
1661 {
1662         union code c = block->opcode_list[offset].c;
1663         jit_state_t *_jit = block->_jit;
1664
1665         jit_name(__func__);
1666         jit_note(__FILE__, __LINE__);
1667
1668         call_to_c_wrapper(state, block, c.opcode, true, C_WRAPPER_CP);
1669 }
1670
1671 static void rec_meta_MOV(struct lightrec_cstate *state,
1672                          const struct block *block, u16 offset)
1673 {
1674         struct regcache *reg_cache = state->reg_cache;
1675         union code c = block->opcode_list[offset].c;
1676         jit_state_t *_jit = block->_jit;
1677         u8 rs, rd;
1678
1679         _jit_name(block->_jit, __func__);
1680         jit_note(__FILE__, __LINE__);
1681         if (c.r.rs)
1682                 rs = lightrec_alloc_reg_in(reg_cache, _jit, c.r.rs, 0);
1683         rd = lightrec_alloc_reg_out(reg_cache, _jit, c.r.rd, REG_EXT);
1684
1685         if (c.r.rs == 0)
1686                 jit_movi(rd, 0);
1687         else
1688                 jit_extr_i(rd, rs);
1689
1690         if (c.r.rs)
1691                 lightrec_free_reg(reg_cache, rs);
1692         lightrec_free_reg(reg_cache, rd);
1693 }
1694
1695 static void rec_meta_EXTC_EXTS(struct lightrec_cstate *state,
1696                                const struct block *block,
1697                                u16 offset)
1698 {
1699         struct regcache *reg_cache = state->reg_cache;
1700         union code c = block->opcode_list[offset].c;
1701         jit_state_t *_jit = block->_jit;
1702         u8 rs, rt;
1703
1704         _jit_name(block->_jit, __func__);
1705         jit_note(__FILE__, __LINE__);
1706
1707         rs = lightrec_alloc_reg_in(reg_cache, _jit, c.i.rs, 0);
1708         rt = lightrec_alloc_reg_out(reg_cache, _jit, c.i.rt, REG_EXT);
1709
1710         if (c.i.op == OP_META_EXTC)
1711                 jit_extr_c(rt, rs);
1712         else
1713                 jit_extr_s(rt, rs);
1714
1715         lightrec_free_reg(reg_cache, rs);
1716         lightrec_free_reg(reg_cache, rt);
1717 }
1718
1719 static const lightrec_rec_func_t rec_standard[64] = {
1720         SET_DEFAULT_ELM(rec_standard, unknown_opcode),
1721         [OP_SPECIAL]            = rec_SPECIAL,
1722         [OP_REGIMM]             = rec_REGIMM,
1723         [OP_J]                  = rec_J,
1724         [OP_JAL]                = rec_JAL,
1725         [OP_BEQ]                = rec_BEQ,
1726         [OP_BNE]                = rec_BNE,
1727         [OP_BLEZ]               = rec_BLEZ,
1728         [OP_BGTZ]               = rec_BGTZ,
1729         [OP_ADDI]               = rec_ADDI,
1730         [OP_ADDIU]              = rec_ADDIU,
1731         [OP_SLTI]               = rec_SLTI,
1732         [OP_SLTIU]              = rec_SLTIU,
1733         [OP_ANDI]               = rec_ANDI,
1734         [OP_ORI]                = rec_ORI,
1735         [OP_XORI]               = rec_XORI,
1736         [OP_LUI]                = rec_LUI,
1737         [OP_CP0]                = rec_CP0,
1738         [OP_CP2]                = rec_CP2,
1739         [OP_LB]                 = rec_LB,
1740         [OP_LH]                 = rec_LH,
1741         [OP_LWL]                = rec_LWL,
1742         [OP_LW]                 = rec_LW,
1743         [OP_LBU]                = rec_LBU,
1744         [OP_LHU]                = rec_LHU,
1745         [OP_LWR]                = rec_LWR,
1746         [OP_SB]                 = rec_SB,
1747         [OP_SH]                 = rec_SH,
1748         [OP_SWL]                = rec_SWL,
1749         [OP_SW]                 = rec_SW,
1750         [OP_SWR]                = rec_SWR,
1751         [OP_LWC2]               = rec_LWC2,
1752         [OP_SWC2]               = rec_SWC2,
1753
1754         [OP_META_MOV]           = rec_meta_MOV,
1755         [OP_META_EXTC]          = rec_meta_EXTC_EXTS,
1756         [OP_META_EXTS]          = rec_meta_EXTC_EXTS,
1757 };
1758
1759 static const lightrec_rec_func_t rec_special[64] = {
1760         SET_DEFAULT_ELM(rec_special, unknown_opcode),
1761         [OP_SPECIAL_SLL]        = rec_special_SLL,
1762         [OP_SPECIAL_SRL]        = rec_special_SRL,
1763         [OP_SPECIAL_SRA]        = rec_special_SRA,
1764         [OP_SPECIAL_SLLV]       = rec_special_SLLV,
1765         [OP_SPECIAL_SRLV]       = rec_special_SRLV,
1766         [OP_SPECIAL_SRAV]       = rec_special_SRAV,
1767         [OP_SPECIAL_JR]         = rec_special_JR,
1768         [OP_SPECIAL_JALR]       = rec_special_JALR,
1769         [OP_SPECIAL_SYSCALL]    = rec_special_SYSCALL,
1770         [OP_SPECIAL_BREAK]      = rec_special_BREAK,
1771         [OP_SPECIAL_MFHI]       = rec_special_MFHI,
1772         [OP_SPECIAL_MTHI]       = rec_special_MTHI,
1773         [OP_SPECIAL_MFLO]       = rec_special_MFLO,
1774         [OP_SPECIAL_MTLO]       = rec_special_MTLO,
1775         [OP_SPECIAL_MULT]       = rec_special_MULT,
1776         [OP_SPECIAL_MULTU]      = rec_special_MULTU,
1777         [OP_SPECIAL_DIV]        = rec_special_DIV,
1778         [OP_SPECIAL_DIVU]       = rec_special_DIVU,
1779         [OP_SPECIAL_ADD]        = rec_special_ADD,
1780         [OP_SPECIAL_ADDU]       = rec_special_ADDU,
1781         [OP_SPECIAL_SUB]        = rec_special_SUB,
1782         [OP_SPECIAL_SUBU]       = rec_special_SUBU,
1783         [OP_SPECIAL_AND]        = rec_special_AND,
1784         [OP_SPECIAL_OR]         = rec_special_OR,
1785         [OP_SPECIAL_XOR]        = rec_special_XOR,
1786         [OP_SPECIAL_NOR]        = rec_special_NOR,
1787         [OP_SPECIAL_SLT]        = rec_special_SLT,
1788         [OP_SPECIAL_SLTU]       = rec_special_SLTU,
1789 };
1790
1791 static const lightrec_rec_func_t rec_regimm[64] = {
1792         SET_DEFAULT_ELM(rec_regimm, unknown_opcode),
1793         [OP_REGIMM_BLTZ]        = rec_regimm_BLTZ,
1794         [OP_REGIMM_BGEZ]        = rec_regimm_BGEZ,
1795         [OP_REGIMM_BLTZAL]      = rec_regimm_BLTZAL,
1796         [OP_REGIMM_BGEZAL]      = rec_regimm_BGEZAL,
1797 };
1798
1799 static const lightrec_rec_func_t rec_cp0[64] = {
1800         SET_DEFAULT_ELM(rec_cp0, rec_CP),
1801         [OP_CP0_MFC0]           = rec_cp0_MFC0,
1802         [OP_CP0_CFC0]           = rec_cp0_CFC0,
1803         [OP_CP0_MTC0]           = rec_cp0_MTC0,
1804         [OP_CP0_CTC0]           = rec_cp0_CTC0,
1805         [OP_CP0_RFE]            = rec_cp0_RFE,
1806 };
1807
1808 static const lightrec_rec_func_t rec_cp2_basic[64] = {
1809         SET_DEFAULT_ELM(rec_cp2_basic, rec_CP),
1810         [OP_CP2_BASIC_MFC2]     = rec_cp2_basic_MFC2,
1811         [OP_CP2_BASIC_CFC2]     = rec_cp2_basic_CFC2,
1812         [OP_CP2_BASIC_MTC2]     = rec_cp2_basic_MTC2,
1813         [OP_CP2_BASIC_CTC2]     = rec_cp2_basic_CTC2,
1814 };
1815
1816 static void rec_SPECIAL(struct lightrec_cstate *state,
1817                         const struct block *block, u16 offset)
1818 {
1819         union code c = block->opcode_list[offset].c;
1820         lightrec_rec_func_t f = rec_special[c.r.op];
1821
1822         if (!HAS_DEFAULT_ELM && unlikely(!f))
1823                 unknown_opcode(state, block, offset);
1824         else
1825                 (*f)(state, block, offset);
1826 }
1827
1828 static void rec_REGIMM(struct lightrec_cstate *state,
1829                        const struct block *block, u16 offset)
1830 {
1831         union code c = block->opcode_list[offset].c;
1832         lightrec_rec_func_t f = rec_regimm[c.r.rt];
1833
1834         if (!HAS_DEFAULT_ELM && unlikely(!f))
1835                 unknown_opcode(state, block, offset);
1836         else
1837                 (*f)(state, block, offset);
1838 }
1839
1840 static void rec_CP0(struct lightrec_cstate *state,
1841                     const struct block *block, u16 offset)
1842 {
1843         union code c = block->opcode_list[offset].c;
1844         lightrec_rec_func_t f = rec_cp0[c.r.rs];
1845
1846         if (!HAS_DEFAULT_ELM && unlikely(!f))
1847                 rec_CP(state, block, offset);
1848         else
1849                 (*f)(state, block, offset);
1850 }
1851
1852 static void rec_CP2(struct lightrec_cstate *state,
1853                     const struct block *block, u16 offset)
1854 {
1855         union code c = block->opcode_list[offset].c;
1856
1857         if (c.r.op == OP_CP2_BASIC) {
1858                 lightrec_rec_func_t f = rec_cp2_basic[c.r.rs];
1859
1860                 if (HAS_DEFAULT_ELM || likely(f)) {
1861                         (*f)(state, block, offset);
1862                         return;
1863                 }
1864         }
1865
1866         rec_CP(state, block, offset);
1867 }
1868
1869 void lightrec_rec_opcode(struct lightrec_cstate *state,
1870                          const struct block *block, u16 offset)
1871 {
1872         struct regcache *reg_cache = state->reg_cache;
1873         struct lightrec_branch_target *target;
1874         const struct opcode *op = &block->opcode_list[offset];
1875         jit_state_t *_jit = block->_jit;
1876         lightrec_rec_func_t f;
1877
1878         if (op->flags & LIGHTREC_SYNC) {
1879                 jit_subi(LIGHTREC_REG_CYCLE, LIGHTREC_REG_CYCLE, state->cycles);
1880                 state->cycles = 0;
1881
1882                 lightrec_storeback_regs(reg_cache, _jit);
1883                 lightrec_regcache_reset(reg_cache);
1884
1885                 pr_debug("Adding branch target at offset 0x%x\n", offset << 2);
1886                 target = &state->targets[state->nb_targets++];
1887                 target->offset = offset;
1888                 target->label = jit_indirect();
1889         }
1890
1891         if (likely(op->opcode)) {
1892                 f = rec_standard[op->i.op];
1893
1894                 if (!HAS_DEFAULT_ELM && unlikely(!f))
1895                         unknown_opcode(state, block, offset);
1896                 else
1897                         (*f)(state, block, offset);
1898         }
1899
1900         if (unlikely(op->flags & LIGHTREC_UNLOAD_RD)) {
1901                 lightrec_clean_reg_if_loaded(reg_cache, _jit, op->r.rd, true);
1902                 pr_debug("Cleaning RD reg %s\n", lightrec_reg_name(op->r.rd));
1903         }
1904         if (unlikely(op->flags & LIGHTREC_UNLOAD_RS)) {
1905                 lightrec_clean_reg_if_loaded(reg_cache, _jit, op->i.rs, true);
1906                 pr_debug("Cleaning RS reg %s\n", lightrec_reg_name(op->i.rt));
1907         }
1908         if (unlikely(op->flags & LIGHTREC_UNLOAD_RT)) {
1909                 lightrec_clean_reg_if_loaded(reg_cache, _jit, op->i.rt, true);
1910                 pr_debug("Cleaning RT reg %s\n", lightrec_reg_name(op->i.rt));
1911         }
1912 }