deps/lightrec/emitter.c

   1 // SPDX-License-Identifier: LGPL-2.1-or-later
   2 /*
   3  * Copyright (C) 2014-2021 Paul Cercueil <paul@crapouillou.net>
   4  */
   5
   6 #include "blockcache.h"
   7 #include "debug.h"
   8 #include "disassembler.h"
   9 #include "emitter.h"
  10 #include "lightning-wrapper.h"
  11 #include "optimizer.h"
  12 #include "regcache.h"
  13
  14 #include <stdbool.h>
  15 #include <stddef.h>
  16
  17 typedef void (*lightrec_rec_func_t)(struct lightrec_cstate *, const struct block *, u16);
  18
  19 /* Forward declarations */
  20 static void rec_SPECIAL(struct lightrec_cstate *state, const struct block *block, u16 offset);
  21 static void rec_REGIMM(struct lightrec_cstate *state, const struct block *block, u16 offset);
  22 static void rec_CP0(struct lightrec_cstate *state, const struct block *block, u16 offset);
  23 static void rec_CP2(struct lightrec_cstate *state, const struct block *block, u16 offset);
  24
  25 static void unknown_opcode(struct lightrec_cstate *state, const struct block *block, u16 offset)
  26 {
  27         pr_warn("Unknown opcode: 0x%08x at PC 0x%08x\n",
  28                 block->opcode_list[offset].c.opcode,
  29                 block->pc + (offset << 2));
  30 }
  31
  32 static void lightrec_emit_end_of_block(struct lightrec_cstate *state,
  33                                        const struct block *block, u16 offset,
  34                                        s8 reg_new_pc, u32 imm, u8 ra_reg,
  35                                        u32 link, bool update_cycles)
  36 {
  37         struct regcache *reg_cache = state->reg_cache;
  38         u32 cycles = state->cycles;
  39         jit_state_t *_jit = block->_jit;
  40         const struct opcode *op = &block->opcode_list[offset],
  41                             *next = &block->opcode_list[offset + 1];
  42
  43         jit_note(__FILE__, __LINE__);
  44
  45         if (link) {
  46                 /* Update the $ra register */
  47                 u8 link_reg = lightrec_alloc_reg_out(reg_cache, _jit, ra_reg, 0);
  48                 jit_movi(link_reg, link);
  49                 lightrec_free_reg(reg_cache, link_reg);
  50         }
  51
  52         if (reg_new_pc < 0) {
  53                 reg_new_pc = lightrec_alloc_reg(reg_cache, _jit, JIT_V0);
  54                 lightrec_lock_reg(reg_cache, _jit, reg_new_pc);
  55
  56                 jit_movi(reg_new_pc, imm);
  57         }
  58
  59         if (has_delay_slot(op->c) &&
  60             !(op->flags & (LIGHTREC_NO_DS | LIGHTREC_LOCAL_BRANCH))) {
  61                 cycles += lightrec_cycles_of_opcode(next->c);
  62
  63                 /* Recompile the delay slot */
  64                 if (next->c.opcode)
  65                         lightrec_rec_opcode(state, block, offset + 1);
  66         }
  67
  68         /* Store back remaining registers */
  69         lightrec_storeback_regs(reg_cache, _jit);
  70
  71         jit_movr(JIT_V0, reg_new_pc);
  72
  73         if (cycles && update_cycles) {
  74                 jit_subi(LIGHTREC_REG_CYCLE, LIGHTREC_REG_CYCLE, cycles);
  75                 pr_debug("EOB: %u cycles\n", cycles);
  76         }
  77
  78         if (offset + !!(op->flags & LIGHTREC_NO_DS) < block->nb_ops - 1)
  79                 state->branches[state->nb_branches++] = jit_b();
  80 }
  81
  82 void lightrec_emit_eob(struct lightrec_cstate *state, const struct block *block,
  83                        u16 offset, bool after_op)
  84 {
  85         struct regcache *reg_cache = state->reg_cache;
  86         jit_state_t *_jit = block->_jit;
  87         union code c = block->opcode_list[offset].c;
  88         u32 cycles = state->cycles;
  89
  90         if (!after_op)
  91                 cycles -= lightrec_cycles_of_opcode(c);
  92
  93         lightrec_storeback_regs(reg_cache, _jit);
  94
  95         jit_movi(JIT_V0, block->pc + (offset << 2));
  96         jit_subi(LIGHTREC_REG_CYCLE, LIGHTREC_REG_CYCLE, cycles);
  97
  98         state->branches[state->nb_branches++] = jit_b();
  99 }
 100
 101 static u8 get_jr_jalr_reg(struct lightrec_cstate *state, const struct block *block, u16 offset)
 102 {
 103         struct regcache *reg_cache = state->reg_cache;
 104         jit_state_t *_jit = block->_jit;
 105         const struct opcode *op = &block->opcode_list[offset];
 106         u8 rs;
 107
 108         rs = lightrec_request_reg_in(reg_cache, _jit, op->r.rs, JIT_V0);
 109         lightrec_lock_reg(reg_cache, _jit, rs);
 110
 111         return rs;
 112 }
 113
 114 static void rec_special_JR(struct lightrec_cstate *state, const struct block *block, u16 offset)
 115 {
 116         u8 rs = get_jr_jalr_reg(state, block, offset);
 117
 118         _jit_name(block->_jit, __func__);
 119         lightrec_emit_end_of_block(state, block, offset, rs, 0, 31, 0, true);
 120 }
 121
 122 static void rec_special_JALR(struct lightrec_cstate *state, const struct block *block, u16 offset)
 123 {
 124         u8 rs = get_jr_jalr_reg(state, block, offset);
 125         union code c = block->opcode_list[offset].c;
 126
 127         _jit_name(block->_jit, __func__);
 128         lightrec_emit_end_of_block(state, block, offset, rs, 0, c.r.rd,
 129                                    get_branch_pc(block, offset, 2), true);
 130 }
 131
 132 static void rec_J(struct lightrec_cstate *state, const struct block *block, u16 offset)
 133 {
 134         union code c = block->opcode_list[offset].c;
 135
 136         _jit_name(block->_jit, __func__);
 137         lightrec_emit_end_of_block(state, block, offset, -1,
 138                                    (block->pc & 0xf0000000) | (c.j.imm << 2),
 139                                    31, 0, true);
 140 }
 141
 142 static void rec_JAL(struct lightrec_cstate *state, const struct block *block, u16 offset)
 143 {
 144         union code c = block->opcode_list[offset].c;
 145
 146         _jit_name(block->_jit, __func__);
 147         lightrec_emit_end_of_block(state, block, offset, -1,
 148                                    (block->pc & 0xf0000000) | (c.j.imm << 2),
 149                                    31, get_branch_pc(block, offset, 2), true);
 150 }
 151
 152 static void rec_b(struct lightrec_cstate *state, const struct block *block, u16 offset,
 153                   jit_code_t code, u32 link, bool unconditional, bool bz)
 154 {
 155         struct regcache *reg_cache = state->reg_cache;
 156         struct native_register *regs_backup;
 157         jit_state_t *_jit = block->_jit;
 158         struct lightrec_branch *branch;
 159         const struct opcode *op = &block->opcode_list[offset],
 160                             *next = &block->opcode_list[offset + 1];
 161         jit_node_t *addr;
 162         u8 link_reg;
 163         u32 target_offset, cycles = state->cycles;
 164         bool is_forward = (s16)op->i.imm >= -1;
 165         u32 next_pc;
 166
 167         jit_note(__FILE__, __LINE__);
 168
 169         if (!(op->flags & LIGHTREC_NO_DS))
 170                 cycles += lightrec_cycles_of_opcode(next->c);
 171
 172         state->cycles = 0;
 173
 174         if (cycles)
 175                 jit_subi(LIGHTREC_REG_CYCLE, LIGHTREC_REG_CYCLE, cycles);
 176
 177         if (!unconditional) {
 178                 u8 rs = lightrec_alloc_reg_in(reg_cache, _jit, op->i.rs, REG_EXT),
 179                    rt = bz ? 0 : lightrec_alloc_reg_in(reg_cache,
 180                                                        _jit, op->i.rt, REG_EXT);
 181
 182                 /* Generate the branch opcode */
 183                 addr = jit_new_node_pww(code, NULL, rs, rt);
 184
 185                 lightrec_free_regs(reg_cache);
 186                 regs_backup = lightrec_regcache_enter_branch(reg_cache);
 187         }
 188
 189         if (op->flags & LIGHTREC_LOCAL_BRANCH) {
 190                 if (next && !(op->flags & LIGHTREC_NO_DS)) {
 191                         /* Recompile the delay slot */
 192                         if (next->opcode)
 193                                 lightrec_rec_opcode(state, block, offset + 1);
 194                 }
 195
 196                 if (link) {
 197                         /* Update the $ra register */
 198                         link_reg = lightrec_alloc_reg_out(reg_cache, _jit, 31, 0);
 199                         jit_movi(link_reg, link);
 200                         lightrec_free_reg(reg_cache, link_reg);
 201                 }
 202
 203                 /* Store back remaining registers */
 204                 lightrec_storeback_regs(reg_cache, _jit);
 205
 206                 target_offset = offset + 1 + (s16)op->i.imm
 207                         - !!(OPT_SWITCH_DELAY_SLOTS && (op->flags & LIGHTREC_NO_DS));
 208                 pr_debug("Adding local branch to offset 0x%x\n",
 209                          target_offset << 2);
 210                 branch = &state->local_branches[
 211                         state->nb_local_branches++];
 212
 213                 branch->target = target_offset;
 214                 if (is_forward)
 215                         branch->branch = jit_b();
 216                 else
 217                         branch->branch = jit_bgti(LIGHTREC_REG_CYCLE, 0);
 218         }
 219
 220         if (!(op->flags & LIGHTREC_LOCAL_BRANCH) || !is_forward) {
 221                 next_pc = get_branch_pc(block, offset, 1 + (s16)op->i.imm);
 222                 lightrec_emit_end_of_block(state, block, offset, -1, next_pc,
 223                                            31, link, false);
 224         }
 225
 226         if (!unconditional) {
 227                 jit_patch(addr);
 228                 lightrec_regcache_leave_branch(reg_cache, regs_backup);
 229
 230                 if (bz && link) {
 231                         /* Update the $ra register */
 232                         link_reg = lightrec_alloc_reg_out(reg_cache, _jit,
 233                                                           31, REG_EXT);
 234                         jit_movi(link_reg, (s32)link);
 235                         lightrec_free_reg(reg_cache, link_reg);
 236                 }
 237
 238                 if (!(op->flags & LIGHTREC_NO_DS) && next->opcode)
 239                         lightrec_rec_opcode(state, block, offset + 1);
 240         }
 241 }
 242
 243 static void rec_BNE(struct lightrec_cstate *state,
 244                     const struct block *block, u16 offset)
 245 {
 246         union code c = block->opcode_list[offset].c;
 247
 248         _jit_name(block->_jit, __func__);
 249
 250         if (c.i.rt == 0)
 251                 rec_b(state, block, offset, jit_code_beqi, 0, false, true);
 252         else
 253                 rec_b(state, block, offset, jit_code_beqr, 0, false, false);
 254 }
 255
 256 static void rec_BEQ(struct lightrec_cstate *state,
 257                     const struct block *block, u16 offset)
 258 {
 259         union code c = block->opcode_list[offset].c;
 260
 261         _jit_name(block->_jit, __func__);
 262
 263         if (c.i.rt == 0)
 264                 rec_b(state, block, offset, jit_code_bnei, 0, c.i.rs == 0, true);
 265         else
 266                 rec_b(state, block, offset, jit_code_bner, 0, c.i.rs == c.i.rt, false);
 267 }
 268
 269 static void rec_BLEZ(struct lightrec_cstate *state,
 270                      const struct block *block, u16 offset)
 271 {
 272         union code c = block->opcode_list[offset].c;
 273
 274         _jit_name(block->_jit, __func__);
 275         rec_b(state, block, offset, jit_code_bgti, 0, c.i.rs == 0, true);
 276 }
 277
 278 static void rec_BGTZ(struct lightrec_cstate *state,
 279                      const struct block *block, u16 offset)
 280 {
 281         _jit_name(block->_jit, __func__);
 282         rec_b(state, block, offset, jit_code_blei, 0, false, true);
 283 }
 284
 285 static void rec_regimm_BLTZ(struct lightrec_cstate *state,
 286                             const struct block *block, u16 offset)
 287 {
 288         _jit_name(block->_jit, __func__);
 289         rec_b(state, block, offset, jit_code_bgei, 0, false, true);
 290 }
 291
 292 static void rec_regimm_BLTZAL(struct lightrec_cstate *state,
 293                               const struct block *block, u16 offset)
 294 {
 295         _jit_name(block->_jit, __func__);
 296         rec_b(state, block, offset, jit_code_bgei,
 297               get_branch_pc(block, offset, 2), false, true);
 298 }
 299
 300 static void rec_regimm_BGEZ(struct lightrec_cstate *state,
 301                             const struct block *block, u16 offset)
 302 {
 303         union code c = block->opcode_list[offset].c;
 304
 305         _jit_name(block->_jit, __func__);
 306         rec_b(state, block, offset, jit_code_blti, 0, !c.i.rs, true);
 307 }
 308
 309 static void rec_regimm_BGEZAL(struct lightrec_cstate *state,
 310                               const struct block *block, u16 offset)
 311 {
 312         const struct opcode *op = &block->opcode_list[offset];
 313         _jit_name(block->_jit, __func__);
 314         rec_b(state, block, offset, jit_code_blti,
 315               get_branch_pc(block, offset, 2),
 316               !op->i.rs, true);
 317 }
 318
 319 static void rec_alu_imm(struct lightrec_cstate *state, const struct block *block,
 320                         u16 offset, jit_code_t code, bool slti)
 321 {
 322         struct regcache *reg_cache = state->reg_cache;
 323         union code c = block->opcode_list[offset].c;
 324         jit_state_t *_jit = block->_jit;
 325         u8 rs, rt, out_flags = REG_EXT;
 326
 327         if (slti)
 328                 out_flags |= REG_ZEXT;
 329
 330         jit_note(__FILE__, __LINE__);
 331         rs = lightrec_alloc_reg_in(reg_cache, _jit, c.i.rs, REG_EXT);
 332         rt = lightrec_alloc_reg_out(reg_cache, _jit, c.i.rt, out_flags);
 333
 334         jit_new_node_www(code, rt, rs, (s32)(s16) c.i.imm);
 335
 336         lightrec_free_reg(reg_cache, rs);
 337         lightrec_free_reg(reg_cache, rt);
 338 }
 339
 340 static void rec_alu_special(struct lightrec_cstate *state, const struct block *block,
 341                             u16 offset, jit_code_t code, bool out_ext)
 342 {
 343         struct regcache *reg_cache = state->reg_cache;
 344         union code c = block->opcode_list[offset].c;
 345         jit_state_t *_jit = block->_jit;
 346         u8 rd, rt, rs;
 347
 348         jit_note(__FILE__, __LINE__);
 349         rs = lightrec_alloc_reg_in(reg_cache, _jit, c.r.rs, REG_EXT);
 350         rt = lightrec_alloc_reg_in(reg_cache, _jit, c.r.rt, REG_EXT);
 351         rd = lightrec_alloc_reg_out(reg_cache, _jit, c.r.rd,
 352                                     out_ext ? REG_EXT | REG_ZEXT : 0);
 353
 354         jit_new_node_www(code, rd, rs, rt);
 355
 356         lightrec_free_reg(reg_cache, rs);
 357         lightrec_free_reg(reg_cache, rt);
 358         lightrec_free_reg(reg_cache, rd);
 359 }
 360
 361 static void rec_alu_shiftv(struct lightrec_cstate *state, const struct block *block,
 362                            u16 offset, jit_code_t code)
 363 {
 364         struct regcache *reg_cache = state->reg_cache;
 365         union code c = block->opcode_list[offset].c;
 366         jit_state_t *_jit = block->_jit;
 367         u8 rd, rt, rs, temp, flags = 0;
 368
 369         jit_note(__FILE__, __LINE__);
 370         rs = lightrec_alloc_reg_in(reg_cache, _jit, c.r.rs, 0);
 371
 372         if (code == jit_code_rshr)
 373                 flags = REG_EXT;
 374         else if (code == jit_code_rshr_u)
 375                 flags = REG_ZEXT;
 376
 377         rt = lightrec_alloc_reg_in(reg_cache, _jit, c.r.rt, flags);
 378         rd = lightrec_alloc_reg_out(reg_cache, _jit, c.r.rd, flags);
 379
 380         if (rs != rd && rt != rd) {
 381                 jit_andi(rd, rs, 0x1f);
 382                 jit_new_node_www(code, rd, rt, rd);
 383         } else {
 384                 temp = lightrec_alloc_reg_temp(reg_cache, _jit);
 385                 jit_andi(temp, rs, 0x1f);
 386                 jit_new_node_www(code, rd, rt, temp);
 387                 lightrec_free_reg(reg_cache, temp);
 388         }
 389
 390         lightrec_free_reg(reg_cache, rs);
 391         lightrec_free_reg(reg_cache, rt);
 392         lightrec_free_reg(reg_cache, rd);
 393 }
 394
 395 static void rec_movi(struct lightrec_cstate *state,
 396                      const struct block *block, u16 offset)
 397 {
 398         struct regcache *reg_cache = state->reg_cache;
 399         union code c = block->opcode_list[offset].c;
 400         jit_state_t *_jit = block->_jit;
 401         u16 flags = REG_EXT;
 402         u8 rt;
 403
 404         if (!(c.i.imm & 0x8000))
 405                 flags |= REG_ZEXT;
 406
 407         rt = lightrec_alloc_reg_out(reg_cache, _jit, c.i.rt, flags);
 408
 409         jit_movi(rt, (s32)(s16) c.i.imm);
 410
 411         lightrec_free_reg(reg_cache, rt);
 412 }
 413
 414 static void rec_ADDIU(struct lightrec_cstate *state,
 415                       const struct block *block, u16 offset)
 416 {
 417         _jit_name(block->_jit, __func__);
 418
 419         if (block->opcode_list[offset].c.i.rs)
 420                 rec_alu_imm(state, block, offset, jit_code_addi, false);
 421         else
 422                 rec_movi(state, block, offset);
 423 }
 424
 425 static void rec_ADDI(struct lightrec_cstate *state,
 426                      const struct block *block, u16 offset)
 427 {
 428         /* TODO: Handle the exception? */
 429         _jit_name(block->_jit, __func__);
 430         rec_ADDIU(state, block, offset);
 431 }
 432
 433 static void rec_SLTIU(struct lightrec_cstate *state,
 434                       const struct block *block, u16 offset)
 435 {
 436         _jit_name(block->_jit, __func__);
 437         rec_alu_imm(state, block, offset, jit_code_lti_u, true);
 438 }
 439
 440 static void rec_SLTI(struct lightrec_cstate *state,
 441                      const struct block *block, u16 offset)
 442 {
 443         _jit_name(block->_jit, __func__);
 444         rec_alu_imm(state, block, offset, jit_code_lti, true);
 445 }
 446
 447 static void rec_ANDI(struct lightrec_cstate *state,
 448                      const struct block *block, u16 offset)
 449 {
 450         struct regcache *reg_cache = state->reg_cache;
 451         union code c = block->opcode_list[offset].c;
 452         jit_state_t *_jit = block->_jit;
 453         u8 rs, rt;
 454
 455         _jit_name(block->_jit, __func__);
 456         jit_note(__FILE__, __LINE__);
 457         rs = lightrec_alloc_reg_in(reg_cache, _jit, c.i.rs, 0);
 458         rt = lightrec_alloc_reg_out(reg_cache, _jit, c.i.rt,
 459                                     REG_EXT | REG_ZEXT);
 460
 461         /* PSX code uses ANDI 0xff / ANDI 0xffff a lot, which are basically
 462          * casts to uint8_t / uint16_t. */
 463         if (c.i.imm == 0xff)
 464                 jit_extr_uc(rt, rs);
 465         else if (c.i.imm == 0xffff)
 466                 jit_extr_us(rt, rs);
 467         else
 468                 jit_andi(rt, rs, (u32)(u16) c.i.imm);
 469
 470         lightrec_free_reg(reg_cache, rs);
 471         lightrec_free_reg(reg_cache, rt);
 472 }
 473
 474 static void rec_alu_or_xor(struct lightrec_cstate *state, const struct block *block,
 475                            u16 offset, jit_code_t code)
 476 {
 477         struct regcache *reg_cache = state->reg_cache;
 478         union code c = block->opcode_list[offset].c;
 479         jit_state_t *_jit = block->_jit;
 480         u8 rs, rt, flags;
 481
 482         jit_note(__FILE__, __LINE__);
 483         rs = lightrec_alloc_reg_in(reg_cache, _jit, c.i.rs, 0);
 484         rt = lightrec_alloc_reg_out(reg_cache, _jit, c.i.rt, 0);
 485
 486         flags = lightrec_get_reg_in_flags(reg_cache, rs);
 487         lightrec_set_reg_out_flags(reg_cache, rt, flags);
 488
 489         jit_new_node_www(code, rt, rs, (u32)(u16) c.i.imm);
 490
 491         lightrec_free_reg(reg_cache, rs);
 492         lightrec_free_reg(reg_cache, rt);
 493 }
 494
 495
 496 static void rec_ORI(struct lightrec_cstate *state,
 497                     const struct block *block, u16 offset)
 498 {
 499         _jit_name(block->_jit, __func__);
 500         rec_alu_or_xor(state, block, offset, jit_code_ori);
 501 }
 502
 503 static void rec_XORI(struct lightrec_cstate *state,
 504                      const struct block *block, u16 offset)
 505 {
 506         _jit_name(block->_jit, __func__);
 507         rec_alu_or_xor(state, block, offset, jit_code_xori);
 508 }
 509
 510 static void rec_LUI(struct lightrec_cstate *state,
 511                     const struct block *block, u16 offset)
 512 {
 513         struct regcache *reg_cache = state->reg_cache;
 514         union code c = block->opcode_list[offset].c;
 515         jit_state_t *_jit = block->_jit;
 516         u8 rt, flags = REG_EXT;
 517
 518         jit_name(__func__);
 519         jit_note(__FILE__, __LINE__);
 520
 521         if (!(c.i.imm & BIT(15)))
 522                 flags |= REG_ZEXT;
 523
 524         rt = lightrec_alloc_reg_out(reg_cache, _jit, c.i.rt, flags);
 525
 526         jit_movi(rt, (s32)(c.i.imm << 16));
 527
 528         lightrec_free_reg(reg_cache, rt);
 529 }
 530
 531 static void rec_special_ADDU(struct lightrec_cstate *state,
 532                              const struct block *block, u16 offset)
 533 {
 534         _jit_name(block->_jit, __func__);
 535         rec_alu_special(state, block, offset, jit_code_addr, false);
 536 }
 537
 538 static void rec_special_ADD(struct lightrec_cstate *state,
 539                             const struct block *block, u16 offset)
 540 {
 541         /* TODO: Handle the exception? */
 542         _jit_name(block->_jit, __func__);
 543         rec_alu_special(state, block, offset, jit_code_addr, false);
 544 }
 545
 546 static void rec_special_SUBU(struct lightrec_cstate *state,
 547                              const struct block *block, u16 offset)
 548 {
 549         _jit_name(block->_jit, __func__);
 550         rec_alu_special(state, block, offset, jit_code_subr, false);
 551 }
 552
 553 static void rec_special_SUB(struct lightrec_cstate *state,
 554                             const struct block *block, u16 offset)
 555 {
 556         /* TODO: Handle the exception? */
 557         _jit_name(block->_jit, __func__);
 558         rec_alu_special(state, block, offset, jit_code_subr, false);
 559 }
 560
 561 static void rec_special_AND(struct lightrec_cstate *state,
 562                             const struct block *block, u16 offset)
 563 {
 564         struct regcache *reg_cache = state->reg_cache;
 565         union code c = block->opcode_list[offset].c;
 566         jit_state_t *_jit = block->_jit;
 567         u8 rd, rt, rs, flags_rs, flags_rt, flags_rd;
 568
 569         _jit_name(block->_jit, __func__);
 570         jit_note(__FILE__, __LINE__);
 571         rs = lightrec_alloc_reg_in(reg_cache, _jit, c.r.rs, 0);
 572         rt = lightrec_alloc_reg_in(reg_cache, _jit, c.r.rt, 0);
 573         rd = lightrec_alloc_reg_out(reg_cache, _jit, c.r.rd, 0);
 574
 575         flags_rs = lightrec_get_reg_in_flags(reg_cache, rs);
 576         flags_rt = lightrec_get_reg_in_flags(reg_cache, rt);
 577
 578         /* Z(rd) = Z(rs) | Z(rt) */
 579         flags_rd = REG_ZEXT & (flags_rs | flags_rt);
 580
 581         /* E(rd) = (E(rt) & Z(rt)) | (E(rs) & Z(rs)) | (E(rs) & E(rt)) */
 582         if (((flags_rs & REG_EXT) && (flags_rt & REG_ZEXT)) ||
 583             ((flags_rt & REG_EXT) && (flags_rs & REG_ZEXT)) ||
 584             (REG_EXT & flags_rs & flags_rt))
 585                 flags_rd |= REG_EXT;
 586
 587         lightrec_set_reg_out_flags(reg_cache, rd, flags_rd);
 588
 589         jit_andr(rd, rs, rt);
 590
 591         lightrec_free_reg(reg_cache, rs);
 592         lightrec_free_reg(reg_cache, rt);
 593         lightrec_free_reg(reg_cache, rd);
 594 }
 595
 596 static void rec_special_or_nor(struct lightrec_cstate *state,
 597                                const struct block *block, u16 offset, bool nor)
 598 {
 599         struct regcache *reg_cache = state->reg_cache;
 600         union code c = block->opcode_list[offset].c;
 601         jit_state_t *_jit = block->_jit;
 602         u8 rd, rt, rs, flags_rs, flags_rt, flags_rd = 0;
 603
 604         jit_note(__FILE__, __LINE__);
 605         rs = lightrec_alloc_reg_in(reg_cache, _jit, c.r.rs, 0);
 606         rt = lightrec_alloc_reg_in(reg_cache, _jit, c.r.rt, 0);
 607         rd = lightrec_alloc_reg_out(reg_cache, _jit, c.r.rd, 0);
 608
 609         flags_rs = lightrec_get_reg_in_flags(reg_cache, rs);
 610         flags_rt = lightrec_get_reg_in_flags(reg_cache, rt);
 611
 612         /* or: Z(rd) = Z(rs) & Z(rt)
 613          * nor: Z(rd) = 0 */
 614         if (!nor)
 615                 flags_rd = REG_ZEXT & flags_rs & flags_rt;
 616
 617         /* E(rd) = (E(rs) & E(rt)) | (E(rt) & !Z(rt)) | (E(rs) & !Z(rs)) */
 618         if ((REG_EXT & flags_rs & flags_rt) ||
 619             (flags_rt & (REG_EXT | REG_ZEXT) == REG_EXT) ||
 620             (flags_rs & (REG_EXT | REG_ZEXT) == REG_EXT))
 621                 flags_rd |= REG_EXT;
 622
 623         lightrec_set_reg_out_flags(reg_cache, rd, flags_rd);
 624
 625         jit_orr(rd, rs, rt);
 626
 627         if (nor)
 628                 jit_comr(rd, rd);
 629
 630         lightrec_free_reg(reg_cache, rs);
 631         lightrec_free_reg(reg_cache, rt);
 632         lightrec_free_reg(reg_cache, rd);
 633 }
 634
 635 static void rec_special_OR(struct lightrec_cstate *state,
 636                            const struct block *block, u16 offset)
 637 {
 638         _jit_name(block->_jit, __func__);
 639         rec_special_or_nor(state, block, offset, false);
 640 }
 641
 642 static void rec_special_NOR(struct lightrec_cstate *state,
 643                             const struct block *block, u16 offset)
 644 {
 645         _jit_name(block->_jit, __func__);
 646         rec_special_or_nor(state, block, offset, true);
 647 }
 648
 649 static void rec_special_XOR(struct lightrec_cstate *state,
 650                             const struct block *block, u16 offset)
 651 {
 652         struct regcache *reg_cache = state->reg_cache;
 653         union code c = block->opcode_list[offset].c;
 654         jit_state_t *_jit = block->_jit;
 655         u8 rd, rt, rs, flags_rs, flags_rt, flags_rd;
 656
 657         _jit_name(block->_jit, __func__);
 658
 659         jit_note(__FILE__, __LINE__);
 660         rs = lightrec_alloc_reg_in(reg_cache, _jit, c.r.rs, 0);
 661         rt = lightrec_alloc_reg_in(reg_cache, _jit, c.r.rt, 0);
 662         rd = lightrec_alloc_reg_out(reg_cache, _jit, c.r.rd, 0);
 663
 664         flags_rs = lightrec_get_reg_in_flags(reg_cache, rs);
 665         flags_rt = lightrec_get_reg_in_flags(reg_cache, rt);
 666
 667         /* Z(rd) = Z(rs) & Z(rt) */
 668         flags_rd = REG_ZEXT & flags_rs & flags_rt;
 669
 670         /* E(rd) = E(rs) & E(rt) */
 671         flags_rd |= REG_EXT & flags_rs & flags_rt;
 672
 673         lightrec_set_reg_out_flags(reg_cache, rd, flags_rd);
 674
 675         jit_xorr(rd, rs, rt);
 676
 677         lightrec_free_reg(reg_cache, rs);
 678         lightrec_free_reg(reg_cache, rt);
 679         lightrec_free_reg(reg_cache, rd);
 680 }
 681
 682 static void rec_special_SLTU(struct lightrec_cstate *state,
 683                              const struct block *block, u16 offset)
 684 {
 685         _jit_name(block->_jit, __func__);
 686         rec_alu_special(state, block, offset, jit_code_ltr_u, true);
 687 }
 688
 689 static void rec_special_SLT(struct lightrec_cstate *state,
 690                             const struct block *block, u16 offset)
 691 {
 692         _jit_name(block->_jit, __func__);
 693         rec_alu_special(state, block, offset, jit_code_ltr, true);
 694 }
 695
 696 static void rec_special_SLLV(struct lightrec_cstate *state,
 697                              const struct block *block, u16 offset)
 698 {
 699         _jit_name(block->_jit, __func__);
 700         rec_alu_shiftv(state, block, offset, jit_code_lshr);
 701 }
 702
 703 static void rec_special_SRLV(struct lightrec_cstate *state,
 704                              const struct block *block, u16 offset)
 705 {
 706         _jit_name(block->_jit, __func__);
 707         rec_alu_shiftv(state, block, offset, jit_code_rshr_u);
 708 }
 709
 710 static void rec_special_SRAV(struct lightrec_cstate *state,
 711                              const struct block *block, u16 offset)
 712 {
 713         _jit_name(block->_jit, __func__);
 714         rec_alu_shiftv(state, block, offset, jit_code_rshr);
 715 }
 716
 717 static void rec_alu_shift(struct lightrec_cstate *state, const struct block *block,
 718                           u16 offset, jit_code_t code)
 719 {
 720         struct regcache *reg_cache = state->reg_cache;
 721         union code c = block->opcode_list[offset].c;
 722         jit_state_t *_jit = block->_jit;
 723         u8 rd, rt, flags = 0;
 724
 725         jit_note(__FILE__, __LINE__);
 726
 727         if (code == jit_code_rshi)
 728                 flags = REG_EXT;
 729         else if (code == jit_code_rshi_u)
 730                 flags = REG_ZEXT;
 731
 732         rt = lightrec_alloc_reg_in(reg_cache, _jit, c.r.rt, flags);
 733
 734         /* Input reg is zero-extended, if we SRL at least by one bit, we know
 735          * the output reg will be both zero-extended and sign-extended. */
 736         if (code == jit_code_rshi_u && c.r.imm)
 737                 flags |= REG_EXT;
 738         rd = lightrec_alloc_reg_out(reg_cache, _jit, c.r.rd, flags);
 739
 740         jit_new_node_www(code, rd, rt, c.r.imm);
 741
 742         lightrec_free_reg(reg_cache, rt);
 743         lightrec_free_reg(reg_cache, rd);
 744 }
 745
 746 static void rec_special_SLL(struct lightrec_cstate *state,
 747                             const struct block *block, u16 offset)
 748 {
 749         _jit_name(block->_jit, __func__);
 750         rec_alu_shift(state, block, offset, jit_code_lshi);
 751 }
 752
 753 static void rec_special_SRL(struct lightrec_cstate *state,
 754                             const struct block *block, u16 offset)
 755 {
 756         _jit_name(block->_jit, __func__);
 757         rec_alu_shift(state, block, offset, jit_code_rshi_u);
 758 }
 759
 760 static void rec_special_SRA(struct lightrec_cstate *state,
 761                             const struct block *block, u16 offset)
 762 {
 763         _jit_name(block->_jit, __func__);
 764         rec_alu_shift(state, block, offset, jit_code_rshi);
 765 }
 766
 767 static void rec_alu_mult(struct lightrec_cstate *state,
 768                          const struct block *block, u16 offset, bool is_signed)
 769 {
 770         struct regcache *reg_cache = state->reg_cache;
 771         union code c = block->opcode_list[offset].c;
 772         u16 flags = block->opcode_list[offset].flags;
 773         u8 reg_lo = get_mult_div_lo(c);
 774         u8 reg_hi = get_mult_div_hi(c);
 775         jit_state_t *_jit = block->_jit;
 776         u8 lo, hi, rs, rt, rflags = 0;
 777
 778         jit_note(__FILE__, __LINE__);
 779
 780         if (is_signed)
 781                 rflags = REG_EXT;
 782         else
 783                 rflags = REG_ZEXT;
 784
 785         rs = lightrec_alloc_reg_in(reg_cache, _jit, c.r.rs, rflags);
 786         rt = lightrec_alloc_reg_in(reg_cache, _jit, c.r.rt, rflags);
 787
 788         if (!(flags & LIGHTREC_NO_LO))
 789                 lo = lightrec_alloc_reg_out(reg_cache, _jit, reg_lo, 0);
 790         else if (__WORDSIZE == 32)
 791                 lo = lightrec_alloc_reg_temp(reg_cache, _jit);
 792
 793         if (!(flags & LIGHTREC_NO_HI))
 794                 hi = lightrec_alloc_reg_out(reg_cache, _jit, reg_hi, REG_EXT);
 795
 796         if (__WORDSIZE == 32) {
 797                 /* On 32-bit systems, do a 32*32->64 bit operation, or a 32*32->32 bit
 798                  * operation if the MULT was detected a 32-bit only. */
 799                 if (!(flags & LIGHTREC_NO_HI)) {
 800                         if (is_signed)
 801                                 jit_qmulr(lo, hi, rs, rt);
 802                         else
 803                                 jit_qmulr_u(lo, hi, rs, rt);
 804                 } else {
 805                         jit_mulr(lo, rs, rt);
 806                 }
 807         } else {
 808                 /* On 64-bit systems, do a 64*64->64 bit operation. */
 809                 if (flags & LIGHTREC_NO_LO) {
 810                         jit_mulr(hi, rs, rt);
 811                         jit_rshi(hi, hi, 32);
 812                 } else {
 813                         jit_mulr(lo, rs, rt);
 814
 815                         /* The 64-bit output value is in $lo, store the upper 32 bits in $hi */
 816                         if (!(flags & LIGHTREC_NO_HI))
 817                                 jit_rshi(hi, lo, 32);
 818                 }
 819         }
 820
 821         lightrec_free_reg(reg_cache, rs);
 822         lightrec_free_reg(reg_cache, rt);
 823         if (!(flags & LIGHTREC_NO_LO) || __WORDSIZE == 32)
 824                 lightrec_free_reg(reg_cache, lo);
 825         if (!(flags & LIGHTREC_NO_HI))
 826                 lightrec_free_reg(reg_cache, hi);
 827 }
 828
 829 static void rec_alu_div(struct lightrec_cstate *state,
 830                         const struct block *block, u16 offset, bool is_signed)
 831 {
 832         struct regcache *reg_cache = state->reg_cache;
 833         union code c = block->opcode_list[offset].c;
 834         u16 flags = block->opcode_list[offset].flags;
 835         bool no_check = flags & LIGHTREC_NO_DIV_CHECK;
 836         u8 reg_lo = get_mult_div_lo(c);
 837         u8 reg_hi = get_mult_div_hi(c);
 838         jit_state_t *_jit = block->_jit;
 839         jit_node_t *branch, *to_end;
 840         u8 lo = 0, hi = 0, rs, rt, rflags = 0;
 841
 842         jit_note(__FILE__, __LINE__);
 843
 844         if (is_signed)
 845                 rflags = REG_EXT;
 846         else
 847                 rflags = REG_ZEXT;
 848
 849         rs = lightrec_alloc_reg_in(reg_cache, _jit, c.r.rs, rflags);
 850         rt = lightrec_alloc_reg_in(reg_cache, _jit, c.r.rt, rflags);
 851
 852         if (!(flags & LIGHTREC_NO_LO))
 853                 lo = lightrec_alloc_reg_out(reg_cache, _jit, reg_lo, 0);
 854
 855         if (!(flags & LIGHTREC_NO_HI))
 856                 hi = lightrec_alloc_reg_out(reg_cache, _jit, reg_hi, 0);
 857
 858         /* Jump to special handler if dividing by zero  */
 859         if (!no_check)
 860                 branch = jit_beqi(rt, 0);
 861
 862         if (flags & LIGHTREC_NO_LO) {
 863                 if (is_signed)
 864                         jit_remr(hi, rs, rt);
 865                 else
 866                         jit_remr_u(hi, rs, rt);
 867         } else if (flags & LIGHTREC_NO_HI) {
 868                 if (is_signed)
 869                         jit_divr(lo, rs, rt);
 870                 else
 871                         jit_divr_u(lo, rs, rt);
 872         } else {
 873                 if (is_signed)
 874                         jit_qdivr(lo, hi, rs, rt);
 875                 else
 876                         jit_qdivr_u(lo, hi, rs, rt);
 877         }
 878
 879         if (!no_check) {
 880                 /* Jump above the div-by-zero handler */
 881                 to_end = jit_b();
 882
 883                 jit_patch(branch);
 884
 885                 if (!(flags & LIGHTREC_NO_LO)) {
 886                         if (is_signed) {
 887                                 jit_lti(lo, rs, 0);
 888                                 jit_lshi(lo, lo, 1);
 889                                 jit_subi(lo, lo, 1);
 890                         } else {
 891                                 jit_movi(lo, 0xffffffff);
 892                         }
 893                 }
 894
 895                 if (!(flags & LIGHTREC_NO_HI))
 896                         jit_movr(hi, rs);
 897
 898                 jit_patch(to_end);
 899         }
 900
 901         lightrec_free_reg(reg_cache, rs);
 902         lightrec_free_reg(reg_cache, rt);
 903
 904         if (!(flags & LIGHTREC_NO_LO))
 905                 lightrec_free_reg(reg_cache, lo);
 906
 907         if (!(flags & LIGHTREC_NO_HI))
 908                 lightrec_free_reg(reg_cache, hi);
 909 }
 910
 911 static void rec_special_MULT(struct lightrec_cstate *state,
 912                              const struct block *block, u16 offset)
 913 {
 914         _jit_name(block->_jit, __func__);
 915         rec_alu_mult(state, block, offset, true);
 916 }
 917
 918 static void rec_special_MULTU(struct lightrec_cstate *state,
 919                               const struct block *block, u16 offset)
 920 {
 921         _jit_name(block->_jit, __func__);
 922         rec_alu_mult(state, block, offset, false);
 923 }
 924
 925 static void rec_special_DIV(struct lightrec_cstate *state,
 926                             const struct block *block, u16 offset)
 927 {
 928         _jit_name(block->_jit, __func__);
 929         rec_alu_div(state, block, offset, true);
 930 }
 931
 932 static void rec_special_DIVU(struct lightrec_cstate *state,
 933                              const struct block *block, u16 offset)
 934 {
 935         _jit_name(block->_jit, __func__);
 936         rec_alu_div(state, block, offset, false);
 937 }
 938
 939 static void rec_alu_mv_lo_hi(struct lightrec_cstate *state,
 940                              const struct block *block, u8 dst, u8 src)
 941 {
 942         struct regcache *reg_cache = state->reg_cache;
 943         jit_state_t *_jit = block->_jit;
 944
 945         jit_note(__FILE__, __LINE__);
 946         src = lightrec_alloc_reg_in(reg_cache, _jit, src, 0);
 947         dst = lightrec_alloc_reg_out(reg_cache, _jit, dst, REG_EXT);
 948
 949         jit_extr_i(dst, src);
 950
 951         lightrec_free_reg(reg_cache, src);
 952         lightrec_free_reg(reg_cache, dst);
 953 }
 954
 955 static void rec_special_MFHI(struct lightrec_cstate *state,
 956                              const struct block *block, u16 offset)
 957 {
 958         union code c = block->opcode_list[offset].c;
 959
 960         _jit_name(block->_jit, __func__);
 961         rec_alu_mv_lo_hi(state, block, c.r.rd, REG_HI);
 962 }
 963
 964 static void rec_special_MTHI(struct lightrec_cstate *state,
 965                              const struct block *block, u16 offset)
 966 {
 967         union code c = block->opcode_list[offset].c;
 968
 969         _jit_name(block->_jit, __func__);
 970         rec_alu_mv_lo_hi(state, block, REG_HI, c.r.rs);
 971 }
 972
 973 static void rec_special_MFLO(struct lightrec_cstate *state,
 974                              const struct block *block, u16 offset)
 975 {
 976         union code c = block->opcode_list[offset].c;
 977
 978         _jit_name(block->_jit, __func__);
 979         rec_alu_mv_lo_hi(state, block, c.r.rd, REG_LO);
 980 }
 981
 982 static void rec_special_MTLO(struct lightrec_cstate *state,
 983                              const struct block *block, u16 offset)
 984 {
 985         union code c = block->opcode_list[offset].c;
 986
 987         _jit_name(block->_jit, __func__);
 988         rec_alu_mv_lo_hi(state, block, REG_LO, c.r.rs);
 989 }
 990
 991 static void call_to_c_wrapper(struct lightrec_cstate *state, const struct block *block,
 992                               u32 arg, bool with_arg, enum c_wrappers wrapper)
 993 {
 994         struct regcache *reg_cache = state->reg_cache;
 995         jit_state_t *_jit = block->_jit;
 996         u8 tmp, tmp2;
 997
 998         tmp = lightrec_alloc_reg_temp(reg_cache, _jit);
 999         jit_ldxi(tmp, LIGHTREC_REG_STATE,
1000                  offsetof(struct lightrec_state, wrappers_eps[wrapper]));
1001
1002         if (with_arg) {
1003                 tmp2 = lightrec_alloc_reg_temp(reg_cache, _jit);
1004                 jit_movi(tmp2, arg);
1005
1006                 jit_stxi_i(offsetof(struct lightrec_state, c_wrapper_arg),
1007                            LIGHTREC_REG_STATE, tmp2);
1008
1009                 lightrec_free_reg(reg_cache, tmp2);
1010         }
1011
1012         lightrec_regcache_mark_live(reg_cache, _jit);
1013         jit_callr(tmp);
1014
1015         lightrec_free_reg(reg_cache, tmp);
1016         lightrec_regcache_mark_live(reg_cache, _jit);
1017 }
1018
1019 static void rec_io(struct lightrec_cstate *state,
1020                    const struct block *block, u16 offset,
1021                    bool load_rt, bool read_rt)
1022 {
1023         struct regcache *reg_cache = state->reg_cache;
1024         jit_state_t *_jit = block->_jit;
1025         union code c = block->opcode_list[offset].c;
1026         u16 flags = block->opcode_list[offset].flags;
1027         bool is_tagged = LIGHTREC_FLAGS_GET_IO_MODE(flags);
1028         u32 lut_entry;
1029
1030         jit_note(__FILE__, __LINE__);
1031
1032         lightrec_clean_reg_if_loaded(reg_cache, _jit, c.i.rs, false);
1033
1034         if (read_rt && likely(c.i.rt))
1035                 lightrec_clean_reg_if_loaded(reg_cache, _jit, c.i.rt, true);
1036         else if (load_rt)
1037                 lightrec_clean_reg_if_loaded(reg_cache, _jit, c.i.rt, false);
1038
1039         if (is_tagged) {
1040                 call_to_c_wrapper(state, block, c.opcode, true, C_WRAPPER_RW);
1041         } else {
1042                 lut_entry = lightrec_get_lut_entry(block);
1043                 call_to_c_wrapper(state, block, (lut_entry << 16) | offset,
1044                                   true, C_WRAPPER_RW_GENERIC);
1045         }
1046 }
1047
1048 static u32 rec_ram_mask(struct lightrec_state *state)
1049 {
1050         return (RAM_SIZE << (state->mirrors_mapped * 2)) - 1;
1051 }
1052
1053 static void rec_store_memory(struct lightrec_cstate *cstate,
1054                              const struct block *block,
1055                              u16 offset, jit_code_t code,
1056                              jit_code_t swap_code,
1057                              uintptr_t addr_offset, u32 addr_mask,
1058                              bool invalidate)
1059 {
1060         const struct lightrec_state *state = cstate->state;
1061         struct regcache *reg_cache = cstate->reg_cache;
1062         struct opcode *op = &block->opcode_list[offset];
1063         jit_state_t *_jit = block->_jit;
1064         union code c = op->c;
1065         u8 rs, rt, tmp, tmp2, tmp3, addr_reg, addr_reg2;
1066         s16 imm = (s16)c.i.imm;
1067         s32 simm = (s32)imm << (1 - lut_is_32bit(state));
1068         s32 lut_offt = offsetof(struct lightrec_state, code_lut);
1069         bool no_mask = op->flags & LIGHTREC_NO_MASK;
1070         bool add_imm = c.i.imm &&
1071                 ((!state->mirrors_mapped && !no_mask) || (invalidate &&
1072                 ((imm & 0x3) || simm + lut_offt != (s16)(simm + lut_offt))));
1073         bool need_tmp = !no_mask || addr_offset || add_imm;
1074         bool need_tmp2 = addr_offset || invalidate;
1075
1076         rt = lightrec_alloc_reg_in(reg_cache, _jit, c.i.rt, 0);
1077         rs = lightrec_alloc_reg_in(reg_cache, _jit, c.i.rs, 0);
1078         if (need_tmp)
1079                 tmp = lightrec_alloc_reg_temp(reg_cache, _jit);
1080
1081         addr_reg = rs;
1082
1083         if (add_imm) {
1084                 jit_addi(tmp, addr_reg, (s16)c.i.imm);
1085                 addr_reg = tmp;
1086                 imm = 0;
1087         } else if (simm) {
1088                 lut_offt += simm;
1089         }
1090
1091         if (!no_mask) {
1092                 jit_andi(tmp, addr_reg, addr_mask);
1093                 addr_reg = tmp;
1094         }
1095
1096         if (need_tmp2)
1097                 tmp2 = lightrec_alloc_reg_temp(reg_cache, _jit);
1098
1099         if (addr_offset) {
1100                 jit_addi(tmp2, addr_reg, addr_offset);
1101                 addr_reg2 = tmp2;
1102         } else {
1103                 addr_reg2 = addr_reg;
1104         }
1105
1106         if (is_big_endian() && swap_code && c.i.rt) {
1107                 tmp3 = lightrec_alloc_reg_temp(reg_cache, _jit);
1108
1109                 jit_new_node_ww(swap_code, tmp3, rt);
1110                 jit_new_node_www(code, imm, addr_reg2, tmp3);
1111
1112                 lightrec_free_reg(reg_cache, tmp3);
1113         } else {
1114                 jit_new_node_www(code, imm, addr_reg2, rt);
1115         }
1116
1117         lightrec_free_reg(reg_cache, rt);
1118
1119         if (invalidate) {
1120                 tmp3 = lightrec_alloc_reg_in(reg_cache, _jit, 0, 0);
1121
1122                 if (c.i.op != OP_SW) {
1123                         jit_andi(tmp2, addr_reg, ~3);
1124                         addr_reg = tmp2;
1125                 }
1126
1127                 if (!lut_is_32bit(state)) {
1128                         jit_lshi(tmp2, addr_reg, 1);
1129                         addr_reg = tmp2;
1130                 }
1131
1132                 if (addr_reg == rs && c.i.rs == 0) {
1133                         addr_reg = LIGHTREC_REG_STATE;
1134                 } else {
1135                         jit_addr(tmp2, addr_reg, LIGHTREC_REG_STATE);
1136                         addr_reg = tmp2;
1137                 }
1138
1139                 if (lut_is_32bit(state))
1140                         jit_stxi_i(lut_offt, addr_reg, tmp3);
1141                 else
1142                         jit_stxi(lut_offt, addr_reg, tmp3);
1143
1144                 lightrec_free_reg(reg_cache, tmp3);
1145         }
1146
1147         if (need_tmp2)
1148                 lightrec_free_reg(reg_cache, tmp2);
1149         if (need_tmp)
1150                 lightrec_free_reg(reg_cache, tmp);
1151         lightrec_free_reg(reg_cache, rs);
1152 }
1153
1154 static void rec_store_ram(struct lightrec_cstate *cstate,
1155                           const struct block *block,
1156                           u16 offset, jit_code_t code,
1157                           jit_code_t swap_code, bool invalidate)
1158 {
1159         struct lightrec_state *state = cstate->state;
1160
1161         _jit_note(block->_jit, __FILE__, __LINE__);
1162
1163         return rec_store_memory(cstate, block, offset, code, swap_code,
1164                                 state->offset_ram, rec_ram_mask(state),
1165                                 invalidate);
1166 }
1167
1168 static void rec_store_scratch(struct lightrec_cstate *cstate,
1169                               const struct block *block, u16 offset,
1170                               jit_code_t code, jit_code_t swap_code)
1171 {
1172         _jit_note(block->_jit, __FILE__, __LINE__);
1173
1174         return rec_store_memory(cstate, block, offset, code, swap_code,
1175                                 cstate->state->offset_scratch,
1176                                 0x1fffffff, false);
1177 }
1178
1179 static void rec_store_direct_no_invalidate(struct lightrec_cstate *cstate,
1180                                            const struct block *block,
1181                                            u16 offset, jit_code_t code,
1182                                            jit_code_t swap_code)
1183 {
1184         struct lightrec_state *state = cstate->state;
1185         struct regcache *reg_cache = cstate->reg_cache;
1186         union code c = block->opcode_list[offset].c;
1187         jit_state_t *_jit = block->_jit;
1188         jit_node_t *to_not_ram, *to_end;
1189         u8 tmp, tmp2, rs, rt;
1190         s16 imm;
1191
1192         jit_note(__FILE__, __LINE__);
1193         rs = lightrec_alloc_reg_in(reg_cache, _jit, c.i.rs, 0);
1194         tmp = lightrec_alloc_reg_temp(reg_cache, _jit);
1195
1196         if (state->offset_ram || state->offset_scratch)
1197                 tmp2 = lightrec_alloc_reg_temp(reg_cache, _jit);
1198
1199         /* Convert to KUNSEG and avoid RAM mirrors */
1200         if (state->mirrors_mapped) {
1201                 imm = (s16)c.i.imm;
1202                 jit_andi(tmp, rs, 0x1f800000 | (4 * RAM_SIZE - 1));
1203         } else if (c.i.imm) {
1204                 imm = 0;
1205                 jit_addi(tmp, rs, (s16)c.i.imm);
1206                 jit_andi(tmp, tmp, 0x1f800000 | (RAM_SIZE - 1));
1207         } else {
1208                 imm = 0;
1209                 jit_andi(tmp, rs, 0x1f800000 | (RAM_SIZE - 1));
1210         }
1211
1212         lightrec_free_reg(reg_cache, rs);
1213
1214         if (state->offset_ram != state->offset_scratch) {
1215                 to_not_ram = jit_bmsi(tmp, BIT(28));
1216
1217                 jit_movi(tmp2, state->offset_ram);
1218
1219                 to_end = jit_b();
1220                 jit_patch(to_not_ram);
1221
1222                 jit_movi(tmp2, state->offset_scratch);
1223                 jit_patch(to_end);
1224         } else if (state->offset_ram) {
1225                 jit_movi(tmp2, state->offset_ram);
1226         }
1227
1228         if (state->offset_ram || state->offset_scratch) {
1229                 jit_addr(tmp, tmp, tmp2);
1230                 lightrec_free_reg(reg_cache, tmp2);
1231         }
1232
1233         rt = lightrec_alloc_reg_in(reg_cache, _jit, c.i.rt, 0);
1234
1235         if (is_big_endian() && swap_code && c.i.rt) {
1236                 tmp2 = lightrec_alloc_reg_temp(reg_cache, _jit);
1237
1238                 jit_new_node_ww(swap_code, tmp2, rt);
1239                 jit_new_node_www(code, imm, tmp, tmp2);
1240
1241                 lightrec_free_reg(reg_cache, tmp2);
1242         } else {
1243                 jit_new_node_www(code, imm, tmp, rt);
1244         }
1245
1246         lightrec_free_reg(reg_cache, rt);
1247         lightrec_free_reg(reg_cache, tmp);
1248 }
1249
1250 static void rec_store_direct(struct lightrec_cstate *cstate, const struct block *block,
1251                              u16 offset, jit_code_t code, jit_code_t swap_code)
1252 {
1253         struct lightrec_state *state = cstate->state;
1254         u32 ram_size = state->mirrors_mapped ? RAM_SIZE * 4 : RAM_SIZE;
1255         struct regcache *reg_cache = cstate->reg_cache;
1256         union code c = block->opcode_list[offset].c;
1257         jit_state_t *_jit = block->_jit;
1258         jit_node_t *to_not_ram, *to_end;
1259         u8 tmp, tmp2, tmp3, rs, rt;
1260
1261         jit_note(__FILE__, __LINE__);
1262
1263         rs = lightrec_alloc_reg_in(reg_cache, _jit, c.i.rs, 0);
1264         tmp2 = lightrec_alloc_reg_temp(reg_cache, _jit);
1265         tmp3 = lightrec_alloc_reg_in(reg_cache, _jit, 0, 0);
1266
1267         /* Convert to KUNSEG and avoid RAM mirrors */
1268         if (c.i.imm) {
1269                 jit_addi(tmp2, rs, (s16)c.i.imm);
1270                 jit_andi(tmp2, tmp2, 0x1f800000 | (ram_size - 1));
1271         } else {
1272                 jit_andi(tmp2, rs, 0x1f800000 | (ram_size - 1));
1273         }
1274
1275         lightrec_free_reg(reg_cache, rs);
1276         tmp = lightrec_alloc_reg_temp(reg_cache, _jit);
1277
1278         to_not_ram = jit_bgti(tmp2, ram_size);
1279
1280         /* Compute the offset to the code LUT */
1281         jit_andi(tmp, tmp2, (RAM_SIZE - 1) & ~3);
1282         if (!lut_is_32bit(state))
1283                 jit_lshi(tmp, tmp, 1);
1284         jit_addr(tmp, LIGHTREC_REG_STATE, tmp);
1285
1286         /* Write NULL to the code LUT to invalidate any block that's there */
1287         if (lut_is_32bit(state))
1288                 jit_stxi_i(offsetof(struct lightrec_state, code_lut), tmp, tmp3);
1289         else
1290                 jit_stxi(offsetof(struct lightrec_state, code_lut), tmp, tmp3);
1291
1292         if (state->offset_ram != state->offset_scratch) {
1293                 jit_movi(tmp, state->offset_ram);
1294
1295                 to_end = jit_b();
1296         }
1297
1298         jit_patch(to_not_ram);
1299
1300         if (state->offset_ram || state->offset_scratch)
1301                 jit_movi(tmp, state->offset_scratch);
1302
1303         if (state->offset_ram != state->offset_scratch)
1304                 jit_patch(to_end);
1305
1306         if (state->offset_ram || state->offset_scratch)
1307                 jit_addr(tmp2, tmp2, tmp);
1308
1309         lightrec_free_reg(reg_cache, tmp);
1310         lightrec_free_reg(reg_cache, tmp3);
1311
1312         rt = lightrec_alloc_reg_in(reg_cache, _jit, c.i.rt, 0);
1313
1314         if (is_big_endian() && swap_code && c.i.rt) {
1315                 tmp = lightrec_alloc_reg_temp(reg_cache, _jit);
1316
1317                 jit_new_node_ww(swap_code, tmp, rt);
1318                 jit_new_node_www(code, 0, tmp2, tmp);
1319
1320                 lightrec_free_reg(reg_cache, tmp);
1321         } else {
1322                 jit_new_node_www(code, 0, tmp2, rt);
1323         }
1324
1325         lightrec_free_reg(reg_cache, rt);
1326         lightrec_free_reg(reg_cache, tmp2);
1327 }
1328
1329 static void rec_store(struct lightrec_cstate *state,
1330                       const struct block *block, u16 offset,
1331                       jit_code_t code, jit_code_t swap_code)
1332 {
1333         u16 flags = block->opcode_list[offset].flags;
1334         bool no_invalidate = (flags & LIGHTREC_NO_INVALIDATE) ||
1335                 state->state->invalidate_from_dma_only;
1336
1337         switch (LIGHTREC_FLAGS_GET_IO_MODE(flags)) {
1338         case LIGHTREC_IO_RAM:
1339                 rec_store_ram(state, block, offset, code,
1340                               swap_code, !no_invalidate);
1341                 break;
1342         case LIGHTREC_IO_SCRATCH:
1343                 rec_store_scratch(state, block, offset, code, swap_code);
1344                 break;
1345         case LIGHTREC_IO_DIRECT:
1346                 if (no_invalidate) {
1347                         rec_store_direct_no_invalidate(state, block, offset,
1348                                                        code, swap_code);
1349                 } else {
1350                         rec_store_direct(state, block, offset, code, swap_code);
1351                 }
1352                 break;
1353         default:
1354                 rec_io(state, block, offset, true, false);
1355                 break;
1356         }
1357 }
1358
1359 static void rec_SB(struct lightrec_cstate *state,
1360                    const struct block *block, u16 offset)
1361 {
1362         _jit_name(block->_jit, __func__);
1363         rec_store(state, block, offset, jit_code_stxi_c, 0);
1364 }
1365
1366 static void rec_SH(struct lightrec_cstate *state,
1367                    const struct block *block, u16 offset)
1368 {
1369         _jit_name(block->_jit, __func__);
1370         rec_store(state, block, offset,
1371                   jit_code_stxi_s, jit_code_bswapr_us);
1372 }
1373
1374 static void rec_SW(struct lightrec_cstate *state,
1375                    const struct block *block, u16 offset)
1376
1377 {
1378         _jit_name(block->_jit, __func__);
1379         rec_store(state, block, offset,
1380                   jit_code_stxi_i, jit_code_bswapr_ui);
1381 }
1382
1383 static void rec_SWL(struct lightrec_cstate *state,
1384                     const struct block *block, u16 offset)
1385 {
1386         _jit_name(block->_jit, __func__);
1387         rec_io(state, block, offset, true, false);
1388 }
1389
1390 static void rec_SWR(struct lightrec_cstate *state,
1391                     const struct block *block, u16 offset)
1392 {
1393         _jit_name(block->_jit, __func__);
1394         rec_io(state, block, offset, true, false);
1395 }
1396
1397 static void rec_SWC2(struct lightrec_cstate *state,
1398                      const struct block *block, u16 offset)
1399 {
1400         _jit_name(block->_jit, __func__);
1401         rec_io(state, block, offset, false, false);
1402 }
1403
1404 static void rec_load_memory(struct lightrec_cstate *cstate,
1405                             const struct block *block, u16 offset,
1406                             jit_code_t code, jit_code_t swap_code, bool is_unsigned,
1407                             uintptr_t addr_offset, u32 addr_mask)
1408 {
1409         struct regcache *reg_cache = cstate->reg_cache;
1410         struct opcode *op = &block->opcode_list[offset];
1411         jit_state_t *_jit = block->_jit;
1412         u8 rs, rt, addr_reg, flags = REG_EXT;
1413         bool no_mask = op->flags & LIGHTREC_NO_MASK;
1414         union code c = op->c;
1415         s16 imm;
1416
1417         if (!c.i.rt)
1418                 return;
1419
1420         if (is_unsigned)
1421                 flags |= REG_ZEXT;
1422
1423         rs = lightrec_alloc_reg_in(reg_cache, _jit, c.i.rs, 0);
1424         rt = lightrec_alloc_reg_out(reg_cache, _jit, c.i.rt, flags);
1425
1426         if (!cstate->state->mirrors_mapped && c.i.imm && !no_mask) {
1427                 jit_addi(rt, rs, (s16)c.i.imm);
1428                 addr_reg = rt;
1429                 imm = 0;
1430         } else {
1431                 addr_reg = rs;
1432                 imm = (s16)c.i.imm;
1433         }
1434
1435         if (!no_mask) {
1436                 jit_andi(rt, addr_reg, addr_mask);
1437                 addr_reg = rt;
1438         }
1439
1440         if (addr_offset) {
1441                 jit_addi(rt, addr_reg, addr_offset);
1442                 addr_reg = rt;
1443         }
1444
1445         jit_new_node_www(code, rt, addr_reg, imm);
1446
1447         if (is_big_endian() && swap_code) {
1448                 jit_new_node_ww(swap_code, rt, rt);
1449
1450                 if (c.i.op == OP_LH)
1451                         jit_extr_s(rt, rt);
1452                 else if (c.i.op == OP_LW && __WORDSIZE == 64)
1453                         jit_extr_i(rt, rt);
1454         }
1455
1456         lightrec_free_reg(reg_cache, rs);
1457         lightrec_free_reg(reg_cache, rt);
1458 }
1459
1460 static void rec_load_ram(struct lightrec_cstate *cstate,
1461                          const struct block *block, u16 offset,
1462                          jit_code_t code, jit_code_t swap_code, bool is_unsigned)
1463 {
1464         _jit_note(block->_jit, __FILE__, __LINE__);
1465
1466         rec_load_memory(cstate, block, offset, code, swap_code, is_unsigned,
1467                         cstate->state->offset_ram, rec_ram_mask(cstate->state));
1468 }
1469
1470 static void rec_load_bios(struct lightrec_cstate *cstate,
1471                           const struct block *block, u16 offset,
1472                           jit_code_t code, jit_code_t swap_code, bool is_unsigned)
1473 {
1474         _jit_note(block->_jit, __FILE__, __LINE__);
1475
1476         rec_load_memory(cstate, block, offset, code, swap_code, is_unsigned,
1477                         cstate->state->offset_bios, 0x1fffffff);
1478 }
1479
1480 static void rec_load_scratch(struct lightrec_cstate *cstate,
1481                              const struct block *block, u16 offset,
1482                              jit_code_t code, jit_code_t swap_code, bool is_unsigned)
1483 {
1484         _jit_note(block->_jit, __FILE__, __LINE__);
1485
1486         rec_load_memory(cstate, block, offset, code, swap_code, is_unsigned,
1487                         cstate->state->offset_scratch, 0x1fffffff);
1488 }
1489
1490 static void rec_load_direct(struct lightrec_cstate *cstate,
1491                             const struct block *block, u16 offset,
1492                             jit_code_t code, jit_code_t swap_code,
1493                             bool is_unsigned)
1494 {
1495         struct lightrec_state *state = cstate->state;
1496         struct regcache *reg_cache = cstate->reg_cache;
1497         union code c = block->opcode_list[offset].c;
1498         jit_state_t *_jit = block->_jit;
1499         jit_node_t *to_not_ram, *to_not_bios, *to_end, *to_end2;
1500         u8 tmp, rs, rt, addr_reg, flags = REG_EXT;
1501         s16 imm;
1502
1503         if (!c.i.rt)
1504                 return;
1505
1506         if (is_unsigned)
1507                 flags |= REG_ZEXT;
1508
1509         jit_note(__FILE__, __LINE__);
1510         rs = lightrec_alloc_reg_in(reg_cache, _jit, c.i.rs, 0);
1511         rt = lightrec_alloc_reg_out(reg_cache, _jit, c.i.rt, flags);
1512
1513         if ((state->offset_ram == state->offset_bios &&
1514             state->offset_ram == state->offset_scratch &&
1515             state->mirrors_mapped) || !c.i.imm) {
1516                 addr_reg = rs;
1517                 imm = (s16)c.i.imm;
1518         } else {
1519                 jit_addi(rt, rs, (s16)c.i.imm);
1520                 addr_reg = rt;
1521                 imm = 0;
1522
1523                 if (c.i.rs != c.i.rt)
1524                         lightrec_free_reg(reg_cache, rs);
1525         }
1526
1527         tmp = lightrec_alloc_reg_temp(reg_cache, _jit);
1528
1529         if (state->offset_ram == state->offset_bios &&
1530             state->offset_ram == state->offset_scratch) {
1531                 if (!state->mirrors_mapped) {
1532                         jit_andi(tmp, addr_reg, BIT(28));
1533                         jit_rshi_u(tmp, tmp, 28 - 22);
1534                         jit_ori(tmp, tmp, 0x1f800000 | (RAM_SIZE - 1));
1535                         jit_andr(rt, addr_reg, tmp);
1536                 } else {
1537                         jit_andi(rt, addr_reg, 0x1fffffff);
1538                 }
1539
1540                 if (state->offset_ram)
1541                         jit_movi(tmp, state->offset_ram);
1542         } else {
1543                 to_not_ram = jit_bmsi(addr_reg, BIT(28));
1544
1545                 /* Convert to KUNSEG and avoid RAM mirrors */
1546                 jit_andi(rt, addr_reg, RAM_SIZE - 1);
1547
1548                 if (state->offset_ram)
1549                         jit_movi(tmp, state->offset_ram);
1550
1551                 to_end = jit_b();
1552
1553                 jit_patch(to_not_ram);
1554
1555                 if (state->offset_bios != state->offset_scratch)
1556                         to_not_bios = jit_bmci(addr_reg, BIT(22));
1557
1558                 /* Convert to KUNSEG */
1559                 jit_andi(rt, addr_reg, 0x1fc00000 | (BIOS_SIZE - 1));
1560
1561                 jit_movi(tmp, state->offset_bios);
1562
1563                 if (state->offset_bios != state->offset_scratch) {
1564                         to_end2 = jit_b();
1565
1566                         jit_patch(to_not_bios);
1567
1568                         /* Convert to KUNSEG */
1569                         jit_andi(rt, addr_reg, 0x1f800fff);
1570
1571                         if (state->offset_scratch)
1572                                 jit_movi(tmp, state->offset_scratch);
1573
1574                         jit_patch(to_end2);
1575                 }
1576
1577                 jit_patch(to_end);
1578         }
1579
1580         if (state->offset_ram || state->offset_bios || state->offset_scratch)
1581                 jit_addr(rt, rt, tmp);
1582
1583         jit_new_node_www(code, rt, rt, imm);
1584
1585         if (is_big_endian() && swap_code) {
1586                 jit_new_node_ww(swap_code, rt, rt);
1587
1588                 if (c.i.op == OP_LH)
1589                         jit_extr_s(rt, rt);
1590                 else if (c.i.op == OP_LW && __WORDSIZE == 64)
1591                         jit_extr_i(rt, rt);
1592         }
1593
1594         lightrec_free_reg(reg_cache, addr_reg);
1595         lightrec_free_reg(reg_cache, rt);
1596         lightrec_free_reg(reg_cache, tmp);
1597 }
1598
1599 static void rec_load(struct lightrec_cstate *state, const struct block *block,
1600                      u16 offset, jit_code_t code, jit_code_t swap_code,
1601                      bool is_unsigned)
1602 {
1603         u16 flags = block->opcode_list[offset].flags;
1604
1605         switch (LIGHTREC_FLAGS_GET_IO_MODE(flags)) {
1606         case LIGHTREC_IO_RAM:
1607                 rec_load_ram(state, block, offset, code, swap_code, is_unsigned);
1608                 break;
1609         case LIGHTREC_IO_BIOS:
1610                 rec_load_bios(state, block, offset, code, swap_code, is_unsigned);
1611                 break;
1612         case LIGHTREC_IO_SCRATCH:
1613                 rec_load_scratch(state, block, offset, code, swap_code, is_unsigned);
1614                 break;
1615         case LIGHTREC_IO_DIRECT:
1616                 rec_load_direct(state, block, offset, code, swap_code, is_unsigned);
1617                 break;
1618         default:
1619                 rec_io(state, block, offset, false, true);
1620                 break;
1621         }
1622 }
1623
1624 static void rec_LB(struct lightrec_cstate *state, const struct block *block, u16 offset)
1625 {
1626         _jit_name(block->_jit, __func__);
1627         rec_load(state, block, offset, jit_code_ldxi_c, 0, false);
1628 }
1629
1630 static void rec_LBU(struct lightrec_cstate *state, const struct block *block, u16 offset)
1631 {
1632         _jit_name(block->_jit, __func__);
1633         rec_load(state, block, offset, jit_code_ldxi_uc, 0, true);
1634 }
1635
1636 static void rec_LH(struct lightrec_cstate *state, const struct block *block, u16 offset)
1637 {
1638         _jit_name(block->_jit, __func__);
1639         rec_load(state, block, offset, jit_code_ldxi_s, jit_code_bswapr_us, false);
1640 }
1641
1642 static void rec_LHU(struct lightrec_cstate *state, const struct block *block, u16 offset)
1643 {
1644         _jit_name(block->_jit, __func__);
1645         rec_load(state, block, offset, jit_code_ldxi_us, jit_code_bswapr_us, true);
1646 }
1647
1648 static void rec_LWL(struct lightrec_cstate *state, const struct block *block, u16 offset)
1649 {
1650         _jit_name(block->_jit, __func__);
1651         rec_io(state, block, offset, true, true);
1652 }
1653
1654 static void rec_LWR(struct lightrec_cstate *state, const struct block *block, u16 offset)
1655 {
1656         _jit_name(block->_jit, __func__);
1657         rec_io(state, block, offset, true, true);
1658 }
1659
1660 static void rec_LW(struct lightrec_cstate *state, const struct block *block, u16 offset)
1661 {
1662         _jit_name(block->_jit, __func__);
1663         rec_load(state, block, offset, jit_code_ldxi_i, jit_code_bswapr_ui, false);
1664 }
1665
1666 static void rec_LWC2(struct lightrec_cstate *state, const struct block *block, u16 offset)
1667 {
1668         _jit_name(block->_jit, __func__);
1669         rec_io(state, block, offset, false, false);
1670 }
1671
1672 static void rec_break_syscall(struct lightrec_cstate *state,
1673                               const struct block *block, u16 offset, bool is_break)
1674 {
1675         _jit_note(block->_jit, __FILE__, __LINE__);
1676
1677         if (is_break)
1678                 call_to_c_wrapper(state, block, 0, false, C_WRAPPER_BREAK);
1679         else
1680                 call_to_c_wrapper(state, block, 0, false, C_WRAPPER_SYSCALL);
1681
1682         /* TODO: the return address should be "pc - 4" if we're a delay slot */
1683         lightrec_emit_end_of_block(state, block, offset, -1,
1684                                    get_ds_pc(block, offset, 0),
1685                                    31, 0, true);
1686 }
1687
1688 static void rec_special_SYSCALL(struct lightrec_cstate *state,
1689                                 const struct block *block, u16 offset)
1690 {
1691         _jit_name(block->_jit, __func__);
1692         rec_break_syscall(state, block, offset, false);
1693 }
1694
1695 static void rec_special_BREAK(struct lightrec_cstate *state,
1696                               const struct block *block, u16 offset)
1697 {
1698         _jit_name(block->_jit, __func__);
1699         rec_break_syscall(state, block, offset, true);
1700 }
1701
1702 static void rec_mtc(struct lightrec_cstate *state, const struct block *block, u16 offset)
1703 {
1704         struct regcache *reg_cache = state->reg_cache;
1705         union code c = block->opcode_list[offset].c;
1706         jit_state_t *_jit = block->_jit;
1707
1708         jit_note(__FILE__, __LINE__);
1709         lightrec_clean_reg_if_loaded(reg_cache, _jit, c.i.rs, false);
1710         lightrec_clean_reg_if_loaded(reg_cache, _jit, c.i.rt, false);
1711
1712         call_to_c_wrapper(state, block, c.opcode, true, C_WRAPPER_MTC);
1713
1714         if (c.i.op == OP_CP0 &&
1715             !(block->opcode_list[offset].flags & LIGHTREC_NO_DS) &&
1716             (c.r.rd == 12 || c.r.rd == 13))
1717                 lightrec_emit_end_of_block(state, block, offset, -1,
1718                                            get_ds_pc(block, offset, 1),
1719                                            0, 0, true);
1720 }
1721
1722 static void
1723 rec_mfc0(struct lightrec_cstate *state, const struct block *block, u16 offset)
1724 {
1725         struct regcache *reg_cache = state->reg_cache;
1726         union code c = block->opcode_list[offset].c;
1727         jit_state_t *_jit = block->_jit;
1728         u8 rt;
1729
1730         jit_note(__FILE__, __LINE__);
1731
1732         rt = lightrec_alloc_reg_out(reg_cache, _jit, c.i.rt, REG_EXT);
1733
1734         jit_ldxi_i(rt, LIGHTREC_REG_STATE,
1735                    offsetof(struct lightrec_state, regs.cp0[c.r.rd]));
1736
1737         lightrec_free_reg(reg_cache, rt);
1738 }
1739
1740 static bool block_in_bios(const struct lightrec_cstate *state,
1741                           const struct block *block)
1742 {
1743         const struct lightrec_mem_map *bios = &state->state->maps[PSX_MAP_BIOS];
1744         u32 pc = kunseg(block->pc);
1745
1746         return pc >= bios->pc && pc < bios->pc + bios->length;
1747 }
1748
1749 static void
1750 rec_mtc0(struct lightrec_cstate *state, const struct block *block, u16 offset)
1751 {
1752         struct regcache *reg_cache = state->reg_cache;
1753         const union code c = block->opcode_list[offset].c;
1754         jit_state_t *_jit = block->_jit;
1755         u8 rt, tmp = 0, tmp2, status;
1756
1757         jit_note(__FILE__, __LINE__);
1758
1759         switch(c.r.rd) {
1760         case 1:
1761         case 4:
1762         case 8:
1763         case 14:
1764         case 15:
1765                 /* Those registers are read-only */
1766                 return;
1767         default:
1768                 break;
1769         }
1770
1771         if (block_in_bios(state, block) && c.r.rd == 12) {
1772                 /* If we are running code from the BIOS, handle writes to the
1773                  * Status register in C. BIOS code may toggle bit 16 which will
1774                  * map/unmap the RAM, while game code cannot do that. */
1775                 rec_mtc(state, block, offset);
1776                 return;
1777         }
1778
1779         rt = lightrec_alloc_reg_in(reg_cache, _jit, c.i.rt, 0);
1780
1781         if (c.r.rd != 13) {
1782                 jit_stxi_i(offsetof(struct lightrec_state, regs.cp0[c.r.rd]),
1783                            LIGHTREC_REG_STATE, rt);
1784         }
1785
1786         if (c.r.rd == 12 || c.r.rd == 13) {
1787                 tmp = lightrec_alloc_reg_temp(reg_cache, _jit);
1788                 jit_ldxi_i(tmp, LIGHTREC_REG_STATE,
1789                            offsetof(struct lightrec_state, regs.cp0[13]));
1790
1791                 tmp2 = lightrec_alloc_reg_temp(reg_cache, _jit);
1792         }
1793
1794         if (c.r.rd == 12) {
1795                 status = rt;
1796         } else if (c.r.rd == 13) {
1797                 /* Cause = (Cause & ~0x0300) | (value & 0x0300) */
1798                 jit_andi(tmp2, rt, 0x0300);
1799                 jit_ori(tmp, tmp, 0x0300);
1800                 jit_xori(tmp, tmp, 0x0300);
1801                 jit_orr(tmp, tmp, tmp2);
1802                 jit_ldxi_i(tmp2, LIGHTREC_REG_STATE,
1803                            offsetof(struct lightrec_state, regs.cp0[12]));
1804                 jit_stxi_i(offsetof(struct lightrec_state, regs.cp0[13]),
1805                            LIGHTREC_REG_STATE, tmp);
1806                 status = tmp2;
1807         }
1808
1809         if (c.r.rd == 12 || c.r.rd == 13) {
1810                 /* Exit dynarec in case there's a software interrupt.
1811                  * exit_flags = !!(status & tmp & 0x0300) & status; */
1812                 jit_andr(tmp, tmp, status);
1813                 jit_andi(tmp, tmp, 0x0300);
1814                 jit_nei(tmp, tmp, 0);
1815                 jit_andr(tmp, tmp, status);
1816         }
1817
1818         if (c.r.rd == 12) {
1819                 /* Exit dynarec in case we unmask a hardware interrupt.
1820                  * exit_flags = !(~status & 0x401) */
1821
1822                 jit_comr(tmp2, status);
1823                 jit_andi(tmp2, tmp2, 0x401);
1824                 jit_eqi(tmp2, tmp2, 0);
1825                 jit_orr(tmp, tmp, tmp2);
1826         }
1827
1828         if (c.r.rd == 12 || c.r.rd == 13) {
1829                 jit_stxi_i(offsetof(struct lightrec_state, exit_flags),
1830                            LIGHTREC_REG_STATE, tmp);
1831
1832                 lightrec_free_reg(reg_cache, tmp);
1833                 lightrec_free_reg(reg_cache, tmp2);
1834         }
1835
1836         lightrec_free_reg(reg_cache, rt);
1837
1838         if (!(block->opcode_list[offset].flags & LIGHTREC_NO_DS) &&
1839             (c.r.rd == 12 || c.r.rd == 13))
1840                 lightrec_emit_eob(state, block, offset + 1, true);
1841 }
1842
1843 static void rec_cp0_MFC0(struct lightrec_cstate *state,
1844                          const struct block *block, u16 offset)
1845 {
1846         _jit_name(block->_jit, __func__);
1847         rec_mfc0(state, block, offset);
1848 }
1849
1850 static void rec_cp0_CFC0(struct lightrec_cstate *state,
1851                          const struct block *block, u16 offset)
1852 {
1853         _jit_name(block->_jit, __func__);
1854         rec_mfc0(state, block, offset);
1855 }
1856
1857 static void rec_cp0_MTC0(struct lightrec_cstate *state,
1858                          const struct block *block, u16 offset)
1859 {
1860         _jit_name(block->_jit, __func__);
1861         rec_mtc0(state, block, offset);
1862 }
1863
1864 static void rec_cp0_CTC0(struct lightrec_cstate *state,
1865                          const struct block *block, u16 offset)
1866 {
1867         _jit_name(block->_jit, __func__);
1868         rec_mtc0(state, block, offset);
1869 }
1870
1871 static unsigned int cp2d_i_offset(u8 reg)
1872 {
1873         return offsetof(struct lightrec_state, regs.cp2d[reg]);
1874 }
1875
1876 static unsigned int cp2d_s_offset(u8 reg)
1877 {
1878         return cp2d_i_offset(reg) + is_big_endian() * 2;
1879 }
1880
1881 static unsigned int cp2c_i_offset(u8 reg)
1882 {
1883         return offsetof(struct lightrec_state, regs.cp2c[reg]);
1884 }
1885
1886 static unsigned int cp2c_s_offset(u8 reg)
1887 {
1888         return cp2c_i_offset(reg) + is_big_endian() * 2;
1889 }
1890
1891 static void rec_cp2_basic_MFC2(struct lightrec_cstate *state,
1892                                const struct block *block, u16 offset)
1893 {
1894         struct regcache *reg_cache = state->reg_cache;
1895         const union code c = block->opcode_list[offset].c;
1896         jit_state_t *_jit = block->_jit;
1897         const u32 zext_regs = 0x300f0080;
1898         u8 rt, tmp, tmp2, tmp3, out, flags;
1899         u8 reg = c.r.rd == 15 ? 14 : c.r.rd;
1900         unsigned int i;
1901
1902         _jit_name(block->_jit, __func__);
1903
1904         flags = (zext_regs & BIT(reg)) ? REG_ZEXT : REG_EXT;
1905         rt = lightrec_alloc_reg_out(reg_cache, _jit, c.r.rt, flags);
1906
1907         switch (reg) {
1908         case 1:
1909         case 3:
1910         case 5:
1911         case 8:
1912         case 9:
1913         case 10:
1914         case 11:
1915                 jit_ldxi_s(rt, LIGHTREC_REG_STATE, cp2d_s_offset(reg));
1916                 break;
1917         case 7:
1918         case 16:
1919         case 17:
1920         case 18:
1921         case 19:
1922                 jit_ldxi_us(rt, LIGHTREC_REG_STATE, cp2d_s_offset(reg));
1923                 break;
1924         case 28:
1925         case 29:
1926                 tmp = lightrec_alloc_reg_temp(reg_cache, _jit);
1927                 tmp2 = lightrec_alloc_reg_temp(reg_cache, _jit);
1928                 tmp3 = lightrec_alloc_reg_temp(reg_cache, _jit);
1929
1930                 for (i = 0; i < 3; i++) {
1931                         out = i == 0 ? rt : tmp;
1932
1933                         jit_ldxi_s(tmp, LIGHTREC_REG_STATE, cp2d_s_offset(9 + i));
1934                         jit_movi(tmp2, 0x1f);
1935                         jit_rshi(out, tmp, 7);
1936
1937                         jit_ltr(tmp3, tmp2, out);
1938                         jit_movnr(out, tmp2, tmp3);
1939
1940                         jit_gei(tmp2, out, 0);
1941                         jit_movzr(out, tmp2, tmp2);
1942
1943                         if (i > 0) {
1944                                 jit_lshi(tmp, tmp, 5 * i);
1945                                 jit_orr(rt, rt, tmp);
1946                         }
1947                 }
1948
1949
1950                 lightrec_free_reg(reg_cache, tmp);
1951                 lightrec_free_reg(reg_cache, tmp2);
1952                 lightrec_free_reg(reg_cache, tmp3);
1953                 break;
1954         default:
1955                 jit_ldxi_i(rt, LIGHTREC_REG_STATE, cp2d_i_offset(reg));
1956                 break;
1957         }
1958
1959         lightrec_free_reg(reg_cache, rt);
1960 }
1961
1962 static void rec_cp2_basic_CFC2(struct lightrec_cstate *state,
1963                                const struct block *block, u16 offset)
1964 {
1965         struct regcache *reg_cache = state->reg_cache;
1966         const union code c = block->opcode_list[offset].c;
1967         jit_state_t *_jit = block->_jit;
1968         u8 rt;
1969
1970         _jit_name(block->_jit, __func__);
1971
1972         switch (c.r.rd) {
1973         case 4:
1974         case 12:
1975         case 20:
1976         case 26:
1977         case 27:
1978         case 29:
1979         case 30:
1980                 rt = lightrec_alloc_reg_out(reg_cache, _jit, c.r.rt, REG_EXT);
1981                 jit_ldxi_s(rt, LIGHTREC_REG_STATE, cp2c_s_offset(c.r.rd));
1982                 break;
1983         default:
1984                 rt = lightrec_alloc_reg_out(reg_cache, _jit, c.r.rt, REG_ZEXT);
1985                 jit_ldxi_i(rt, LIGHTREC_REG_STATE, cp2c_i_offset(c.r.rd));
1986                 break;
1987         }
1988
1989         lightrec_free_reg(reg_cache, rt);
1990 }
1991
1992 static void rec_cp2_basic_MTC2(struct lightrec_cstate *state,
1993                                const struct block *block, u16 offset)
1994 {
1995         struct regcache *reg_cache = state->reg_cache;
1996         const union code c = block->opcode_list[offset].c;
1997         jit_state_t *_jit = block->_jit;
1998         jit_node_t *loop, *to_loop;
1999         u8 rt, tmp, tmp2, flags = 0;
2000
2001         _jit_name(block->_jit, __func__);
2002
2003         if (c.r.rd == 31)
2004                 return;
2005
2006         if (c.r.rd == 30)
2007                 flags |= REG_EXT;
2008
2009         rt = lightrec_alloc_reg_in(reg_cache, _jit, c.r.rt, flags);
2010
2011         switch (c.r.rd) {
2012         case 15:
2013                 tmp = lightrec_alloc_reg_temp(reg_cache, _jit);
2014                 jit_ldxi_i(tmp, LIGHTREC_REG_STATE, cp2d_i_offset(13));
2015
2016                 tmp2 = lightrec_alloc_reg_temp(reg_cache, _jit);
2017                 jit_ldxi_i(tmp2, LIGHTREC_REG_STATE, cp2d_i_offset(14));
2018
2019                 jit_stxi_i(cp2d_i_offset(12), LIGHTREC_REG_STATE, tmp);
2020                 jit_stxi_i(cp2d_i_offset(13), LIGHTREC_REG_STATE, tmp2);
2021                 jit_stxi_i(cp2d_i_offset(14), LIGHTREC_REG_STATE, rt);
2022
2023                 lightrec_free_reg(reg_cache, tmp);
2024                 lightrec_free_reg(reg_cache, tmp2);
2025                 break;
2026         case 28:
2027                 tmp = lightrec_alloc_reg_temp(reg_cache, _jit);
2028
2029                 jit_lshi(tmp, rt, 7);
2030                 jit_andi(tmp, tmp, 0xf80);
2031                 jit_stxi_s(cp2d_s_offset(9), LIGHTREC_REG_STATE, tmp);
2032
2033                 jit_lshi(tmp, rt, 2);
2034                 jit_andi(tmp, tmp, 0xf80);
2035                 jit_stxi_s(cp2d_s_offset(10), LIGHTREC_REG_STATE, tmp);
2036
2037                 jit_rshi(tmp, rt, 3);
2038                 jit_andi(tmp, tmp, 0xf80);
2039                 jit_stxi_s(cp2d_s_offset(11), LIGHTREC_REG_STATE, tmp);
2040
2041                 lightrec_free_reg(reg_cache, tmp);
2042                 break;
2043         case 30:
2044                 tmp = lightrec_alloc_reg_temp(reg_cache, _jit);
2045                 tmp2 = lightrec_alloc_reg_temp(reg_cache, _jit);
2046
2047                 /* if (rt < 0) rt = ~rt; */
2048                 jit_rshi(tmp, rt, 31);
2049                 jit_xorr(tmp, rt, tmp);
2050
2051                 /* We know the sign bit is 0. Left-shift by 1 to start the algorithm */
2052                 jit_lshi(tmp, tmp, 1);
2053                 jit_movi(tmp2, 33);
2054
2055                 /* Decrement tmp2 and right-shift the value by 1 until it equals zero */
2056                 loop = jit_label();
2057                 jit_subi(tmp2, tmp2, 1);
2058                 jit_rshi_u(tmp, tmp, 1);
2059                 to_loop = jit_bnei(tmp, 0);
2060
2061                 jit_patch_at(to_loop, loop);
2062
2063                 jit_stxi_i(cp2d_i_offset(31), LIGHTREC_REG_STATE, tmp2);
2064                 jit_stxi_i(cp2d_i_offset(30), LIGHTREC_REG_STATE, rt);
2065
2066                 lightrec_free_reg(reg_cache, tmp);
2067                 lightrec_free_reg(reg_cache, tmp2);
2068                 break;
2069         default:
2070                 jit_stxi_i(cp2d_i_offset(c.r.rd), LIGHTREC_REG_STATE, rt);
2071                 break;
2072         }
2073
2074         lightrec_free_reg(reg_cache, rt);
2075 }
2076
2077 static void rec_cp2_basic_CTC2(struct lightrec_cstate *state,
2078                                const struct block *block, u16 offset)
2079 {
2080         struct regcache *reg_cache = state->reg_cache;
2081         const union code c = block->opcode_list[offset].c;
2082         jit_state_t *_jit = block->_jit;
2083         u8 rt, tmp, tmp2;
2084
2085         _jit_name(block->_jit, __func__);
2086
2087         rt = lightrec_alloc_reg_in(reg_cache, _jit, c.r.rt, 0);
2088
2089         switch (c.r.rd) {
2090         case 4:
2091         case 12:
2092         case 20:
2093         case 26:
2094         case 27:
2095         case 29:
2096         case 30:
2097                 jit_stxi_s(cp2c_s_offset(c.r.rd), LIGHTREC_REG_STATE, rt);
2098                 break;
2099         case 31:
2100                 tmp = lightrec_alloc_reg_temp(reg_cache, _jit);
2101                 tmp2 = lightrec_alloc_reg_temp(reg_cache, _jit);
2102
2103                 jit_andi(tmp, rt, 0x7f87e000);
2104                 jit_nei(tmp, tmp, 0);
2105                 jit_lshi(tmp, tmp, 31);
2106
2107                 jit_andi(tmp2, rt, 0x7ffff000);
2108                 jit_orr(tmp, tmp2, tmp);
2109
2110                 jit_stxi_i(cp2c_i_offset(31), LIGHTREC_REG_STATE, tmp);
2111
2112                 lightrec_free_reg(reg_cache, tmp);
2113                 lightrec_free_reg(reg_cache, tmp2);
2114                 break;
2115
2116         default:
2117                 jit_stxi_i(cp2c_i_offset(c.r.rd), LIGHTREC_REG_STATE, rt);
2118         }
2119
2120         lightrec_free_reg(reg_cache, rt);
2121 }
2122
2123 static void rec_cp0_RFE(struct lightrec_cstate *state,
2124                         const struct block *block, u16 offset)
2125 {
2126         struct regcache *reg_cache = state->reg_cache;
2127         jit_state_t *_jit = block->_jit;
2128         u8 status, tmp;
2129
2130         jit_name(__func__);
2131         jit_note(__FILE__, __LINE__);
2132
2133         status = lightrec_alloc_reg_temp(reg_cache, _jit);
2134         jit_ldxi_i(status, LIGHTREC_REG_STATE,
2135                    offsetof(struct lightrec_state, regs.cp0[12]));
2136
2137         tmp = lightrec_alloc_reg_temp(reg_cache, _jit);
2138
2139         /* status = ((status >> 2) & 0xf) | status & ~0xf; */
2140         jit_rshi(tmp, status, 2);
2141         jit_andi(tmp, tmp, 0xf);
2142         jit_andi(status, status, ~0xful);
2143         jit_orr(status, status, tmp);
2144
2145         jit_ldxi_i(tmp, LIGHTREC_REG_STATE,
2146                    offsetof(struct lightrec_state, regs.cp0[13]));
2147         jit_stxi_i(offsetof(struct lightrec_state, regs.cp0[12]),
2148                    LIGHTREC_REG_STATE, status);
2149
2150         /* Exit dynarec in case there's a software interrupt.
2151          * exit_flags = !!(status & cause & 0x0300) & status; */
2152         jit_andr(tmp, tmp, status);
2153         jit_andi(tmp, tmp, 0x0300);
2154         jit_nei(tmp, tmp, 0);
2155         jit_andr(tmp, tmp, status);
2156         jit_stxi_i(offsetof(struct lightrec_state, exit_flags),
2157                    LIGHTREC_REG_STATE, tmp);
2158
2159         lightrec_free_reg(reg_cache, status);
2160         lightrec_free_reg(reg_cache, tmp);
2161 }
2162
2163 static void rec_CP(struct lightrec_cstate *state,
2164                    const struct block *block, u16 offset)
2165 {
2166         union code c = block->opcode_list[offset].c;
2167         jit_state_t *_jit = block->_jit;
2168
2169         jit_name(__func__);
2170         jit_note(__FILE__, __LINE__);
2171
2172         call_to_c_wrapper(state, block, c.opcode, true, C_WRAPPER_CP);
2173 }
2174
2175 static void rec_meta_MOV(struct lightrec_cstate *state,
2176                          const struct block *block, u16 offset)
2177 {
2178         struct regcache *reg_cache = state->reg_cache;
2179         union code c = block->opcode_list[offset].c;
2180         jit_state_t *_jit = block->_jit;
2181         u8 rs, rd;
2182
2183         _jit_name(block->_jit, __func__);
2184         jit_note(__FILE__, __LINE__);
2185         if (c.r.rs)
2186                 rs = lightrec_alloc_reg_in(reg_cache, _jit, c.r.rs, 0);
2187         rd = lightrec_alloc_reg_out(reg_cache, _jit, c.r.rd, REG_EXT);
2188
2189         if (c.r.rs == 0)
2190                 jit_movi(rd, 0);
2191         else
2192                 jit_extr_i(rd, rs);
2193
2194         if (c.r.rs)
2195                 lightrec_free_reg(reg_cache, rs);
2196         lightrec_free_reg(reg_cache, rd);
2197 }
2198
2199 static void rec_meta_EXTC_EXTS(struct lightrec_cstate *state,
2200                                const struct block *block,
2201                                u16 offset)
2202 {
2203         struct regcache *reg_cache = state->reg_cache;
2204         union code c = block->opcode_list[offset].c;
2205         jit_state_t *_jit = block->_jit;
2206         u8 rs, rt;
2207
2208         _jit_name(block->_jit, __func__);
2209         jit_note(__FILE__, __LINE__);
2210
2211         rs = lightrec_alloc_reg_in(reg_cache, _jit, c.i.rs, 0);
2212         rt = lightrec_alloc_reg_out(reg_cache, _jit, c.i.rt, REG_EXT);
2213
2214         if (c.i.op == OP_META_EXTC)
2215                 jit_extr_c(rt, rs);
2216         else
2217                 jit_extr_s(rt, rs);
2218
2219         lightrec_free_reg(reg_cache, rs);
2220         lightrec_free_reg(reg_cache, rt);
2221 }
2222
2223 static const lightrec_rec_func_t rec_standard[64] = {
2224         SET_DEFAULT_ELM(rec_standard, unknown_opcode),
2225         [OP_SPECIAL]            = rec_SPECIAL,
2226         [OP_REGIMM]             = rec_REGIMM,
2227         [OP_J]                  = rec_J,
2228         [OP_JAL]                = rec_JAL,
2229         [OP_BEQ]                = rec_BEQ,
2230         [OP_BNE]                = rec_BNE,
2231         [OP_BLEZ]               = rec_BLEZ,
2232         [OP_BGTZ]               = rec_BGTZ,
2233         [OP_ADDI]               = rec_ADDI,
2234         [OP_ADDIU]              = rec_ADDIU,
2235         [OP_SLTI]               = rec_SLTI,
2236         [OP_SLTIU]              = rec_SLTIU,
2237         [OP_ANDI]               = rec_ANDI,
2238         [OP_ORI]                = rec_ORI,
2239         [OP_XORI]               = rec_XORI,
2240         [OP_LUI]                = rec_LUI,
2241         [OP_CP0]                = rec_CP0,
2242         [OP_CP2]                = rec_CP2,
2243         [OP_LB]                 = rec_LB,
2244         [OP_LH]                 = rec_LH,
2245         [OP_LWL]                = rec_LWL,
2246         [OP_LW]                 = rec_LW,
2247         [OP_LBU]                = rec_LBU,
2248         [OP_LHU]                = rec_LHU,
2249         [OP_LWR]                = rec_LWR,
2250         [OP_SB]                 = rec_SB,
2251         [OP_SH]                 = rec_SH,
2252         [OP_SWL]                = rec_SWL,
2253         [OP_SW]                 = rec_SW,
2254         [OP_SWR]                = rec_SWR,
2255         [OP_LWC2]               = rec_LWC2,
2256         [OP_SWC2]               = rec_SWC2,
2257
2258         [OP_META_MOV]           = rec_meta_MOV,
2259         [OP_META_EXTC]          = rec_meta_EXTC_EXTS,
2260         [OP_META_EXTS]          = rec_meta_EXTC_EXTS,
2261 };
2262
2263 static const lightrec_rec_func_t rec_special[64] = {
2264         SET_DEFAULT_ELM(rec_special, unknown_opcode),
2265         [OP_SPECIAL_SLL]        = rec_special_SLL,
2266         [OP_SPECIAL_SRL]        = rec_special_SRL,
2267         [OP_SPECIAL_SRA]        = rec_special_SRA,
2268         [OP_SPECIAL_SLLV]       = rec_special_SLLV,
2269         [OP_SPECIAL_SRLV]       = rec_special_SRLV,
2270         [OP_SPECIAL_SRAV]       = rec_special_SRAV,
2271         [OP_SPECIAL_JR]         = rec_special_JR,
2272         [OP_SPECIAL_JALR]       = rec_special_JALR,
2273         [OP_SPECIAL_SYSCALL]    = rec_special_SYSCALL,
2274         [OP_SPECIAL_BREAK]      = rec_special_BREAK,
2275         [OP_SPECIAL_MFHI]       = rec_special_MFHI,
2276         [OP_SPECIAL_MTHI]       = rec_special_MTHI,
2277         [OP_SPECIAL_MFLO]       = rec_special_MFLO,
2278         [OP_SPECIAL_MTLO]       = rec_special_MTLO,
2279         [OP_SPECIAL_MULT]       = rec_special_MULT,
2280         [OP_SPECIAL_MULTU]      = rec_special_MULTU,
2281         [OP_SPECIAL_DIV]        = rec_special_DIV,
2282         [OP_SPECIAL_DIVU]       = rec_special_DIVU,
2283         [OP_SPECIAL_ADD]        = rec_special_ADD,
2284         [OP_SPECIAL_ADDU]       = rec_special_ADDU,
2285         [OP_SPECIAL_SUB]        = rec_special_SUB,
2286         [OP_SPECIAL_SUBU]       = rec_special_SUBU,
2287         [OP_SPECIAL_AND]        = rec_special_AND,
2288         [OP_SPECIAL_OR]         = rec_special_OR,
2289         [OP_SPECIAL_XOR]        = rec_special_XOR,
2290         [OP_SPECIAL_NOR]        = rec_special_NOR,
2291         [OP_SPECIAL_SLT]        = rec_special_SLT,
2292         [OP_SPECIAL_SLTU]       = rec_special_SLTU,
2293 };
2294
2295 static const lightrec_rec_func_t rec_regimm[64] = {
2296         SET_DEFAULT_ELM(rec_regimm, unknown_opcode),
2297         [OP_REGIMM_BLTZ]        = rec_regimm_BLTZ,
2298         [OP_REGIMM_BGEZ]        = rec_regimm_BGEZ,
2299         [OP_REGIMM_BLTZAL]      = rec_regimm_BLTZAL,
2300         [OP_REGIMM_BGEZAL]      = rec_regimm_BGEZAL,
2301 };
2302
2303 static const lightrec_rec_func_t rec_cp0[64] = {
2304         SET_DEFAULT_ELM(rec_cp0, rec_CP),
2305         [OP_CP0_MFC0]           = rec_cp0_MFC0,
2306         [OP_CP0_CFC0]           = rec_cp0_CFC0,
2307         [OP_CP0_MTC0]           = rec_cp0_MTC0,
2308         [OP_CP0_CTC0]           = rec_cp0_CTC0,
2309         [OP_CP0_RFE]            = rec_cp0_RFE,
2310 };
2311
2312 static const lightrec_rec_func_t rec_cp2_basic[64] = {
2313         SET_DEFAULT_ELM(rec_cp2_basic, rec_CP),
2314         [OP_CP2_BASIC_MFC2]     = rec_cp2_basic_MFC2,
2315         [OP_CP2_BASIC_CFC2]     = rec_cp2_basic_CFC2,
2316         [OP_CP2_BASIC_MTC2]     = rec_cp2_basic_MTC2,
2317         [OP_CP2_BASIC_CTC2]     = rec_cp2_basic_CTC2,
2318 };
2319
2320 static void rec_SPECIAL(struct lightrec_cstate *state,
2321                         const struct block *block, u16 offset)
2322 {
2323         union code c = block->opcode_list[offset].c;
2324         lightrec_rec_func_t f = rec_special[c.r.op];
2325
2326         if (!HAS_DEFAULT_ELM && unlikely(!f))
2327                 unknown_opcode(state, block, offset);
2328         else
2329                 (*f)(state, block, offset);
2330 }
2331
2332 static void rec_REGIMM(struct lightrec_cstate *state,
2333                        const struct block *block, u16 offset)
2334 {
2335         union code c = block->opcode_list[offset].c;
2336         lightrec_rec_func_t f = rec_regimm[c.r.rt];
2337
2338         if (!HAS_DEFAULT_ELM && unlikely(!f))
2339                 unknown_opcode(state, block, offset);
2340         else
2341                 (*f)(state, block, offset);
2342 }
2343
2344 static void rec_CP0(struct lightrec_cstate *state,
2345                     const struct block *block, u16 offset)
2346 {
2347         union code c = block->opcode_list[offset].c;
2348         lightrec_rec_func_t f = rec_cp0[c.r.rs];
2349
2350         if (!HAS_DEFAULT_ELM && unlikely(!f))
2351                 rec_CP(state, block, offset);
2352         else
2353                 (*f)(state, block, offset);
2354 }
2355
2356 static void rec_CP2(struct lightrec_cstate *state,
2357                     const struct block *block, u16 offset)
2358 {
2359         union code c = block->opcode_list[offset].c;
2360
2361         if (c.r.op == OP_CP2_BASIC) {
2362                 lightrec_rec_func_t f = rec_cp2_basic[c.r.rs];
2363
2364                 if (HAS_DEFAULT_ELM || likely(f)) {
2365                         (*f)(state, block, offset);
2366                         return;
2367                 }
2368         }
2369
2370         rec_CP(state, block, offset);
2371 }
2372
2373 void lightrec_rec_opcode(struct lightrec_cstate *state,
2374                          const struct block *block, u16 offset)
2375 {
2376         struct regcache *reg_cache = state->reg_cache;
2377         struct lightrec_branch_target *target;
2378         const struct opcode *op = &block->opcode_list[offset];
2379         jit_state_t *_jit = block->_jit;
2380         lightrec_rec_func_t f;
2381
2382         if (op->flags & LIGHTREC_SYNC) {
2383                 jit_subi(LIGHTREC_REG_CYCLE, LIGHTREC_REG_CYCLE, state->cycles);
2384                 state->cycles = 0;
2385
2386                 lightrec_storeback_regs(reg_cache, _jit);
2387                 lightrec_regcache_reset(reg_cache);
2388
2389                 pr_debug("Adding branch target at offset 0x%x\n", offset << 2);
2390                 target = &state->targets[state->nb_targets++];
2391                 target->offset = offset;
2392                 target->label = jit_indirect();
2393         }
2394
2395         if (likely(op->opcode)) {
2396                 f = rec_standard[op->i.op];
2397
2398                 if (!HAS_DEFAULT_ELM && unlikely(!f))
2399                         unknown_opcode(state, block, offset);
2400                 else
2401                         (*f)(state, block, offset);
2402         }
2403
2404         if (unlikely(op->flags & LIGHTREC_UNLOAD_RD)) {
2405                 lightrec_clean_reg_if_loaded(reg_cache, _jit, op->r.rd, true);
2406                 pr_debug("Cleaning RD reg %s\n", lightrec_reg_name(op->r.rd));
2407         }
2408         if (unlikely(op->flags & LIGHTREC_UNLOAD_RS)) {
2409                 lightrec_clean_reg_if_loaded(reg_cache, _jit, op->i.rs, true);
2410                 pr_debug("Cleaning RS reg %s\n", lightrec_reg_name(op->i.rt));
2411         }
2412         if (unlikely(op->flags & LIGHTREC_UNLOAD_RT)) {
2413                 lightrec_clean_reg_if_loaded(reg_cache, _jit, op->i.rt, true);
2414                 pr_debug("Cleaning RT reg %s\n", lightrec_reg_name(op->i.rt));
2415         }
2416 }