deps/lightrec/emitter.c

   1 // SPDX-License-Identifier: LGPL-2.1-or-later
   2 /*
   3  * Copyright (C) 2014-2021 Paul Cercueil <paul@crapouillou.net>
   4  */
   5
   6 #include "blockcache.h"
   7 #include "debug.h"
   8 #include "disassembler.h"
   9 #include "emitter.h"
  10 #include "lightning-wrapper.h"
  11 #include "optimizer.h"
  12 #include "regcache.h"
  13
  14 #include <stdbool.h>
  15 #include <stddef.h>
  16
  17 typedef void (*lightrec_rec_func_t)(struct lightrec_cstate *, const struct block *, u16);
  18
  19 /* Forward declarations */
  20 static void rec_SPECIAL(struct lightrec_cstate *state, const struct block *block, u16 offset);
  21 static void rec_REGIMM(struct lightrec_cstate *state, const struct block *block, u16 offset);
  22 static void rec_CP0(struct lightrec_cstate *state, const struct block *block, u16 offset);
  23 static void rec_CP2(struct lightrec_cstate *state, const struct block *block, u16 offset);
  24
  25 static void unknown_opcode(struct lightrec_cstate *state, const struct block *block, u16 offset)
  26 {
  27         pr_warn("Unknown opcode: 0x%08x at PC 0x%08x\n",
  28                 block->opcode_list[offset].c.opcode,
  29                 block->pc + (offset << 2));
  30 }
  31
  32 static void lightrec_emit_end_of_block(struct lightrec_cstate *state,
  33                                        const struct block *block, u16 offset,
  34                                        s8 reg_new_pc, u32 imm, u8 ra_reg,
  35                                        u32 link, bool update_cycles)
  36 {
  37         struct regcache *reg_cache = state->reg_cache;
  38         jit_state_t *_jit = block->_jit;
  39         const struct opcode *op = &block->opcode_list[offset],
  40                             *next = &block->opcode_list[offset + 1];
  41         u32 cycles = state->cycles + lightrec_cycles_of_opcode(op->c);
  42         u16 offset_after_eob;
  43
  44         jit_note(__FILE__, __LINE__);
  45
  46         if (link) {
  47                 /* Update the $ra register */
  48                 u8 link_reg = lightrec_alloc_reg_out(reg_cache, _jit, ra_reg, 0);
  49                 jit_movi(link_reg, link);
  50                 lightrec_free_reg(reg_cache, link_reg);
  51         }
  52
  53         if (reg_new_pc < 0) {
  54                 reg_new_pc = lightrec_alloc_reg(reg_cache, _jit, JIT_V0);
  55                 lightrec_lock_reg(reg_cache, _jit, reg_new_pc);
  56
  57                 jit_movi(reg_new_pc, imm);
  58         }
  59
  60         if (has_delay_slot(op->c) &&
  61             !op_flag_no_ds(op->flags) && !op_flag_local_branch(op->flags)) {
  62                 cycles += lightrec_cycles_of_opcode(next->c);
  63
  64                 /* Recompile the delay slot */
  65                 if (next->c.opcode)
  66                         lightrec_rec_opcode(state, block, offset + 1);
  67         }
  68
  69         /* Clean the remaining registers */
  70         lightrec_clean_regs(reg_cache, _jit);
  71
  72         jit_movr(JIT_V0, reg_new_pc);
  73
  74         if (cycles && update_cycles) {
  75                 jit_subi(LIGHTREC_REG_CYCLE, LIGHTREC_REG_CYCLE, cycles);
  76                 pr_debug("EOB: %u cycles\n", cycles);
  77         }
  78
  79         offset_after_eob = offset + 1 +
  80                 (has_delay_slot(op->c) && !op_flag_no_ds(op->flags));
  81
  82         if (offset_after_eob < block->nb_ops)
  83                 state->branches[state->nb_branches++] = jit_b();
  84 }
  85
  86 void lightrec_emit_eob(struct lightrec_cstate *state, const struct block *block,
  87                        u16 offset, bool after_op)
  88 {
  89         struct regcache *reg_cache = state->reg_cache;
  90         jit_state_t *_jit = block->_jit;
  91         union code c = block->opcode_list[offset].c;
  92         u32 cycles = state->cycles;
  93
  94         if (after_op)
  95                 cycles += lightrec_cycles_of_opcode(c);
  96
  97         lightrec_clean_regs(reg_cache, _jit);
  98
  99         jit_movi(JIT_V0, block->pc + (offset << 2));
 100         jit_subi(LIGHTREC_REG_CYCLE, LIGHTREC_REG_CYCLE, cycles);
 101
 102         state->branches[state->nb_branches++] = jit_b();
 103 }
 104
 105 static u8 get_jr_jalr_reg(struct lightrec_cstate *state, const struct block *block, u16 offset)
 106 {
 107         struct regcache *reg_cache = state->reg_cache;
 108         jit_state_t *_jit = block->_jit;
 109         const struct opcode *op = &block->opcode_list[offset];
 110         u8 rs;
 111
 112         rs = lightrec_request_reg_in(reg_cache, _jit, op->r.rs, JIT_V0);
 113         lightrec_lock_reg(reg_cache, _jit, rs);
 114
 115         return rs;
 116 }
 117
 118 static void rec_special_JR(struct lightrec_cstate *state, const struct block *block, u16 offset)
 119 {
 120         u8 rs = get_jr_jalr_reg(state, block, offset);
 121
 122         _jit_name(block->_jit, __func__);
 123         lightrec_emit_end_of_block(state, block, offset, rs, 0, 31, 0, true);
 124 }
 125
 126 static void rec_special_JALR(struct lightrec_cstate *state, const struct block *block, u16 offset)
 127 {
 128         u8 rs = get_jr_jalr_reg(state, block, offset);
 129         union code c = block->opcode_list[offset].c;
 130
 131         _jit_name(block->_jit, __func__);
 132         lightrec_emit_end_of_block(state, block, offset, rs, 0, c.r.rd,
 133                                    get_branch_pc(block, offset, 2), true);
 134 }
 135
 136 static void rec_J(struct lightrec_cstate *state, const struct block *block, u16 offset)
 137 {
 138         union code c = block->opcode_list[offset].c;
 139
 140         _jit_name(block->_jit, __func__);
 141         lightrec_emit_end_of_block(state, block, offset, -1,
 142                                    (block->pc & 0xf0000000) | (c.j.imm << 2),
 143                                    31, 0, true);
 144 }
 145
 146 static void rec_JAL(struct lightrec_cstate *state, const struct block *block, u16 offset)
 147 {
 148         union code c = block->opcode_list[offset].c;
 149
 150         _jit_name(block->_jit, __func__);
 151         lightrec_emit_end_of_block(state, block, offset, -1,
 152                                    (block->pc & 0xf0000000) | (c.j.imm << 2),
 153                                    31, get_branch_pc(block, offset, 2), true);
 154 }
 155
 156 static void lightrec_do_early_unload(struct lightrec_cstate *state,
 157                                      const struct block *block, u16 offset)
 158 {
 159         struct regcache *reg_cache = state->reg_cache;
 160         const struct opcode *op = &block->opcode_list[offset];
 161         jit_state_t *_jit = block->_jit;
 162         unsigned int i;
 163         u8 reg;
 164         struct {
 165                 u8 reg, op;
 166         } reg_ops[3] = {
 167                 { op->r.rd, LIGHTREC_FLAGS_GET_RD(op->flags), },
 168                 { op->i.rt, LIGHTREC_FLAGS_GET_RT(op->flags), },
 169                 { op->i.rs, LIGHTREC_FLAGS_GET_RS(op->flags), },
 170         };
 171
 172         for (i = 0; i < ARRAY_SIZE(reg_ops); i++) {
 173                 reg = reg_ops[i].reg;
 174
 175                 switch (reg_ops[i].op) {
 176                 case LIGHTREC_REG_UNLOAD:
 177                         lightrec_clean_reg_if_loaded(reg_cache, _jit, reg, true);
 178                         break;
 179
 180                 case LIGHTREC_REG_DISCARD:
 181                         lightrec_discard_reg_if_loaded(reg_cache, reg);
 182                         break;
 183
 184                 case LIGHTREC_REG_CLEAN:
 185                         lightrec_clean_reg_if_loaded(reg_cache, _jit, reg, false);
 186                         break;
 187                 default:
 188                         break;
 189                 };
 190         }
 191 }
 192
 193 static void rec_b(struct lightrec_cstate *state, const struct block *block, u16 offset,
 194                   jit_code_t code, u32 link, bool unconditional, bool bz)
 195 {
 196         struct regcache *reg_cache = state->reg_cache;
 197         struct native_register *regs_backup;
 198         jit_state_t *_jit = block->_jit;
 199         struct lightrec_branch *branch;
 200         const struct opcode *op = &block->opcode_list[offset],
 201                             *next = &block->opcode_list[offset + 1];
 202         jit_node_t *addr;
 203         u8 link_reg, rs, rt;
 204         bool is_forward = (s16)op->i.imm >= -1;
 205         int op_cycles = lightrec_cycles_of_opcode(op->c);
 206         u32 target_offset, cycles = state->cycles + op_cycles;
 207         u32 next_pc;
 208
 209         jit_note(__FILE__, __LINE__);
 210
 211         if (!op_flag_no_ds(op->flags))
 212                 cycles += lightrec_cycles_of_opcode(next->c);
 213
 214         state->cycles = -op_cycles;
 215
 216         if (!unconditional) {
 217                 rs = lightrec_alloc_reg_in(reg_cache, _jit, op->i.rs, REG_EXT);
 218                 rt = bz ? 0 : lightrec_alloc_reg_in(reg_cache,
 219                                                     _jit, op->i.rt, REG_EXT);
 220
 221                 /* Unload dead registers before evaluating the branch */
 222                 if (OPT_EARLY_UNLOAD)
 223                         lightrec_do_early_unload(state, block, offset);
 224         }
 225
 226         if (cycles)
 227                 jit_subi(LIGHTREC_REG_CYCLE, LIGHTREC_REG_CYCLE, cycles);
 228
 229         if (!unconditional) {
 230                 /* Generate the branch opcode */
 231                 addr = jit_new_node_pww(code, NULL, rs, rt);
 232
 233                 lightrec_free_regs(reg_cache);
 234                 regs_backup = lightrec_regcache_enter_branch(reg_cache);
 235         }
 236
 237         if (op_flag_local_branch(op->flags)) {
 238                 /* Recompile the delay slot */
 239                 if (next && next->opcode && !op_flag_no_ds(op->flags))
 240                         lightrec_rec_opcode(state, block, offset + 1);
 241
 242                 if (link) {
 243                         /* Update the $ra register */
 244                         link_reg = lightrec_alloc_reg_out(reg_cache, _jit, 31, 0);
 245                         jit_movi(link_reg, link);
 246                         lightrec_free_reg(reg_cache, link_reg);
 247                 }
 248
 249                 /* Clean remaining registers */
 250                 lightrec_clean_regs(reg_cache, _jit);
 251
 252                 target_offset = offset + 1 + (s16)op->i.imm
 253                         - !!op_flag_no_ds(op->flags);
 254                 pr_debug("Adding local branch to offset 0x%x\n",
 255                          target_offset << 2);
 256                 branch = &state->local_branches[
 257                         state->nb_local_branches++];
 258
 259                 branch->target = target_offset;
 260                 if (is_forward)
 261                         branch->branch = jit_b();
 262                 else
 263                         branch->branch = jit_bgti(LIGHTREC_REG_CYCLE, 0);
 264         }
 265
 266         if (!op_flag_local_branch(op->flags) || !is_forward) {
 267                 next_pc = get_branch_pc(block, offset, 1 + (s16)op->i.imm);
 268                 lightrec_emit_end_of_block(state, block, offset, -1, next_pc,
 269                                            31, link, false);
 270         }
 271
 272         if (!unconditional) {
 273                 jit_patch(addr);
 274                 lightrec_regcache_leave_branch(reg_cache, regs_backup);
 275
 276                 if (bz && link) {
 277                         /* Update the $ra register */
 278                         link_reg = lightrec_alloc_reg_out(reg_cache, _jit,
 279                                                           31, REG_EXT);
 280                         jit_movi(link_reg, (s32)link);
 281                         lightrec_free_reg(reg_cache, link_reg);
 282                 }
 283
 284                 if (!op_flag_no_ds(op->flags) && next->opcode)
 285                         lightrec_rec_opcode(state, block, offset + 1);
 286         }
 287 }
 288
 289 static void rec_BNE(struct lightrec_cstate *state,
 290                     const struct block *block, u16 offset)
 291 {
 292         union code c = block->opcode_list[offset].c;
 293
 294         _jit_name(block->_jit, __func__);
 295
 296         if (c.i.rt == 0)
 297                 rec_b(state, block, offset, jit_code_beqi, 0, false, true);
 298         else
 299                 rec_b(state, block, offset, jit_code_beqr, 0, false, false);
 300 }
 301
 302 static void rec_BEQ(struct lightrec_cstate *state,
 303                     const struct block *block, u16 offset)
 304 {
 305         union code c = block->opcode_list[offset].c;
 306
 307         _jit_name(block->_jit, __func__);
 308
 309         if (c.i.rt == 0)
 310                 rec_b(state, block, offset, jit_code_bnei, 0, c.i.rs == 0, true);
 311         else
 312                 rec_b(state, block, offset, jit_code_bner, 0, c.i.rs == c.i.rt, false);
 313 }
 314
 315 static void rec_BLEZ(struct lightrec_cstate *state,
 316                      const struct block *block, u16 offset)
 317 {
 318         union code c = block->opcode_list[offset].c;
 319
 320         _jit_name(block->_jit, __func__);
 321         rec_b(state, block, offset, jit_code_bgti, 0, c.i.rs == 0, true);
 322 }
 323
 324 static void rec_BGTZ(struct lightrec_cstate *state,
 325                      const struct block *block, u16 offset)
 326 {
 327         _jit_name(block->_jit, __func__);
 328         rec_b(state, block, offset, jit_code_blei, 0, false, true);
 329 }
 330
 331 static void rec_regimm_BLTZ(struct lightrec_cstate *state,
 332                             const struct block *block, u16 offset)
 333 {
 334         _jit_name(block->_jit, __func__);
 335         rec_b(state, block, offset, jit_code_bgei, 0, false, true);
 336 }
 337
 338 static void rec_regimm_BLTZAL(struct lightrec_cstate *state,
 339                               const struct block *block, u16 offset)
 340 {
 341         _jit_name(block->_jit, __func__);
 342         rec_b(state, block, offset, jit_code_bgei,
 343               get_branch_pc(block, offset, 2), false, true);
 344 }
 345
 346 static void rec_regimm_BGEZ(struct lightrec_cstate *state,
 347                             const struct block *block, u16 offset)
 348 {
 349         union code c = block->opcode_list[offset].c;
 350
 351         _jit_name(block->_jit, __func__);
 352         rec_b(state, block, offset, jit_code_blti, 0, !c.i.rs, true);
 353 }
 354
 355 static void rec_regimm_BGEZAL(struct lightrec_cstate *state,
 356                               const struct block *block, u16 offset)
 357 {
 358         const struct opcode *op = &block->opcode_list[offset];
 359         _jit_name(block->_jit, __func__);
 360         rec_b(state, block, offset, jit_code_blti,
 361               get_branch_pc(block, offset, 2),
 362               !op->i.rs, true);
 363 }
 364
 365 static void rec_alu_imm(struct lightrec_cstate *state, const struct block *block,
 366                         u16 offset, jit_code_t code, bool slti)
 367 {
 368         struct regcache *reg_cache = state->reg_cache;
 369         union code c = block->opcode_list[offset].c;
 370         jit_state_t *_jit = block->_jit;
 371         u8 rs, rt, out_flags = REG_EXT;
 372
 373         if (slti)
 374                 out_flags |= REG_ZEXT;
 375
 376         jit_note(__FILE__, __LINE__);
 377         rs = lightrec_alloc_reg_in(reg_cache, _jit, c.i.rs, REG_EXT);
 378         rt = lightrec_alloc_reg_out(reg_cache, _jit, c.i.rt, out_flags);
 379
 380         jit_new_node_www(code, rt, rs, (s32)(s16) c.i.imm);
 381
 382         lightrec_free_reg(reg_cache, rs);
 383         lightrec_free_reg(reg_cache, rt);
 384 }
 385
 386 static void rec_alu_special(struct lightrec_cstate *state, const struct block *block,
 387                             u16 offset, jit_code_t code, bool out_ext)
 388 {
 389         struct regcache *reg_cache = state->reg_cache;
 390         union code c = block->opcode_list[offset].c;
 391         jit_state_t *_jit = block->_jit;
 392         u8 rd, rt, rs;
 393
 394         jit_note(__FILE__, __LINE__);
 395         rs = lightrec_alloc_reg_in(reg_cache, _jit, c.r.rs, REG_EXT);
 396         rt = lightrec_alloc_reg_in(reg_cache, _jit, c.r.rt, REG_EXT);
 397         rd = lightrec_alloc_reg_out(reg_cache, _jit, c.r.rd,
 398                                     out_ext ? REG_EXT | REG_ZEXT : 0);
 399
 400         jit_new_node_www(code, rd, rs, rt);
 401
 402         lightrec_free_reg(reg_cache, rs);
 403         lightrec_free_reg(reg_cache, rt);
 404         lightrec_free_reg(reg_cache, rd);
 405 }
 406
 407 static void rec_alu_shiftv(struct lightrec_cstate *state, const struct block *block,
 408                            u16 offset, jit_code_t code)
 409 {
 410         struct regcache *reg_cache = state->reg_cache;
 411         union code c = block->opcode_list[offset].c;
 412         jit_state_t *_jit = block->_jit;
 413         u8 rd, rt, rs, temp, flags = 0;
 414
 415         jit_note(__FILE__, __LINE__);
 416         rs = lightrec_alloc_reg_in(reg_cache, _jit, c.r.rs, 0);
 417
 418         if (code == jit_code_rshr)
 419                 flags = REG_EXT;
 420         else if (code == jit_code_rshr_u)
 421                 flags = REG_ZEXT;
 422
 423         rt = lightrec_alloc_reg_in(reg_cache, _jit, c.r.rt, flags);
 424         rd = lightrec_alloc_reg_out(reg_cache, _jit, c.r.rd, flags);
 425
 426         if (rs != rd && rt != rd) {
 427                 jit_andi(rd, rs, 0x1f);
 428                 jit_new_node_www(code, rd, rt, rd);
 429         } else {
 430                 temp = lightrec_alloc_reg_temp(reg_cache, _jit);
 431                 jit_andi(temp, rs, 0x1f);
 432                 jit_new_node_www(code, rd, rt, temp);
 433                 lightrec_free_reg(reg_cache, temp);
 434         }
 435
 436         lightrec_free_reg(reg_cache, rs);
 437         lightrec_free_reg(reg_cache, rt);
 438         lightrec_free_reg(reg_cache, rd);
 439 }
 440
 441 static void rec_movi(struct lightrec_cstate *state,
 442                      const struct block *block, u16 offset)
 443 {
 444         struct regcache *reg_cache = state->reg_cache;
 445         union code c = block->opcode_list[offset].c;
 446         jit_state_t *_jit = block->_jit;
 447         u16 flags = REG_EXT;
 448         u8 rt;
 449
 450         if (!(c.i.imm & 0x8000))
 451                 flags |= REG_ZEXT;
 452
 453         rt = lightrec_alloc_reg_out(reg_cache, _jit, c.i.rt, flags);
 454
 455         jit_movi(rt, (s32)(s16) c.i.imm);
 456
 457         lightrec_free_reg(reg_cache, rt);
 458 }
 459
 460 static void rec_ADDIU(struct lightrec_cstate *state,
 461                       const struct block *block, u16 offset)
 462 {
 463         _jit_name(block->_jit, __func__);
 464
 465         if (block->opcode_list[offset].c.i.rs)
 466                 rec_alu_imm(state, block, offset, jit_code_addi, false);
 467         else
 468                 rec_movi(state, block, offset);
 469 }
 470
 471 static void rec_ADDI(struct lightrec_cstate *state,
 472                      const struct block *block, u16 offset)
 473 {
 474         /* TODO: Handle the exception? */
 475         _jit_name(block->_jit, __func__);
 476         rec_ADDIU(state, block, offset);
 477 }
 478
 479 static void rec_SLTIU(struct lightrec_cstate *state,
 480                       const struct block *block, u16 offset)
 481 {
 482         _jit_name(block->_jit, __func__);
 483         rec_alu_imm(state, block, offset, jit_code_lti_u, true);
 484 }
 485
 486 static void rec_SLTI(struct lightrec_cstate *state,
 487                      const struct block *block, u16 offset)
 488 {
 489         _jit_name(block->_jit, __func__);
 490         rec_alu_imm(state, block, offset, jit_code_lti, true);
 491 }
 492
 493 static void rec_ANDI(struct lightrec_cstate *state,
 494                      const struct block *block, u16 offset)
 495 {
 496         struct regcache *reg_cache = state->reg_cache;
 497         union code c = block->opcode_list[offset].c;
 498         jit_state_t *_jit = block->_jit;
 499         u8 rs, rt;
 500
 501         _jit_name(block->_jit, __func__);
 502         jit_note(__FILE__, __LINE__);
 503         rs = lightrec_alloc_reg_in(reg_cache, _jit, c.i.rs, 0);
 504         rt = lightrec_alloc_reg_out(reg_cache, _jit, c.i.rt,
 505                                     REG_EXT | REG_ZEXT);
 506
 507         /* PSX code uses ANDI 0xff / ANDI 0xffff a lot, which are basically
 508          * casts to uint8_t / uint16_t. */
 509         if (c.i.imm == 0xff)
 510                 jit_extr_uc(rt, rs);
 511         else if (c.i.imm == 0xffff)
 512                 jit_extr_us(rt, rs);
 513         else
 514                 jit_andi(rt, rs, (u32)(u16) c.i.imm);
 515
 516         lightrec_free_reg(reg_cache, rs);
 517         lightrec_free_reg(reg_cache, rt);
 518 }
 519
 520 static void rec_alu_or_xor(struct lightrec_cstate *state, const struct block *block,
 521                            u16 offset, jit_code_t code)
 522 {
 523         struct regcache *reg_cache = state->reg_cache;
 524         union code c = block->opcode_list[offset].c;
 525         jit_state_t *_jit = block->_jit;
 526         u8 rs, rt, flags;
 527
 528         jit_note(__FILE__, __LINE__);
 529         rs = lightrec_alloc_reg_in(reg_cache, _jit, c.i.rs, 0);
 530         rt = lightrec_alloc_reg_out(reg_cache, _jit, c.i.rt, 0);
 531
 532         flags = lightrec_get_reg_in_flags(reg_cache, rs);
 533         lightrec_set_reg_out_flags(reg_cache, rt, flags);
 534
 535         jit_new_node_www(code, rt, rs, (u32)(u16) c.i.imm);
 536
 537         lightrec_free_reg(reg_cache, rs);
 538         lightrec_free_reg(reg_cache, rt);
 539 }
 540
 541
 542 static void rec_ORI(struct lightrec_cstate *state,
 543                     const struct block *block, u16 offset)
 544 {
 545         _jit_name(block->_jit, __func__);
 546         rec_alu_or_xor(state, block, offset, jit_code_ori);
 547 }
 548
 549 static void rec_XORI(struct lightrec_cstate *state,
 550                      const struct block *block, u16 offset)
 551 {
 552         _jit_name(block->_jit, __func__);
 553         rec_alu_or_xor(state, block, offset, jit_code_xori);
 554 }
 555
 556 static void rec_LUI(struct lightrec_cstate *state,
 557                     const struct block *block, u16 offset)
 558 {
 559         struct regcache *reg_cache = state->reg_cache;
 560         union code c = block->opcode_list[offset].c;
 561         jit_state_t *_jit = block->_jit;
 562         u8 rt, flags = REG_EXT;
 563
 564         jit_name(__func__);
 565         jit_note(__FILE__, __LINE__);
 566
 567         if (!(c.i.imm & BIT(15)))
 568                 flags |= REG_ZEXT;
 569
 570         rt = lightrec_alloc_reg_out(reg_cache, _jit, c.i.rt, flags);
 571
 572         jit_movi(rt, (s32)(c.i.imm << 16));
 573
 574         lightrec_free_reg(reg_cache, rt);
 575 }
 576
 577 static void rec_special_ADDU(struct lightrec_cstate *state,
 578                              const struct block *block, u16 offset)
 579 {
 580         _jit_name(block->_jit, __func__);
 581         rec_alu_special(state, block, offset, jit_code_addr, false);
 582 }
 583
 584 static void rec_special_ADD(struct lightrec_cstate *state,
 585                             const struct block *block, u16 offset)
 586 {
 587         /* TODO: Handle the exception? */
 588         _jit_name(block->_jit, __func__);
 589         rec_alu_special(state, block, offset, jit_code_addr, false);
 590 }
 591
 592 static void rec_special_SUBU(struct lightrec_cstate *state,
 593                              const struct block *block, u16 offset)
 594 {
 595         _jit_name(block->_jit, __func__);
 596         rec_alu_special(state, block, offset, jit_code_subr, false);
 597 }
 598
 599 static void rec_special_SUB(struct lightrec_cstate *state,
 600                             const struct block *block, u16 offset)
 601 {
 602         /* TODO: Handle the exception? */
 603         _jit_name(block->_jit, __func__);
 604         rec_alu_special(state, block, offset, jit_code_subr, false);
 605 }
 606
 607 static void rec_special_AND(struct lightrec_cstate *state,
 608                             const struct block *block, u16 offset)
 609 {
 610         struct regcache *reg_cache = state->reg_cache;
 611         union code c = block->opcode_list[offset].c;
 612         jit_state_t *_jit = block->_jit;
 613         u8 rd, rt, rs, flags_rs, flags_rt, flags_rd;
 614
 615         _jit_name(block->_jit, __func__);
 616         jit_note(__FILE__, __LINE__);
 617         rs = lightrec_alloc_reg_in(reg_cache, _jit, c.r.rs, 0);
 618         rt = lightrec_alloc_reg_in(reg_cache, _jit, c.r.rt, 0);
 619         rd = lightrec_alloc_reg_out(reg_cache, _jit, c.r.rd, 0);
 620
 621         flags_rs = lightrec_get_reg_in_flags(reg_cache, rs);
 622         flags_rt = lightrec_get_reg_in_flags(reg_cache, rt);
 623
 624         /* Z(rd) = Z(rs) | Z(rt) */
 625         flags_rd = REG_ZEXT & (flags_rs | flags_rt);
 626
 627         /* E(rd) = (E(rt) & Z(rt)) | (E(rs) & Z(rs)) | (E(rs) & E(rt)) */
 628         if (((flags_rs & REG_EXT) && (flags_rt & REG_ZEXT)) ||
 629             ((flags_rt & REG_EXT) && (flags_rs & REG_ZEXT)) ||
 630             (REG_EXT & flags_rs & flags_rt))
 631                 flags_rd |= REG_EXT;
 632
 633         lightrec_set_reg_out_flags(reg_cache, rd, flags_rd);
 634
 635         jit_andr(rd, rs, rt);
 636
 637         lightrec_free_reg(reg_cache, rs);
 638         lightrec_free_reg(reg_cache, rt);
 639         lightrec_free_reg(reg_cache, rd);
 640 }
 641
 642 static void rec_special_or_nor(struct lightrec_cstate *state,
 643                                const struct block *block, u16 offset, bool nor)
 644 {
 645         struct regcache *reg_cache = state->reg_cache;
 646         union code c = block->opcode_list[offset].c;
 647         jit_state_t *_jit = block->_jit;
 648         u8 rd, rt, rs, flags_rs, flags_rt, flags_rd = 0;
 649
 650         jit_note(__FILE__, __LINE__);
 651         rs = lightrec_alloc_reg_in(reg_cache, _jit, c.r.rs, 0);
 652         rt = lightrec_alloc_reg_in(reg_cache, _jit, c.r.rt, 0);
 653         rd = lightrec_alloc_reg_out(reg_cache, _jit, c.r.rd, 0);
 654
 655         flags_rs = lightrec_get_reg_in_flags(reg_cache, rs);
 656         flags_rt = lightrec_get_reg_in_flags(reg_cache, rt);
 657
 658         /* or: Z(rd) = Z(rs) & Z(rt)
 659          * nor: Z(rd) = 0 */
 660         if (!nor)
 661                 flags_rd = REG_ZEXT & flags_rs & flags_rt;
 662
 663         /* E(rd) = (E(rs) & E(rt)) | (E(rt) & !Z(rt)) | (E(rs) & !Z(rs)) */
 664         if ((REG_EXT & flags_rs & flags_rt) ||
 665             (flags_rt & (REG_EXT | REG_ZEXT) == REG_EXT) ||
 666             (flags_rs & (REG_EXT | REG_ZEXT) == REG_EXT))
 667                 flags_rd |= REG_EXT;
 668
 669         lightrec_set_reg_out_flags(reg_cache, rd, flags_rd);
 670
 671         jit_orr(rd, rs, rt);
 672
 673         if (nor)
 674                 jit_comr(rd, rd);
 675
 676         lightrec_free_reg(reg_cache, rs);
 677         lightrec_free_reg(reg_cache, rt);
 678         lightrec_free_reg(reg_cache, rd);
 679 }
 680
 681 static void rec_special_OR(struct lightrec_cstate *state,
 682                            const struct block *block, u16 offset)
 683 {
 684         _jit_name(block->_jit, __func__);
 685         rec_special_or_nor(state, block, offset, false);
 686 }
 687
 688 static void rec_special_NOR(struct lightrec_cstate *state,
 689                             const struct block *block, u16 offset)
 690 {
 691         _jit_name(block->_jit, __func__);
 692         rec_special_or_nor(state, block, offset, true);
 693 }
 694
 695 static void rec_special_XOR(struct lightrec_cstate *state,
 696                             const struct block *block, u16 offset)
 697 {
 698         struct regcache *reg_cache = state->reg_cache;
 699         union code c = block->opcode_list[offset].c;
 700         jit_state_t *_jit = block->_jit;
 701         u8 rd, rt, rs, flags_rs, flags_rt, flags_rd;
 702
 703         _jit_name(block->_jit, __func__);
 704
 705         jit_note(__FILE__, __LINE__);
 706         rs = lightrec_alloc_reg_in(reg_cache, _jit, c.r.rs, 0);
 707         rt = lightrec_alloc_reg_in(reg_cache, _jit, c.r.rt, 0);
 708         rd = lightrec_alloc_reg_out(reg_cache, _jit, c.r.rd, 0);
 709
 710         flags_rs = lightrec_get_reg_in_flags(reg_cache, rs);
 711         flags_rt = lightrec_get_reg_in_flags(reg_cache, rt);
 712
 713         /* Z(rd) = Z(rs) & Z(rt) */
 714         flags_rd = REG_ZEXT & flags_rs & flags_rt;
 715
 716         /* E(rd) = E(rs) & E(rt) */
 717         flags_rd |= REG_EXT & flags_rs & flags_rt;
 718
 719         lightrec_set_reg_out_flags(reg_cache, rd, flags_rd);
 720
 721         jit_xorr(rd, rs, rt);
 722
 723         lightrec_free_reg(reg_cache, rs);
 724         lightrec_free_reg(reg_cache, rt);
 725         lightrec_free_reg(reg_cache, rd);
 726 }
 727
 728 static void rec_special_SLTU(struct lightrec_cstate *state,
 729                              const struct block *block, u16 offset)
 730 {
 731         _jit_name(block->_jit, __func__);
 732         rec_alu_special(state, block, offset, jit_code_ltr_u, true);
 733 }
 734
 735 static void rec_special_SLT(struct lightrec_cstate *state,
 736                             const struct block *block, u16 offset)
 737 {
 738         _jit_name(block->_jit, __func__);
 739         rec_alu_special(state, block, offset, jit_code_ltr, true);
 740 }
 741
 742 static void rec_special_SLLV(struct lightrec_cstate *state,
 743                              const struct block *block, u16 offset)
 744 {
 745         _jit_name(block->_jit, __func__);
 746         rec_alu_shiftv(state, block, offset, jit_code_lshr);
 747 }
 748
 749 static void rec_special_SRLV(struct lightrec_cstate *state,
 750                              const struct block *block, u16 offset)
 751 {
 752         _jit_name(block->_jit, __func__);
 753         rec_alu_shiftv(state, block, offset, jit_code_rshr_u);
 754 }
 755
 756 static void rec_special_SRAV(struct lightrec_cstate *state,
 757                              const struct block *block, u16 offset)
 758 {
 759         _jit_name(block->_jit, __func__);
 760         rec_alu_shiftv(state, block, offset, jit_code_rshr);
 761 }
 762
 763 static void rec_alu_shift(struct lightrec_cstate *state, const struct block *block,
 764                           u16 offset, jit_code_t code)
 765 {
 766         struct regcache *reg_cache = state->reg_cache;
 767         union code c = block->opcode_list[offset].c;
 768         jit_state_t *_jit = block->_jit;
 769         u8 rd, rt, flags = 0;
 770
 771         jit_note(__FILE__, __LINE__);
 772
 773         if (code == jit_code_rshi)
 774                 flags = REG_EXT;
 775         else if (code == jit_code_rshi_u)
 776                 flags = REG_ZEXT;
 777
 778         rt = lightrec_alloc_reg_in(reg_cache, _jit, c.r.rt, flags);
 779
 780         /* Input reg is zero-extended, if we SRL at least by one bit, we know
 781          * the output reg will be both zero-extended and sign-extended. */
 782         if (code == jit_code_rshi_u && c.r.imm)
 783                 flags |= REG_EXT;
 784         rd = lightrec_alloc_reg_out(reg_cache, _jit, c.r.rd, flags);
 785
 786         jit_new_node_www(code, rd, rt, c.r.imm);
 787
 788         lightrec_free_reg(reg_cache, rt);
 789         lightrec_free_reg(reg_cache, rd);
 790 }
 791
 792 static void rec_special_SLL(struct lightrec_cstate *state,
 793                             const struct block *block, u16 offset)
 794 {
 795         _jit_name(block->_jit, __func__);
 796         rec_alu_shift(state, block, offset, jit_code_lshi);
 797 }
 798
 799 static void rec_special_SRL(struct lightrec_cstate *state,
 800                             const struct block *block, u16 offset)
 801 {
 802         _jit_name(block->_jit, __func__);
 803         rec_alu_shift(state, block, offset, jit_code_rshi_u);
 804 }
 805
 806 static void rec_special_SRA(struct lightrec_cstate *state,
 807                             const struct block *block, u16 offset)
 808 {
 809         _jit_name(block->_jit, __func__);
 810         rec_alu_shift(state, block, offset, jit_code_rshi);
 811 }
 812
 813 static void rec_alu_mult(struct lightrec_cstate *state,
 814                          const struct block *block, u16 offset, bool is_signed)
 815 {
 816         struct regcache *reg_cache = state->reg_cache;
 817         union code c = block->opcode_list[offset].c;
 818         u32 flags = block->opcode_list[offset].flags;
 819         u8 reg_lo = get_mult_div_lo(c);
 820         u8 reg_hi = get_mult_div_hi(c);
 821         jit_state_t *_jit = block->_jit;
 822         u8 lo, hi, rs, rt, rflags = 0;
 823
 824         jit_note(__FILE__, __LINE__);
 825
 826         if (is_signed)
 827                 rflags = REG_EXT;
 828         else
 829                 rflags = REG_ZEXT;
 830
 831         rs = lightrec_alloc_reg_in(reg_cache, _jit, c.r.rs, rflags);
 832         rt = lightrec_alloc_reg_in(reg_cache, _jit, c.r.rt, rflags);
 833
 834         if (!op_flag_no_lo(flags))
 835                 lo = lightrec_alloc_reg_out(reg_cache, _jit, reg_lo, 0);
 836         else if (__WORDSIZE == 32)
 837                 lo = lightrec_alloc_reg_temp(reg_cache, _jit);
 838
 839         if (!op_flag_no_hi(flags))
 840                 hi = lightrec_alloc_reg_out(reg_cache, _jit, reg_hi, REG_EXT);
 841
 842         if (__WORDSIZE == 32) {
 843                 /* On 32-bit systems, do a 32*32->64 bit operation, or a 32*32->32 bit
 844                  * operation if the MULT was detected a 32-bit only. */
 845                 if (!op_flag_no_hi(flags)) {
 846                         if (is_signed)
 847                                 jit_qmulr(lo, hi, rs, rt);
 848                         else
 849                                 jit_qmulr_u(lo, hi, rs, rt);
 850                 } else {
 851                         jit_mulr(lo, rs, rt);
 852                 }
 853         } else {
 854                 /* On 64-bit systems, do a 64*64->64 bit operation. */
 855                 if (op_flag_no_lo(flags)) {
 856                         jit_mulr(hi, rs, rt);
 857                         jit_rshi(hi, hi, 32);
 858                 } else {
 859                         jit_mulr(lo, rs, rt);
 860
 861                         /* The 64-bit output value is in $lo, store the upper 32 bits in $hi */
 862                         if (!op_flag_no_hi(flags))
 863                                 jit_rshi(hi, lo, 32);
 864                 }
 865         }
 866
 867         lightrec_free_reg(reg_cache, rs);
 868         lightrec_free_reg(reg_cache, rt);
 869         if (!op_flag_no_lo(flags) || __WORDSIZE == 32)
 870                 lightrec_free_reg(reg_cache, lo);
 871         if (!op_flag_no_hi(flags))
 872                 lightrec_free_reg(reg_cache, hi);
 873 }
 874
 875 static void rec_alu_div(struct lightrec_cstate *state,
 876                         const struct block *block, u16 offset, bool is_signed)
 877 {
 878         struct regcache *reg_cache = state->reg_cache;
 879         union code c = block->opcode_list[offset].c;
 880         u32 flags = block->opcode_list[offset].flags;
 881         bool no_check = op_flag_no_div_check(flags);
 882         u8 reg_lo = get_mult_div_lo(c);
 883         u8 reg_hi = get_mult_div_hi(c);
 884         jit_state_t *_jit = block->_jit;
 885         jit_node_t *branch, *to_end;
 886         u8 lo = 0, hi = 0, rs, rt, rflags = 0;
 887
 888         jit_note(__FILE__, __LINE__);
 889
 890         if (is_signed)
 891                 rflags = REG_EXT;
 892         else
 893                 rflags = REG_ZEXT;
 894
 895         rs = lightrec_alloc_reg_in(reg_cache, _jit, c.r.rs, rflags);
 896         rt = lightrec_alloc_reg_in(reg_cache, _jit, c.r.rt, rflags);
 897
 898         if (!op_flag_no_lo(flags))
 899                 lo = lightrec_alloc_reg_out(reg_cache, _jit, reg_lo, 0);
 900
 901         if (!op_flag_no_hi(flags))
 902                 hi = lightrec_alloc_reg_out(reg_cache, _jit, reg_hi, 0);
 903
 904         /* Jump to special handler if dividing by zero  */
 905         if (!no_check)
 906                 branch = jit_beqi(rt, 0);
 907
 908         if (op_flag_no_lo(flags)) {
 909                 if (is_signed)
 910                         jit_remr(hi, rs, rt);
 911                 else
 912                         jit_remr_u(hi, rs, rt);
 913         } else if (op_flag_no_hi(flags)) {
 914                 if (is_signed)
 915                         jit_divr(lo, rs, rt);
 916                 else
 917                         jit_divr_u(lo, rs, rt);
 918         } else {
 919                 if (is_signed)
 920                         jit_qdivr(lo, hi, rs, rt);
 921                 else
 922                         jit_qdivr_u(lo, hi, rs, rt);
 923         }
 924
 925         if (!no_check) {
 926                 /* Jump above the div-by-zero handler */
 927                 to_end = jit_b();
 928
 929                 jit_patch(branch);
 930
 931                 if (!op_flag_no_lo(flags)) {
 932                         if (is_signed) {
 933                                 jit_lti(lo, rs, 0);
 934                                 jit_lshi(lo, lo, 1);
 935                                 jit_subi(lo, lo, 1);
 936                         } else {
 937                                 jit_movi(lo, 0xffffffff);
 938                         }
 939                 }
 940
 941                 if (!op_flag_no_hi(flags))
 942                         jit_movr(hi, rs);
 943
 944                 jit_patch(to_end);
 945         }
 946
 947         lightrec_free_reg(reg_cache, rs);
 948         lightrec_free_reg(reg_cache, rt);
 949
 950         if (!op_flag_no_lo(flags))
 951                 lightrec_free_reg(reg_cache, lo);
 952
 953         if (!op_flag_no_hi(flags))
 954                 lightrec_free_reg(reg_cache, hi);
 955 }
 956
 957 static void rec_special_MULT(struct lightrec_cstate *state,
 958                              const struct block *block, u16 offset)
 959 {
 960         _jit_name(block->_jit, __func__);
 961         rec_alu_mult(state, block, offset, true);
 962 }
 963
 964 static void rec_special_MULTU(struct lightrec_cstate *state,
 965                               const struct block *block, u16 offset)
 966 {
 967         _jit_name(block->_jit, __func__);
 968         rec_alu_mult(state, block, offset, false);
 969 }
 970
 971 static void rec_special_DIV(struct lightrec_cstate *state,
 972                             const struct block *block, u16 offset)
 973 {
 974         _jit_name(block->_jit, __func__);
 975         rec_alu_div(state, block, offset, true);
 976 }
 977
 978 static void rec_special_DIVU(struct lightrec_cstate *state,
 979                              const struct block *block, u16 offset)
 980 {
 981         _jit_name(block->_jit, __func__);
 982         rec_alu_div(state, block, offset, false);
 983 }
 984
 985 static void rec_alu_mv_lo_hi(struct lightrec_cstate *state,
 986                              const struct block *block, u8 dst, u8 src)
 987 {
 988         struct regcache *reg_cache = state->reg_cache;
 989         jit_state_t *_jit = block->_jit;
 990
 991         jit_note(__FILE__, __LINE__);
 992         src = lightrec_alloc_reg_in(reg_cache, _jit, src, 0);
 993         dst = lightrec_alloc_reg_out(reg_cache, _jit, dst, REG_EXT);
 994
 995         jit_extr_i(dst, src);
 996
 997         lightrec_free_reg(reg_cache, src);
 998         lightrec_free_reg(reg_cache, dst);
 999 }
1000
1001 static void rec_special_MFHI(struct lightrec_cstate *state,
1002                              const struct block *block, u16 offset)
1003 {
1004         union code c = block->opcode_list[offset].c;
1005
1006         _jit_name(block->_jit, __func__);
1007         rec_alu_mv_lo_hi(state, block, c.r.rd, REG_HI);
1008 }
1009
1010 static void rec_special_MTHI(struct lightrec_cstate *state,
1011                              const struct block *block, u16 offset)
1012 {
1013         union code c = block->opcode_list[offset].c;
1014
1015         _jit_name(block->_jit, __func__);
1016         rec_alu_mv_lo_hi(state, block, REG_HI, c.r.rs);
1017 }
1018
1019 static void rec_special_MFLO(struct lightrec_cstate *state,
1020                              const struct block *block, u16 offset)
1021 {
1022         union code c = block->opcode_list[offset].c;
1023
1024         _jit_name(block->_jit, __func__);
1025         rec_alu_mv_lo_hi(state, block, c.r.rd, REG_LO);
1026 }
1027
1028 static void rec_special_MTLO(struct lightrec_cstate *state,
1029                              const struct block *block, u16 offset)
1030 {
1031         union code c = block->opcode_list[offset].c;
1032
1033         _jit_name(block->_jit, __func__);
1034         rec_alu_mv_lo_hi(state, block, REG_LO, c.r.rs);
1035 }
1036
1037 static void call_to_c_wrapper(struct lightrec_cstate *state, const struct block *block,
1038                               u32 arg, bool with_arg, enum c_wrappers wrapper)
1039 {
1040         struct regcache *reg_cache = state->reg_cache;
1041         jit_state_t *_jit = block->_jit;
1042         u8 tmp;
1043
1044         tmp = lightrec_alloc_reg_temp(reg_cache, _jit);
1045         jit_ldxi(tmp, LIGHTREC_REG_STATE,
1046                  offsetof(struct lightrec_state, wrappers_eps[wrapper]));
1047
1048         if (with_arg) {
1049                 jit_prepare();
1050                 jit_pushargi(arg);
1051         }
1052
1053         lightrec_regcache_mark_live(reg_cache, _jit);
1054         jit_callr(tmp);
1055
1056         lightrec_free_reg(reg_cache, tmp);
1057         lightrec_regcache_mark_live(reg_cache, _jit);
1058 }
1059
1060 static void rec_io(struct lightrec_cstate *state,
1061                    const struct block *block, u16 offset,
1062                    bool load_rt, bool read_rt)
1063 {
1064         struct regcache *reg_cache = state->reg_cache;
1065         jit_state_t *_jit = block->_jit;
1066         union code c = block->opcode_list[offset].c;
1067         u32 flags = block->opcode_list[offset].flags;
1068         bool is_tagged = LIGHTREC_FLAGS_GET_IO_MODE(flags);
1069         u32 lut_entry;
1070
1071         jit_note(__FILE__, __LINE__);
1072
1073         lightrec_clean_reg_if_loaded(reg_cache, _jit, c.i.rs, false);
1074
1075         if (read_rt && likely(c.i.rt))
1076                 lightrec_clean_reg_if_loaded(reg_cache, _jit, c.i.rt, true);
1077         else if (load_rt)
1078                 lightrec_clean_reg_if_loaded(reg_cache, _jit, c.i.rt, false);
1079
1080         if (is_tagged) {
1081                 call_to_c_wrapper(state, block, c.opcode, true, C_WRAPPER_RW);
1082         } else {
1083                 lut_entry = lightrec_get_lut_entry(block);
1084                 call_to_c_wrapper(state, block, (lut_entry << 16) | offset,
1085                                   true, C_WRAPPER_RW_GENERIC);
1086         }
1087 }
1088
1089 static u32 rec_ram_mask(struct lightrec_state *state)
1090 {
1091         return (RAM_SIZE << (state->mirrors_mapped * 2)) - 1;
1092 }
1093
1094 static void rec_store_memory(struct lightrec_cstate *cstate,
1095                              const struct block *block,
1096                              u16 offset, jit_code_t code,
1097                              jit_code_t swap_code,
1098                              uintptr_t addr_offset, u32 addr_mask,
1099                              bool invalidate)
1100 {
1101         const struct lightrec_state *state = cstate->state;
1102         struct regcache *reg_cache = cstate->reg_cache;
1103         struct opcode *op = &block->opcode_list[offset];
1104         jit_state_t *_jit = block->_jit;
1105         union code c = op->c;
1106         u8 rs, rt, tmp, tmp2, tmp3, addr_reg, addr_reg2;
1107         s16 imm = (s16)c.i.imm;
1108         s32 simm = (s32)imm << (1 - lut_is_32bit(state));
1109         s32 lut_offt = offsetof(struct lightrec_state, code_lut);
1110         bool no_mask = op_flag_no_mask(op->flags);
1111         bool add_imm = c.i.imm &&
1112                 ((!state->mirrors_mapped && !no_mask) || (invalidate &&
1113                 ((imm & 0x3) || simm + lut_offt != (s16)(simm + lut_offt))));
1114         bool need_tmp = !no_mask || addr_offset || add_imm;
1115         bool need_tmp2 = addr_offset || invalidate;
1116
1117         rt = lightrec_alloc_reg_in(reg_cache, _jit, c.i.rt, 0);
1118         rs = lightrec_alloc_reg_in(reg_cache, _jit, c.i.rs, 0);
1119         if (need_tmp)
1120                 tmp = lightrec_alloc_reg_temp(reg_cache, _jit);
1121
1122         addr_reg = rs;
1123
1124         if (add_imm) {
1125                 jit_addi(tmp, addr_reg, (s16)c.i.imm);
1126                 addr_reg = tmp;
1127                 imm = 0;
1128         } else if (simm) {
1129                 lut_offt += simm;
1130         }
1131
1132         if (!no_mask) {
1133                 jit_andi(tmp, addr_reg, addr_mask);
1134                 addr_reg = tmp;
1135         }
1136
1137         if (need_tmp2)
1138                 tmp2 = lightrec_alloc_reg_temp(reg_cache, _jit);
1139
1140         if (addr_offset) {
1141                 jit_addi(tmp2, addr_reg, addr_offset);
1142                 addr_reg2 = tmp2;
1143         } else {
1144                 addr_reg2 = addr_reg;
1145         }
1146
1147         if (is_big_endian() && swap_code && c.i.rt) {
1148                 tmp3 = lightrec_alloc_reg_temp(reg_cache, _jit);
1149
1150                 jit_new_node_ww(swap_code, tmp3, rt);
1151                 jit_new_node_www(code, imm, addr_reg2, tmp3);
1152
1153                 lightrec_free_reg(reg_cache, tmp3);
1154         } else {
1155                 jit_new_node_www(code, imm, addr_reg2, rt);
1156         }
1157
1158         lightrec_free_reg(reg_cache, rt);
1159
1160         if (invalidate) {
1161                 tmp3 = lightrec_alloc_reg_in(reg_cache, _jit, 0, 0);
1162
1163                 if (c.i.op != OP_SW) {
1164                         jit_andi(tmp2, addr_reg, ~3);
1165                         addr_reg = tmp2;
1166                 }
1167
1168                 if (!lut_is_32bit(state)) {
1169                         jit_lshi(tmp2, addr_reg, 1);
1170                         addr_reg = tmp2;
1171                 }
1172
1173                 if (addr_reg == rs && c.i.rs == 0) {
1174                         addr_reg = LIGHTREC_REG_STATE;
1175                 } else {
1176                         jit_addr(tmp2, addr_reg, LIGHTREC_REG_STATE);
1177                         addr_reg = tmp2;
1178                 }
1179
1180                 if (lut_is_32bit(state))
1181                         jit_stxi_i(lut_offt, addr_reg, tmp3);
1182                 else
1183                         jit_stxi(lut_offt, addr_reg, tmp3);
1184
1185                 lightrec_free_reg(reg_cache, tmp3);
1186         }
1187
1188         if (need_tmp2)
1189                 lightrec_free_reg(reg_cache, tmp2);
1190         if (need_tmp)
1191                 lightrec_free_reg(reg_cache, tmp);
1192         lightrec_free_reg(reg_cache, rs);
1193 }
1194
1195 static void rec_store_ram(struct lightrec_cstate *cstate,
1196                           const struct block *block,
1197                           u16 offset, jit_code_t code,
1198                           jit_code_t swap_code, bool invalidate)
1199 {
1200         struct lightrec_state *state = cstate->state;
1201
1202         _jit_note(block->_jit, __FILE__, __LINE__);
1203
1204         return rec_store_memory(cstate, block, offset, code, swap_code,
1205                                 state->offset_ram, rec_ram_mask(state),
1206                                 invalidate);
1207 }
1208
1209 static void rec_store_scratch(struct lightrec_cstate *cstate,
1210                               const struct block *block, u16 offset,
1211                               jit_code_t code, jit_code_t swap_code)
1212 {
1213         _jit_note(block->_jit, __FILE__, __LINE__);
1214
1215         return rec_store_memory(cstate, block, offset, code, swap_code,
1216                                 cstate->state->offset_scratch,
1217                                 0x1fffffff, false);
1218 }
1219
1220 static void rec_store_direct_no_invalidate(struct lightrec_cstate *cstate,
1221                                            const struct block *block,
1222                                            u16 offset, jit_code_t code,
1223                                            jit_code_t swap_code)
1224 {
1225         struct lightrec_state *state = cstate->state;
1226         struct regcache *reg_cache = cstate->reg_cache;
1227         union code c = block->opcode_list[offset].c;
1228         jit_state_t *_jit = block->_jit;
1229         jit_node_t *to_not_ram, *to_end;
1230         u8 tmp, tmp2, rs, rt;
1231         s16 imm;
1232
1233         jit_note(__FILE__, __LINE__);
1234         rs = lightrec_alloc_reg_in(reg_cache, _jit, c.i.rs, 0);
1235         rt = lightrec_alloc_reg_in(reg_cache, _jit, c.i.rt, 0);
1236         tmp = lightrec_alloc_reg_temp(reg_cache, _jit);
1237
1238         if (state->offset_ram || state->offset_scratch)
1239                 tmp2 = lightrec_alloc_reg_temp(reg_cache, _jit);
1240
1241         /* Convert to KUNSEG and avoid RAM mirrors */
1242         if (state->mirrors_mapped) {
1243                 imm = (s16)c.i.imm;
1244                 jit_andi(tmp, rs, 0x1f800000 | (4 * RAM_SIZE - 1));
1245         } else if (c.i.imm) {
1246                 imm = 0;
1247                 jit_addi(tmp, rs, (s16)c.i.imm);
1248                 jit_andi(tmp, tmp, 0x1f800000 | (RAM_SIZE - 1));
1249         } else {
1250                 imm = 0;
1251                 jit_andi(tmp, rs, 0x1f800000 | (RAM_SIZE - 1));
1252         }
1253
1254         lightrec_free_reg(reg_cache, rs);
1255
1256         if (state->offset_ram != state->offset_scratch) {
1257                 to_not_ram = jit_bmsi(tmp, BIT(28));
1258
1259                 jit_movi(tmp2, state->offset_ram);
1260
1261                 to_end = jit_b();
1262                 jit_patch(to_not_ram);
1263
1264                 jit_movi(tmp2, state->offset_scratch);
1265                 jit_patch(to_end);
1266         } else if (state->offset_ram) {
1267                 jit_movi(tmp2, state->offset_ram);
1268         }
1269
1270         if (state->offset_ram || state->offset_scratch) {
1271                 jit_addr(tmp, tmp, tmp2);
1272                 lightrec_free_reg(reg_cache, tmp2);
1273         }
1274
1275         if (is_big_endian() && swap_code && c.i.rt) {
1276                 tmp2 = lightrec_alloc_reg_temp(reg_cache, _jit);
1277
1278                 jit_new_node_ww(swap_code, tmp2, rt);
1279                 jit_new_node_www(code, imm, tmp, tmp2);
1280
1281                 lightrec_free_reg(reg_cache, tmp2);
1282         } else {
1283                 jit_new_node_www(code, imm, tmp, rt);
1284         }
1285
1286         lightrec_free_reg(reg_cache, rt);
1287         lightrec_free_reg(reg_cache, tmp);
1288 }
1289
1290 static void rec_store_direct(struct lightrec_cstate *cstate, const struct block *block,
1291                              u16 offset, jit_code_t code, jit_code_t swap_code)
1292 {
1293         struct lightrec_state *state = cstate->state;
1294         u32 ram_size = state->mirrors_mapped ? RAM_SIZE * 4 : RAM_SIZE;
1295         struct regcache *reg_cache = cstate->reg_cache;
1296         union code c = block->opcode_list[offset].c;
1297         jit_state_t *_jit = block->_jit;
1298         jit_node_t *to_not_ram, *to_end;
1299         u8 tmp, tmp2, tmp3, rs, rt;
1300
1301         jit_note(__FILE__, __LINE__);
1302
1303         rs = lightrec_alloc_reg_in(reg_cache, _jit, c.i.rs, 0);
1304         tmp2 = lightrec_alloc_reg_temp(reg_cache, _jit);
1305         tmp3 = lightrec_alloc_reg_in(reg_cache, _jit, 0, 0);
1306
1307         /* Convert to KUNSEG and avoid RAM mirrors */
1308         if (c.i.imm) {
1309                 jit_addi(tmp2, rs, (s16)c.i.imm);
1310                 jit_andi(tmp2, tmp2, 0x1f800000 | (ram_size - 1));
1311         } else {
1312                 jit_andi(tmp2, rs, 0x1f800000 | (ram_size - 1));
1313         }
1314
1315         lightrec_free_reg(reg_cache, rs);
1316         tmp = lightrec_alloc_reg_temp(reg_cache, _jit);
1317
1318         to_not_ram = jit_bgti(tmp2, ram_size);
1319
1320         /* Compute the offset to the code LUT */
1321         jit_andi(tmp, tmp2, (RAM_SIZE - 1) & ~3);
1322         if (!lut_is_32bit(state))
1323                 jit_lshi(tmp, tmp, 1);
1324         jit_addr(tmp, LIGHTREC_REG_STATE, tmp);
1325
1326         /* Write NULL to the code LUT to invalidate any block that's there */
1327         if (lut_is_32bit(state))
1328                 jit_stxi_i(offsetof(struct lightrec_state, code_lut), tmp, tmp3);
1329         else
1330                 jit_stxi(offsetof(struct lightrec_state, code_lut), tmp, tmp3);
1331
1332         if (state->offset_ram != state->offset_scratch) {
1333                 jit_movi(tmp, state->offset_ram);
1334
1335                 to_end = jit_b();
1336         }
1337
1338         jit_patch(to_not_ram);
1339
1340         if (state->offset_ram || state->offset_scratch)
1341                 jit_movi(tmp, state->offset_scratch);
1342
1343         if (state->offset_ram != state->offset_scratch)
1344                 jit_patch(to_end);
1345
1346         if (state->offset_ram || state->offset_scratch)
1347                 jit_addr(tmp2, tmp2, tmp);
1348
1349         lightrec_free_reg(reg_cache, tmp);
1350         lightrec_free_reg(reg_cache, tmp3);
1351
1352         rt = lightrec_alloc_reg_in(reg_cache, _jit, c.i.rt, 0);
1353
1354         if (is_big_endian() && swap_code && c.i.rt) {
1355                 tmp = lightrec_alloc_reg_temp(reg_cache, _jit);
1356
1357                 jit_new_node_ww(swap_code, tmp, rt);
1358                 jit_new_node_www(code, 0, tmp2, tmp);
1359
1360                 lightrec_free_reg(reg_cache, tmp);
1361         } else {
1362                 jit_new_node_www(code, 0, tmp2, rt);
1363         }
1364
1365         lightrec_free_reg(reg_cache, rt);
1366         lightrec_free_reg(reg_cache, tmp2);
1367 }
1368
1369 static void rec_store(struct lightrec_cstate *state,
1370                       const struct block *block, u16 offset,
1371                       jit_code_t code, jit_code_t swap_code)
1372 {
1373         u32 flags = block->opcode_list[offset].flags;
1374         bool no_invalidate = op_flag_no_invalidate(flags) ||
1375                 state->state->invalidate_from_dma_only;
1376
1377         switch (LIGHTREC_FLAGS_GET_IO_MODE(flags)) {
1378         case LIGHTREC_IO_RAM:
1379                 rec_store_ram(state, block, offset, code,
1380                               swap_code, !no_invalidate);
1381                 break;
1382         case LIGHTREC_IO_SCRATCH:
1383                 rec_store_scratch(state, block, offset, code, swap_code);
1384                 break;
1385         case LIGHTREC_IO_DIRECT:
1386                 if (no_invalidate) {
1387                         rec_store_direct_no_invalidate(state, block, offset,
1388                                                        code, swap_code);
1389                 } else {
1390                         rec_store_direct(state, block, offset, code, swap_code);
1391                 }
1392                 break;
1393         default:
1394                 rec_io(state, block, offset, true, false);
1395                 break;
1396         }
1397 }
1398
1399 static void rec_SB(struct lightrec_cstate *state,
1400                    const struct block *block, u16 offset)
1401 {
1402         _jit_name(block->_jit, __func__);
1403         rec_store(state, block, offset, jit_code_stxi_c, 0);
1404 }
1405
1406 static void rec_SH(struct lightrec_cstate *state,
1407                    const struct block *block, u16 offset)
1408 {
1409         _jit_name(block->_jit, __func__);
1410         rec_store(state, block, offset,
1411                   jit_code_stxi_s, jit_code_bswapr_us);
1412 }
1413
1414 static void rec_SW(struct lightrec_cstate *state,
1415                    const struct block *block, u16 offset)
1416
1417 {
1418         _jit_name(block->_jit, __func__);
1419         rec_store(state, block, offset,
1420                   jit_code_stxi_i, jit_code_bswapr_ui);
1421 }
1422
1423 static void rec_SWL(struct lightrec_cstate *state,
1424                     const struct block *block, u16 offset)
1425 {
1426         _jit_name(block->_jit, __func__);
1427         rec_io(state, block, offset, true, false);
1428 }
1429
1430 static void rec_SWR(struct lightrec_cstate *state,
1431                     const struct block *block, u16 offset)
1432 {
1433         _jit_name(block->_jit, __func__);
1434         rec_io(state, block, offset, true, false);
1435 }
1436
1437 static void rec_SWC2(struct lightrec_cstate *state,
1438                      const struct block *block, u16 offset)
1439 {
1440         _jit_name(block->_jit, __func__);
1441         rec_io(state, block, offset, false, false);
1442 }
1443
1444 static void rec_load_memory(struct lightrec_cstate *cstate,
1445                             const struct block *block, u16 offset,
1446                             jit_code_t code, jit_code_t swap_code, bool is_unsigned,
1447                             uintptr_t addr_offset, u32 addr_mask)
1448 {
1449         struct regcache *reg_cache = cstate->reg_cache;
1450         struct opcode *op = &block->opcode_list[offset];
1451         jit_state_t *_jit = block->_jit;
1452         u8 rs, rt, addr_reg, flags = REG_EXT;
1453         bool no_mask = op_flag_no_mask(op->flags);
1454         union code c = op->c;
1455         s16 imm;
1456
1457         if (!c.i.rt)
1458                 return;
1459
1460         if (is_unsigned)
1461                 flags |= REG_ZEXT;
1462
1463         rs = lightrec_alloc_reg_in(reg_cache, _jit, c.i.rs, 0);
1464         rt = lightrec_alloc_reg_out(reg_cache, _jit, c.i.rt, flags);
1465
1466         if (!cstate->state->mirrors_mapped && c.i.imm && !no_mask) {
1467                 jit_addi(rt, rs, (s16)c.i.imm);
1468                 addr_reg = rt;
1469                 imm = 0;
1470         } else {
1471                 addr_reg = rs;
1472                 imm = (s16)c.i.imm;
1473         }
1474
1475         if (!no_mask) {
1476                 jit_andi(rt, addr_reg, addr_mask);
1477                 addr_reg = rt;
1478         }
1479
1480         if (addr_offset) {
1481                 jit_addi(rt, addr_reg, addr_offset);
1482                 addr_reg = rt;
1483         }
1484
1485         jit_new_node_www(code, rt, addr_reg, imm);
1486
1487         if (is_big_endian() && swap_code) {
1488                 jit_new_node_ww(swap_code, rt, rt);
1489
1490                 if (c.i.op == OP_LH)
1491                         jit_extr_s(rt, rt);
1492                 else if (c.i.op == OP_LW && __WORDSIZE == 64)
1493                         jit_extr_i(rt, rt);
1494         }
1495
1496         lightrec_free_reg(reg_cache, rs);
1497         lightrec_free_reg(reg_cache, rt);
1498 }
1499
1500 static void rec_load_ram(struct lightrec_cstate *cstate,
1501                          const struct block *block, u16 offset,
1502                          jit_code_t code, jit_code_t swap_code, bool is_unsigned)
1503 {
1504         _jit_note(block->_jit, __FILE__, __LINE__);
1505
1506         rec_load_memory(cstate, block, offset, code, swap_code, is_unsigned,
1507                         cstate->state->offset_ram, rec_ram_mask(cstate->state));
1508 }
1509
1510 static void rec_load_bios(struct lightrec_cstate *cstate,
1511                           const struct block *block, u16 offset,
1512                           jit_code_t code, jit_code_t swap_code, bool is_unsigned)
1513 {
1514         _jit_note(block->_jit, __FILE__, __LINE__);
1515
1516         rec_load_memory(cstate, block, offset, code, swap_code, is_unsigned,
1517                         cstate->state->offset_bios, 0x1fffffff);
1518 }
1519
1520 static void rec_load_scratch(struct lightrec_cstate *cstate,
1521                              const struct block *block, u16 offset,
1522                              jit_code_t code, jit_code_t swap_code, bool is_unsigned)
1523 {
1524         _jit_note(block->_jit, __FILE__, __LINE__);
1525
1526         rec_load_memory(cstate, block, offset, code, swap_code, is_unsigned,
1527                         cstate->state->offset_scratch, 0x1fffffff);
1528 }
1529
1530 static void rec_load_direct(struct lightrec_cstate *cstate,
1531                             const struct block *block, u16 offset,
1532                             jit_code_t code, jit_code_t swap_code,
1533                             bool is_unsigned)
1534 {
1535         struct lightrec_state *state = cstate->state;
1536         struct regcache *reg_cache = cstate->reg_cache;
1537         union code c = block->opcode_list[offset].c;
1538         jit_state_t *_jit = block->_jit;
1539         jit_node_t *to_not_ram, *to_not_bios, *to_end, *to_end2;
1540         u8 tmp, rs, rt, addr_reg, flags = REG_EXT;
1541         s16 imm;
1542
1543         if (!c.i.rt)
1544                 return;
1545
1546         if (is_unsigned)
1547                 flags |= REG_ZEXT;
1548
1549         jit_note(__FILE__, __LINE__);
1550         rs = lightrec_alloc_reg_in(reg_cache, _jit, c.i.rs, 0);
1551         rt = lightrec_alloc_reg_out(reg_cache, _jit, c.i.rt, flags);
1552
1553         if ((state->offset_ram == state->offset_bios &&
1554             state->offset_ram == state->offset_scratch &&
1555             state->mirrors_mapped) || !c.i.imm) {
1556                 addr_reg = rs;
1557                 imm = (s16)c.i.imm;
1558         } else {
1559                 jit_addi(rt, rs, (s16)c.i.imm);
1560                 addr_reg = rt;
1561                 imm = 0;
1562
1563                 if (c.i.rs != c.i.rt)
1564                         lightrec_free_reg(reg_cache, rs);
1565         }
1566
1567         tmp = lightrec_alloc_reg_temp(reg_cache, _jit);
1568
1569         if (state->offset_ram == state->offset_bios &&
1570             state->offset_ram == state->offset_scratch) {
1571                 if (!state->mirrors_mapped) {
1572                         jit_andi(tmp, addr_reg, BIT(28));
1573                         jit_rshi_u(tmp, tmp, 28 - 22);
1574                         jit_ori(tmp, tmp, 0x1f800000 | (RAM_SIZE - 1));
1575                         jit_andr(rt, addr_reg, tmp);
1576                 } else {
1577                         jit_andi(rt, addr_reg, 0x1fffffff);
1578                 }
1579
1580                 if (state->offset_ram)
1581                         jit_movi(tmp, state->offset_ram);
1582         } else {
1583                 to_not_ram = jit_bmsi(addr_reg, BIT(28));
1584
1585                 /* Convert to KUNSEG and avoid RAM mirrors */
1586                 jit_andi(rt, addr_reg, RAM_SIZE - 1);
1587
1588                 if (state->offset_ram)
1589                         jit_movi(tmp, state->offset_ram);
1590
1591                 to_end = jit_b();
1592
1593                 jit_patch(to_not_ram);
1594
1595                 if (state->offset_bios != state->offset_scratch)
1596                         to_not_bios = jit_bmci(addr_reg, BIT(22));
1597
1598                 /* Convert to KUNSEG */
1599                 jit_andi(rt, addr_reg, 0x1fc00000 | (BIOS_SIZE - 1));
1600
1601                 jit_movi(tmp, state->offset_bios);
1602
1603                 if (state->offset_bios != state->offset_scratch) {
1604                         to_end2 = jit_b();
1605
1606                         jit_patch(to_not_bios);
1607
1608                         /* Convert to KUNSEG */
1609                         jit_andi(rt, addr_reg, 0x1f800fff);
1610
1611                         if (state->offset_scratch)
1612                                 jit_movi(tmp, state->offset_scratch);
1613
1614                         jit_patch(to_end2);
1615                 }
1616
1617                 jit_patch(to_end);
1618         }
1619
1620         if (state->offset_ram || state->offset_bios || state->offset_scratch)
1621                 jit_addr(rt, rt, tmp);
1622
1623         jit_new_node_www(code, rt, rt, imm);
1624
1625         if (is_big_endian() && swap_code) {
1626                 jit_new_node_ww(swap_code, rt, rt);
1627
1628                 if (c.i.op == OP_LH)
1629                         jit_extr_s(rt, rt);
1630                 else if (c.i.op == OP_LW && __WORDSIZE == 64)
1631                         jit_extr_i(rt, rt);
1632         }
1633
1634         lightrec_free_reg(reg_cache, addr_reg);
1635         lightrec_free_reg(reg_cache, rt);
1636         lightrec_free_reg(reg_cache, tmp);
1637 }
1638
1639 static void rec_load(struct lightrec_cstate *state, const struct block *block,
1640                      u16 offset, jit_code_t code, jit_code_t swap_code,
1641                      bool is_unsigned)
1642 {
1643         u32 flags = block->opcode_list[offset].flags;
1644
1645         switch (LIGHTREC_FLAGS_GET_IO_MODE(flags)) {
1646         case LIGHTREC_IO_RAM:
1647                 rec_load_ram(state, block, offset, code, swap_code, is_unsigned);
1648                 break;
1649         case LIGHTREC_IO_BIOS:
1650                 rec_load_bios(state, block, offset, code, swap_code, is_unsigned);
1651                 break;
1652         case LIGHTREC_IO_SCRATCH:
1653                 rec_load_scratch(state, block, offset, code, swap_code, is_unsigned);
1654                 break;
1655         case LIGHTREC_IO_DIRECT:
1656                 rec_load_direct(state, block, offset, code, swap_code, is_unsigned);
1657                 break;
1658         default:
1659                 rec_io(state, block, offset, false, true);
1660                 break;
1661         }
1662 }
1663
1664 static void rec_LB(struct lightrec_cstate *state, const struct block *block, u16 offset)
1665 {
1666         _jit_name(block->_jit, __func__);
1667         rec_load(state, block, offset, jit_code_ldxi_c, 0, false);
1668 }
1669
1670 static void rec_LBU(struct lightrec_cstate *state, const struct block *block, u16 offset)
1671 {
1672         _jit_name(block->_jit, __func__);
1673         rec_load(state, block, offset, jit_code_ldxi_uc, 0, true);
1674 }
1675
1676 static void rec_LH(struct lightrec_cstate *state, const struct block *block, u16 offset)
1677 {
1678         _jit_name(block->_jit, __func__);
1679         rec_load(state, block, offset, jit_code_ldxi_s, jit_code_bswapr_us, false);
1680 }
1681
1682 static void rec_LHU(struct lightrec_cstate *state, const struct block *block, u16 offset)
1683 {
1684         _jit_name(block->_jit, __func__);
1685         rec_load(state, block, offset, jit_code_ldxi_us, jit_code_bswapr_us, true);
1686 }
1687
1688 static void rec_LWL(struct lightrec_cstate *state, const struct block *block, u16 offset)
1689 {
1690         _jit_name(block->_jit, __func__);
1691         rec_io(state, block, offset, true, true);
1692 }
1693
1694 static void rec_LWR(struct lightrec_cstate *state, const struct block *block, u16 offset)
1695 {
1696         _jit_name(block->_jit, __func__);
1697         rec_io(state, block, offset, true, true);
1698 }
1699
1700 static void rec_LW(struct lightrec_cstate *state, const struct block *block, u16 offset)
1701 {
1702         _jit_name(block->_jit, __func__);
1703         rec_load(state, block, offset, jit_code_ldxi_i, jit_code_bswapr_ui, false);
1704 }
1705
1706 static void rec_LWC2(struct lightrec_cstate *state, const struct block *block, u16 offset)
1707 {
1708         _jit_name(block->_jit, __func__);
1709         rec_io(state, block, offset, false, false);
1710 }
1711
1712 static void rec_break_syscall(struct lightrec_cstate *state,
1713                               const struct block *block, u16 offset, bool is_break)
1714 {
1715         _jit_note(block->_jit, __FILE__, __LINE__);
1716
1717         if (is_break)
1718                 call_to_c_wrapper(state, block, 0, false, C_WRAPPER_BREAK);
1719         else
1720                 call_to_c_wrapper(state, block, 0, false, C_WRAPPER_SYSCALL);
1721
1722         /* TODO: the return address should be "pc - 4" if we're a delay slot */
1723         lightrec_emit_end_of_block(state, block, offset, -1,
1724                                    get_ds_pc(block, offset, 0),
1725                                    31, 0, true);
1726 }
1727
1728 static void rec_special_SYSCALL(struct lightrec_cstate *state,
1729                                 const struct block *block, u16 offset)
1730 {
1731         _jit_name(block->_jit, __func__);
1732         rec_break_syscall(state, block, offset, false);
1733 }
1734
1735 static void rec_special_BREAK(struct lightrec_cstate *state,
1736                               const struct block *block, u16 offset)
1737 {
1738         _jit_name(block->_jit, __func__);
1739         rec_break_syscall(state, block, offset, true);
1740 }
1741
1742 static void rec_mtc(struct lightrec_cstate *state, const struct block *block, u16 offset)
1743 {
1744         struct regcache *reg_cache = state->reg_cache;
1745         union code c = block->opcode_list[offset].c;
1746         jit_state_t *_jit = block->_jit;
1747
1748         jit_note(__FILE__, __LINE__);
1749         lightrec_clean_reg_if_loaded(reg_cache, _jit, c.i.rs, false);
1750         lightrec_clean_reg_if_loaded(reg_cache, _jit, c.i.rt, false);
1751
1752         call_to_c_wrapper(state, block, c.opcode, true, C_WRAPPER_MTC);
1753
1754         if (c.i.op == OP_CP0 &&
1755             !op_flag_no_ds(block->opcode_list[offset].flags) &&
1756             (c.r.rd == 12 || c.r.rd == 13))
1757                 lightrec_emit_end_of_block(state, block, offset, -1,
1758                                            get_ds_pc(block, offset, 1),
1759                                            0, 0, true);
1760 }
1761
1762 static void
1763 rec_mfc0(struct lightrec_cstate *state, const struct block *block, u16 offset)
1764 {
1765         struct regcache *reg_cache = state->reg_cache;
1766         union code c = block->opcode_list[offset].c;
1767         jit_state_t *_jit = block->_jit;
1768         u8 rt;
1769
1770         jit_note(__FILE__, __LINE__);
1771
1772         rt = lightrec_alloc_reg_out(reg_cache, _jit, c.i.rt, REG_EXT);
1773
1774         jit_ldxi_i(rt, LIGHTREC_REG_STATE,
1775                    offsetof(struct lightrec_state, regs.cp0[c.r.rd]));
1776
1777         lightrec_free_reg(reg_cache, rt);
1778 }
1779
1780 static bool block_in_bios(const struct lightrec_cstate *state,
1781                           const struct block *block)
1782 {
1783         const struct lightrec_mem_map *bios = &state->state->maps[PSX_MAP_BIOS];
1784         u32 pc = kunseg(block->pc);
1785
1786         return pc >= bios->pc && pc < bios->pc + bios->length;
1787 }
1788
1789 static void
1790 rec_mtc0(struct lightrec_cstate *state, const struct block *block, u16 offset)
1791 {
1792         struct regcache *reg_cache = state->reg_cache;
1793         const union code c = block->opcode_list[offset].c;
1794         jit_state_t *_jit = block->_jit;
1795         u8 rt, tmp = 0, tmp2, status;
1796
1797         jit_note(__FILE__, __LINE__);
1798
1799         switch(c.r.rd) {
1800         case 1:
1801         case 4:
1802         case 8:
1803         case 14:
1804         case 15:
1805                 /* Those registers are read-only */
1806                 return;
1807         default:
1808                 break;
1809         }
1810
1811         if (block_in_bios(state, block) && c.r.rd == 12) {
1812                 /* If we are running code from the BIOS, handle writes to the
1813                  * Status register in C. BIOS code may toggle bit 16 which will
1814                  * map/unmap the RAM, while game code cannot do that. */
1815                 rec_mtc(state, block, offset);
1816                 return;
1817         }
1818
1819         rt = lightrec_alloc_reg_in(reg_cache, _jit, c.i.rt, 0);
1820
1821         if (c.r.rd != 13) {
1822                 jit_stxi_i(offsetof(struct lightrec_state, regs.cp0[c.r.rd]),
1823                            LIGHTREC_REG_STATE, rt);
1824         }
1825
1826         if (c.r.rd == 12 || c.r.rd == 13) {
1827                 tmp = lightrec_alloc_reg_temp(reg_cache, _jit);
1828                 jit_ldxi_i(tmp, LIGHTREC_REG_STATE,
1829                            offsetof(struct lightrec_state, regs.cp0[13]));
1830
1831                 tmp2 = lightrec_alloc_reg_temp(reg_cache, _jit);
1832         }
1833
1834         if (c.r.rd == 12) {
1835                 status = rt;
1836         } else if (c.r.rd == 13) {
1837                 /* Cause = (Cause & ~0x0300) | (value & 0x0300) */
1838                 jit_andi(tmp2, rt, 0x0300);
1839                 jit_ori(tmp, tmp, 0x0300);
1840                 jit_xori(tmp, tmp, 0x0300);
1841                 jit_orr(tmp, tmp, tmp2);
1842                 jit_ldxi_i(tmp2, LIGHTREC_REG_STATE,
1843                            offsetof(struct lightrec_state, regs.cp0[12]));
1844                 jit_stxi_i(offsetof(struct lightrec_state, regs.cp0[13]),
1845                            LIGHTREC_REG_STATE, tmp);
1846                 status = tmp2;
1847         }
1848
1849         if (c.r.rd == 12 || c.r.rd == 13) {
1850                 /* Exit dynarec in case there's a software interrupt.
1851                  * exit_flags = !!(status & tmp & 0x0300) & status; */
1852                 jit_andr(tmp, tmp, status);
1853                 jit_andi(tmp, tmp, 0x0300);
1854                 jit_nei(tmp, tmp, 0);
1855                 jit_andr(tmp, tmp, status);
1856         }
1857
1858         if (c.r.rd == 12) {
1859                 /* Exit dynarec in case we unmask a hardware interrupt.
1860                  * exit_flags = !(~status & 0x401) */
1861
1862                 jit_comr(tmp2, status);
1863                 jit_andi(tmp2, tmp2, 0x401);
1864                 jit_eqi(tmp2, tmp2, 0);
1865                 jit_orr(tmp, tmp, tmp2);
1866         }
1867
1868         if (c.r.rd == 12 || c.r.rd == 13) {
1869                 jit_stxi_i(offsetof(struct lightrec_state, exit_flags),
1870                            LIGHTREC_REG_STATE, tmp);
1871
1872                 lightrec_free_reg(reg_cache, tmp);
1873                 lightrec_free_reg(reg_cache, tmp2);
1874         }
1875
1876         lightrec_free_reg(reg_cache, rt);
1877
1878         if (!op_flag_no_ds(block->opcode_list[offset].flags) &&
1879             (c.r.rd == 12 || c.r.rd == 13))
1880                 lightrec_emit_eob(state, block, offset + 1, true);
1881 }
1882
1883 static void rec_cp0_MFC0(struct lightrec_cstate *state,
1884                          const struct block *block, u16 offset)
1885 {
1886         _jit_name(block->_jit, __func__);
1887         rec_mfc0(state, block, offset);
1888 }
1889
1890 static void rec_cp0_CFC0(struct lightrec_cstate *state,
1891                          const struct block *block, u16 offset)
1892 {
1893         _jit_name(block->_jit, __func__);
1894         rec_mfc0(state, block, offset);
1895 }
1896
1897 static void rec_cp0_MTC0(struct lightrec_cstate *state,
1898                          const struct block *block, u16 offset)
1899 {
1900         _jit_name(block->_jit, __func__);
1901         rec_mtc0(state, block, offset);
1902 }
1903
1904 static void rec_cp0_CTC0(struct lightrec_cstate *state,
1905                          const struct block *block, u16 offset)
1906 {
1907         _jit_name(block->_jit, __func__);
1908         rec_mtc0(state, block, offset);
1909 }
1910
1911 static unsigned int cp2d_i_offset(u8 reg)
1912 {
1913         return offsetof(struct lightrec_state, regs.cp2d[reg]);
1914 }
1915
1916 static unsigned int cp2d_s_offset(u8 reg)
1917 {
1918         return cp2d_i_offset(reg) + is_big_endian() * 2;
1919 }
1920
1921 static unsigned int cp2c_i_offset(u8 reg)
1922 {
1923         return offsetof(struct lightrec_state, regs.cp2c[reg]);
1924 }
1925
1926 static unsigned int cp2c_s_offset(u8 reg)
1927 {
1928         return cp2c_i_offset(reg) + is_big_endian() * 2;
1929 }
1930
1931 static void rec_cp2_basic_MFC2(struct lightrec_cstate *state,
1932                                const struct block *block, u16 offset)
1933 {
1934         struct regcache *reg_cache = state->reg_cache;
1935         const union code c = block->opcode_list[offset].c;
1936         jit_state_t *_jit = block->_jit;
1937         const u32 zext_regs = 0x300f0080;
1938         u8 rt, tmp, tmp2, tmp3, out, flags;
1939         u8 reg = c.r.rd == 15 ? 14 : c.r.rd;
1940         unsigned int i;
1941
1942         _jit_name(block->_jit, __func__);
1943
1944         flags = (zext_regs & BIT(reg)) ? REG_ZEXT : REG_EXT;
1945         rt = lightrec_alloc_reg_out(reg_cache, _jit, c.r.rt, flags);
1946
1947         switch (reg) {
1948         case 1:
1949         case 3:
1950         case 5:
1951         case 8:
1952         case 9:
1953         case 10:
1954         case 11:
1955                 jit_ldxi_s(rt, LIGHTREC_REG_STATE, cp2d_s_offset(reg));
1956                 break;
1957         case 7:
1958         case 16:
1959         case 17:
1960         case 18:
1961         case 19:
1962                 jit_ldxi_us(rt, LIGHTREC_REG_STATE, cp2d_s_offset(reg));
1963                 break;
1964         case 28:
1965         case 29:
1966                 tmp = lightrec_alloc_reg_temp(reg_cache, _jit);
1967                 tmp2 = lightrec_alloc_reg_temp(reg_cache, _jit);
1968                 tmp3 = lightrec_alloc_reg_temp(reg_cache, _jit);
1969
1970                 for (i = 0; i < 3; i++) {
1971                         out = i == 0 ? rt : tmp;
1972
1973                         jit_ldxi_s(tmp, LIGHTREC_REG_STATE, cp2d_s_offset(9 + i));
1974                         jit_movi(tmp2, 0x1f);
1975                         jit_rshi(out, tmp, 7);
1976
1977                         jit_ltr(tmp3, tmp2, out);
1978                         jit_movnr(out, tmp2, tmp3);
1979
1980                         jit_gei(tmp2, out, 0);
1981                         jit_movzr(out, tmp2, tmp2);
1982
1983                         if (i > 0) {
1984                                 jit_lshi(tmp, tmp, 5 * i);
1985                                 jit_orr(rt, rt, tmp);
1986                         }
1987                 }
1988
1989
1990                 lightrec_free_reg(reg_cache, tmp);
1991                 lightrec_free_reg(reg_cache, tmp2);
1992                 lightrec_free_reg(reg_cache, tmp3);
1993                 break;
1994         default:
1995                 jit_ldxi_i(rt, LIGHTREC_REG_STATE, cp2d_i_offset(reg));
1996                 break;
1997         }
1998
1999         lightrec_free_reg(reg_cache, rt);
2000 }
2001
2002 static void rec_cp2_basic_CFC2(struct lightrec_cstate *state,
2003                                const struct block *block, u16 offset)
2004 {
2005         struct regcache *reg_cache = state->reg_cache;
2006         const union code c = block->opcode_list[offset].c;
2007         jit_state_t *_jit = block->_jit;
2008         u8 rt;
2009
2010         _jit_name(block->_jit, __func__);
2011
2012         switch (c.r.rd) {
2013         case 4:
2014         case 12:
2015         case 20:
2016         case 26:
2017         case 27:
2018         case 29:
2019         case 30:
2020                 rt = lightrec_alloc_reg_out(reg_cache, _jit, c.r.rt, REG_EXT);
2021                 jit_ldxi_s(rt, LIGHTREC_REG_STATE, cp2c_s_offset(c.r.rd));
2022                 break;
2023         default:
2024                 rt = lightrec_alloc_reg_out(reg_cache, _jit, c.r.rt, REG_ZEXT);
2025                 jit_ldxi_i(rt, LIGHTREC_REG_STATE, cp2c_i_offset(c.r.rd));
2026                 break;
2027         }
2028
2029         lightrec_free_reg(reg_cache, rt);
2030 }
2031
2032 static void rec_cp2_basic_MTC2(struct lightrec_cstate *state,
2033                                const struct block *block, u16 offset)
2034 {
2035         struct regcache *reg_cache = state->reg_cache;
2036         const union code c = block->opcode_list[offset].c;
2037         jit_state_t *_jit = block->_jit;
2038         jit_node_t *loop, *to_loop;
2039         u8 rt, tmp, tmp2, flags = 0;
2040
2041         _jit_name(block->_jit, __func__);
2042
2043         if (c.r.rd == 31)
2044                 return;
2045
2046         if (c.r.rd == 30)
2047                 flags |= REG_EXT;
2048
2049         rt = lightrec_alloc_reg_in(reg_cache, _jit, c.r.rt, flags);
2050
2051         switch (c.r.rd) {
2052         case 15:
2053                 tmp = lightrec_alloc_reg_temp(reg_cache, _jit);
2054                 jit_ldxi_i(tmp, LIGHTREC_REG_STATE, cp2d_i_offset(13));
2055
2056                 tmp2 = lightrec_alloc_reg_temp(reg_cache, _jit);
2057                 jit_ldxi_i(tmp2, LIGHTREC_REG_STATE, cp2d_i_offset(14));
2058
2059                 jit_stxi_i(cp2d_i_offset(12), LIGHTREC_REG_STATE, tmp);
2060                 jit_stxi_i(cp2d_i_offset(13), LIGHTREC_REG_STATE, tmp2);
2061                 jit_stxi_i(cp2d_i_offset(14), LIGHTREC_REG_STATE, rt);
2062
2063                 lightrec_free_reg(reg_cache, tmp);
2064                 lightrec_free_reg(reg_cache, tmp2);
2065                 break;
2066         case 28:
2067                 tmp = lightrec_alloc_reg_temp(reg_cache, _jit);
2068
2069                 jit_lshi(tmp, rt, 7);
2070                 jit_andi(tmp, tmp, 0xf80);
2071                 jit_stxi_s(cp2d_s_offset(9), LIGHTREC_REG_STATE, tmp);
2072
2073                 jit_lshi(tmp, rt, 2);
2074                 jit_andi(tmp, tmp, 0xf80);
2075                 jit_stxi_s(cp2d_s_offset(10), LIGHTREC_REG_STATE, tmp);
2076
2077                 jit_rshi(tmp, rt, 3);
2078                 jit_andi(tmp, tmp, 0xf80);
2079                 jit_stxi_s(cp2d_s_offset(11), LIGHTREC_REG_STATE, tmp);
2080
2081                 lightrec_free_reg(reg_cache, tmp);
2082                 break;
2083         case 30:
2084                 tmp = lightrec_alloc_reg_temp(reg_cache, _jit);
2085                 tmp2 = lightrec_alloc_reg_temp(reg_cache, _jit);
2086
2087                 /* if (rt < 0) rt = ~rt; */
2088                 jit_rshi(tmp, rt, 31);
2089                 jit_xorr(tmp, rt, tmp);
2090
2091                 /* We know the sign bit is 0. Left-shift by 1 to start the algorithm */
2092                 jit_lshi(tmp, tmp, 1);
2093                 jit_movi(tmp2, 33);
2094
2095                 /* Decrement tmp2 and right-shift the value by 1 until it equals zero */
2096                 loop = jit_label();
2097                 jit_subi(tmp2, tmp2, 1);
2098                 jit_rshi_u(tmp, tmp, 1);
2099                 to_loop = jit_bnei(tmp, 0);
2100
2101                 jit_patch_at(to_loop, loop);
2102
2103                 jit_stxi_i(cp2d_i_offset(31), LIGHTREC_REG_STATE, tmp2);
2104                 jit_stxi_i(cp2d_i_offset(30), LIGHTREC_REG_STATE, rt);
2105
2106                 lightrec_free_reg(reg_cache, tmp);
2107                 lightrec_free_reg(reg_cache, tmp2);
2108                 break;
2109         default:
2110                 jit_stxi_i(cp2d_i_offset(c.r.rd), LIGHTREC_REG_STATE, rt);
2111                 break;
2112         }
2113
2114         lightrec_free_reg(reg_cache, rt);
2115 }
2116
2117 static void rec_cp2_basic_CTC2(struct lightrec_cstate *state,
2118                                const struct block *block, u16 offset)
2119 {
2120         struct regcache *reg_cache = state->reg_cache;
2121         const union code c = block->opcode_list[offset].c;
2122         jit_state_t *_jit = block->_jit;
2123         u8 rt, tmp, tmp2;
2124
2125         _jit_name(block->_jit, __func__);
2126
2127         rt = lightrec_alloc_reg_in(reg_cache, _jit, c.r.rt, 0);
2128
2129         switch (c.r.rd) {
2130         case 4:
2131         case 12:
2132         case 20:
2133         case 26:
2134         case 27:
2135         case 29:
2136         case 30:
2137                 jit_stxi_s(cp2c_s_offset(c.r.rd), LIGHTREC_REG_STATE, rt);
2138                 break;
2139         case 31:
2140                 tmp = lightrec_alloc_reg_temp(reg_cache, _jit);
2141                 tmp2 = lightrec_alloc_reg_temp(reg_cache, _jit);
2142
2143                 jit_andi(tmp, rt, 0x7f87e000);
2144                 jit_nei(tmp, tmp, 0);
2145                 jit_lshi(tmp, tmp, 31);
2146
2147                 jit_andi(tmp2, rt, 0x7ffff000);
2148                 jit_orr(tmp, tmp2, tmp);
2149
2150                 jit_stxi_i(cp2c_i_offset(31), LIGHTREC_REG_STATE, tmp);
2151
2152                 lightrec_free_reg(reg_cache, tmp);
2153                 lightrec_free_reg(reg_cache, tmp2);
2154                 break;
2155
2156         default:
2157                 jit_stxi_i(cp2c_i_offset(c.r.rd), LIGHTREC_REG_STATE, rt);
2158         }
2159
2160         lightrec_free_reg(reg_cache, rt);
2161 }
2162
2163 static void rec_cp0_RFE(struct lightrec_cstate *state,
2164                         const struct block *block, u16 offset)
2165 {
2166         struct regcache *reg_cache = state->reg_cache;
2167         jit_state_t *_jit = block->_jit;
2168         u8 status, tmp;
2169
2170         jit_name(__func__);
2171         jit_note(__FILE__, __LINE__);
2172
2173         status = lightrec_alloc_reg_temp(reg_cache, _jit);
2174         jit_ldxi_i(status, LIGHTREC_REG_STATE,
2175                    offsetof(struct lightrec_state, regs.cp0[12]));
2176
2177         tmp = lightrec_alloc_reg_temp(reg_cache, _jit);
2178
2179         /* status = ((status >> 2) & 0xf) | status & ~0xf; */
2180         jit_rshi(tmp, status, 2);
2181         jit_andi(tmp, tmp, 0xf);
2182         jit_andi(status, status, ~0xful);
2183         jit_orr(status, status, tmp);
2184
2185         jit_ldxi_i(tmp, LIGHTREC_REG_STATE,
2186                    offsetof(struct lightrec_state, regs.cp0[13]));
2187         jit_stxi_i(offsetof(struct lightrec_state, regs.cp0[12]),
2188                    LIGHTREC_REG_STATE, status);
2189
2190         /* Exit dynarec in case there's a software interrupt.
2191          * exit_flags = !!(status & cause & 0x0300) & status; */
2192         jit_andr(tmp, tmp, status);
2193         jit_andi(tmp, tmp, 0x0300);
2194         jit_nei(tmp, tmp, 0);
2195         jit_andr(tmp, tmp, status);
2196         jit_stxi_i(offsetof(struct lightrec_state, exit_flags),
2197                    LIGHTREC_REG_STATE, tmp);
2198
2199         lightrec_free_reg(reg_cache, status);
2200         lightrec_free_reg(reg_cache, tmp);
2201 }
2202
2203 static void rec_CP(struct lightrec_cstate *state,
2204                    const struct block *block, u16 offset)
2205 {
2206         union code c = block->opcode_list[offset].c;
2207         jit_state_t *_jit = block->_jit;
2208
2209         jit_name(__func__);
2210         jit_note(__FILE__, __LINE__);
2211
2212         call_to_c_wrapper(state, block, c.opcode, true, C_WRAPPER_CP);
2213 }
2214
2215 static void rec_meta_MOV(struct lightrec_cstate *state,
2216                          const struct block *block, u16 offset)
2217 {
2218         struct regcache *reg_cache = state->reg_cache;
2219         union code c = block->opcode_list[offset].c;
2220         jit_state_t *_jit = block->_jit;
2221         u8 rs, rd;
2222
2223         _jit_name(block->_jit, __func__);
2224         jit_note(__FILE__, __LINE__);
2225         if (c.r.rs)
2226                 rs = lightrec_alloc_reg_in(reg_cache, _jit, c.r.rs, 0);
2227         rd = lightrec_alloc_reg_out(reg_cache, _jit, c.r.rd, REG_EXT);
2228
2229         if (c.r.rs == 0)
2230                 jit_movi(rd, 0);
2231         else
2232                 jit_extr_i(rd, rs);
2233
2234         if (c.r.rs)
2235                 lightrec_free_reg(reg_cache, rs);
2236         lightrec_free_reg(reg_cache, rd);
2237 }
2238
2239 static void rec_meta_EXTC_EXTS(struct lightrec_cstate *state,
2240                                const struct block *block,
2241                                u16 offset)
2242 {
2243         struct regcache *reg_cache = state->reg_cache;
2244         union code c = block->opcode_list[offset].c;
2245         jit_state_t *_jit = block->_jit;
2246         u8 rs, rt;
2247
2248         _jit_name(block->_jit, __func__);
2249         jit_note(__FILE__, __LINE__);
2250
2251         rs = lightrec_alloc_reg_in(reg_cache, _jit, c.i.rs, 0);
2252         rt = lightrec_alloc_reg_out(reg_cache, _jit, c.i.rt, REG_EXT);
2253
2254         if (c.i.op == OP_META_EXTC)
2255                 jit_extr_c(rt, rs);
2256         else
2257                 jit_extr_s(rt, rs);
2258
2259         lightrec_free_reg(reg_cache, rs);
2260         lightrec_free_reg(reg_cache, rt);
2261 }
2262
2263 static const lightrec_rec_func_t rec_standard[64] = {
2264         SET_DEFAULT_ELM(rec_standard, unknown_opcode),
2265         [OP_SPECIAL]            = rec_SPECIAL,
2266         [OP_REGIMM]             = rec_REGIMM,
2267         [OP_J]                  = rec_J,
2268         [OP_JAL]                = rec_JAL,
2269         [OP_BEQ]                = rec_BEQ,
2270         [OP_BNE]                = rec_BNE,
2271         [OP_BLEZ]               = rec_BLEZ,
2272         [OP_BGTZ]               = rec_BGTZ,
2273         [OP_ADDI]               = rec_ADDI,
2274         [OP_ADDIU]              = rec_ADDIU,
2275         [OP_SLTI]               = rec_SLTI,
2276         [OP_SLTIU]              = rec_SLTIU,
2277         [OP_ANDI]               = rec_ANDI,
2278         [OP_ORI]                = rec_ORI,
2279         [OP_XORI]               = rec_XORI,
2280         [OP_LUI]                = rec_LUI,
2281         [OP_CP0]                = rec_CP0,
2282         [OP_CP2]                = rec_CP2,
2283         [OP_LB]                 = rec_LB,
2284         [OP_LH]                 = rec_LH,
2285         [OP_LWL]                = rec_LWL,
2286         [OP_LW]                 = rec_LW,
2287         [OP_LBU]                = rec_LBU,
2288         [OP_LHU]                = rec_LHU,
2289         [OP_LWR]                = rec_LWR,
2290         [OP_SB]                 = rec_SB,
2291         [OP_SH]                 = rec_SH,
2292         [OP_SWL]                = rec_SWL,
2293         [OP_SW]                 = rec_SW,
2294         [OP_SWR]                = rec_SWR,
2295         [OP_LWC2]               = rec_LWC2,
2296         [OP_SWC2]               = rec_SWC2,
2297
2298         [OP_META_MOV]           = rec_meta_MOV,
2299         [OP_META_EXTC]          = rec_meta_EXTC_EXTS,
2300         [OP_META_EXTS]          = rec_meta_EXTC_EXTS,
2301 };
2302
2303 static const lightrec_rec_func_t rec_special[64] = {
2304         SET_DEFAULT_ELM(rec_special, unknown_opcode),
2305         [OP_SPECIAL_SLL]        = rec_special_SLL,
2306         [OP_SPECIAL_SRL]        = rec_special_SRL,
2307         [OP_SPECIAL_SRA]        = rec_special_SRA,
2308         [OP_SPECIAL_SLLV]       = rec_special_SLLV,
2309         [OP_SPECIAL_SRLV]       = rec_special_SRLV,
2310         [OP_SPECIAL_SRAV]       = rec_special_SRAV,
2311         [OP_SPECIAL_JR]         = rec_special_JR,
2312         [OP_SPECIAL_JALR]       = rec_special_JALR,
2313         [OP_SPECIAL_SYSCALL]    = rec_special_SYSCALL,
2314         [OP_SPECIAL_BREAK]      = rec_special_BREAK,
2315         [OP_SPECIAL_MFHI]       = rec_special_MFHI,
2316         [OP_SPECIAL_MTHI]       = rec_special_MTHI,
2317         [OP_SPECIAL_MFLO]       = rec_special_MFLO,
2318         [OP_SPECIAL_MTLO]       = rec_special_MTLO,
2319         [OP_SPECIAL_MULT]       = rec_special_MULT,
2320         [OP_SPECIAL_MULTU]      = rec_special_MULTU,
2321         [OP_SPECIAL_DIV]        = rec_special_DIV,
2322         [OP_SPECIAL_DIVU]       = rec_special_DIVU,
2323         [OP_SPECIAL_ADD]        = rec_special_ADD,
2324         [OP_SPECIAL_ADDU]       = rec_special_ADDU,
2325         [OP_SPECIAL_SUB]        = rec_special_SUB,
2326         [OP_SPECIAL_SUBU]       = rec_special_SUBU,
2327         [OP_SPECIAL_AND]        = rec_special_AND,
2328         [OP_SPECIAL_OR]         = rec_special_OR,
2329         [OP_SPECIAL_XOR]        = rec_special_XOR,
2330         [OP_SPECIAL_NOR]        = rec_special_NOR,
2331         [OP_SPECIAL_SLT]        = rec_special_SLT,
2332         [OP_SPECIAL_SLTU]       = rec_special_SLTU,
2333 };
2334
2335 static const lightrec_rec_func_t rec_regimm[64] = {
2336         SET_DEFAULT_ELM(rec_regimm, unknown_opcode),
2337         [OP_REGIMM_BLTZ]        = rec_regimm_BLTZ,
2338         [OP_REGIMM_BGEZ]        = rec_regimm_BGEZ,
2339         [OP_REGIMM_BLTZAL]      = rec_regimm_BLTZAL,
2340         [OP_REGIMM_BGEZAL]      = rec_regimm_BGEZAL,
2341 };
2342
2343 static const lightrec_rec_func_t rec_cp0[64] = {
2344         SET_DEFAULT_ELM(rec_cp0, rec_CP),
2345         [OP_CP0_MFC0]           = rec_cp0_MFC0,
2346         [OP_CP0_CFC0]           = rec_cp0_CFC0,
2347         [OP_CP0_MTC0]           = rec_cp0_MTC0,
2348         [OP_CP0_CTC0]           = rec_cp0_CTC0,
2349         [OP_CP0_RFE]            = rec_cp0_RFE,
2350 };
2351
2352 static const lightrec_rec_func_t rec_cp2_basic[64] = {
2353         SET_DEFAULT_ELM(rec_cp2_basic, rec_CP),
2354         [OP_CP2_BASIC_MFC2]     = rec_cp2_basic_MFC2,
2355         [OP_CP2_BASIC_CFC2]     = rec_cp2_basic_CFC2,
2356         [OP_CP2_BASIC_MTC2]     = rec_cp2_basic_MTC2,
2357         [OP_CP2_BASIC_CTC2]     = rec_cp2_basic_CTC2,
2358 };
2359
2360 static void rec_SPECIAL(struct lightrec_cstate *state,
2361                         const struct block *block, u16 offset)
2362 {
2363         union code c = block->opcode_list[offset].c;
2364         lightrec_rec_func_t f = rec_special[c.r.op];
2365
2366         if (!HAS_DEFAULT_ELM && unlikely(!f))
2367                 unknown_opcode(state, block, offset);
2368         else
2369                 (*f)(state, block, offset);
2370 }
2371
2372 static void rec_REGIMM(struct lightrec_cstate *state,
2373                        const struct block *block, u16 offset)
2374 {
2375         union code c = block->opcode_list[offset].c;
2376         lightrec_rec_func_t f = rec_regimm[c.r.rt];
2377
2378         if (!HAS_DEFAULT_ELM && unlikely(!f))
2379                 unknown_opcode(state, block, offset);
2380         else
2381                 (*f)(state, block, offset);
2382 }
2383
2384 static void rec_CP0(struct lightrec_cstate *state,
2385                     const struct block *block, u16 offset)
2386 {
2387         union code c = block->opcode_list[offset].c;
2388         lightrec_rec_func_t f = rec_cp0[c.r.rs];
2389
2390         if (!HAS_DEFAULT_ELM && unlikely(!f))
2391                 rec_CP(state, block, offset);
2392         else
2393                 (*f)(state, block, offset);
2394 }
2395
2396 static void rec_CP2(struct lightrec_cstate *state,
2397                     const struct block *block, u16 offset)
2398 {
2399         union code c = block->opcode_list[offset].c;
2400
2401         if (c.r.op == OP_CP2_BASIC) {
2402                 lightrec_rec_func_t f = rec_cp2_basic[c.r.rs];
2403
2404                 if (HAS_DEFAULT_ELM || likely(f)) {
2405                         (*f)(state, block, offset);
2406                         return;
2407                 }
2408         }
2409
2410         rec_CP(state, block, offset);
2411 }
2412
2413 void lightrec_rec_opcode(struct lightrec_cstate *state,
2414                          const struct block *block, u16 offset)
2415 {
2416         struct regcache *reg_cache = state->reg_cache;
2417         struct lightrec_branch_target *target;
2418         const struct opcode *op = &block->opcode_list[offset];
2419         jit_state_t *_jit = block->_jit;
2420         lightrec_rec_func_t f;
2421         u16 unload_offset;
2422
2423         if (op_flag_sync(op->flags)) {
2424                 if (state->cycles)
2425                         jit_subi(LIGHTREC_REG_CYCLE, LIGHTREC_REG_CYCLE, state->cycles);
2426                 state->cycles = 0;
2427
2428                 lightrec_storeback_regs(reg_cache, _jit);
2429                 lightrec_regcache_reset(reg_cache);
2430
2431                 pr_debug("Adding branch target at offset 0x%x\n", offset << 2);
2432                 target = &state->targets[state->nb_targets++];
2433                 target->offset = offset;
2434                 target->label = jit_indirect();
2435         }
2436
2437         if (likely(op->opcode)) {
2438                 f = rec_standard[op->i.op];
2439
2440                 if (!HAS_DEFAULT_ELM && unlikely(!f))
2441                         unknown_opcode(state, block, offset);
2442                 else
2443                         (*f)(state, block, offset);
2444         }
2445
2446         if (OPT_EARLY_UNLOAD) {
2447                 unload_offset = offset +
2448                         (has_delay_slot(op->c) && !op_flag_no_ds(op->flags));
2449
2450                 lightrec_do_early_unload(state, block, unload_offset);
2451         }
2452 }